drivers/mtd/nand/qcom_nandc.c

   1 /*
   2  * Copyright (c) 2016, The Linux Foundation. All rights reserved.
   3  *
   4  * This software is licensed under the terms of the GNU General Public
   5  * License version 2, as published by the Free Software Foundation, and
   6  * may be copied, distributed, and modified under those terms.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  */
  13
  14 #include <linux/clk.h>
  15 #include <linux/slab.h>
  16 #include <linux/bitops.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/dmaengine.h>
  19 #include <linux/module.h>
  20 #include <linux/mtd/rawnand.h>
  21 #include <linux/mtd/partitions.h>
  22 #include <linux/of.h>
  23 #include <linux/of_device.h>
  24 #include <linux/delay.h>
  25
  26 /* NANDc reg offsets */
  27 #define NAND_FLASH_CMD                  0x00
  28 #define NAND_ADDR0                      0x04
  29 #define NAND_ADDR1                      0x08
  30 #define NAND_FLASH_CHIP_SELECT          0x0c
  31 #define NAND_EXEC_CMD                   0x10
  32 #define NAND_FLASH_STATUS               0x14
  33 #define NAND_BUFFER_STATUS              0x18
  34 #define NAND_DEV0_CFG0                  0x20
  35 #define NAND_DEV0_CFG1                  0x24
  36 #define NAND_DEV0_ECC_CFG               0x28
  37 #define NAND_DEV1_ECC_CFG               0x2c
  38 #define NAND_DEV1_CFG0                  0x30
  39 #define NAND_DEV1_CFG1                  0x34
  40 #define NAND_READ_ID                    0x40
  41 #define NAND_READ_STATUS                0x44
  42 #define NAND_DEV_CMD0                   0xa0
  43 #define NAND_DEV_CMD1                   0xa4
  44 #define NAND_DEV_CMD2                   0xa8
  45 #define NAND_DEV_CMD_VLD                0xac
  46 #define SFLASHC_BURST_CFG               0xe0
  47 #define NAND_ERASED_CW_DETECT_CFG       0xe8
  48 #define NAND_ERASED_CW_DETECT_STATUS    0xec
  49 #define NAND_EBI2_ECC_BUF_CFG           0xf0
  50 #define FLASH_BUF_ACC                   0x100
  51
  52 #define NAND_CTRL                       0xf00
  53 #define NAND_VERSION                    0xf08
  54 #define NAND_READ_LOCATION_0            0xf20
  55 #define NAND_READ_LOCATION_1            0xf24
  56 #define NAND_READ_LOCATION_2            0xf28
  57 #define NAND_READ_LOCATION_3            0xf2c
  58
  59 /* dummy register offsets, used by write_reg_dma */
  60 #define NAND_DEV_CMD1_RESTORE           0xdead
  61 #define NAND_DEV_CMD_VLD_RESTORE        0xbeef
  62
  63 /* NAND_FLASH_CMD bits */
  64 #define PAGE_ACC                        BIT(4)
  65 #define LAST_PAGE                       BIT(5)
  66
  67 /* NAND_FLASH_CHIP_SELECT bits */
  68 #define NAND_DEV_SEL                    0
  69 #define DM_EN                           BIT(2)
  70
  71 /* NAND_FLASH_STATUS bits */
  72 #define FS_OP_ERR                       BIT(4)
  73 #define FS_READY_BSY_N                  BIT(5)
  74 #define FS_MPU_ERR                      BIT(8)
  75 #define FS_DEVICE_STS_ERR               BIT(16)
  76 #define FS_DEVICE_WP                    BIT(23)
  77
  78 /* NAND_BUFFER_STATUS bits */
  79 #define BS_UNCORRECTABLE_BIT            BIT(8)
  80 #define BS_CORRECTABLE_ERR_MSK          0x1f
  81
  82 /* NAND_DEVn_CFG0 bits */
  83 #define DISABLE_STATUS_AFTER_WRITE      4
  84 #define CW_PER_PAGE                     6
  85 #define UD_SIZE_BYTES                   9
  86 #define ECC_PARITY_SIZE_BYTES_RS        19
  87 #define SPARE_SIZE_BYTES                23
  88 #define NUM_ADDR_CYCLES                 27
  89 #define STATUS_BFR_READ                 30
  90 #define SET_RD_MODE_AFTER_STATUS        31
  91
  92 /* NAND_DEVn_CFG0 bits */
  93 #define DEV0_CFG1_ECC_DISABLE           0
  94 #define WIDE_FLASH                      1
  95 #define NAND_RECOVERY_CYCLES            2
  96 #define CS_ACTIVE_BSY                   5
  97 #define BAD_BLOCK_BYTE_NUM              6
  98 #define BAD_BLOCK_IN_SPARE_AREA         16
  99 #define WR_RD_BSY_GAP                   17
 100 #define ENABLE_BCH_ECC                  27
 101
 102 /* NAND_DEV0_ECC_CFG bits */
 103 #define ECC_CFG_ECC_DISABLE             0
 104 #define ECC_SW_RESET                    1
 105 #define ECC_MODE                        4
 106 #define ECC_PARITY_SIZE_BYTES_BCH       8
 107 #define ECC_NUM_DATA_BYTES              16
 108 #define ECC_FORCE_CLK_OPEN              30
 109
 110 /* NAND_DEV_CMD1 bits */
 111 #define READ_ADDR                       0
 112
 113 /* NAND_DEV_CMD_VLD bits */
 114 #define READ_START_VLD                  BIT(0)
 115 #define READ_STOP_VLD                   BIT(1)
 116 #define WRITE_START_VLD                 BIT(2)
 117 #define ERASE_START_VLD                 BIT(3)
 118 #define SEQ_READ_START_VLD              BIT(4)
 119
 120 /* NAND_EBI2_ECC_BUF_CFG bits */
 121 #define NUM_STEPS                       0
 122
 123 /* NAND_ERASED_CW_DETECT_CFG bits */
 124 #define ERASED_CW_ECC_MASK              1
 125 #define AUTO_DETECT_RES                 0
 126 #define MASK_ECC                        (1 << ERASED_CW_ECC_MASK)
 127 #define RESET_ERASED_DET                (1 << AUTO_DETECT_RES)
 128 #define ACTIVE_ERASED_DET               (0 << AUTO_DETECT_RES)
 129 #define CLR_ERASED_PAGE_DET             (RESET_ERASED_DET | MASK_ECC)
 130 #define SET_ERASED_PAGE_DET             (ACTIVE_ERASED_DET | MASK_ECC)
 131
 132 /* NAND_ERASED_CW_DETECT_STATUS bits */
 133 #define PAGE_ALL_ERASED                 BIT(7)
 134 #define CODEWORD_ALL_ERASED             BIT(6)
 135 #define PAGE_ERASED                     BIT(5)
 136 #define CODEWORD_ERASED                 BIT(4)
 137 #define ERASED_PAGE                     (PAGE_ALL_ERASED | PAGE_ERASED)
 138 #define ERASED_CW                       (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
 139
 140 /* NAND_READ_LOCATION_n bits */
 141 #define READ_LOCATION_OFFSET            0
 142 #define READ_LOCATION_SIZE              16
 143 #define READ_LOCATION_LAST              31
 144
 145 /* Version Mask */
 146 #define NAND_VERSION_MAJOR_MASK         0xf0000000
 147 #define NAND_VERSION_MAJOR_SHIFT        28
 148 #define NAND_VERSION_MINOR_MASK         0x0fff0000
 149 #define NAND_VERSION_MINOR_SHIFT        16
 150
 151 /* NAND OP_CMDs */
 152 #define OP_PAGE_READ                    0x2
 153 #define OP_PAGE_READ_WITH_ECC           0x3
 154 #define OP_PAGE_READ_WITH_ECC_SPARE     0x4
 155 #define OP_PROGRAM_PAGE                 0x6
 156 #define OP_PAGE_PROGRAM_WITH_ECC        0x7
 157 #define OP_PROGRAM_PAGE_SPARE           0x9
 158 #define OP_BLOCK_ERASE                  0xa
 159 #define OP_FETCH_ID                     0xb
 160 #define OP_RESET_DEVICE                 0xd
 161
 162 /* Default Value for NAND_DEV_CMD_VLD */
 163 #define NAND_DEV_CMD_VLD_VAL            (READ_START_VLD | WRITE_START_VLD | \
 164                                          ERASE_START_VLD | SEQ_READ_START_VLD)
 165
 166 /* NAND_CTRL bits */
 167 #define BAM_MODE_EN                     BIT(0)
 168
 169 /*
 170  * the NAND controller performs reads/writes with ECC in 516 byte chunks.
 171  * the driver calls the chunks 'step' or 'codeword' interchangeably
 172  */
 173 #define NANDC_STEP_SIZE                 512
 174
 175 /*
 176  * the largest page size we support is 8K, this will have 16 steps/codewords
 177  * of 512 bytes each
 178  */
 179 #define MAX_NUM_STEPS                   (SZ_8K / NANDC_STEP_SIZE)
 180
 181 /* we read at most 3 registers per codeword scan */
 182 #define MAX_REG_RD                      (3 * MAX_NUM_STEPS)
 183
 184 /* ECC modes supported by the controller */
 185 #define ECC_NONE        BIT(0)
 186 #define ECC_RS_4BIT     BIT(1)
 187 #define ECC_BCH_4BIT    BIT(2)
 188 #define ECC_BCH_8BIT    BIT(3)
 189
 190 #define nandc_set_read_loc(nandc, reg, offset, size, is_last)   \
 191 nandc_set_reg(nandc, NAND_READ_LOCATION_##reg,                  \
 192               ((offset) << READ_LOCATION_OFFSET) |              \
 193               ((size) << READ_LOCATION_SIZE) |                  \
 194               ((is_last) << READ_LOCATION_LAST))
 195
 196 /*
 197  * Returns the actual register address for all NAND_DEV_ registers
 198  * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
 199  */
 200 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
 201
 202 #define QPIC_PER_CW_CMD_SGL             32
 203 #define QPIC_PER_CW_DATA_SGL            8
 204
 205 /*
 206  * Flags used in DMA descriptor preparation helper functions
 207  * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
 208  */
 209 /* Don't set the EOT in current tx BAM sgl */
 210 #define NAND_BAM_NO_EOT                 BIT(0)
 211 /* Set the NWD flag in current BAM sgl */
 212 #define NAND_BAM_NWD                    BIT(1)
 213 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
 214 #define NAND_BAM_NEXT_SGL               BIT(2)
 215 /*
 216  * Erased codeword status is being used two times in single transfer so this
 217  * flag will determine the current value of erased codeword status register
 218  */
 219 #define NAND_ERASED_CW_SET              BIT(4)
 220
 221 /*
 222  * This data type corresponds to the BAM transaction which will be used for all
 223  * NAND transfers.
 224  * @cmd_sgl - sgl for NAND BAM command pipe
 225  * @data_sgl - sgl for NAND BAM consumer/producer pipe
 226  * @cmd_sgl_pos - current index in command sgl.
 227  * @cmd_sgl_start - start index in command sgl.
 228  * @tx_sgl_pos - current index in data sgl for tx.
 229  * @tx_sgl_start - start index in data sgl for tx.
 230  * @rx_sgl_pos - current index in data sgl for rx.
 231  * @rx_sgl_start - start index in data sgl for rx.
 232  */
 233 struct bam_transaction {
 234         struct scatterlist *cmd_sgl;
 235         struct scatterlist *data_sgl;
 236         u32 cmd_sgl_pos;
 237         u32 cmd_sgl_start;
 238         u32 tx_sgl_pos;
 239         u32 tx_sgl_start;
 240         u32 rx_sgl_pos;
 241         u32 rx_sgl_start;
 242 };
 243
 244 /*
 245  * This data type corresponds to the nand dma descriptor
 246  * @list - list for desc_info
 247  * @dir - DMA transfer direction
 248  * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
 249  *            ADM
 250  * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
 251  * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
 252  * @dma_desc - low level DMA engine descriptor
 253  */
 254 struct desc_info {
 255         struct list_head node;
 256
 257         enum dma_data_direction dir;
 258         union {
 259                 struct scatterlist adm_sgl;
 260                 struct {
 261                         struct scatterlist *bam_sgl;
 262                         int sgl_cnt;
 263                 };
 264         };
 265         struct dma_async_tx_descriptor *dma_desc;
 266 };
 267
 268 /*
 269  * holds the current register values that we want to write. acts as a contiguous
 270  * chunk of memory which we use to write the controller registers through DMA.
 271  */
 272 struct nandc_regs {
 273         __le32 cmd;
 274         __le32 addr0;
 275         __le32 addr1;
 276         __le32 chip_sel;
 277         __le32 exec;
 278
 279         __le32 cfg0;
 280         __le32 cfg1;
 281         __le32 ecc_bch_cfg;
 282
 283         __le32 clrflashstatus;
 284         __le32 clrreadstatus;
 285
 286         __le32 cmd1;
 287         __le32 vld;
 288
 289         __le32 orig_cmd1;
 290         __le32 orig_vld;
 291
 292         __le32 ecc_buf_cfg;
 293         __le32 read_location0;
 294         __le32 read_location1;
 295         __le32 read_location2;
 296         __le32 read_location3;
 297
 298         __le32 erased_cw_detect_cfg_clr;
 299         __le32 erased_cw_detect_cfg_set;
 300 };
 301
 302 /*
 303  * NAND controller data struct
 304  *
 305  * @controller:                 base controller structure
 306  * @host_list:                  list containing all the chips attached to the
 307  *                              controller
 308  * @dev:                        parent device
 309  * @base:                       MMIO base
 310  * @base_dma:                   physical base address of controller registers
 311  * @core_clk:                   controller clock
 312  * @aon_clk:                    another controller clock
 313  *
 314  * @chan:                       dma channel
 315  * @cmd_crci:                   ADM DMA CRCI for command flow control
 316  * @data_crci:                  ADM DMA CRCI for data flow control
 317  * @desc_list:                  DMA descriptor list (list of desc_infos)
 318  *
 319  * @data_buffer:                our local DMA buffer for page read/writes,
 320  *                              used when we can't use the buffer provided
 321  *                              by upper layers directly
 322  * @buf_size/count/start:       markers for chip->read_buf/write_buf functions
 323  * @reg_read_buf:               local buffer for reading back registers via DMA
 324  * @reg_read_dma:               contains dma address for register read buffer
 325  * @reg_read_pos:               marker for data read in reg_read_buf
 326  *
 327  * @regs:                       a contiguous chunk of memory for DMA register
 328  *                              writes. contains the register values to be
 329  *                              written to controller
 330  * @cmd1/vld:                   some fixed controller register values
 331  * @props:                      properties of current NAND controller,
 332  *                              initialized via DT match data
 333  * @max_cwperpage:              maximum QPIC codewords required. calculated
 334  *                              from all connected NAND devices pagesize
 335  */
 336 struct qcom_nand_controller {
 337         struct nand_hw_control controller;
 338         struct list_head host_list;
 339
 340         struct device *dev;
 341
 342         void __iomem *base;
 343         dma_addr_t base_dma;
 344
 345         struct clk *core_clk;
 346         struct clk *aon_clk;
 347
 348         union {
 349                 /* will be used only by QPIC for BAM DMA */
 350                 struct {
 351                         struct dma_chan *tx_chan;
 352                         struct dma_chan *rx_chan;
 353                         struct dma_chan *cmd_chan;
 354                 };
 355
 356                 /* will be used only by EBI2 for ADM DMA */
 357                 struct {
 358                         struct dma_chan *chan;
 359                         unsigned int cmd_crci;
 360                         unsigned int data_crci;
 361                 };
 362         };
 363
 364         struct list_head desc_list;
 365         struct bam_transaction *bam_txn;
 366
 367         u8              *data_buffer;
 368         int             buf_size;
 369         int             buf_count;
 370         int             buf_start;
 371         unsigned int    max_cwperpage;
 372
 373         __le32 *reg_read_buf;
 374         dma_addr_t reg_read_dma;
 375         int reg_read_pos;
 376
 377         struct nandc_regs *regs;
 378
 379         u32 cmd1, vld;
 380         const struct qcom_nandc_props *props;
 381 };
 382
 383 /*
 384  * NAND chip structure
 385  *
 386  * @chip:                       base NAND chip structure
 387  * @node:                       list node to add itself to host_list in
 388  *                              qcom_nand_controller
 389  *
 390  * @cs:                         chip select value for this chip
 391  * @cw_size:                    the number of bytes in a single step/codeword
 392  *                              of a page, consisting of all data, ecc, spare
 393  *                              and reserved bytes
 394  * @cw_data:                    the number of bytes within a codeword protected
 395  *                              by ECC
 396  * @use_ecc:                    request the controller to use ECC for the
 397  *                              upcoming read/write
 398  * @bch_enabled:                flag to tell whether BCH ECC mode is used
 399  * @ecc_bytes_hw:               ECC bytes used by controller hardware for this
 400  *                              chip
 401  * @status:                     value to be returned if NAND_CMD_STATUS command
 402  *                              is executed
 403  * @last_command:               keeps track of last command on this chip. used
 404  *                              for reading correct status
 405  *
 406  * @cfg0, cfg1, cfg0_raw..:     NANDc register configurations needed for
 407  *                              ecc/non-ecc mode for the current nand flash
 408  *                              device
 409  */
 410 struct qcom_nand_host {
 411         struct nand_chip chip;
 412         struct list_head node;
 413
 414         int cs;
 415         int cw_size;
 416         int cw_data;
 417         bool use_ecc;
 418         bool bch_enabled;
 419         int ecc_bytes_hw;
 420         int spare_bytes;
 421         int bbm_size;
 422         u8 status;
 423         int last_command;
 424
 425         u32 cfg0, cfg1;
 426         u32 cfg0_raw, cfg1_raw;
 427         u32 ecc_buf_cfg;
 428         u32 ecc_bch_cfg;
 429         u32 clrflashstatus;
 430         u32 clrreadstatus;
 431 };
 432
 433 /*
 434  * This data type corresponds to the NAND controller properties which varies
 435  * among different NAND controllers.
 436  * @ecc_modes - ecc mode for NAND
 437  * @is_bam - whether NAND controller is using BAM
 438  * @is_qpic - whether NAND CTRL is part of qpic IP
 439  * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
 440  */
 441 struct qcom_nandc_props {
 442         u32 ecc_modes;
 443         bool is_bam;
 444         bool is_qpic;
 445         u32 dev_cmd_reg_start;
 446 };
 447
 448 /* Frees the BAM transaction memory */
 449 static void free_bam_transaction(struct qcom_nand_controller *nandc)
 450 {
 451         struct bam_transaction *bam_txn = nandc->bam_txn;
 452
 453         devm_kfree(nandc->dev, bam_txn);
 454 }
 455
 456 /* Allocates and Initializes the BAM transaction */
 457 static struct bam_transaction *
 458 alloc_bam_transaction(struct qcom_nand_controller *nandc)
 459 {
 460         struct bam_transaction *bam_txn;
 461         size_t bam_txn_size;
 462         unsigned int num_cw = nandc->max_cwperpage;
 463         void *bam_txn_buf;
 464
 465         bam_txn_size =
 466                 sizeof(*bam_txn) + num_cw *
 467                 ((sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
 468                 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
 469
 470         bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
 471         if (!bam_txn_buf)
 472                 return NULL;
 473
 474         bam_txn = bam_txn_buf;
 475         bam_txn_buf += sizeof(*bam_txn);
 476
 477         bam_txn->cmd_sgl = bam_txn_buf;
 478         bam_txn_buf +=
 479                 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
 480
 481         bam_txn->data_sgl = bam_txn_buf;
 482
 483         return bam_txn;
 484 }
 485
 486 /* Clears the BAM transaction indexes */
 487 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
 488 {
 489         struct bam_transaction *bam_txn = nandc->bam_txn;
 490
 491         if (!nandc->props->is_bam)
 492                 return;
 493
 494         bam_txn->cmd_sgl_pos = 0;
 495         bam_txn->cmd_sgl_start = 0;
 496         bam_txn->tx_sgl_pos = 0;
 497         bam_txn->tx_sgl_start = 0;
 498         bam_txn->rx_sgl_pos = 0;
 499         bam_txn->rx_sgl_start = 0;
 500
 501         sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
 502                       QPIC_PER_CW_CMD_SGL);
 503         sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
 504                       QPIC_PER_CW_DATA_SGL);
 505 }
 506
 507 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
 508 {
 509         return container_of(chip, struct qcom_nand_host, chip);
 510 }
 511
 512 static inline struct qcom_nand_controller *
 513 get_qcom_nand_controller(struct nand_chip *chip)
 514 {
 515         return container_of(chip->controller, struct qcom_nand_controller,
 516                             controller);
 517 }
 518
 519 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
 520 {
 521         return ioread32(nandc->base + offset);
 522 }
 523
 524 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
 525                                u32 val)
 526 {
 527         iowrite32(val, nandc->base + offset);
 528 }
 529
 530 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
 531                                           bool is_cpu)
 532 {
 533         if (!nandc->props->is_bam)
 534                 return;
 535
 536         if (is_cpu)
 537                 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
 538                                         MAX_REG_RD *
 539                                         sizeof(*nandc->reg_read_buf),
 540                                         DMA_FROM_DEVICE);
 541         else
 542                 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
 543                                            MAX_REG_RD *
 544                                            sizeof(*nandc->reg_read_buf),
 545                                            DMA_FROM_DEVICE);
 546 }
 547
 548 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
 549 {
 550         switch (offset) {
 551         case NAND_FLASH_CMD:
 552                 return &regs->cmd;
 553         case NAND_ADDR0:
 554                 return &regs->addr0;
 555         case NAND_ADDR1:
 556                 return &regs->addr1;
 557         case NAND_FLASH_CHIP_SELECT:
 558                 return &regs->chip_sel;
 559         case NAND_EXEC_CMD:
 560                 return &regs->exec;
 561         case NAND_FLASH_STATUS:
 562                 return &regs->clrflashstatus;
 563         case NAND_DEV0_CFG0:
 564                 return &regs->cfg0;
 565         case NAND_DEV0_CFG1:
 566                 return &regs->cfg1;
 567         case NAND_DEV0_ECC_CFG:
 568                 return &regs->ecc_bch_cfg;
 569         case NAND_READ_STATUS:
 570                 return &regs->clrreadstatus;
 571         case NAND_DEV_CMD1:
 572                 return &regs->cmd1;
 573         case NAND_DEV_CMD1_RESTORE:
 574                 return &regs->orig_cmd1;
 575         case NAND_DEV_CMD_VLD:
 576                 return &regs->vld;
 577         case NAND_DEV_CMD_VLD_RESTORE:
 578                 return &regs->orig_vld;
 579         case NAND_EBI2_ECC_BUF_CFG:
 580                 return &regs->ecc_buf_cfg;
 581         case NAND_READ_LOCATION_0:
 582                 return &regs->read_location0;
 583         case NAND_READ_LOCATION_1:
 584                 return &regs->read_location1;
 585         case NAND_READ_LOCATION_2:
 586                 return &regs->read_location2;
 587         case NAND_READ_LOCATION_3:
 588                 return &regs->read_location3;
 589         default:
 590                 return NULL;
 591         }
 592 }
 593
 594 static void nandc_set_reg(struct qcom_nand_controller *nandc, int offset,
 595                           u32 val)
 596 {
 597         struct nandc_regs *regs = nandc->regs;
 598         __le32 *reg;
 599
 600         reg = offset_to_nandc_reg(regs, offset);
 601
 602         if (reg)
 603                 *reg = cpu_to_le32(val);
 604 }
 605
 606 /* helper to configure address register values */
 607 static void set_address(struct qcom_nand_host *host, u16 column, int page)
 608 {
 609         struct nand_chip *chip = &host->chip;
 610         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 611
 612         if (chip->options & NAND_BUSWIDTH_16)
 613                 column >>= 1;
 614
 615         nandc_set_reg(nandc, NAND_ADDR0, page << 16 | column);
 616         nandc_set_reg(nandc, NAND_ADDR1, page >> 16 & 0xff);
 617 }
 618
 619 /*
 620  * update_rw_regs:      set up read/write register values, these will be
 621  *                      written to the NAND controller registers via DMA
 622  *
 623  * @num_cw:             number of steps for the read/write operation
 624  * @read:               read or write operation
 625  */
 626 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read)
 627 {
 628         struct nand_chip *chip = &host->chip;
 629         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
 630         u32 cmd, cfg0, cfg1, ecc_bch_cfg;
 631
 632         if (read) {
 633                 if (host->use_ecc)
 634                         cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
 635                 else
 636                         cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
 637         } else {
 638                 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
 639         }
 640
 641         if (host->use_ecc) {
 642                 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
 643                                 (num_cw - 1) << CW_PER_PAGE;
 644
 645                 cfg1 = host->cfg1;
 646                 ecc_bch_cfg = host->ecc_bch_cfg;
 647         } else {
 648                 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
 649                                 (num_cw - 1) << CW_PER_PAGE;
 650
 651                 cfg1 = host->cfg1_raw;
 652                 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
 653         }
 654
 655         nandc_set_reg(nandc, NAND_FLASH_CMD, cmd);
 656         nandc_set_reg(nandc, NAND_DEV0_CFG0, cfg0);
 657         nandc_set_reg(nandc, NAND_DEV0_CFG1, cfg1);
 658         nandc_set_reg(nandc, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
 659         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
 660         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
 661         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
 662         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
 663
 664         if (read)
 665                 nandc_set_read_loc(nandc, 0, 0, host->use_ecc ?
 666                                    host->cw_data : host->cw_size, 1);
 667 }
 668
 669 /*
 670  * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
 671  * for BAM. This descriptor will be added in the NAND DMA descriptor queue
 672  * which will be submitted to DMA engine.
 673  */
 674 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
 675                                   struct dma_chan *chan,
 676                                   unsigned long flags)
 677 {
 678         struct desc_info *desc;
 679         struct scatterlist *sgl;
 680         unsigned int sgl_cnt;
 681         int ret;
 682         struct bam_transaction *bam_txn = nandc->bam_txn;
 683         enum dma_transfer_direction dir_eng;
 684         struct dma_async_tx_descriptor *dma_desc;
 685
 686         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 687         if (!desc)
 688                 return -ENOMEM;
 689
 690         if (chan == nandc->cmd_chan) {
 691                 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
 692                 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
 693                 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
 694                 dir_eng = DMA_MEM_TO_DEV;
 695                 desc->dir = DMA_TO_DEVICE;
 696         } else if (chan == nandc->tx_chan) {
 697                 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
 698                 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
 699                 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
 700                 dir_eng = DMA_MEM_TO_DEV;
 701                 desc->dir = DMA_TO_DEVICE;
 702         } else {
 703                 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
 704                 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
 705                 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
 706                 dir_eng = DMA_DEV_TO_MEM;
 707                 desc->dir = DMA_FROM_DEVICE;
 708         }
 709
 710         sg_mark_end(sgl + sgl_cnt - 1);
 711         ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 712         if (ret == 0) {
 713                 dev_err(nandc->dev, "failure in mapping desc\n");
 714                 kfree(desc);
 715                 return -ENOMEM;
 716         }
 717
 718         desc->sgl_cnt = sgl_cnt;
 719         desc->bam_sgl = sgl;
 720
 721         dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
 722                                            flags);
 723
 724         if (!dma_desc) {
 725                 dev_err(nandc->dev, "failure in prep desc\n");
 726                 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
 727                 kfree(desc);
 728                 return -EINVAL;
 729         }
 730
 731         desc->dma_desc = dma_desc;
 732
 733         list_add_tail(&desc->node, &nandc->desc_list);
 734
 735         return 0;
 736 }
 737
 738 /*
 739  * Prepares the data descriptor for BAM DMA which will be used for NAND
 740  * data reads and writes.
 741  */
 742 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
 743                                   const void *vaddr,
 744                                   int size, unsigned int flags)
 745 {
 746         int ret;
 747         struct bam_transaction *bam_txn = nandc->bam_txn;
 748
 749         if (read) {
 750                 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
 751                            vaddr, size);
 752                 bam_txn->rx_sgl_pos++;
 753         } else {
 754                 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
 755                            vaddr, size);
 756                 bam_txn->tx_sgl_pos++;
 757
 758                 /*
 759                  * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
 760                  * is not set, form the DMA descriptor
 761                  */
 762                 if (!(flags & NAND_BAM_NO_EOT)) {
 763                         ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
 764                                                      DMA_PREP_INTERRUPT);
 765                         if (ret)
 766                                 return ret;
 767                 }
 768         }
 769
 770         return 0;
 771 }
 772
 773 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
 774                              int reg_off, const void *vaddr, int size,
 775                              bool flow_control)
 776 {
 777         struct desc_info *desc;
 778         struct dma_async_tx_descriptor *dma_desc;
 779         struct scatterlist *sgl;
 780         struct dma_slave_config slave_conf;
 781         enum dma_transfer_direction dir_eng;
 782         int ret;
 783
 784         desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 785         if (!desc)
 786                 return -ENOMEM;
 787
 788         sgl = &desc->adm_sgl;
 789
 790         sg_init_one(sgl, vaddr, size);
 791
 792         if (read) {
 793                 dir_eng = DMA_DEV_TO_MEM;
 794                 desc->dir = DMA_FROM_DEVICE;
 795         } else {
 796                 dir_eng = DMA_MEM_TO_DEV;
 797                 desc->dir = DMA_TO_DEVICE;
 798         }
 799
 800         ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
 801         if (ret == 0) {
 802                 ret = -ENOMEM;
 803                 goto err;
 804         }
 805
 806         memset(&slave_conf, 0x00, sizeof(slave_conf));
 807
 808         slave_conf.device_fc = flow_control;
 809         if (read) {
 810                 slave_conf.src_maxburst = 16;
 811                 slave_conf.src_addr = nandc->base_dma + reg_off;
 812                 slave_conf.slave_id = nandc->data_crci;
 813         } else {
 814                 slave_conf.dst_maxburst = 16;
 815                 slave_conf.dst_addr = nandc->base_dma + reg_off;
 816                 slave_conf.slave_id = nandc->cmd_crci;
 817         }
 818
 819         ret = dmaengine_slave_config(nandc->chan, &slave_conf);
 820         if (ret) {
 821                 dev_err(nandc->dev, "failed to configure dma channel\n");
 822                 goto err;
 823         }
 824
 825         dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
 826         if (!dma_desc) {
 827                 dev_err(nandc->dev, "failed to prepare desc\n");
 828                 ret = -EINVAL;
 829                 goto err;
 830         }
 831
 832         desc->dma_desc = dma_desc;
 833
 834         list_add_tail(&desc->node, &nandc->desc_list);
 835
 836         return 0;
 837 err:
 838         kfree(desc);
 839
 840         return ret;
 841 }
 842
 843 /*
 844  * read_reg_dma:        prepares a descriptor to read a given number of
 845  *                      contiguous registers to the reg_read_buf pointer
 846  *
 847  * @first:              offset of the first register in the contiguous block
 848  * @num_regs:           number of registers to read
 849  * @flags:              flags to control DMA descriptor preparation
 850  */
 851 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
 852                         int num_regs, unsigned int flags)
 853 {
 854         bool flow_control = false;
 855         void *vaddr;
 856         int size;
 857
 858         if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
 859                 flow_control = true;
 860
 861         if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
 862                 first = dev_cmd_reg_addr(nandc, first);
 863
 864         size = num_regs * sizeof(u32);
 865         vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
 866         nandc->reg_read_pos += num_regs;
 867
 868         return prep_adm_dma_desc(nandc, true, first, vaddr, size, flow_control);
 869 }
 870
 871 /*
 872  * write_reg_dma:       prepares a descriptor to write a given number of
 873  *                      contiguous registers
 874  *
 875  * @first:              offset of the first register in the contiguous block
 876  * @num_regs:           number of registers to write
 877  * @flags:              flags to control DMA descriptor preparation
 878  */
 879 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
 880                          int num_regs, unsigned int flags)
 881 {
 882         bool flow_control = false;
 883         struct nandc_regs *regs = nandc->regs;
 884         void *vaddr;
 885         int size;
 886
 887         vaddr = offset_to_nandc_reg(regs, first);
 888
 889         if (first == NAND_FLASH_CMD)
 890                 flow_control = true;
 891
 892         if (first == NAND_ERASED_CW_DETECT_CFG) {
 893                 if (flags & NAND_ERASED_CW_SET)
 894                         vaddr = &regs->erased_cw_detect_cfg_set;
 895                 else
 896                         vaddr = &regs->erased_cw_detect_cfg_clr;
 897         }
 898
 899         if (first == NAND_EXEC_CMD)
 900                 flags |= NAND_BAM_NWD;
 901
 902         if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
 903                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
 904
 905         if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
 906                 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
 907
 908         size = num_regs * sizeof(u32);
 909
 910         return prep_adm_dma_desc(nandc, false, first, vaddr, size,
 911                                  flow_control);
 912 }
 913
 914 /*
 915  * read_data_dma:       prepares a DMA descriptor to transfer data from the
 916  *                      controller's internal buffer to the buffer 'vaddr'
 917  *
 918  * @reg_off:            offset within the controller's data buffer
 919  * @vaddr:              virtual address of the buffer we want to write to
 920  * @size:               DMA transaction size in bytes
 921  * @flags:              flags to control DMA descriptor preparation
 922  */
 923 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 924                          const u8 *vaddr, int size, unsigned int flags)
 925 {
 926         if (nandc->props->is_bam)
 927                 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
 928
 929         return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
 930 }
 931
 932 /*
 933  * write_data_dma:      prepares a DMA descriptor to transfer data from
 934  *                      'vaddr' to the controller's internal buffer
 935  *
 936  * @reg_off:            offset within the controller's data buffer
 937  * @vaddr:              virtual address of the buffer we want to read from
 938  * @size:               DMA transaction size in bytes
 939  * @flags:              flags to control DMA descriptor preparation
 940  */
 941 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
 942                           const u8 *vaddr, int size, unsigned int flags)
 943 {
 944         if (nandc->props->is_bam)
 945                 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
 946
 947         return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
 948 }
 949
 950 /*
 951  * Helper to prepare DMA descriptors for configuring registers
 952  * before reading a NAND page.
 953  */
 954 static void config_nand_page_read(struct qcom_nand_controller *nandc)
 955 {
 956         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
 957         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
 958         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
 959         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
 960         write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
 961                       NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
 962 }
 963
 964 /*
 965  * Helper to prepare DMA descriptors for configuring registers
 966  * before reading each codeword in NAND page.
 967  */
 968 static void config_nand_cw_read(struct qcom_nand_controller *nandc)
 969 {
 970         if (nandc->props->is_bam)
 971                 write_reg_dma(nandc, NAND_READ_LOCATION_0, 4,
 972                               NAND_BAM_NEXT_SGL);
 973
 974         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
 975         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
 976
 977         read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
 978         read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
 979                      NAND_BAM_NEXT_SGL);
 980 }
 981
 982 /*
 983  * Helper to prepare dma descriptors to configure registers needed for reading a
 984  * single codeword in page
 985  */
 986 static void config_nand_single_cw_page_read(struct qcom_nand_controller *nandc)
 987 {
 988         config_nand_page_read(nandc);
 989         config_nand_cw_read(nandc);
 990 }
 991
 992 /*
 993  * Helper to prepare DMA descriptors used to configure registers needed for
 994  * before writing a NAND page.
 995  */
 996 static void config_nand_page_write(struct qcom_nand_controller *nandc)
 997 {
 998         write_reg_dma(nandc, NAND_ADDR0, 2, 0);
 999         write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1000         write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1001                       NAND_BAM_NEXT_SGL);
1002 }
1003
1004 /*
1005  * Helper to prepare DMA descriptors for configuring registers
1006  * before writing each codeword in NAND page.
1007  */
1008 static void config_nand_cw_write(struct qcom_nand_controller *nandc)
1009 {
1010         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1011         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1012
1013         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1014
1015         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1016         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1017 }
1018
1019 /*
1020  * the following functions are used within chip->cmdfunc() to perform different
1021  * NAND_CMD_* commands
1022  */
1023
1024 /* sets up descriptors for NAND_CMD_PARAM */
1025 static int nandc_param(struct qcom_nand_host *host)
1026 {
1027         struct nand_chip *chip = &host->chip;
1028         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1029
1030         /*
1031          * NAND_CMD_PARAM is called before we know much about the FLASH chip
1032          * in use. we configure the controller to perform a raw read of 512
1033          * bytes to read onfi params
1034          */
1035         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
1036         nandc_set_reg(nandc, NAND_ADDR0, 0);
1037         nandc_set_reg(nandc, NAND_ADDR1, 0);
1038         nandc_set_reg(nandc, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1039                                         | 512 << UD_SIZE_BYTES
1040                                         | 5 << NUM_ADDR_CYCLES
1041                                         | 0 << SPARE_SIZE_BYTES);
1042         nandc_set_reg(nandc, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1043                                         | 0 << CS_ACTIVE_BSY
1044                                         | 17 << BAD_BLOCK_BYTE_NUM
1045                                         | 1 << BAD_BLOCK_IN_SPARE_AREA
1046                                         | 2 << WR_RD_BSY_GAP
1047                                         | 0 << WIDE_FLASH
1048                                         | 1 << DEV0_CFG1_ECC_DISABLE);
1049         nandc_set_reg(nandc, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1050
1051         /* configure CMD1 and VLD for ONFI param probing */
1052         nandc_set_reg(nandc, NAND_DEV_CMD_VLD,
1053                       (nandc->vld & ~READ_START_VLD));
1054         nandc_set_reg(nandc, NAND_DEV_CMD1,
1055                       (nandc->cmd1 & ~(0xFF << READ_ADDR))
1056                       | NAND_CMD_PARAM << READ_ADDR);
1057
1058         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1059
1060         nandc_set_reg(nandc, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1061         nandc_set_reg(nandc, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1062         nandc_set_read_loc(nandc, 0, 0, 512, 1);
1063
1064         write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1065         write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1066
1067         nandc->buf_count = 512;
1068         memset(nandc->data_buffer, 0xff, nandc->buf_count);
1069
1070         config_nand_single_cw_page_read(nandc);
1071
1072         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1073                       nandc->buf_count, 0);
1074
1075         /* restore CMD1 and VLD regs */
1076         write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1077         write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1078
1079         return 0;
1080 }
1081
1082 /* sets up descriptors for NAND_CMD_ERASE1 */
1083 static int erase_block(struct qcom_nand_host *host, int page_addr)
1084 {
1085         struct nand_chip *chip = &host->chip;
1086         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1087
1088         nandc_set_reg(nandc, NAND_FLASH_CMD,
1089                       OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1090         nandc_set_reg(nandc, NAND_ADDR0, page_addr);
1091         nandc_set_reg(nandc, NAND_ADDR1, 0);
1092         nandc_set_reg(nandc, NAND_DEV0_CFG0,
1093                       host->cfg0_raw & ~(7 << CW_PER_PAGE));
1094         nandc_set_reg(nandc, NAND_DEV0_CFG1, host->cfg1_raw);
1095         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1096         nandc_set_reg(nandc, NAND_FLASH_STATUS, host->clrflashstatus);
1097         nandc_set_reg(nandc, NAND_READ_STATUS, host->clrreadstatus);
1098
1099         write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1100         write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1101         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1102
1103         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1104
1105         write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1106         write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1107
1108         return 0;
1109 }
1110
1111 /* sets up descriptors for NAND_CMD_READID */
1112 static int read_id(struct qcom_nand_host *host, int column)
1113 {
1114         struct nand_chip *chip = &host->chip;
1115         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1116
1117         if (column == -1)
1118                 return 0;
1119
1120         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_FETCH_ID);
1121         nandc_set_reg(nandc, NAND_ADDR0, column);
1122         nandc_set_reg(nandc, NAND_ADDR1, 0);
1123         nandc_set_reg(nandc, NAND_FLASH_CHIP_SELECT,
1124                       nandc->props->is_bam ? 0 : DM_EN);
1125         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1126
1127         write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1128         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1129
1130         read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1131
1132         return 0;
1133 }
1134
1135 /* sets up descriptors for NAND_CMD_RESET */
1136 static int reset(struct qcom_nand_host *host)
1137 {
1138         struct nand_chip *chip = &host->chip;
1139         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1140
1141         nandc_set_reg(nandc, NAND_FLASH_CMD, OP_RESET_DEVICE);
1142         nandc_set_reg(nandc, NAND_EXEC_CMD, 1);
1143
1144         write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1145         write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1146
1147         read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1148
1149         return 0;
1150 }
1151
1152 /* helpers to submit/free our list of dma descriptors */
1153 static int submit_descs(struct qcom_nand_controller *nandc)
1154 {
1155         struct desc_info *desc;
1156         dma_cookie_t cookie = 0;
1157         struct bam_transaction *bam_txn = nandc->bam_txn;
1158         int r;
1159
1160         if (nandc->props->is_bam) {
1161                 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1162                         r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1163                         if (r)
1164                                 return r;
1165                 }
1166
1167                 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1168                         r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1169                                                    DMA_PREP_INTERRUPT);
1170                         if (r)
1171                                 return r;
1172                 }
1173
1174                 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1175                         r = prepare_bam_async_desc(nandc, nandc->cmd_chan, 0);
1176                         if (r)
1177                                 return r;
1178                 }
1179         }
1180
1181         list_for_each_entry(desc, &nandc->desc_list, node)
1182                 cookie = dmaengine_submit(desc->dma_desc);
1183
1184         if (nandc->props->is_bam) {
1185                 dma_async_issue_pending(nandc->tx_chan);
1186                 dma_async_issue_pending(nandc->rx_chan);
1187
1188                 if (dma_sync_wait(nandc->cmd_chan, cookie) != DMA_COMPLETE)
1189                         return -ETIMEDOUT;
1190         } else {
1191                 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1192                         return -ETIMEDOUT;
1193         }
1194
1195         return 0;
1196 }
1197
1198 static void free_descs(struct qcom_nand_controller *nandc)
1199 {
1200         struct desc_info *desc, *n;
1201
1202         list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1203                 list_del(&desc->node);
1204
1205                 if (nandc->props->is_bam)
1206                         dma_unmap_sg(nandc->dev, desc->bam_sgl,
1207                                      desc->sgl_cnt, desc->dir);
1208                 else
1209                         dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1210                                      desc->dir);
1211
1212                 kfree(desc);
1213         }
1214 }
1215
1216 /* reset the register read buffer for next NAND operation */
1217 static void clear_read_regs(struct qcom_nand_controller *nandc)
1218 {
1219         nandc->reg_read_pos = 0;
1220         nandc_read_buffer_sync(nandc, false);
1221 }
1222
1223 static void pre_command(struct qcom_nand_host *host, int command)
1224 {
1225         struct nand_chip *chip = &host->chip;
1226         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1227
1228         nandc->buf_count = 0;
1229         nandc->buf_start = 0;
1230         host->use_ecc = false;
1231         host->last_command = command;
1232
1233         clear_read_regs(nandc);
1234
1235         if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1236             command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1237                 clear_bam_transaction(nandc);
1238 }
1239
1240 /*
1241  * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1242  * privately maintained status byte, this status byte can be read after
1243  * NAND_CMD_STATUS is called
1244  */
1245 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1246 {
1247         struct nand_chip *chip = &host->chip;
1248         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1249         struct nand_ecc_ctrl *ecc = &chip->ecc;
1250         int num_cw;
1251         int i;
1252
1253         num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1254         nandc_read_buffer_sync(nandc, true);
1255
1256         for (i = 0; i < num_cw; i++) {
1257                 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1258
1259                 if (flash_status & FS_MPU_ERR)
1260                         host->status &= ~NAND_STATUS_WP;
1261
1262                 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1263                                                  (flash_status &
1264                                                   FS_DEVICE_STS_ERR)))
1265                         host->status |= NAND_STATUS_FAIL;
1266         }
1267 }
1268
1269 static void post_command(struct qcom_nand_host *host, int command)
1270 {
1271         struct nand_chip *chip = &host->chip;
1272         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1273
1274         switch (command) {
1275         case NAND_CMD_READID:
1276                 nandc_read_buffer_sync(nandc, true);
1277                 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1278                        nandc->buf_count);
1279                 break;
1280         case NAND_CMD_PAGEPROG:
1281         case NAND_CMD_ERASE1:
1282                 parse_erase_write_errors(host, command);
1283                 break;
1284         default:
1285                 break;
1286         }
1287 }
1288
1289 /*
1290  * Implements chip->cmdfunc. It's  only used for a limited set of commands.
1291  * The rest of the commands wouldn't be called by upper layers. For example,
1292  * NAND_CMD_READOOB would never be called because we have our own versions
1293  * of read_oob ops for nand_ecc_ctrl.
1294  */
1295 static void qcom_nandc_command(struct mtd_info *mtd, unsigned int command,
1296                                int column, int page_addr)
1297 {
1298         struct nand_chip *chip = mtd_to_nand(mtd);
1299         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1300         struct nand_ecc_ctrl *ecc = &chip->ecc;
1301         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1302         bool wait = false;
1303         int ret = 0;
1304
1305         pre_command(host, command);
1306
1307         switch (command) {
1308         case NAND_CMD_RESET:
1309                 ret = reset(host);
1310                 wait = true;
1311                 break;
1312
1313         case NAND_CMD_READID:
1314                 nandc->buf_count = 4;
1315                 ret = read_id(host, column);
1316                 wait = true;
1317                 break;
1318
1319         case NAND_CMD_PARAM:
1320                 ret = nandc_param(host);
1321                 wait = true;
1322                 break;
1323
1324         case NAND_CMD_ERASE1:
1325                 ret = erase_block(host, page_addr);
1326                 wait = true;
1327                 break;
1328
1329         case NAND_CMD_READ0:
1330                 /* we read the entire page for now */
1331                 WARN_ON(column != 0);
1332
1333                 host->use_ecc = true;
1334                 set_address(host, 0, page_addr);
1335                 update_rw_regs(host, ecc->steps, true);
1336                 break;
1337
1338         case NAND_CMD_SEQIN:
1339                 WARN_ON(column != 0);
1340                 set_address(host, 0, page_addr);
1341                 break;
1342
1343         case NAND_CMD_PAGEPROG:
1344         case NAND_CMD_STATUS:
1345         case NAND_CMD_NONE:
1346         default:
1347                 break;
1348         }
1349
1350         if (ret) {
1351                 dev_err(nandc->dev, "failure executing command %d\n",
1352                         command);
1353                 free_descs(nandc);
1354                 return;
1355         }
1356
1357         if (wait) {
1358                 ret = submit_descs(nandc);
1359                 if (ret)
1360                         dev_err(nandc->dev,
1361                                 "failure submitting descs for command %d\n",
1362                                 command);
1363         }
1364
1365         free_descs(nandc);
1366
1367         post_command(host, command);
1368 }
1369
1370 /*
1371  * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1372  * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1373  *
1374  * when using RS ECC, the HW reports the same erros when reading an erased CW,
1375  * but it notifies that it is an erased CW by placing special characters at
1376  * certain offsets in the buffer.
1377  *
1378  * verify if the page is erased or not, and fix up the page for RS ECC by
1379  * replacing the special characters with 0xff.
1380  */
1381 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1382 {
1383         u8 empty1, empty2;
1384
1385         /*
1386          * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1387          * is erased by looking for 0x54s at offsets 3 and 175 from the
1388          * beginning of each codeword
1389          */
1390
1391         empty1 = data_buf[3];
1392         empty2 = data_buf[175];
1393
1394         /*
1395          * if the erased codework markers, if they exist override them with
1396          * 0xffs
1397          */
1398         if ((empty1 == 0x54 && empty2 == 0xff) ||
1399             (empty1 == 0xff && empty2 == 0x54)) {
1400                 data_buf[3] = 0xff;
1401                 data_buf[175] = 0xff;
1402         }
1403
1404         /*
1405          * check if the entire chunk contains 0xffs or not. if it doesn't, then
1406          * restore the original values at the special offsets
1407          */
1408         if (memchr_inv(data_buf, 0xff, data_len)) {
1409                 data_buf[3] = empty1;
1410                 data_buf[175] = empty2;
1411
1412                 return false;
1413         }
1414
1415         return true;
1416 }
1417
1418 struct read_stats {
1419         __le32 flash;
1420         __le32 buffer;
1421         __le32 erased_cw;
1422 };
1423
1424 /*
1425  * reads back status registers set by the controller to notify page read
1426  * errors. this is equivalent to what 'ecc->correct()' would do.
1427  */
1428 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1429                              u8 *oob_buf)
1430 {
1431         struct nand_chip *chip = &host->chip;
1432         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1433         struct mtd_info *mtd = nand_to_mtd(chip);
1434         struct nand_ecc_ctrl *ecc = &chip->ecc;
1435         unsigned int max_bitflips = 0;
1436         struct read_stats *buf;
1437         int i;
1438
1439         buf = (struct read_stats *)nandc->reg_read_buf;
1440         nandc_read_buffer_sync(nandc, true);
1441
1442         for (i = 0; i < ecc->steps; i++, buf++) {
1443                 u32 flash, buffer, erased_cw;
1444                 int data_len, oob_len;
1445
1446                 if (i == (ecc->steps - 1)) {
1447                         data_len = ecc->size - ((ecc->steps - 1) << 2);
1448                         oob_len = ecc->steps << 2;
1449                 } else {
1450                         data_len = host->cw_data;
1451                         oob_len = 0;
1452                 }
1453
1454                 flash = le32_to_cpu(buf->flash);
1455                 buffer = le32_to_cpu(buf->buffer);
1456                 erased_cw = le32_to_cpu(buf->erased_cw);
1457
1458                 if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1459                         bool erased;
1460
1461                         /* ignore erased codeword errors */
1462                         if (host->bch_enabled) {
1463                                 erased = (erased_cw & ERASED_CW) == ERASED_CW ?
1464                                          true : false;
1465                         } else {
1466                                 erased = erased_chunk_check_and_fixup(data_buf,
1467                                                                       data_len);
1468                         }
1469
1470                         if (erased) {
1471                                 data_buf += data_len;
1472                                 if (oob_buf)
1473                                         oob_buf += oob_len + ecc->bytes;
1474                                 continue;
1475                         }
1476
1477                         if (buffer & BS_UNCORRECTABLE_BIT) {
1478                                 int ret, ecclen, extraooblen;
1479                                 void *eccbuf;
1480
1481                                 eccbuf = oob_buf ? oob_buf + oob_len : NULL;
1482                                 ecclen = oob_buf ? host->ecc_bytes_hw : 0;
1483                                 extraooblen = oob_buf ? oob_len : 0;
1484
1485                                 /*
1486                                  * make sure it isn't an erased page reported
1487                                  * as not-erased by HW because of a few bitflips
1488                                  */
1489                                 ret = nand_check_erased_ecc_chunk(data_buf,
1490                                         data_len, eccbuf, ecclen, oob_buf,
1491                                         extraooblen, ecc->strength);
1492                                 if (ret < 0) {
1493                                         mtd->ecc_stats.failed++;
1494                                 } else {
1495                                         mtd->ecc_stats.corrected += ret;
1496                                         max_bitflips =
1497                                                 max_t(unsigned int, max_bitflips, ret);
1498                                 }
1499                         }
1500                 } else {
1501                         unsigned int stat;
1502
1503                         stat = buffer & BS_CORRECTABLE_ERR_MSK;
1504                         mtd->ecc_stats.corrected += stat;
1505                         max_bitflips = max(max_bitflips, stat);
1506                 }
1507
1508                 data_buf += data_len;
1509                 if (oob_buf)
1510                         oob_buf += oob_len + ecc->bytes;
1511         }
1512
1513         return max_bitflips;
1514 }
1515
1516 /*
1517  * helper to perform the actual page read operation, used by ecc->read_page(),
1518  * ecc->read_oob()
1519  */
1520 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1521                          u8 *oob_buf)
1522 {
1523         struct nand_chip *chip = &host->chip;
1524         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1525         struct nand_ecc_ctrl *ecc = &chip->ecc;
1526         int i, ret;
1527
1528         config_nand_page_read(nandc);
1529
1530         /* queue cmd descs for each codeword */
1531         for (i = 0; i < ecc->steps; i++) {
1532                 int data_size, oob_size;
1533
1534                 if (i == (ecc->steps - 1)) {
1535                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1536                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1537                                    host->spare_bytes;
1538                 } else {
1539                         data_size = host->cw_data;
1540                         oob_size = host->ecc_bytes_hw + host->spare_bytes;
1541                 }
1542
1543                 if (nandc->props->is_bam) {
1544                         if (data_buf && oob_buf) {
1545                                 nandc_set_read_loc(nandc, 0, 0, data_size, 0);
1546                                 nandc_set_read_loc(nandc, 1, data_size,
1547                                                    oob_size, 1);
1548                         } else if (data_buf) {
1549                                 nandc_set_read_loc(nandc, 0, 0, data_size, 1);
1550                         } else {
1551                                 nandc_set_read_loc(nandc, 0, data_size,
1552                                                    oob_size, 1);
1553                         }
1554                 }
1555
1556                 config_nand_cw_read(nandc);
1557
1558                 if (data_buf)
1559                         read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1560                                       data_size, 0);
1561
1562                 /*
1563                  * when ecc is enabled, the controller doesn't read the real
1564                  * or dummy bad block markers in each chunk. To maintain a
1565                  * consistent layout across RAW and ECC reads, we just
1566                  * leave the real/dummy BBM offsets empty (i.e, filled with
1567                  * 0xffs)
1568                  */
1569                 if (oob_buf) {
1570                         int j;
1571
1572                         for (j = 0; j < host->bbm_size; j++)
1573                                 *oob_buf++ = 0xff;
1574
1575                         read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1576                                       oob_buf, oob_size, 0);
1577                 }
1578
1579                 if (data_buf)
1580                         data_buf += data_size;
1581                 if (oob_buf)
1582                         oob_buf += oob_size;
1583         }
1584
1585         ret = submit_descs(nandc);
1586         if (ret)
1587                 dev_err(nandc->dev, "failure to read page/oob\n");
1588
1589         free_descs(nandc);
1590
1591         return ret;
1592 }
1593
1594 /*
1595  * a helper that copies the last step/codeword of a page (containing free oob)
1596  * into our local buffer
1597  */
1598 static int copy_last_cw(struct qcom_nand_host *host, int page)
1599 {
1600         struct nand_chip *chip = &host->chip;
1601         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1602         struct nand_ecc_ctrl *ecc = &chip->ecc;
1603         int size;
1604         int ret;
1605
1606         clear_read_regs(nandc);
1607
1608         size = host->use_ecc ? host->cw_data : host->cw_size;
1609
1610         /* prepare a clean read buffer */
1611         memset(nandc->data_buffer, 0xff, size);
1612
1613         set_address(host, host->cw_size * (ecc->steps - 1), page);
1614         update_rw_regs(host, 1, true);
1615
1616         config_nand_single_cw_page_read(nandc);
1617
1618         read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
1619
1620         ret = submit_descs(nandc);
1621         if (ret)
1622                 dev_err(nandc->dev, "failed to copy last codeword\n");
1623
1624         free_descs(nandc);
1625
1626         return ret;
1627 }
1628
1629 /* implements ecc->read_page() */
1630 static int qcom_nandc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
1631                                 uint8_t *buf, int oob_required, int page)
1632 {
1633         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1634         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1635         u8 *data_buf, *oob_buf = NULL;
1636         int ret;
1637
1638         data_buf = buf;
1639         oob_buf = oob_required ? chip->oob_poi : NULL;
1640
1641         clear_bam_transaction(nandc);
1642         ret = read_page_ecc(host, data_buf, oob_buf);
1643         if (ret) {
1644                 dev_err(nandc->dev, "failure to read page\n");
1645                 return ret;
1646         }
1647
1648         return parse_read_errors(host, data_buf, oob_buf);
1649 }
1650
1651 /* implements ecc->read_page_raw() */
1652 static int qcom_nandc_read_page_raw(struct mtd_info *mtd,
1653                                     struct nand_chip *chip, uint8_t *buf,
1654                                     int oob_required, int page)
1655 {
1656         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1657         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1658         u8 *data_buf, *oob_buf;
1659         struct nand_ecc_ctrl *ecc = &chip->ecc;
1660         int i, ret;
1661         int read_loc;
1662
1663         data_buf = buf;
1664         oob_buf = chip->oob_poi;
1665
1666         host->use_ecc = false;
1667
1668         clear_bam_transaction(nandc);
1669         update_rw_regs(host, ecc->steps, true);
1670         config_nand_page_read(nandc);
1671
1672         for (i = 0; i < ecc->steps; i++) {
1673                 int data_size1, data_size2, oob_size1, oob_size2;
1674                 int reg_off = FLASH_BUF_ACC;
1675
1676                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1677                 oob_size1 = host->bbm_size;
1678
1679                 if (i == (ecc->steps - 1)) {
1680                         data_size2 = ecc->size - data_size1 -
1681                                      ((ecc->steps - 1) << 2);
1682                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1683                                     host->spare_bytes;
1684                 } else {
1685                         data_size2 = host->cw_data - data_size1;
1686                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1687                 }
1688
1689                 if (nandc->props->is_bam) {
1690                         read_loc = 0;
1691                         nandc_set_read_loc(nandc, 0, read_loc, data_size1, 0);
1692                         read_loc += data_size1;
1693
1694                         nandc_set_read_loc(nandc, 1, read_loc, oob_size1, 0);
1695                         read_loc += oob_size1;
1696
1697                         nandc_set_read_loc(nandc, 2, read_loc, data_size2, 0);
1698                         read_loc += data_size2;
1699
1700                         nandc_set_read_loc(nandc, 3, read_loc, oob_size2, 1);
1701                 }
1702
1703                 config_nand_cw_read(nandc);
1704
1705                 read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1706                 reg_off += data_size1;
1707                 data_buf += data_size1;
1708
1709                 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1710                 reg_off += oob_size1;
1711                 oob_buf += oob_size1;
1712
1713                 read_data_dma(nandc, reg_off, data_buf, data_size2, 0);
1714                 reg_off += data_size2;
1715                 data_buf += data_size2;
1716
1717                 read_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1718                 oob_buf += oob_size2;
1719         }
1720
1721         ret = submit_descs(nandc);
1722         if (ret)
1723                 dev_err(nandc->dev, "failure to read raw page\n");
1724
1725         free_descs(nandc);
1726
1727         return 0;
1728 }
1729
1730 /* implements ecc->read_oob() */
1731 static int qcom_nandc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1732                                int page)
1733 {
1734         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1735         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1736         struct nand_ecc_ctrl *ecc = &chip->ecc;
1737         int ret;
1738
1739         clear_read_regs(nandc);
1740         clear_bam_transaction(nandc);
1741
1742         host->use_ecc = true;
1743         set_address(host, 0, page);
1744         update_rw_regs(host, ecc->steps, true);
1745
1746         ret = read_page_ecc(host, NULL, chip->oob_poi);
1747         if (ret)
1748                 dev_err(nandc->dev, "failure to read oob\n");
1749
1750         return ret;
1751 }
1752
1753 /* implements ecc->write_page() */
1754 static int qcom_nandc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1755                                  const uint8_t *buf, int oob_required, int page)
1756 {
1757         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1758         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1759         struct nand_ecc_ctrl *ecc = &chip->ecc;
1760         u8 *data_buf, *oob_buf;
1761         int i, ret;
1762
1763         clear_read_regs(nandc);
1764         clear_bam_transaction(nandc);
1765
1766         data_buf = (u8 *)buf;
1767         oob_buf = chip->oob_poi;
1768
1769         host->use_ecc = true;
1770         update_rw_regs(host, ecc->steps, false);
1771         config_nand_page_write(nandc);
1772
1773         for (i = 0; i < ecc->steps; i++) {
1774                 int data_size, oob_size;
1775
1776                 if (i == (ecc->steps - 1)) {
1777                         data_size = ecc->size - ((ecc->steps - 1) << 2);
1778                         oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1779                                    host->spare_bytes;
1780                 } else {
1781                         data_size = host->cw_data;
1782                         oob_size = ecc->bytes;
1783                 }
1784
1785
1786                 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
1787                                i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
1788
1789                 /*
1790                  * when ECC is enabled, we don't really need to write anything
1791                  * to oob for the first n - 1 codewords since these oob regions
1792                  * just contain ECC bytes that's written by the controller
1793                  * itself. For the last codeword, we skip the bbm positions and
1794                  * write to the free oob area.
1795                  */
1796                 if (i == (ecc->steps - 1)) {
1797                         oob_buf += host->bbm_size;
1798
1799                         write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1800                                        oob_buf, oob_size, 0);
1801                 }
1802
1803                 config_nand_cw_write(nandc);
1804
1805                 data_buf += data_size;
1806                 oob_buf += oob_size;
1807         }
1808
1809         ret = submit_descs(nandc);
1810         if (ret)
1811                 dev_err(nandc->dev, "failure to write page\n");
1812
1813         free_descs(nandc);
1814
1815         return ret;
1816 }
1817
1818 /* implements ecc->write_page_raw() */
1819 static int qcom_nandc_write_page_raw(struct mtd_info *mtd,
1820                                      struct nand_chip *chip, const uint8_t *buf,
1821                                      int oob_required, int page)
1822 {
1823         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1824         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1825         struct nand_ecc_ctrl *ecc = &chip->ecc;
1826         u8 *data_buf, *oob_buf;
1827         int i, ret;
1828
1829         clear_read_regs(nandc);
1830         clear_bam_transaction(nandc);
1831
1832         data_buf = (u8 *)buf;
1833         oob_buf = chip->oob_poi;
1834
1835         host->use_ecc = false;
1836         update_rw_regs(host, ecc->steps, false);
1837         config_nand_page_write(nandc);
1838
1839         for (i = 0; i < ecc->steps; i++) {
1840                 int data_size1, data_size2, oob_size1, oob_size2;
1841                 int reg_off = FLASH_BUF_ACC;
1842
1843                 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1844                 oob_size1 = host->bbm_size;
1845
1846                 if (i == (ecc->steps - 1)) {
1847                         data_size2 = ecc->size - data_size1 -
1848                                      ((ecc->steps - 1) << 2);
1849                         oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1850                                     host->spare_bytes;
1851                 } else {
1852                         data_size2 = host->cw_data - data_size1;
1853                         oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1854                 }
1855
1856                 write_data_dma(nandc, reg_off, data_buf, data_size1,
1857                                NAND_BAM_NO_EOT);
1858                 reg_off += data_size1;
1859                 data_buf += data_size1;
1860
1861                 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
1862                                NAND_BAM_NO_EOT);
1863                 reg_off += oob_size1;
1864                 oob_buf += oob_size1;
1865
1866                 write_data_dma(nandc, reg_off, data_buf, data_size2,
1867                                NAND_BAM_NO_EOT);
1868                 reg_off += data_size2;
1869                 data_buf += data_size2;
1870
1871                 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
1872                 oob_buf += oob_size2;
1873
1874                 config_nand_cw_write(nandc);
1875         }
1876
1877         ret = submit_descs(nandc);
1878         if (ret)
1879                 dev_err(nandc->dev, "failure to write raw page\n");
1880
1881         free_descs(nandc);
1882
1883         return ret;
1884 }
1885
1886 /*
1887  * implements ecc->write_oob()
1888  *
1889  * the NAND controller cannot write only data or only oob within a codeword,
1890  * since ecc is calculated for the combined codeword. we first copy the
1891  * entire contents for the last codeword(data + oob), replace the old oob
1892  * with the new one in chip->oob_poi, and then write the entire codeword.
1893  * this read-copy-write operation results in a slight performance loss.
1894  */
1895 static int qcom_nandc_write_oob(struct mtd_info *mtd, struct nand_chip *chip,
1896                                 int page)
1897 {
1898         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1899         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1900         struct nand_ecc_ctrl *ecc = &chip->ecc;
1901         u8 *oob = chip->oob_poi;
1902         int data_size, oob_size;
1903         int ret, status = 0;
1904
1905         host->use_ecc = true;
1906
1907         clear_bam_transaction(nandc);
1908         ret = copy_last_cw(host, page);
1909         if (ret)
1910                 return ret;
1911
1912         clear_read_regs(nandc);
1913         clear_bam_transaction(nandc);
1914
1915         /* calculate the data and oob size for the last codeword/step */
1916         data_size = ecc->size - ((ecc->steps - 1) << 2);
1917         oob_size = mtd->oobavail;
1918
1919         /* override new oob content to last codeword */
1920         mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
1921                                     0, mtd->oobavail);
1922
1923         set_address(host, host->cw_size * (ecc->steps - 1), page);
1924         update_rw_regs(host, 1, false);
1925
1926         config_nand_page_write(nandc);
1927         write_data_dma(nandc, FLASH_BUF_ACC,
1928                        nandc->data_buffer, data_size + oob_size, 0);
1929         config_nand_cw_write(nandc);
1930
1931         ret = submit_descs(nandc);
1932
1933         free_descs(nandc);
1934
1935         if (ret) {
1936                 dev_err(nandc->dev, "failure to write oob\n");
1937                 return -EIO;
1938         }
1939
1940         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1941
1942         status = chip->waitfunc(mtd, chip);
1943
1944         return status & NAND_STATUS_FAIL ? -EIO : 0;
1945 }
1946
1947 static int qcom_nandc_block_bad(struct mtd_info *mtd, loff_t ofs)
1948 {
1949         struct nand_chip *chip = mtd_to_nand(mtd);
1950         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1951         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1952         struct nand_ecc_ctrl *ecc = &chip->ecc;
1953         int page, ret, bbpos, bad = 0;
1954         u32 flash_status;
1955
1956         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1957
1958         /*
1959          * configure registers for a raw sub page read, the address is set to
1960          * the beginning of the last codeword, we don't care about reading ecc
1961          * portion of oob. we just want the first few bytes from this codeword
1962          * that contains the BBM
1963          */
1964         host->use_ecc = false;
1965
1966         clear_bam_transaction(nandc);
1967         ret = copy_last_cw(host, page);
1968         if (ret)
1969                 goto err;
1970
1971         flash_status = le32_to_cpu(nandc->reg_read_buf[0]);
1972
1973         if (flash_status & (FS_OP_ERR | FS_MPU_ERR)) {
1974                 dev_warn(nandc->dev, "error when trying to read BBM\n");
1975                 goto err;
1976         }
1977
1978         bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1979
1980         bad = nandc->data_buffer[bbpos] != 0xff;
1981
1982         if (chip->options & NAND_BUSWIDTH_16)
1983                 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1984 err:
1985         return bad;
1986 }
1987
1988 static int qcom_nandc_block_markbad(struct mtd_info *mtd, loff_t ofs)
1989 {
1990         struct nand_chip *chip = mtd_to_nand(mtd);
1991         struct qcom_nand_host *host = to_qcom_nand_host(chip);
1992         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1993         struct nand_ecc_ctrl *ecc = &chip->ecc;
1994         int page, ret, status = 0;
1995
1996         clear_read_regs(nandc);
1997         clear_bam_transaction(nandc);
1998
1999         /*
2000          * to mark the BBM as bad, we flash the entire last codeword with 0s.
2001          * we don't care about the rest of the content in the codeword since
2002          * we aren't going to use this block again
2003          */
2004         memset(nandc->data_buffer, 0x00, host->cw_size);
2005
2006         page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2007
2008         /* prepare write */
2009         host->use_ecc = false;
2010         set_address(host, host->cw_size * (ecc->steps - 1), page);
2011         update_rw_regs(host, 1, false);
2012
2013         config_nand_page_write(nandc);
2014         write_data_dma(nandc, FLASH_BUF_ACC,
2015                        nandc->data_buffer, host->cw_size, 0);
2016         config_nand_cw_write(nandc);
2017
2018         ret = submit_descs(nandc);
2019
2020         free_descs(nandc);
2021
2022         if (ret) {
2023                 dev_err(nandc->dev, "failure to update BBM\n");
2024                 return -EIO;
2025         }
2026
2027         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
2028
2029         status = chip->waitfunc(mtd, chip);
2030
2031         return status & NAND_STATUS_FAIL ? -EIO : 0;
2032 }
2033
2034 /*
2035  * the three functions below implement chip->read_byte(), chip->read_buf()
2036  * and chip->write_buf() respectively. these aren't used for
2037  * reading/writing page data, they are used for smaller data like reading
2038  * id, status etc
2039  */
2040 static uint8_t qcom_nandc_read_byte(struct mtd_info *mtd)
2041 {
2042         struct nand_chip *chip = mtd_to_nand(mtd);
2043         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2044         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2045         u8 *buf = nandc->data_buffer;
2046         u8 ret = 0x0;
2047
2048         if (host->last_command == NAND_CMD_STATUS) {
2049                 ret = host->status;
2050
2051                 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2052
2053                 return ret;
2054         }
2055
2056         if (nandc->buf_start < nandc->buf_count)
2057                 ret = buf[nandc->buf_start++];
2058
2059         return ret;
2060 }
2061
2062 static void qcom_nandc_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
2063 {
2064         struct nand_chip *chip = mtd_to_nand(mtd);
2065         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2066         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2067
2068         memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2069         nandc->buf_start += real_len;
2070 }
2071
2072 static void qcom_nandc_write_buf(struct mtd_info *mtd, const uint8_t *buf,
2073                                  int len)
2074 {
2075         struct nand_chip *chip = mtd_to_nand(mtd);
2076         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2077         int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2078
2079         memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2080
2081         nandc->buf_start += real_len;
2082 }
2083
2084 /* we support only one external chip for now */
2085 static void qcom_nandc_select_chip(struct mtd_info *mtd, int chipnr)
2086 {
2087         struct nand_chip *chip = mtd_to_nand(mtd);
2088         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2089
2090         if (chipnr <= 0)
2091                 return;
2092
2093         dev_warn(nandc->dev, "invalid chip select\n");
2094 }
2095
2096 /*
2097  * NAND controller page layout info
2098  *
2099  * Layout with ECC enabled:
2100  *
2101  * |----------------------|  |---------------------------------|
2102  * |           xx.......yy|  |             *********xx.......yy|
2103  * |    DATA   xx..ECC..yy|  |    DATA     **SPARE**xx..ECC..yy|
2104  * |   (516)   xx.......yy|  |  (516-n*4)  **(n*4)**xx.......yy|
2105  * |           xx.......yy|  |             *********xx.......yy|
2106  * |----------------------|  |---------------------------------|
2107  *     codeword 1,2..n-1                  codeword n
2108  *  <---(528/532 Bytes)-->    <-------(528/532 Bytes)--------->
2109  *
2110  * n = Number of codewords in the page
2111  * . = ECC bytes
2112  * * = Spare/free bytes
2113  * x = Unused byte(s)
2114  * y = Reserved byte(s)
2115  *
2116  * 2K page: n = 4, spare = 16 bytes
2117  * 4K page: n = 8, spare = 32 bytes
2118  * 8K page: n = 16, spare = 64 bytes
2119  *
2120  * the qcom nand controller operates at a sub page/codeword level. each
2121  * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2122  * the number of ECC bytes vary based on the ECC strength and the bus width.
2123  *
2124  * the first n - 1 codewords contains 516 bytes of user data, the remaining
2125  * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2126  * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2127  *
2128  * When we access a page with ECC enabled, the reserved bytes(s) are not
2129  * accessible at all. When reading, we fill up these unreadable positions
2130  * with 0xffs. When writing, the controller skips writing the inaccessible
2131  * bytes.
2132  *
2133  * Layout with ECC disabled:
2134  *
2135  * |------------------------------|  |---------------------------------------|
2136  * |         yy          xx.......|  |         bb          *********xx.......|
2137  * |  DATA1  yy  DATA2   xx..ECC..|  |  DATA1  bb  DATA2   **SPARE**xx..ECC..|
2138  * | (size1) yy (size2)  xx.......|  | (size1) bb (size2)  **(n*4)**xx.......|
2139  * |         yy          xx.......|  |         bb          *********xx.......|
2140  * |------------------------------|  |---------------------------------------|
2141  *         codeword 1,2..n-1                        codeword n
2142  *  <-------(528/532 Bytes)------>    <-----------(528/532 Bytes)----------->
2143  *
2144  * n = Number of codewords in the page
2145  * . = ECC bytes
2146  * * = Spare/free bytes
2147  * x = Unused byte(s)
2148  * y = Dummy Bad Bock byte(s)
2149  * b = Real Bad Block byte(s)
2150  * size1/size2 = function of codeword size and 'n'
2151  *
2152  * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2153  * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2154  * Block Markers. In the last codeword, this position contains the real BBM
2155  *
2156  * In order to have a consistent layout between RAW and ECC modes, we assume
2157  * the following OOB layout arrangement:
2158  *
2159  * |-----------|  |--------------------|
2160  * |yyxx.......|  |bb*********xx.......|
2161  * |yyxx..ECC..|  |bb*FREEOOB*xx..ECC..|
2162  * |yyxx.......|  |bb*********xx.......|
2163  * |yyxx.......|  |bb*********xx.......|
2164  * |-----------|  |--------------------|
2165  *  first n - 1       nth OOB region
2166  *  OOB regions
2167  *
2168  * n = Number of codewords in the page
2169  * . = ECC bytes
2170  * * = FREE OOB bytes
2171  * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2172  * x = Unused byte(s)
2173  * b = Real bad block byte(s) (inaccessible when ECC enabled)
2174  *
2175  * This layout is read as is when ECC is disabled. When ECC is enabled, the
2176  * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2177  * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2178  * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2179  * the sum of the three).
2180  */
2181 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2182                                    struct mtd_oob_region *oobregion)
2183 {
2184         struct nand_chip *chip = mtd_to_nand(mtd);
2185         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2186         struct nand_ecc_ctrl *ecc = &chip->ecc;
2187
2188         if (section > 1)
2189                 return -ERANGE;
2190
2191         if (!section) {
2192                 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2193                                     host->bbm_size;
2194                 oobregion->offset = 0;
2195         } else {
2196                 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2197                 oobregion->offset = mtd->oobsize - oobregion->length;
2198         }
2199
2200         return 0;
2201 }
2202
2203 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2204                                      struct mtd_oob_region *oobregion)
2205 {
2206         struct nand_chip *chip = mtd_to_nand(mtd);
2207         struct qcom_nand_host *host = to_qcom_nand_host(chip);
2208         struct nand_ecc_ctrl *ecc = &chip->ecc;
2209
2210         if (section)
2211                 return -ERANGE;
2212
2213         oobregion->length = ecc->steps * 4;
2214         oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2215
2216         return 0;
2217 }
2218
2219 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2220         .ecc = qcom_nand_ooblayout_ecc,
2221         .free = qcom_nand_ooblayout_free,
2222 };
2223
2224 static int qcom_nand_host_setup(struct qcom_nand_host *host)
2225 {
2226         struct nand_chip *chip = &host->chip;
2227         struct mtd_info *mtd = nand_to_mtd(chip);
2228         struct nand_ecc_ctrl *ecc = &chip->ecc;
2229         struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2230         int cwperpage, bad_block_byte;
2231         bool wide_bus;
2232         int ecc_mode = 1;
2233
2234         /*
2235          * the controller requires each step consists of 512 bytes of data.
2236          * bail out if DT has populated a wrong step size.
2237          */
2238         if (ecc->size != NANDC_STEP_SIZE) {
2239                 dev_err(nandc->dev, "invalid ecc size\n");
2240                 return -EINVAL;
2241         }
2242
2243         wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2244
2245         if (ecc->strength >= 8) {
2246                 /* 8 bit ECC defaults to BCH ECC on all platforms */
2247                 host->bch_enabled = true;
2248                 ecc_mode = 1;
2249
2250                 if (wide_bus) {
2251                         host->ecc_bytes_hw = 14;
2252                         host->spare_bytes = 0;
2253                         host->bbm_size = 2;
2254                 } else {
2255                         host->ecc_bytes_hw = 13;
2256                         host->spare_bytes = 2;
2257                         host->bbm_size = 1;
2258                 }
2259         } else {
2260                 /*
2261                  * if the controller supports BCH for 4 bit ECC, the controller
2262                  * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2263                  * always 10 bytes
2264                  */
2265                 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2266                         /* BCH */
2267                         host->bch_enabled = true;
2268                         ecc_mode = 0;
2269
2270                         if (wide_bus) {
2271                                 host->ecc_bytes_hw = 8;
2272                                 host->spare_bytes = 2;
2273                                 host->bbm_size = 2;
2274                         } else {
2275                                 host->ecc_bytes_hw = 7;
2276                                 host->spare_bytes = 4;
2277                                 host->bbm_size = 1;
2278                         }
2279                 } else {
2280                         /* RS */
2281                         host->ecc_bytes_hw = 10;
2282
2283                         if (wide_bus) {
2284                                 host->spare_bytes = 0;
2285                                 host->bbm_size = 2;
2286                         } else {
2287                                 host->spare_bytes = 1;
2288                                 host->bbm_size = 1;
2289                         }
2290                 }
2291         }
2292
2293         /*
2294          * we consider ecc->bytes as the sum of all the non-data content in a
2295          * step. It gives us a clean representation of the oob area (even if
2296          * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2297          * ECC and 12 bytes for 4 bit ECC
2298          */
2299         ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2300
2301         ecc->read_page          = qcom_nandc_read_page;
2302         ecc->read_page_raw      = qcom_nandc_read_page_raw;
2303         ecc->read_oob           = qcom_nandc_read_oob;
2304         ecc->write_page         = qcom_nandc_write_page;
2305         ecc->write_page_raw     = qcom_nandc_write_page_raw;
2306         ecc->write_oob          = qcom_nandc_write_oob;
2307
2308         ecc->mode = NAND_ECC_HW;
2309
2310         mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2311
2312         cwperpage = mtd->writesize / ecc->size;
2313         nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2314                                      cwperpage);
2315
2316         /*
2317          * DATA_UD_BYTES varies based on whether the read/write command protects
2318          * spare data with ECC too. We protect spare data by default, so we set
2319          * it to main + spare data, which are 512 and 4 bytes respectively.
2320          */
2321         host->cw_data = 516;
2322
2323         /*
2324          * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2325          * for 8 bit ECC
2326          */
2327         host->cw_size = host->cw_data + ecc->bytes;
2328
2329         if (ecc->bytes * (mtd->writesize / ecc->size) > mtd->oobsize) {
2330                 dev_err(nandc->dev, "ecc data doesn't fit in OOB area\n");
2331                 return -EINVAL;
2332         }
2333
2334         bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2335
2336         host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2337                                 | host->cw_data << UD_SIZE_BYTES
2338                                 | 0 << DISABLE_STATUS_AFTER_WRITE
2339                                 | 5 << NUM_ADDR_CYCLES
2340                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2341                                 | 0 << STATUS_BFR_READ
2342                                 | 1 << SET_RD_MODE_AFTER_STATUS
2343                                 | host->spare_bytes << SPARE_SIZE_BYTES;
2344
2345         host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2346                                 | 0 <<  CS_ACTIVE_BSY
2347                                 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2348                                 | 0 << BAD_BLOCK_IN_SPARE_AREA
2349                                 | 2 << WR_RD_BSY_GAP
2350                                 | wide_bus << WIDE_FLASH
2351                                 | host->bch_enabled << ENABLE_BCH_ECC;
2352
2353         host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2354                                 | host->cw_size << UD_SIZE_BYTES
2355                                 | 5 << NUM_ADDR_CYCLES
2356                                 | 0 << SPARE_SIZE_BYTES;
2357
2358         host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2359                                 | 0 << CS_ACTIVE_BSY
2360                                 | 17 << BAD_BLOCK_BYTE_NUM
2361                                 | 1 << BAD_BLOCK_IN_SPARE_AREA
2362                                 | 2 << WR_RD_BSY_GAP
2363                                 | wide_bus << WIDE_FLASH
2364                                 | 1 << DEV0_CFG1_ECC_DISABLE;
2365
2366         host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2367                                 | 0 << ECC_SW_RESET
2368                                 | host->cw_data << ECC_NUM_DATA_BYTES
2369                                 | 1 << ECC_FORCE_CLK_OPEN
2370                                 | ecc_mode << ECC_MODE
2371                                 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2372
2373         host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2374
2375         host->clrflashstatus = FS_READY_BSY_N;
2376         host->clrreadstatus = 0xc0;
2377         nandc->regs->erased_cw_detect_cfg_clr =
2378                 cpu_to_le32(CLR_ERASED_PAGE_DET);
2379         nandc->regs->erased_cw_detect_cfg_set =
2380                 cpu_to_le32(SET_ERASED_PAGE_DET);
2381
2382         dev_dbg(nandc->dev,
2383                 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2384                 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2385                 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2386                 cwperpage);
2387
2388         return 0;
2389 }
2390
2391 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2392 {
2393         int ret;
2394
2395         ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2396         if (ret) {
2397                 dev_err(nandc->dev, "failed to set DMA mask\n");
2398                 return ret;
2399         }
2400
2401         /*
2402          * we use the internal buffer for reading ONFI params, reading small
2403          * data like ID and status, and preforming read-copy-write operations
2404          * when writing to a codeword partially. 532 is the maximum possible
2405          * size of a codeword for our nand controller
2406          */
2407         nandc->buf_size = 532;
2408
2409         nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2410                                         GFP_KERNEL);
2411         if (!nandc->data_buffer)
2412                 return -ENOMEM;
2413
2414         nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2415                                         GFP_KERNEL);
2416         if (!nandc->regs)
2417                 return -ENOMEM;
2418
2419         nandc->reg_read_buf = devm_kzalloc(nandc->dev,
2420                                 MAX_REG_RD * sizeof(*nandc->reg_read_buf),
2421                                 GFP_KERNEL);
2422         if (!nandc->reg_read_buf)
2423                 return -ENOMEM;
2424
2425         if (nandc->props->is_bam) {
2426                 nandc->reg_read_dma =
2427                         dma_map_single(nandc->dev, nandc->reg_read_buf,
2428                                        MAX_REG_RD *
2429                                        sizeof(*nandc->reg_read_buf),
2430                                        DMA_FROM_DEVICE);
2431                 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2432                         dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2433                         return -EIO;
2434                 }
2435
2436                 nandc->tx_chan = dma_request_slave_channel(nandc->dev, "tx");
2437                 if (!nandc->tx_chan) {
2438                         dev_err(nandc->dev, "failed to request tx channel\n");
2439                         return -ENODEV;
2440                 }
2441
2442                 nandc->rx_chan = dma_request_slave_channel(nandc->dev, "rx");
2443                 if (!nandc->rx_chan) {
2444                         dev_err(nandc->dev, "failed to request rx channel\n");
2445                         return -ENODEV;
2446                 }
2447
2448                 nandc->cmd_chan = dma_request_slave_channel(nandc->dev, "cmd");
2449                 if (!nandc->cmd_chan) {
2450                         dev_err(nandc->dev, "failed to request cmd channel\n");
2451                         return -ENODEV;
2452                 }
2453
2454                 /*
2455                  * Initially allocate BAM transaction to read ONFI param page.
2456                  * After detecting all the devices, this BAM transaction will
2457                  * be freed and the next BAM tranasction will be allocated with
2458                  * maximum codeword size
2459                  */
2460                 nandc->max_cwperpage = 1;
2461                 nandc->bam_txn = alloc_bam_transaction(nandc);
2462                 if (!nandc->bam_txn) {
2463                         dev_err(nandc->dev,
2464                                 "failed to allocate bam transaction\n");
2465                         return -ENOMEM;
2466                 }
2467         } else {
2468                 nandc->chan = dma_request_slave_channel(nandc->dev, "rxtx");
2469                 if (!nandc->chan) {
2470                         dev_err(nandc->dev,
2471                                 "failed to request slave channel\n");
2472                         return -ENODEV;
2473                 }
2474         }
2475
2476         INIT_LIST_HEAD(&nandc->desc_list);
2477         INIT_LIST_HEAD(&nandc->host_list);
2478
2479         nand_hw_control_init(&nandc->controller);
2480
2481         return 0;
2482 }
2483
2484 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2485 {
2486         if (nandc->props->is_bam) {
2487                 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2488                         dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2489                                          MAX_REG_RD *
2490                                          sizeof(*nandc->reg_read_buf),
2491                                          DMA_FROM_DEVICE);
2492
2493                 if (nandc->tx_chan)
2494                         dma_release_channel(nandc->tx_chan);
2495
2496                 if (nandc->rx_chan)
2497                         dma_release_channel(nandc->rx_chan);
2498
2499                 if (nandc->cmd_chan)
2500                         dma_release_channel(nandc->cmd_chan);
2501         } else {
2502                 if (nandc->chan)
2503                         dma_release_channel(nandc->chan);
2504         }
2505 }
2506
2507 /* one time setup of a few nand controller registers */
2508 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2509 {
2510         u32 nand_ctrl;
2511
2512         /* kill onenand */
2513         if (!nandc->props->is_qpic)
2514                 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2515         nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2516                     NAND_DEV_CMD_VLD_VAL);
2517
2518         /* enable ADM or BAM DMA */
2519         if (nandc->props->is_bam) {
2520                 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2521                 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2522         } else {
2523                 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2524         }
2525
2526         /* save the original values of these registers */
2527         nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2528         nandc->vld = NAND_DEV_CMD_VLD_VAL;
2529
2530         return 0;
2531 }
2532
2533 static int qcom_nand_host_init(struct qcom_nand_controller *nandc,
2534                                struct qcom_nand_host *host,
2535                                struct device_node *dn)
2536 {
2537         struct nand_chip *chip = &host->chip;
2538         struct mtd_info *mtd = nand_to_mtd(chip);
2539         struct device *dev = nandc->dev;
2540         int ret;
2541
2542         ret = of_property_read_u32(dn, "reg", &host->cs);
2543         if (ret) {
2544                 dev_err(dev, "can't get chip-select\n");
2545                 return -ENXIO;
2546         }
2547
2548         nand_set_flash_node(chip, dn);
2549         mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2550         if (!mtd->name)
2551                 return -ENOMEM;
2552
2553         mtd->owner = THIS_MODULE;
2554         mtd->dev.parent = dev;
2555
2556         chip->cmdfunc           = qcom_nandc_command;
2557         chip->select_chip       = qcom_nandc_select_chip;
2558         chip->read_byte         = qcom_nandc_read_byte;
2559         chip->read_buf          = qcom_nandc_read_buf;
2560         chip->write_buf         = qcom_nandc_write_buf;
2561         chip->onfi_set_features = nand_onfi_get_set_features_notsupp;
2562         chip->onfi_get_features = nand_onfi_get_set_features_notsupp;
2563
2564         /*
2565          * the bad block marker is readable only when we read the last codeword
2566          * of a page with ECC disabled. currently, the nand_base and nand_bbt
2567          * helpers don't allow us to read BB from a nand chip with ECC
2568          * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2569          * and block_markbad helpers until we permanently switch to using
2570          * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2571          */
2572         chip->block_bad         = qcom_nandc_block_bad;
2573         chip->block_markbad     = qcom_nandc_block_markbad;
2574
2575         chip->controller = &nandc->controller;
2576         chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USE_BOUNCE_BUFFER |
2577                          NAND_SKIP_BBTSCAN;
2578
2579         /* set up initial status value */
2580         host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2581
2582         ret = nand_scan_ident(mtd, 1, NULL);
2583         if (ret)
2584                 return ret;
2585
2586         ret = qcom_nand_host_setup(host);
2587
2588         return ret;
2589 }
2590
2591 static int qcom_nand_mtd_register(struct qcom_nand_controller *nandc,
2592                                   struct qcom_nand_host *host,
2593                                   struct device_node *dn)
2594 {
2595         struct nand_chip *chip = &host->chip;
2596         struct mtd_info *mtd = nand_to_mtd(chip);
2597         int ret;
2598
2599         ret = nand_scan_tail(mtd);
2600         if (ret)
2601                 return ret;
2602
2603         ret = mtd_device_register(mtd, NULL, 0);
2604         if (ret)
2605                 nand_cleanup(mtd_to_nand(mtd));
2606
2607         return ret;
2608 }
2609
2610 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2611 {
2612         struct device *dev = nandc->dev;
2613         struct device_node *dn = dev->of_node, *child;
2614         struct qcom_nand_host *host, *tmp;
2615         int ret;
2616
2617         for_each_available_child_of_node(dn, child) {
2618                 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2619                 if (!host) {
2620                         of_node_put(child);
2621                         return -ENOMEM;
2622                 }
2623
2624                 ret = qcom_nand_host_init(nandc, host, child);
2625                 if (ret) {
2626                         devm_kfree(dev, host);
2627                         continue;
2628                 }
2629
2630                 list_add_tail(&host->node, &nandc->host_list);
2631         }
2632
2633         if (list_empty(&nandc->host_list))
2634                 return -ENODEV;
2635
2636         if (nandc->props->is_bam) {
2637                 free_bam_transaction(nandc);
2638                 nandc->bam_txn = alloc_bam_transaction(nandc);
2639                 if (!nandc->bam_txn) {
2640                         dev_err(nandc->dev,
2641                                 "failed to allocate bam transaction\n");
2642                         return -ENOMEM;
2643                 }
2644         }
2645
2646         list_for_each_entry_safe(host, tmp, &nandc->host_list, node) {
2647                 ret = qcom_nand_mtd_register(nandc, host, child);
2648                 if (ret) {
2649                         list_del(&host->node);
2650                         devm_kfree(dev, host);
2651                 }
2652         }
2653
2654         if (list_empty(&nandc->host_list))
2655                 return -ENODEV;
2656
2657         return 0;
2658 }
2659
2660 /* parse custom DT properties here */
2661 static int qcom_nandc_parse_dt(struct platform_device *pdev)
2662 {
2663         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2664         struct device_node *np = nandc->dev->of_node;
2665         int ret;
2666
2667         if (!nandc->props->is_bam) {
2668                 ret = of_property_read_u32(np, "qcom,cmd-crci",
2669                                            &nandc->cmd_crci);
2670                 if (ret) {
2671                         dev_err(nandc->dev, "command CRCI unspecified\n");
2672                         return ret;
2673                 }
2674
2675                 ret = of_property_read_u32(np, "qcom,data-crci",
2676                                            &nandc->data_crci);
2677                 if (ret) {
2678                         dev_err(nandc->dev, "data CRCI unspecified\n");
2679                         return ret;
2680                 }
2681         }
2682
2683         return 0;
2684 }
2685
2686 static int qcom_nandc_probe(struct platform_device *pdev)
2687 {
2688         struct qcom_nand_controller *nandc;
2689         const void *dev_data;
2690         struct device *dev = &pdev->dev;
2691         struct resource *res;
2692         int ret;
2693
2694         nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
2695         if (!nandc)
2696                 return -ENOMEM;
2697
2698         platform_set_drvdata(pdev, nandc);
2699         nandc->dev = dev;
2700
2701         dev_data = of_device_get_match_data(dev);
2702         if (!dev_data) {
2703                 dev_err(&pdev->dev, "failed to get device data\n");
2704                 return -ENODEV;
2705         }
2706
2707         nandc->props = dev_data;
2708
2709         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2710         nandc->base = devm_ioremap_resource(dev, res);
2711         if (IS_ERR(nandc->base))
2712                 return PTR_ERR(nandc->base);
2713
2714         nandc->base_dma = phys_to_dma(dev, (phys_addr_t)res->start);
2715
2716         nandc->core_clk = devm_clk_get(dev, "core");
2717         if (IS_ERR(nandc->core_clk))
2718                 return PTR_ERR(nandc->core_clk);
2719
2720         nandc->aon_clk = devm_clk_get(dev, "aon");
2721         if (IS_ERR(nandc->aon_clk))
2722                 return PTR_ERR(nandc->aon_clk);
2723
2724         ret = qcom_nandc_parse_dt(pdev);
2725         if (ret)
2726                 return ret;
2727
2728         ret = qcom_nandc_alloc(nandc);
2729         if (ret)
2730                 goto err_core_clk;
2731
2732         ret = clk_prepare_enable(nandc->core_clk);
2733         if (ret)
2734                 goto err_core_clk;
2735
2736         ret = clk_prepare_enable(nandc->aon_clk);
2737         if (ret)
2738                 goto err_aon_clk;
2739
2740         ret = qcom_nandc_setup(nandc);
2741         if (ret)
2742                 goto err_setup;
2743
2744         ret = qcom_probe_nand_devices(nandc);
2745         if (ret)
2746                 goto err_setup;
2747
2748         return 0;
2749
2750 err_setup:
2751         clk_disable_unprepare(nandc->aon_clk);
2752 err_aon_clk:
2753         clk_disable_unprepare(nandc->core_clk);
2754 err_core_clk:
2755         qcom_nandc_unalloc(nandc);
2756
2757         return ret;
2758 }
2759
2760 static int qcom_nandc_remove(struct platform_device *pdev)
2761 {
2762         struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2763         struct qcom_nand_host *host;
2764
2765         list_for_each_entry(host, &nandc->host_list, node)
2766                 nand_release(&host->chip);
2767
2768         qcom_nandc_unalloc(nandc);
2769
2770         clk_disable_unprepare(nandc->aon_clk);
2771         clk_disable_unprepare(nandc->core_clk);
2772
2773         return 0;
2774 }
2775
2776 static const struct qcom_nandc_props ipq806x_nandc_props = {
2777         .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
2778         .is_bam = false,
2779         .dev_cmd_reg_start = 0x0,
2780 };
2781
2782 static const struct qcom_nandc_props ipq4019_nandc_props = {
2783         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2784         .is_bam = true,
2785         .is_qpic = true,
2786         .dev_cmd_reg_start = 0x0,
2787 };
2788
2789 static const struct qcom_nandc_props ipq8074_nandc_props = {
2790         .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2791         .is_bam = true,
2792         .is_qpic = true,
2793         .dev_cmd_reg_start = 0x7000,
2794 };
2795
2796 /*
2797  * data will hold a struct pointer containing more differences once we support
2798  * more controller variants
2799  */
2800 static const struct of_device_id qcom_nandc_of_match[] = {
2801         {
2802                 .compatible = "qcom,ipq806x-nand",
2803                 .data = &ipq806x_nandc_props,
2804         },
2805         {
2806                 .compatible = "qcom,ipq4019-nand",
2807                 .data = &ipq4019_nandc_props,
2808         },
2809         {
2810                 .compatible = "qcom,ipq8074-nand",
2811                 .data = &ipq8074_nandc_props,
2812         },
2813         {}
2814 };
2815 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2816
2817 static struct platform_driver qcom_nandc_driver = {
2818         .driver = {
2819                 .name = "qcom-nandc",
2820                 .of_match_table = qcom_nandc_of_match,
2821         },
2822         .probe   = qcom_nandc_probe,
2823         .remove  = qcom_nandc_remove,
2824 };
2825 module_platform_driver(qcom_nandc_driver);
2826
2827 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2828 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2829 MODULE_LICENSE("GPL v2");