drivers/tty/serial/msm_serial.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Driver for msm7k serial device and console
   4  *
   5  * Copyright (C) 2007 Google, Inc.
   6  * Author: Robert Love <rlove@google.com>
   7  * Copyright (c) 2011, Code Aurora Forum. All rights reserved.
   8  */
   9
  10 #if defined(CONFIG_SERIAL_MSM_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
  11 # define SUPPORT_SYSRQ
  12 #endif
  13
  14 #include <linux/kernel.h>
  15 #include <linux/atomic.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/dmaengine.h>
  18 #include <linux/module.h>
  19 #include <linux/io.h>
  20 #include <linux/ioport.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/init.h>
  23 #include <linux/console.h>
  24 #include <linux/tty.h>
  25 #include <linux/tty_flip.h>
  26 #include <linux/serial_core.h>
  27 #include <linux/slab.h>
  28 #include <linux/clk.h>
  29 #include <linux/platform_device.h>
  30 #include <linux/delay.h>
  31 #include <linux/of.h>
  32 #include <linux/of_device.h>
  33 #include <linux/wait.h>
  34
  35 #define UART_MR1                        0x0000
  36
  37 #define UART_MR1_AUTO_RFR_LEVEL0        0x3F
  38 #define UART_MR1_AUTO_RFR_LEVEL1        0x3FF00
  39 #define UART_DM_MR1_AUTO_RFR_LEVEL1     0xFFFFFF00
  40 #define UART_MR1_RX_RDY_CTL             BIT(7)
  41 #define UART_MR1_CTS_CTL                BIT(6)
  42
  43 #define UART_MR2                        0x0004
  44 #define UART_MR2_ERROR_MODE             BIT(6)
  45 #define UART_MR2_BITS_PER_CHAR          0x30
  46 #define UART_MR2_BITS_PER_CHAR_5        (0x0 << 4)
  47 #define UART_MR2_BITS_PER_CHAR_6        (0x1 << 4)
  48 #define UART_MR2_BITS_PER_CHAR_7        (0x2 << 4)
  49 #define UART_MR2_BITS_PER_CHAR_8        (0x3 << 4)
  50 #define UART_MR2_STOP_BIT_LEN_ONE       (0x1 << 2)
  51 #define UART_MR2_STOP_BIT_LEN_TWO       (0x3 << 2)
  52 #define UART_MR2_PARITY_MODE_NONE       0x0
  53 #define UART_MR2_PARITY_MODE_ODD        0x1
  54 #define UART_MR2_PARITY_MODE_EVEN       0x2
  55 #define UART_MR2_PARITY_MODE_SPACE      0x3
  56 #define UART_MR2_PARITY_MODE            0x3
  57
  58 #define UART_CSR                        0x0008
  59
  60 #define UART_TF                         0x000C
  61 #define UARTDM_TF                       0x0070
  62
  63 #define UART_CR                         0x0010
  64 #define UART_CR_CMD_NULL                (0 << 4)
  65 #define UART_CR_CMD_RESET_RX            (1 << 4)
  66 #define UART_CR_CMD_RESET_TX            (2 << 4)
  67 #define UART_CR_CMD_RESET_ERR           (3 << 4)
  68 #define UART_CR_CMD_RESET_BREAK_INT     (4 << 4)
  69 #define UART_CR_CMD_START_BREAK         (5 << 4)
  70 #define UART_CR_CMD_STOP_BREAK          (6 << 4)
  71 #define UART_CR_CMD_RESET_CTS           (7 << 4)
  72 #define UART_CR_CMD_RESET_STALE_INT     (8 << 4)
  73 #define UART_CR_CMD_PACKET_MODE         (9 << 4)
  74 #define UART_CR_CMD_MODE_RESET          (12 << 4)
  75 #define UART_CR_CMD_SET_RFR             (13 << 4)
  76 #define UART_CR_CMD_RESET_RFR           (14 << 4)
  77 #define UART_CR_CMD_PROTECTION_EN       (16 << 4)
  78 #define UART_CR_CMD_STALE_EVENT_DISABLE (6 << 8)
  79 #define UART_CR_CMD_STALE_EVENT_ENABLE  (80 << 4)
  80 #define UART_CR_CMD_FORCE_STALE         (4 << 8)
  81 #define UART_CR_CMD_RESET_TX_READY      (3 << 8)
  82 #define UART_CR_TX_DISABLE              BIT(3)
  83 #define UART_CR_TX_ENABLE               BIT(2)
  84 #define UART_CR_RX_DISABLE              BIT(1)
  85 #define UART_CR_RX_ENABLE               BIT(0)
  86 #define UART_CR_CMD_RESET_RXBREAK_START ((1 << 11) | (2 << 4))
  87
  88 #define UART_IMR                        0x0014
  89 #define UART_IMR_TXLEV                  BIT(0)
  90 #define UART_IMR_RXSTALE                BIT(3)
  91 #define UART_IMR_RXLEV                  BIT(4)
  92 #define UART_IMR_DELTA_CTS              BIT(5)
  93 #define UART_IMR_CURRENT_CTS            BIT(6)
  94 #define UART_IMR_RXBREAK_START          BIT(10)
  95
  96 #define UART_IPR_RXSTALE_LAST           0x20
  97 #define UART_IPR_STALE_LSB              0x1F
  98 #define UART_IPR_STALE_TIMEOUT_MSB      0x3FF80
  99 #define UART_DM_IPR_STALE_TIMEOUT_MSB   0xFFFFFF80
 100
 101 #define UART_IPR                        0x0018
 102 #define UART_TFWR                       0x001C
 103 #define UART_RFWR                       0x0020
 104 #define UART_HCR                        0x0024
 105
 106 #define UART_MREG                       0x0028
 107 #define UART_NREG                       0x002C
 108 #define UART_DREG                       0x0030
 109 #define UART_MNDREG                     0x0034
 110 #define UART_IRDA                       0x0038
 111 #define UART_MISR_MODE                  0x0040
 112 #define UART_MISR_RESET                 0x0044
 113 #define UART_MISR_EXPORT                0x0048
 114 #define UART_MISR_VAL                   0x004C
 115 #define UART_TEST_CTRL                  0x0050
 116
 117 #define UART_SR                         0x0008
 118 #define UART_SR_HUNT_CHAR               BIT(7)
 119 #define UART_SR_RX_BREAK                BIT(6)
 120 #define UART_SR_PAR_FRAME_ERR           BIT(5)
 121 #define UART_SR_OVERRUN                 BIT(4)
 122 #define UART_SR_TX_EMPTY                BIT(3)
 123 #define UART_SR_TX_READY                BIT(2)
 124 #define UART_SR_RX_FULL                 BIT(1)
 125 #define UART_SR_RX_READY                BIT(0)
 126
 127 #define UART_RF                         0x000C
 128 #define UARTDM_RF                       0x0070
 129 #define UART_MISR                       0x0010
 130 #define UART_ISR                        0x0014
 131 #define UART_ISR_TX_READY               BIT(7)
 132
 133 #define UARTDM_RXFS                     0x50
 134 #define UARTDM_RXFS_BUF_SHIFT           0x7
 135 #define UARTDM_RXFS_BUF_MASK            0x7
 136
 137 #define UARTDM_DMEN                     0x3C
 138 #define UARTDM_DMEN_RX_SC_ENABLE        BIT(5)
 139 #define UARTDM_DMEN_TX_SC_ENABLE        BIT(4)
 140
 141 #define UARTDM_DMEN_TX_BAM_ENABLE       BIT(2)  /* UARTDM_1P4 */
 142 #define UARTDM_DMEN_TX_DM_ENABLE        BIT(0)  /* < UARTDM_1P4 */
 143
 144 #define UARTDM_DMEN_RX_BAM_ENABLE       BIT(3)  /* UARTDM_1P4 */
 145 #define UARTDM_DMEN_RX_DM_ENABLE        BIT(1)  /* < UARTDM_1P4 */
 146
 147 #define UARTDM_DMRX                     0x34
 148 #define UARTDM_NCF_TX                   0x40
 149 #define UARTDM_RX_TOTAL_SNAP            0x38
 150
 151 #define UARTDM_BURST_SIZE               16   /* in bytes */
 152 #define UARTDM_TX_AIGN(x)               ((x) & ~0x3) /* valid for > 1p3 */
 153 #define UARTDM_TX_MAX                   256   /* in bytes, valid for <= 1p3 */
 154 #define UARTDM_RX_SIZE                  (UART_XMIT_SIZE / 4)
 155
 156 enum {
 157         UARTDM_1P1 = 1,
 158         UARTDM_1P2,
 159         UARTDM_1P3,
 160         UARTDM_1P4,
 161 };
 162
 163 struct msm_dma {
 164         struct dma_chan         *chan;
 165         enum dma_data_direction dir;
 166         dma_addr_t              phys;
 167         unsigned char           *virt;
 168         dma_cookie_t            cookie;
 169         u32                     enable_bit;
 170         unsigned int            count;
 171         struct dma_async_tx_descriptor  *desc;
 172 };
 173
 174 struct msm_port {
 175         struct uart_port        uart;
 176         char                    name[16];
 177         struct clk              *clk;
 178         struct clk              *pclk;
 179         unsigned int            imr;
 180         int                     is_uartdm;
 181         unsigned int            old_snap_state;
 182         bool                    break_detected;
 183         struct msm_dma          tx_dma;
 184         struct msm_dma          rx_dma;
 185 };
 186
 187 #define UART_TO_MSM(uart_port)  container_of(uart_port, struct msm_port, uart)
 188
 189 static
 190 void msm_write(struct uart_port *port, unsigned int val, unsigned int off)
 191 {
 192         writel_relaxed(val, port->membase + off);
 193 }
 194
 195 static
 196 unsigned int msm_read(struct uart_port *port, unsigned int off)
 197 {
 198         return readl_relaxed(port->membase + off);
 199 }
 200
 201 /*
 202  * Setup the MND registers to use the TCXO clock.
 203  */
 204 static void msm_serial_set_mnd_regs_tcxo(struct uart_port *port)
 205 {
 206         msm_write(port, 0x06, UART_MREG);
 207         msm_write(port, 0xF1, UART_NREG);
 208         msm_write(port, 0x0F, UART_DREG);
 209         msm_write(port, 0x1A, UART_MNDREG);
 210         port->uartclk = 1843200;
 211 }
 212
 213 /*
 214  * Setup the MND registers to use the TCXO clock divided by 4.
 215  */
 216 static void msm_serial_set_mnd_regs_tcxoby4(struct uart_port *port)
 217 {
 218         msm_write(port, 0x18, UART_MREG);
 219         msm_write(port, 0xF6, UART_NREG);
 220         msm_write(port, 0x0F, UART_DREG);
 221         msm_write(port, 0x0A, UART_MNDREG);
 222         port->uartclk = 1843200;
 223 }
 224
 225 static void msm_serial_set_mnd_regs(struct uart_port *port)
 226 {
 227         struct msm_port *msm_port = UART_TO_MSM(port);
 228
 229         /*
 230          * These registers don't exist so we change the clk input rate
 231          * on uartdm hardware instead
 232          */
 233         if (msm_port->is_uartdm)
 234                 return;
 235
 236         if (port->uartclk == 19200000)
 237                 msm_serial_set_mnd_regs_tcxo(port);
 238         else if (port->uartclk == 4800000)
 239                 msm_serial_set_mnd_regs_tcxoby4(port);
 240 }
 241
 242 static void msm_handle_tx(struct uart_port *port);
 243 static void msm_start_rx_dma(struct msm_port *msm_port);
 244
 245 static void msm_stop_dma(struct uart_port *port, struct msm_dma *dma)
 246 {
 247         struct device *dev = port->dev;
 248         unsigned int mapped;
 249         u32 val;
 250
 251         mapped = dma->count;
 252         dma->count = 0;
 253
 254         dmaengine_terminate_all(dma->chan);
 255
 256         /*
 257          * DMA Stall happens if enqueue and flush command happens concurrently.
 258          * For example before changing the baud rate/protocol configuration and
 259          * sending flush command to ADM, disable the channel of UARTDM.
 260          * Note: should not reset the receiver here immediately as it is not
 261          * suggested to do disable/reset or reset/disable at the same time.
 262          */
 263         val = msm_read(port, UARTDM_DMEN);
 264         val &= ~dma->enable_bit;
 265         msm_write(port, val, UARTDM_DMEN);
 266
 267         if (mapped)
 268                 dma_unmap_single(dev, dma->phys, mapped, dma->dir);
 269 }
 270
 271 static void msm_release_dma(struct msm_port *msm_port)
 272 {
 273         struct msm_dma *dma;
 274
 275         dma = &msm_port->tx_dma;
 276         if (dma->chan) {
 277                 msm_stop_dma(&msm_port->uart, dma);
 278                 dma_release_channel(dma->chan);
 279         }
 280
 281         memset(dma, 0, sizeof(*dma));
 282
 283         dma = &msm_port->rx_dma;
 284         if (dma->chan) {
 285                 msm_stop_dma(&msm_port->uart, dma);
 286                 dma_release_channel(dma->chan);
 287                 kfree(dma->virt);
 288         }
 289
 290         memset(dma, 0, sizeof(*dma));
 291 }
 292
 293 static void msm_request_tx_dma(struct msm_port *msm_port, resource_size_t base)
 294 {
 295         struct device *dev = msm_port->uart.dev;
 296         struct dma_slave_config conf;
 297         struct msm_dma *dma;
 298         u32 crci = 0;
 299         int ret;
 300
 301         dma = &msm_port->tx_dma;
 302
 303         /* allocate DMA resources, if available */
 304         dma->chan = dma_request_slave_channel_reason(dev, "tx");
 305         if (IS_ERR(dma->chan))
 306                 goto no_tx;
 307
 308         of_property_read_u32(dev->of_node, "qcom,tx-crci", &crci);
 309
 310         memset(&conf, 0, sizeof(conf));
 311         conf.direction = DMA_MEM_TO_DEV;
 312         conf.device_fc = true;
 313         conf.dst_addr = base + UARTDM_TF;
 314         conf.dst_maxburst = UARTDM_BURST_SIZE;
 315         conf.slave_id = crci;
 316
 317         ret = dmaengine_slave_config(dma->chan, &conf);
 318         if (ret)
 319                 goto rel_tx;
 320
 321         dma->dir = DMA_TO_DEVICE;
 322
 323         if (msm_port->is_uartdm < UARTDM_1P4)
 324                 dma->enable_bit = UARTDM_DMEN_TX_DM_ENABLE;
 325         else
 326                 dma->enable_bit = UARTDM_DMEN_TX_BAM_ENABLE;
 327
 328         return;
 329
 330 rel_tx:
 331         dma_release_channel(dma->chan);
 332 no_tx:
 333         memset(dma, 0, sizeof(*dma));
 334 }
 335
 336 static void msm_request_rx_dma(struct msm_port *msm_port, resource_size_t base)
 337 {
 338         struct device *dev = msm_port->uart.dev;
 339         struct dma_slave_config conf;
 340         struct msm_dma *dma;
 341         u32 crci = 0;
 342         int ret;
 343
 344         dma = &msm_port->rx_dma;
 345
 346         /* allocate DMA resources, if available */
 347         dma->chan = dma_request_slave_channel_reason(dev, "rx");
 348         if (IS_ERR(dma->chan))
 349                 goto no_rx;
 350
 351         of_property_read_u32(dev->of_node, "qcom,rx-crci", &crci);
 352
 353         dma->virt = kzalloc(UARTDM_RX_SIZE, GFP_KERNEL);
 354         if (!dma->virt)
 355                 goto rel_rx;
 356
 357         memset(&conf, 0, sizeof(conf));
 358         conf.direction = DMA_DEV_TO_MEM;
 359         conf.device_fc = true;
 360         conf.src_addr = base + UARTDM_RF;
 361         conf.src_maxburst = UARTDM_BURST_SIZE;
 362         conf.slave_id = crci;
 363
 364         ret = dmaengine_slave_config(dma->chan, &conf);
 365         if (ret)
 366                 goto err;
 367
 368         dma->dir = DMA_FROM_DEVICE;
 369
 370         if (msm_port->is_uartdm < UARTDM_1P4)
 371                 dma->enable_bit = UARTDM_DMEN_RX_DM_ENABLE;
 372         else
 373                 dma->enable_bit = UARTDM_DMEN_RX_BAM_ENABLE;
 374
 375         return;
 376 err:
 377         kfree(dma->virt);
 378 rel_rx:
 379         dma_release_channel(dma->chan);
 380 no_rx:
 381         memset(dma, 0, sizeof(*dma));
 382 }
 383
 384 static inline void msm_wait_for_xmitr(struct uart_port *port)
 385 {
 386         unsigned int timeout = 500000;
 387
 388         while (!(msm_read(port, UART_SR) & UART_SR_TX_EMPTY)) {
 389                 if (msm_read(port, UART_ISR) & UART_ISR_TX_READY)
 390                         break;
 391                 udelay(1);
 392                 if (!timeout--)
 393                         break;
 394         }
 395         msm_write(port, UART_CR_CMD_RESET_TX_READY, UART_CR);
 396 }
 397
 398 static void msm_stop_tx(struct uart_port *port)
 399 {
 400         struct msm_port *msm_port = UART_TO_MSM(port);
 401
 402         msm_port->imr &= ~UART_IMR_TXLEV;
 403         msm_write(port, msm_port->imr, UART_IMR);
 404 }
 405
 406 static void msm_start_tx(struct uart_port *port)
 407 {
 408         struct msm_port *msm_port = UART_TO_MSM(port);
 409         struct msm_dma *dma = &msm_port->tx_dma;
 410
 411         /* Already started in DMA mode */
 412         if (dma->count)
 413                 return;
 414
 415         msm_port->imr |= UART_IMR_TXLEV;
 416         msm_write(port, msm_port->imr, UART_IMR);
 417 }
 418
 419 static void msm_reset_dm_count(struct uart_port *port, int count)
 420 {
 421         msm_wait_for_xmitr(port);
 422         msm_write(port, count, UARTDM_NCF_TX);
 423         msm_read(port, UARTDM_NCF_TX);
 424 }
 425
 426 static void msm_complete_tx_dma(void *args)
 427 {
 428         struct msm_port *msm_port = args;
 429         struct uart_port *port = &msm_port->uart;
 430         struct circ_buf *xmit = &port->state->xmit;
 431         struct msm_dma *dma = &msm_port->tx_dma;
 432         struct dma_tx_state state;
 433         enum dma_status status;
 434         unsigned long flags;
 435         unsigned int count;
 436         u32 val;
 437
 438         spin_lock_irqsave(&port->lock, flags);
 439
 440         /* Already stopped */
 441         if (!dma->count)
 442                 goto done;
 443
 444         status = dmaengine_tx_status(dma->chan, dma->cookie, &state);
 445
 446         dma_unmap_single(port->dev, dma->phys, dma->count, dma->dir);
 447
 448         val = msm_read(port, UARTDM_DMEN);
 449         val &= ~dma->enable_bit;
 450         msm_write(port, val, UARTDM_DMEN);
 451
 452         if (msm_port->is_uartdm > UARTDM_1P3) {
 453                 msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 454                 msm_write(port, UART_CR_TX_ENABLE, UART_CR);
 455         }
 456
 457         count = dma->count - state.residue;
 458         port->icount.tx += count;
 459         dma->count = 0;
 460
 461         xmit->tail += count;
 462         xmit->tail &= UART_XMIT_SIZE - 1;
 463
 464         /* Restore "Tx FIFO below watermark" interrupt */
 465         msm_port->imr |= UART_IMR_TXLEV;
 466         msm_write(port, msm_port->imr, UART_IMR);
 467
 468         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 469                 uart_write_wakeup(port);
 470
 471         msm_handle_tx(port);
 472 done:
 473         spin_unlock_irqrestore(&port->lock, flags);
 474 }
 475
 476 static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
 477 {
 478         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 479         struct uart_port *port = &msm_port->uart;
 480         struct msm_dma *dma = &msm_port->tx_dma;
 481         void *cpu_addr;
 482         int ret;
 483         u32 val;
 484
 485         cpu_addr = &xmit->buf[xmit->tail];
 486
 487         dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
 488         ret = dma_mapping_error(port->dev, dma->phys);
 489         if (ret)
 490                 return ret;
 491
 492         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 493                                                 count, DMA_MEM_TO_DEV,
 494                                                 DMA_PREP_INTERRUPT |
 495                                                 DMA_PREP_FENCE);
 496         if (!dma->desc) {
 497                 ret = -EIO;
 498                 goto unmap;
 499         }
 500
 501         dma->desc->callback = msm_complete_tx_dma;
 502         dma->desc->callback_param = msm_port;
 503
 504         dma->cookie = dmaengine_submit(dma->desc);
 505         ret = dma_submit_error(dma->cookie);
 506         if (ret)
 507                 goto unmap;
 508
 509         /*
 510          * Using DMA complete for Tx FIFO reload, no need for
 511          * "Tx FIFO below watermark" one, disable it
 512          */
 513         msm_port->imr &= ~UART_IMR_TXLEV;
 514         msm_write(port, msm_port->imr, UART_IMR);
 515
 516         dma->count = count;
 517
 518         val = msm_read(port, UARTDM_DMEN);
 519         val |= dma->enable_bit;
 520
 521         if (msm_port->is_uartdm < UARTDM_1P4)
 522                 msm_write(port, val, UARTDM_DMEN);
 523
 524         msm_reset_dm_count(port, count);
 525
 526         if (msm_port->is_uartdm > UARTDM_1P3)
 527                 msm_write(port, val, UARTDM_DMEN);
 528
 529         dma_async_issue_pending(dma->chan);
 530         return 0;
 531 unmap:
 532         dma_unmap_single(port->dev, dma->phys, count, dma->dir);
 533         return ret;
 534 }
 535
 536 static void msm_complete_rx_dma(void *args)
 537 {
 538         struct msm_port *msm_port = args;
 539         struct uart_port *port = &msm_port->uart;
 540         struct tty_port *tport = &port->state->port;
 541         struct msm_dma *dma = &msm_port->rx_dma;
 542         int count = 0, i, sysrq;
 543         unsigned long flags;
 544         u32 val;
 545
 546         spin_lock_irqsave(&port->lock, flags);
 547
 548         /* Already stopped */
 549         if (!dma->count)
 550                 goto done;
 551
 552         val = msm_read(port, UARTDM_DMEN);
 553         val &= ~dma->enable_bit;
 554         msm_write(port, val, UARTDM_DMEN);
 555
 556         if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
 557                 port->icount.overrun++;
 558                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 559                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 560         }
 561
 562         count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
 563
 564         port->icount.rx += count;
 565
 566         dma->count = 0;
 567
 568         dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 569
 570         for (i = 0; i < count; i++) {
 571                 char flag = TTY_NORMAL;
 572
 573                 if (msm_port->break_detected && dma->virt[i] == 0) {
 574                         port->icount.brk++;
 575                         flag = TTY_BREAK;
 576                         msm_port->break_detected = false;
 577                         if (uart_handle_break(port))
 578                                 continue;
 579                 }
 580
 581                 if (!(port->read_status_mask & UART_SR_RX_BREAK))
 582                         flag = TTY_NORMAL;
 583
 584                 spin_unlock_irqrestore(&port->lock, flags);
 585                 sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
 586                 spin_lock_irqsave(&port->lock, flags);
 587                 if (!sysrq)
 588                         tty_insert_flip_char(tport, dma->virt[i], flag);
 589         }
 590
 591         msm_start_rx_dma(msm_port);
 592 done:
 593         spin_unlock_irqrestore(&port->lock, flags);
 594
 595         if (count)
 596                 tty_flip_buffer_push(tport);
 597 }
 598
 599 static void msm_start_rx_dma(struct msm_port *msm_port)
 600 {
 601         struct msm_dma *dma = &msm_port->rx_dma;
 602         struct uart_port *uart = &msm_port->uart;
 603         u32 val;
 604         int ret;
 605
 606         if (IS_ENABLED(CONFIG_CONSOLE_POLL))
 607                 return;
 608
 609         if (!dma->chan)
 610                 return;
 611
 612         dma->phys = dma_map_single(uart->dev, dma->virt,
 613                                    UARTDM_RX_SIZE, dma->dir);
 614         ret = dma_mapping_error(uart->dev, dma->phys);
 615         if (ret)
 616                 return;
 617
 618         dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
 619                                                 UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
 620                                                 DMA_PREP_INTERRUPT);
 621         if (!dma->desc)
 622                 goto unmap;
 623
 624         dma->desc->callback = msm_complete_rx_dma;
 625         dma->desc->callback_param = msm_port;
 626
 627         dma->cookie = dmaengine_submit(dma->desc);
 628         ret = dma_submit_error(dma->cookie);
 629         if (ret)
 630                 goto unmap;
 631         /*
 632          * Using DMA for FIFO off-load, no need for "Rx FIFO over
 633          * watermark" or "stale" interrupts, disable them
 634          */
 635         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 636
 637         /*
 638          * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
 639          * we need RXSTALE to flush input DMA fifo to memory
 640          */
 641         if (msm_port->is_uartdm < UARTDM_1P4)
 642                 msm_port->imr |= UART_IMR_RXSTALE;
 643
 644         msm_write(uart, msm_port->imr, UART_IMR);
 645
 646         dma->count = UARTDM_RX_SIZE;
 647
 648         dma_async_issue_pending(dma->chan);
 649
 650         msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 651         msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 652
 653         val = msm_read(uart, UARTDM_DMEN);
 654         val |= dma->enable_bit;
 655
 656         if (msm_port->is_uartdm < UARTDM_1P4)
 657                 msm_write(uart, val, UARTDM_DMEN);
 658
 659         msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
 660
 661         if (msm_port->is_uartdm > UARTDM_1P3)
 662                 msm_write(uart, val, UARTDM_DMEN);
 663
 664         return;
 665 unmap:
 666         dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
 667 }
 668
 669 static void msm_stop_rx(struct uart_port *port)
 670 {
 671         struct msm_port *msm_port = UART_TO_MSM(port);
 672         struct msm_dma *dma = &msm_port->rx_dma;
 673
 674         msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
 675         msm_write(port, msm_port->imr, UART_IMR);
 676
 677         if (dma->chan)
 678                 msm_stop_dma(port, dma);
 679 }
 680
 681 static void msm_enable_ms(struct uart_port *port)
 682 {
 683         struct msm_port *msm_port = UART_TO_MSM(port);
 684
 685         msm_port->imr |= UART_IMR_DELTA_CTS;
 686         msm_write(port, msm_port->imr, UART_IMR);
 687 }
 688
 689 static void msm_handle_rx_dm(struct uart_port *port, unsigned int misr)
 690 {
 691         struct tty_port *tport = &port->state->port;
 692         unsigned int sr;
 693         int count = 0;
 694         struct msm_port *msm_port = UART_TO_MSM(port);
 695
 696         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 697                 port->icount.overrun++;
 698                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 699                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 700         }
 701
 702         if (misr & UART_IMR_RXSTALE) {
 703                 count = msm_read(port, UARTDM_RX_TOTAL_SNAP) -
 704                         msm_port->old_snap_state;
 705                 msm_port->old_snap_state = 0;
 706         } else {
 707                 count = 4 * (msm_read(port, UART_RFWR));
 708                 msm_port->old_snap_state += count;
 709         }
 710
 711         /* TODO: Precise error reporting */
 712
 713         port->icount.rx += count;
 714
 715         while (count > 0) {
 716                 unsigned char buf[4];
 717                 int sysrq, r_count, i;
 718
 719                 sr = msm_read(port, UART_SR);
 720                 if ((sr & UART_SR_RX_READY) == 0) {
 721                         msm_port->old_snap_state -= count;
 722                         break;
 723                 }
 724
 725                 ioread32_rep(port->membase + UARTDM_RF, buf, 1);
 726                 r_count = min_t(int, count, sizeof(buf));
 727
 728                 for (i = 0; i < r_count; i++) {
 729                         char flag = TTY_NORMAL;
 730
 731                         if (msm_port->break_detected && buf[i] == 0) {
 732                                 port->icount.brk++;
 733                                 flag = TTY_BREAK;
 734                                 msm_port->break_detected = false;
 735                                 if (uart_handle_break(port))
 736                                         continue;
 737                         }
 738
 739                         if (!(port->read_status_mask & UART_SR_RX_BREAK))
 740                                 flag = TTY_NORMAL;
 741
 742                         spin_unlock(&port->lock);
 743                         sysrq = uart_handle_sysrq_char(port, buf[i]);
 744                         spin_lock(&port->lock);
 745                         if (!sysrq)
 746                                 tty_insert_flip_char(tport, buf[i], flag);
 747                 }
 748                 count -= r_count;
 749         }
 750
 751         spin_unlock(&port->lock);
 752         tty_flip_buffer_push(tport);
 753         spin_lock(&port->lock);
 754
 755         if (misr & (UART_IMR_RXSTALE))
 756                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
 757         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
 758         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
 759
 760         /* Try to use DMA */
 761         msm_start_rx_dma(msm_port);
 762 }
 763
 764 static void msm_handle_rx(struct uart_port *port)
 765 {
 766         struct tty_port *tport = &port->state->port;
 767         unsigned int sr;
 768
 769         /*
 770          * Handle overrun. My understanding of the hardware is that overrun
 771          * is not tied to the RX buffer, so we handle the case out of band.
 772          */
 773         if ((msm_read(port, UART_SR) & UART_SR_OVERRUN)) {
 774                 port->icount.overrun++;
 775                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
 776                 msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 777         }
 778
 779         /* and now the main RX loop */
 780         while ((sr = msm_read(port, UART_SR)) & UART_SR_RX_READY) {
 781                 unsigned int c;
 782                 char flag = TTY_NORMAL;
 783                 int sysrq;
 784
 785                 c = msm_read(port, UART_RF);
 786
 787                 if (sr & UART_SR_RX_BREAK) {
 788                         port->icount.brk++;
 789                         if (uart_handle_break(port))
 790                                 continue;
 791                 } else if (sr & UART_SR_PAR_FRAME_ERR) {
 792                         port->icount.frame++;
 793                 } else {
 794                         port->icount.rx++;
 795                 }
 796
 797                 /* Mask conditions we're ignorning. */
 798                 sr &= port->read_status_mask;
 799
 800                 if (sr & UART_SR_RX_BREAK)
 801                         flag = TTY_BREAK;
 802                 else if (sr & UART_SR_PAR_FRAME_ERR)
 803                         flag = TTY_FRAME;
 804
 805                 spin_unlock(&port->lock);
 806                 sysrq = uart_handle_sysrq_char(port, c);
 807                 spin_lock(&port->lock);
 808                 if (!sysrq)
 809                         tty_insert_flip_char(tport, c, flag);
 810         }
 811
 812         spin_unlock(&port->lock);
 813         tty_flip_buffer_push(tport);
 814         spin_lock(&port->lock);
 815 }
 816
 817 static void msm_handle_tx_pio(struct uart_port *port, unsigned int tx_count)
 818 {
 819         struct circ_buf *xmit = &port->state->xmit;
 820         struct msm_port *msm_port = UART_TO_MSM(port);
 821         unsigned int num_chars;
 822         unsigned int tf_pointer = 0;
 823         void __iomem *tf;
 824
 825         if (msm_port->is_uartdm)
 826                 tf = port->membase + UARTDM_TF;
 827         else
 828                 tf = port->membase + UART_TF;
 829
 830         if (tx_count && msm_port->is_uartdm)
 831                 msm_reset_dm_count(port, tx_count);
 832
 833         while (tf_pointer < tx_count) {
 834                 int i;
 835                 char buf[4] = { 0 };
 836
 837                 if (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
 838                         break;
 839
 840                 if (msm_port->is_uartdm)
 841                         num_chars = min(tx_count - tf_pointer,
 842                                         (unsigned int)sizeof(buf));
 843                 else
 844                         num_chars = 1;
 845
 846                 for (i = 0; i < num_chars; i++) {
 847                         buf[i] = xmit->buf[xmit->tail + i];
 848                         port->icount.tx++;
 849                 }
 850
 851                 iowrite32_rep(tf, buf, 1);
 852                 xmit->tail = (xmit->tail + num_chars) & (UART_XMIT_SIZE - 1);
 853                 tf_pointer += num_chars;
 854         }
 855
 856         /* disable tx interrupts if nothing more to send */
 857         if (uart_circ_empty(xmit))
 858                 msm_stop_tx(port);
 859
 860         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 861                 uart_write_wakeup(port);
 862 }
 863
 864 static void msm_handle_tx(struct uart_port *port)
 865 {
 866         struct msm_port *msm_port = UART_TO_MSM(port);
 867         struct circ_buf *xmit = &msm_port->uart.state->xmit;
 868         struct msm_dma *dma = &msm_port->tx_dma;
 869         unsigned int pio_count, dma_count, dma_min;
 870         char buf[4] = { 0 };
 871         void __iomem *tf;
 872         int err = 0;
 873
 874         if (port->x_char) {
 875                 if (msm_port->is_uartdm)
 876                         tf = port->membase + UARTDM_TF;
 877                 else
 878                         tf = port->membase + UART_TF;
 879
 880                 buf[0] = port->x_char;
 881
 882                 if (msm_port->is_uartdm)
 883                         msm_reset_dm_count(port, 1);
 884
 885                 iowrite32_rep(tf, buf, 1);
 886                 port->icount.tx++;
 887                 port->x_char = 0;
 888                 return;
 889         }
 890
 891         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 892                 msm_stop_tx(port);
 893                 return;
 894         }
 895
 896         pio_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 897         dma_count = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 898
 899         dma_min = 1;    /* Always DMA */
 900         if (msm_port->is_uartdm > UARTDM_1P3) {
 901                 dma_count = UARTDM_TX_AIGN(dma_count);
 902                 dma_min = UARTDM_BURST_SIZE;
 903         } else {
 904                 if (dma_count > UARTDM_TX_MAX)
 905                         dma_count = UARTDM_TX_MAX;
 906         }
 907
 908         if (pio_count > port->fifosize)
 909                 pio_count = port->fifosize;
 910
 911         if (!dma->chan || dma_count < dma_min)
 912                 msm_handle_tx_pio(port, pio_count);
 913         else
 914                 err = msm_handle_tx_dma(msm_port, dma_count);
 915
 916         if (err)        /* fall back to PIO mode */
 917                 msm_handle_tx_pio(port, pio_count);
 918 }
 919
 920 static void msm_handle_delta_cts(struct uart_port *port)
 921 {
 922         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 923         port->icount.cts++;
 924         wake_up_interruptible(&port->state->port.delta_msr_wait);
 925 }
 926
 927 static irqreturn_t msm_uart_irq(int irq, void *dev_id)
 928 {
 929         struct uart_port *port = dev_id;
 930         struct msm_port *msm_port = UART_TO_MSM(port);
 931         struct msm_dma *dma = &msm_port->rx_dma;
 932         unsigned long flags;
 933         unsigned int misr;
 934         u32 val;
 935
 936         spin_lock_irqsave(&port->lock, flags);
 937         misr = msm_read(port, UART_MISR);
 938         msm_write(port, 0, UART_IMR); /* disable interrupt */
 939
 940         if (misr & UART_IMR_RXBREAK_START) {
 941                 msm_port->break_detected = true;
 942                 msm_write(port, UART_CR_CMD_RESET_RXBREAK_START, UART_CR);
 943         }
 944
 945         if (misr & (UART_IMR_RXLEV | UART_IMR_RXSTALE)) {
 946                 if (dma->count) {
 947                         val = UART_CR_CMD_STALE_EVENT_DISABLE;
 948                         msm_write(port, val, UART_CR);
 949                         val = UART_CR_CMD_RESET_STALE_INT;
 950                         msm_write(port, val, UART_CR);
 951                         /*
 952                          * Flush DMA input fifo to memory, this will also
 953                          * trigger DMA RX completion
 954                          */
 955                         dmaengine_terminate_all(dma->chan);
 956                 } else if (msm_port->is_uartdm) {
 957                         msm_handle_rx_dm(port, misr);
 958                 } else {
 959                         msm_handle_rx(port);
 960                 }
 961         }
 962         if (misr & UART_IMR_TXLEV)
 963                 msm_handle_tx(port);
 964         if (misr & UART_IMR_DELTA_CTS)
 965                 msm_handle_delta_cts(port);
 966
 967         msm_write(port, msm_port->imr, UART_IMR); /* restore interrupt */
 968         spin_unlock_irqrestore(&port->lock, flags);
 969
 970         return IRQ_HANDLED;
 971 }
 972
 973 static unsigned int msm_tx_empty(struct uart_port *port)
 974 {
 975         return (msm_read(port, UART_SR) & UART_SR_TX_EMPTY) ? TIOCSER_TEMT : 0;
 976 }
 977
 978 static unsigned int msm_get_mctrl(struct uart_port *port)
 979 {
 980         return TIOCM_CAR | TIOCM_CTS | TIOCM_DSR | TIOCM_RTS;
 981 }
 982
 983 static void msm_reset(struct uart_port *port)
 984 {
 985         struct msm_port *msm_port = UART_TO_MSM(port);
 986         unsigned int mr;
 987
 988         /* reset everything */
 989         msm_write(port, UART_CR_CMD_RESET_RX, UART_CR);
 990         msm_write(port, UART_CR_CMD_RESET_TX, UART_CR);
 991         msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
 992         msm_write(port, UART_CR_CMD_RESET_BREAK_INT, UART_CR);
 993         msm_write(port, UART_CR_CMD_RESET_CTS, UART_CR);
 994         msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
 995         mr = msm_read(port, UART_MR1);
 996         mr &= ~UART_MR1_RX_RDY_CTL;
 997         msm_write(port, mr, UART_MR1);
 998
 999         /* Disable DM modes */
1000         if (msm_port->is_uartdm)
1001                 msm_write(port, 0, UARTDM_DMEN);
1002 }
1003
1004 static void msm_set_mctrl(struct uart_port *port, unsigned int mctrl)
1005 {
1006         unsigned int mr;
1007
1008         mr = msm_read(port, UART_MR1);
1009
1010         if (!(mctrl & TIOCM_RTS)) {
1011                 mr &= ~UART_MR1_RX_RDY_CTL;
1012                 msm_write(port, mr, UART_MR1);
1013                 msm_write(port, UART_CR_CMD_RESET_RFR, UART_CR);
1014         } else {
1015                 mr |= UART_MR1_RX_RDY_CTL;
1016                 msm_write(port, mr, UART_MR1);
1017         }
1018 }
1019
1020 static void msm_break_ctl(struct uart_port *port, int break_ctl)
1021 {
1022         if (break_ctl)
1023                 msm_write(port, UART_CR_CMD_START_BREAK, UART_CR);
1024         else
1025                 msm_write(port, UART_CR_CMD_STOP_BREAK, UART_CR);
1026 }
1027
1028 struct msm_baud_map {
1029         u16     divisor;
1030         u8      code;
1031         u8      rxstale;
1032 };
1033
1034 static const struct msm_baud_map *
1035 msm_find_best_baud(struct uart_port *port, unsigned int baud,
1036                    unsigned long *rate)
1037 {
1038         struct msm_port *msm_port = UART_TO_MSM(port);
1039         unsigned int divisor, result;
1040         unsigned long target, old, best_rate = 0, diff, best_diff = ULONG_MAX;
1041         const struct msm_baud_map *entry, *end, *best;
1042         static const struct msm_baud_map table[] = {
1043                 {    1, 0xff, 31 },
1044                 {    2, 0xee, 16 },
1045                 {    3, 0xdd,  8 },
1046                 {    4, 0xcc,  6 },
1047                 {    6, 0xbb,  6 },
1048                 {    8, 0xaa,  6 },
1049                 {   12, 0x99,  6 },
1050                 {   16, 0x88,  1 },
1051                 {   24, 0x77,  1 },
1052                 {   32, 0x66,  1 },
1053                 {   48, 0x55,  1 },
1054                 {   96, 0x44,  1 },
1055                 {  192, 0x33,  1 },
1056                 {  384, 0x22,  1 },
1057                 {  768, 0x11,  1 },
1058                 { 1536, 0x00,  1 },
1059         };
1060
1061         best = table; /* Default to smallest divider */
1062         target = clk_round_rate(msm_port->clk, 16 * baud);
1063         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1064
1065         end = table + ARRAY_SIZE(table);
1066         entry = table;
1067         while (entry < end) {
1068                 if (entry->divisor <= divisor) {
1069                         result = target / entry->divisor / 16;
1070                         diff = abs(result - baud);
1071
1072                         /* Keep track of best entry */
1073                         if (diff < best_diff) {
1074                                 best_diff = diff;
1075                                 best = entry;
1076                                 best_rate = target;
1077                         }
1078
1079                         if (result == baud)
1080                                 break;
1081                 } else if (entry->divisor > divisor) {
1082                         old = target;
1083                         target = clk_round_rate(msm_port->clk, old + 1);
1084                         /*
1085                          * The rate didn't get any faster so we can't do
1086                          * better at dividing it down
1087                          */
1088                         if (target == old)
1089                                 break;
1090
1091                         /* Start the divisor search over at this new rate */
1092                         entry = table;
1093                         divisor = DIV_ROUND_CLOSEST(target, 16 * baud);
1094                         continue;
1095                 }
1096                 entry++;
1097         }
1098
1099         *rate = best_rate;
1100         return best;
1101 }
1102
1103 static int msm_set_baud_rate(struct uart_port *port, unsigned int baud,
1104                              unsigned long *saved_flags)
1105 {
1106         unsigned int rxstale, watermark, mask;
1107         struct msm_port *msm_port = UART_TO_MSM(port);
1108         const struct msm_baud_map *entry;
1109         unsigned long flags, rate;
1110
1111         flags = *saved_flags;
1112         spin_unlock_irqrestore(&port->lock, flags);
1113
1114         entry = msm_find_best_baud(port, baud, &rate);
1115         clk_set_rate(msm_port->clk, rate);
1116         baud = rate / 16 / entry->divisor;
1117
1118         spin_lock_irqsave(&port->lock, flags);
1119         *saved_flags = flags;
1120         port->uartclk = rate;
1121
1122         msm_write(port, entry->code, UART_CSR);
1123
1124         /* RX stale watermark */
1125         rxstale = entry->rxstale;
1126         watermark = UART_IPR_STALE_LSB & rxstale;
1127         if (msm_port->is_uartdm) {
1128                 mask = UART_DM_IPR_STALE_TIMEOUT_MSB;
1129         } else {
1130                 watermark |= UART_IPR_RXSTALE_LAST;
1131                 mask = UART_IPR_STALE_TIMEOUT_MSB;
1132         }
1133
1134         watermark |= mask & (rxstale << 2);
1135
1136         msm_write(port, watermark, UART_IPR);
1137
1138         /* set RX watermark */
1139         watermark = (port->fifosize * 3) / 4;
1140         msm_write(port, watermark, UART_RFWR);
1141
1142         /* set TX watermark */
1143         msm_write(port, 10, UART_TFWR);
1144
1145         msm_write(port, UART_CR_CMD_PROTECTION_EN, UART_CR);
1146         msm_reset(port);
1147
1148         /* Enable RX and TX */
1149         msm_write(port, UART_CR_TX_ENABLE | UART_CR_RX_ENABLE, UART_CR);
1150
1151         /* turn on RX and CTS interrupts */
1152         msm_port->imr = UART_IMR_RXLEV | UART_IMR_RXSTALE |
1153                         UART_IMR_CURRENT_CTS | UART_IMR_RXBREAK_START;
1154
1155         msm_write(port, msm_port->imr, UART_IMR);
1156
1157         if (msm_port->is_uartdm) {
1158                 msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1159                 msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1160                 msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
1161         }
1162
1163         return baud;
1164 }
1165
1166 static void msm_init_clock(struct uart_port *port)
1167 {
1168         struct msm_port *msm_port = UART_TO_MSM(port);
1169
1170         clk_prepare_enable(msm_port->clk);
1171         clk_prepare_enable(msm_port->pclk);
1172         msm_serial_set_mnd_regs(port);
1173 }
1174
1175 static int msm_startup(struct uart_port *port)
1176 {
1177         struct msm_port *msm_port = UART_TO_MSM(port);
1178         unsigned int data, rfr_level, mask;
1179         int ret;
1180
1181         snprintf(msm_port->name, sizeof(msm_port->name),
1182                  "msm_serial%d", port->line);
1183
1184         msm_init_clock(port);
1185
1186         if (likely(port->fifosize > 12))
1187                 rfr_level = port->fifosize - 12;
1188         else
1189                 rfr_level = port->fifosize;
1190
1191         /* set automatic RFR level */
1192         data = msm_read(port, UART_MR1);
1193
1194         if (msm_port->is_uartdm)
1195                 mask = UART_DM_MR1_AUTO_RFR_LEVEL1;
1196         else
1197                 mask = UART_MR1_AUTO_RFR_LEVEL1;
1198
1199         data &= ~mask;
1200         data &= ~UART_MR1_AUTO_RFR_LEVEL0;
1201         data |= mask & (rfr_level << 2);
1202         data |= UART_MR1_AUTO_RFR_LEVEL0 & rfr_level;
1203         msm_write(port, data, UART_MR1);
1204
1205         if (msm_port->is_uartdm) {
1206                 msm_request_tx_dma(msm_port, msm_port->uart.mapbase);
1207                 msm_request_rx_dma(msm_port, msm_port->uart.mapbase);
1208         }
1209
1210         ret = request_irq(port->irq, msm_uart_irq, IRQF_TRIGGER_HIGH,
1211                           msm_port->name, port);
1212         if (unlikely(ret))
1213                 goto err_irq;
1214
1215         return 0;
1216
1217 err_irq:
1218         if (msm_port->is_uartdm)
1219                 msm_release_dma(msm_port);
1220
1221         clk_disable_unprepare(msm_port->pclk);
1222         clk_disable_unprepare(msm_port->clk);
1223
1224         return ret;
1225 }
1226
1227 static void msm_shutdown(struct uart_port *port)
1228 {
1229         struct msm_port *msm_port = UART_TO_MSM(port);
1230
1231         msm_port->imr = 0;
1232         msm_write(port, 0, UART_IMR); /* disable interrupts */
1233
1234         if (msm_port->is_uartdm)
1235                 msm_release_dma(msm_port);
1236
1237         clk_disable_unprepare(msm_port->clk);
1238
1239         free_irq(port->irq, port);
1240 }
1241
1242 static void msm_set_termios(struct uart_port *port, struct ktermios *termios,
1243                             struct ktermios *old)
1244 {
1245         struct msm_port *msm_port = UART_TO_MSM(port);
1246         struct msm_dma *dma = &msm_port->rx_dma;
1247         unsigned long flags;
1248         unsigned int baud, mr;
1249
1250         spin_lock_irqsave(&port->lock, flags);
1251
1252         if (dma->chan) /* Terminate if any */
1253                 msm_stop_dma(port, dma);
1254
1255         /* calculate and set baud rate */
1256         baud = uart_get_baud_rate(port, termios, old, 300, 4000000);
1257         baud = msm_set_baud_rate(port, baud, &flags);
1258         if (tty_termios_baud_rate(termios))
1259                 tty_termios_encode_baud_rate(termios, baud, baud);
1260
1261         /* calculate parity */
1262         mr = msm_read(port, UART_MR2);
1263         mr &= ~UART_MR2_PARITY_MODE;
1264         if (termios->c_cflag & PARENB) {
1265                 if (termios->c_cflag & PARODD)
1266                         mr |= UART_MR2_PARITY_MODE_ODD;
1267                 else if (termios->c_cflag & CMSPAR)
1268                         mr |= UART_MR2_PARITY_MODE_SPACE;
1269                 else
1270                         mr |= UART_MR2_PARITY_MODE_EVEN;
1271         }
1272
1273         /* calculate bits per char */
1274         mr &= ~UART_MR2_BITS_PER_CHAR;
1275         switch (termios->c_cflag & CSIZE) {
1276         case CS5:
1277                 mr |= UART_MR2_BITS_PER_CHAR_5;
1278                 break;
1279         case CS6:
1280                 mr |= UART_MR2_BITS_PER_CHAR_6;
1281                 break;
1282         case CS7:
1283                 mr |= UART_MR2_BITS_PER_CHAR_7;
1284                 break;
1285         case CS8:
1286         default:
1287                 mr |= UART_MR2_BITS_PER_CHAR_8;
1288                 break;
1289         }
1290
1291         /* calculate stop bits */
1292         mr &= ~(UART_MR2_STOP_BIT_LEN_ONE | UART_MR2_STOP_BIT_LEN_TWO);
1293         if (termios->c_cflag & CSTOPB)
1294                 mr |= UART_MR2_STOP_BIT_LEN_TWO;
1295         else
1296                 mr |= UART_MR2_STOP_BIT_LEN_ONE;
1297
1298         /* set parity, bits per char, and stop bit */
1299         msm_write(port, mr, UART_MR2);
1300
1301         /* calculate and set hardware flow control */
1302         mr = msm_read(port, UART_MR1);
1303         mr &= ~(UART_MR1_CTS_CTL | UART_MR1_RX_RDY_CTL);
1304         if (termios->c_cflag & CRTSCTS) {
1305                 mr |= UART_MR1_CTS_CTL;
1306                 mr |= UART_MR1_RX_RDY_CTL;
1307         }
1308         msm_write(port, mr, UART_MR1);
1309
1310         /* Configure status bits to ignore based on termio flags. */
1311         port->read_status_mask = 0;
1312         if (termios->c_iflag & INPCK)
1313                 port->read_status_mask |= UART_SR_PAR_FRAME_ERR;
1314         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1315                 port->read_status_mask |= UART_SR_RX_BREAK;
1316
1317         uart_update_timeout(port, termios->c_cflag, baud);
1318
1319         /* Try to use DMA */
1320         msm_start_rx_dma(msm_port);
1321
1322         spin_unlock_irqrestore(&port->lock, flags);
1323 }
1324
1325 static const char *msm_type(struct uart_port *port)
1326 {
1327         return "MSM";
1328 }
1329
1330 static void msm_release_port(struct uart_port *port)
1331 {
1332         struct platform_device *pdev = to_platform_device(port->dev);
1333         struct resource *uart_resource;
1334         resource_size_t size;
1335
1336         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1337         if (unlikely(!uart_resource))
1338                 return;
1339         size = resource_size(uart_resource);
1340
1341         release_mem_region(port->mapbase, size);
1342         iounmap(port->membase);
1343         port->membase = NULL;
1344 }
1345
1346 static int msm_request_port(struct uart_port *port)
1347 {
1348         struct platform_device *pdev = to_platform_device(port->dev);
1349         struct resource *uart_resource;
1350         resource_size_t size;
1351         int ret;
1352
1353         uart_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1354         if (unlikely(!uart_resource))
1355                 return -ENXIO;
1356
1357         size = resource_size(uart_resource);
1358
1359         if (!request_mem_region(port->mapbase, size, "msm_serial"))
1360                 return -EBUSY;
1361
1362         port->membase = ioremap(port->mapbase, size);
1363         if (!port->membase) {
1364                 ret = -EBUSY;
1365                 goto fail_release_port;
1366         }
1367
1368         return 0;
1369
1370 fail_release_port:
1371         release_mem_region(port->mapbase, size);
1372         return ret;
1373 }
1374
1375 static void msm_config_port(struct uart_port *port, int flags)
1376 {
1377         int ret;
1378
1379         if (flags & UART_CONFIG_TYPE) {
1380                 port->type = PORT_MSM;
1381                 ret = msm_request_port(port);
1382                 if (ret)
1383                         return;
1384         }
1385 }
1386
1387 static int msm_verify_port(struct uart_port *port, struct serial_struct *ser)
1388 {
1389         if (unlikely(ser->type != PORT_UNKNOWN && ser->type != PORT_MSM))
1390                 return -EINVAL;
1391         if (unlikely(port->irq != ser->irq))
1392                 return -EINVAL;
1393         return 0;
1394 }
1395
1396 static void msm_power(struct uart_port *port, unsigned int state,
1397                       unsigned int oldstate)
1398 {
1399         struct msm_port *msm_port = UART_TO_MSM(port);
1400
1401         switch (state) {
1402         case 0:
1403                 clk_prepare_enable(msm_port->clk);
1404                 clk_prepare_enable(msm_port->pclk);
1405                 break;
1406         case 3:
1407                 clk_disable_unprepare(msm_port->clk);
1408                 clk_disable_unprepare(msm_port->pclk);
1409                 break;
1410         default:
1411                 pr_err("msm_serial: Unknown PM state %d\n", state);
1412         }
1413 }
1414
1415 #ifdef CONFIG_CONSOLE_POLL
1416 static int msm_poll_get_char_single(struct uart_port *port)
1417 {
1418         struct msm_port *msm_port = UART_TO_MSM(port);
1419         unsigned int rf_reg = msm_port->is_uartdm ? UARTDM_RF : UART_RF;
1420
1421         if (!(msm_read(port, UART_SR) & UART_SR_RX_READY))
1422                 return NO_POLL_CHAR;
1423
1424         return msm_read(port, rf_reg) & 0xff;
1425 }
1426
1427 static int msm_poll_get_char_dm(struct uart_port *port)
1428 {
1429         int c;
1430         static u32 slop;
1431         static int count;
1432         unsigned char *sp = (unsigned char *)&slop;
1433
1434         /* Check if a previous read had more than one char */
1435         if (count) {
1436                 c = sp[sizeof(slop) - count];
1437                 count--;
1438         /* Or if FIFO is empty */
1439         } else if (!(msm_read(port, UART_SR) & UART_SR_RX_READY)) {
1440                 /*
1441                  * If RX packing buffer has less than a word, force stale to
1442                  * push contents into RX FIFO
1443                  */
1444                 count = msm_read(port, UARTDM_RXFS);
1445                 count = (count >> UARTDM_RXFS_BUF_SHIFT) & UARTDM_RXFS_BUF_MASK;
1446                 if (count) {
1447                         msm_write(port, UART_CR_CMD_FORCE_STALE, UART_CR);
1448                         slop = msm_read(port, UARTDM_RF);
1449                         c = sp[0];
1450                         count--;
1451                         msm_write(port, UART_CR_CMD_RESET_STALE_INT, UART_CR);
1452                         msm_write(port, 0xFFFFFF, UARTDM_DMRX);
1453                         msm_write(port, UART_CR_CMD_STALE_EVENT_ENABLE,
1454                                   UART_CR);
1455                 } else {
1456                         c = NO_POLL_CHAR;
1457                 }
1458         /* FIFO has a word */
1459         } else {
1460                 slop = msm_read(port, UARTDM_RF);
1461                 c = sp[0];
1462                 count = sizeof(slop) - 1;
1463         }
1464
1465         return c;
1466 }
1467
1468 static int msm_poll_get_char(struct uart_port *port)
1469 {
1470         u32 imr;
1471         int c;
1472         struct msm_port *msm_port = UART_TO_MSM(port);
1473
1474         /* Disable all interrupts */
1475         imr = msm_read(port, UART_IMR);
1476         msm_write(port, 0, UART_IMR);
1477
1478         if (msm_port->is_uartdm)
1479                 c = msm_poll_get_char_dm(port);
1480         else
1481                 c = msm_poll_get_char_single(port);
1482
1483         /* Enable interrupts */
1484         msm_write(port, imr, UART_IMR);
1485
1486         return c;
1487 }
1488
1489 static void msm_poll_put_char(struct uart_port *port, unsigned char c)
1490 {
1491         u32 imr;
1492         struct msm_port *msm_port = UART_TO_MSM(port);
1493
1494         /* Disable all interrupts */
1495         imr = msm_read(port, UART_IMR);
1496         msm_write(port, 0, UART_IMR);
1497
1498         if (msm_port->is_uartdm)
1499                 msm_reset_dm_count(port, 1);
1500
1501         /* Wait until FIFO is empty */
1502         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1503                 cpu_relax();
1504
1505         /* Write a character */
1506         msm_write(port, c, msm_port->is_uartdm ? UARTDM_TF : UART_TF);
1507
1508         /* Wait until FIFO is empty */
1509         while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1510                 cpu_relax();
1511
1512         /* Enable interrupts */
1513         msm_write(port, imr, UART_IMR);
1514 }
1515 #endif
1516
1517 static struct uart_ops msm_uart_pops = {
1518         .tx_empty = msm_tx_empty,
1519         .set_mctrl = msm_set_mctrl,
1520         .get_mctrl = msm_get_mctrl,
1521         .stop_tx = msm_stop_tx,
1522         .start_tx = msm_start_tx,
1523         .stop_rx = msm_stop_rx,
1524         .enable_ms = msm_enable_ms,
1525         .break_ctl = msm_break_ctl,
1526         .startup = msm_startup,
1527         .shutdown = msm_shutdown,
1528         .set_termios = msm_set_termios,
1529         .type = msm_type,
1530         .release_port = msm_release_port,
1531         .request_port = msm_request_port,
1532         .config_port = msm_config_port,
1533         .verify_port = msm_verify_port,
1534         .pm = msm_power,
1535 #ifdef CONFIG_CONSOLE_POLL
1536         .poll_get_char  = msm_poll_get_char,
1537         .poll_put_char  = msm_poll_put_char,
1538 #endif
1539 };
1540
1541 static struct msm_port msm_uart_ports[] = {
1542         {
1543                 .uart = {
1544                         .iotype = UPIO_MEM,
1545                         .ops = &msm_uart_pops,
1546                         .flags = UPF_BOOT_AUTOCONF,
1547                         .fifosize = 64,
1548                         .line = 0,
1549                 },
1550         },
1551         {
1552                 .uart = {
1553                         .iotype = UPIO_MEM,
1554                         .ops = &msm_uart_pops,
1555                         .flags = UPF_BOOT_AUTOCONF,
1556                         .fifosize = 64,
1557                         .line = 1,
1558                 },
1559         },
1560         {
1561                 .uart = {
1562                         .iotype = UPIO_MEM,
1563                         .ops = &msm_uart_pops,
1564                         .flags = UPF_BOOT_AUTOCONF,
1565                         .fifosize = 64,
1566                         .line = 2,
1567                 },
1568         },
1569 };
1570
1571 #define UART_NR ARRAY_SIZE(msm_uart_ports)
1572
1573 static inline struct uart_port *msm_get_port_from_line(unsigned int line)
1574 {
1575         return &msm_uart_ports[line].uart;
1576 }
1577
1578 #ifdef CONFIG_SERIAL_MSM_CONSOLE
1579 static void __msm_console_write(struct uart_port *port, const char *s,
1580                                 unsigned int count, bool is_uartdm)
1581 {
1582         unsigned long flags;
1583         int i;
1584         int num_newlines = 0;
1585         bool replaced = false;
1586         void __iomem *tf;
1587         int locked = 1;
1588
1589         if (is_uartdm)
1590                 tf = port->membase + UARTDM_TF;
1591         else
1592                 tf = port->membase + UART_TF;
1593
1594         /* Account for newlines that will get a carriage return added */
1595         for (i = 0; i < count; i++)
1596                 if (s[i] == '\n')
1597                         num_newlines++;
1598         count += num_newlines;
1599
1600         local_irq_save(flags);
1601
1602         if (port->sysrq)
1603                 locked = 0;
1604         else if (oops_in_progress)
1605                 locked = spin_trylock(&port->lock);
1606         else
1607                 spin_lock(&port->lock);
1608
1609         if (is_uartdm)
1610                 msm_reset_dm_count(port, count);
1611
1612         i = 0;
1613         while (i < count) {
1614                 int j;
1615                 unsigned int num_chars;
1616                 char buf[4] = { 0 };
1617
1618                 if (is_uartdm)
1619                         num_chars = min(count - i, (unsigned int)sizeof(buf));
1620                 else
1621                         num_chars = 1;
1622
1623                 for (j = 0; j < num_chars; j++) {
1624                         char c = *s;
1625
1626                         if (c == '\n' && !replaced) {
1627                                 buf[j] = '\r';
1628                                 j++;
1629                                 replaced = true;
1630                         }
1631                         if (j < num_chars) {
1632                                 buf[j] = c;
1633                                 s++;
1634                                 replaced = false;
1635                         }
1636                 }
1637
1638                 while (!(msm_read(port, UART_SR) & UART_SR_TX_READY))
1639                         cpu_relax();
1640
1641                 iowrite32_rep(tf, buf, 1);
1642                 i += num_chars;
1643         }
1644
1645         if (locked)
1646                 spin_unlock(&port->lock);
1647
1648         local_irq_restore(flags);
1649 }
1650
1651 static void msm_console_write(struct console *co, const char *s,
1652                               unsigned int count)
1653 {
1654         struct uart_port *port;
1655         struct msm_port *msm_port;
1656
1657         BUG_ON(co->index < 0 || co->index >= UART_NR);
1658
1659         port = msm_get_port_from_line(co->index);
1660         msm_port = UART_TO_MSM(port);
1661
1662         __msm_console_write(port, s, count, msm_port->is_uartdm);
1663 }
1664
1665 static int __init msm_console_setup(struct console *co, char *options)
1666 {
1667         struct uart_port *port;
1668         int baud = 115200;
1669         int bits = 8;
1670         int parity = 'n';
1671         int flow = 'n';
1672
1673         if (unlikely(co->index >= UART_NR || co->index < 0))
1674                 return -ENXIO;
1675
1676         port = msm_get_port_from_line(co->index);
1677
1678         if (unlikely(!port->membase))
1679                 return -ENXIO;
1680
1681         msm_init_clock(port);
1682
1683         if (options)
1684                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1685
1686         pr_info("msm_serial: console setup on port #%d\n", port->line);
1687
1688         return uart_set_options(port, co, baud, parity, bits, flow);
1689 }
1690
1691 static void
1692 msm_serial_early_write(struct console *con, const char *s, unsigned n)
1693 {
1694         struct earlycon_device *dev = con->data;
1695
1696         __msm_console_write(&dev->port, s, n, false);
1697 }
1698
1699 static int __init
1700 msm_serial_early_console_setup(struct earlycon_device *device, const char *opt)
1701 {
1702         if (!device->port.membase)
1703                 return -ENODEV;
1704
1705         device->con->write = msm_serial_early_write;
1706         return 0;
1707 }
1708 OF_EARLYCON_DECLARE(msm_serial, "qcom,msm-uart",
1709                     msm_serial_early_console_setup);
1710
1711 static void
1712 msm_serial_early_write_dm(struct console *con, const char *s, unsigned n)
1713 {
1714         struct earlycon_device *dev = con->data;
1715
1716         __msm_console_write(&dev->port, s, n, true);
1717 }
1718
1719 static int __init
1720 msm_serial_early_console_setup_dm(struct earlycon_device *device,
1721                                   const char *opt)
1722 {
1723         if (!device->port.membase)
1724                 return -ENODEV;
1725
1726         device->con->write = msm_serial_early_write_dm;
1727         return 0;
1728 }
1729 OF_EARLYCON_DECLARE(msm_serial_dm, "qcom,msm-uartdm",
1730                     msm_serial_early_console_setup_dm);
1731
1732 static struct uart_driver msm_uart_driver;
1733
1734 static struct console msm_console = {
1735         .name = "ttyMSM",
1736         .write = msm_console_write,
1737         .device = uart_console_device,
1738         .setup = msm_console_setup,
1739         .flags = CON_PRINTBUFFER,
1740         .index = -1,
1741         .data = &msm_uart_driver,
1742 };
1743
1744 #define MSM_CONSOLE     (&msm_console)
1745
1746 #else
1747 #define MSM_CONSOLE     NULL
1748 #endif
1749
1750 static struct uart_driver msm_uart_driver = {
1751         .owner = THIS_MODULE,
1752         .driver_name = "msm_serial",
1753         .dev_name = "ttyMSM",
1754         .nr = UART_NR,
1755         .cons = MSM_CONSOLE,
1756 };
1757
1758 static atomic_t msm_uart_next_id = ATOMIC_INIT(0);
1759
1760 static const struct of_device_id msm_uartdm_table[] = {
1761         { .compatible = "qcom,msm-uartdm-v1.1", .data = (void *)UARTDM_1P1 },
1762         { .compatible = "qcom,msm-uartdm-v1.2", .data = (void *)UARTDM_1P2 },
1763         { .compatible = "qcom,msm-uartdm-v1.3", .data = (void *)UARTDM_1P3 },
1764         { .compatible = "qcom,msm-uartdm-v1.4", .data = (void *)UARTDM_1P4 },
1765         { }
1766 };
1767
1768 static int msm_serial_probe(struct platform_device *pdev)
1769 {
1770         struct msm_port *msm_port;
1771         struct resource *resource;
1772         struct uart_port *port;
1773         const struct of_device_id *id;
1774         int irq, line;
1775
1776         if (pdev->dev.of_node)
1777                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1778         else
1779                 line = pdev->id;
1780
1781         if (line < 0)
1782                 line = atomic_inc_return(&msm_uart_next_id) - 1;
1783
1784         if (unlikely(line < 0 || line >= UART_NR))
1785                 return -ENXIO;
1786
1787         dev_info(&pdev->dev, "msm_serial: detected port #%d\n", line);
1788
1789         port = msm_get_port_from_line(line);
1790         port->dev = &pdev->dev;
1791         msm_port = UART_TO_MSM(port);
1792
1793         id = of_match_device(msm_uartdm_table, &pdev->dev);
1794         if (id)
1795                 msm_port->is_uartdm = (unsigned long)id->data;
1796         else
1797                 msm_port->is_uartdm = 0;
1798
1799         msm_port->clk = devm_clk_get(&pdev->dev, "core");
1800         if (IS_ERR(msm_port->clk))
1801                 return PTR_ERR(msm_port->clk);
1802
1803         if (msm_port->is_uartdm) {
1804                 msm_port->pclk = devm_clk_get(&pdev->dev, "iface");
1805                 if (IS_ERR(msm_port->pclk))
1806                         return PTR_ERR(msm_port->pclk);
1807         }
1808
1809         port->uartclk = clk_get_rate(msm_port->clk);
1810         dev_info(&pdev->dev, "uartclk = %d\n", port->uartclk);
1811
1812         resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1813         if (unlikely(!resource))
1814                 return -ENXIO;
1815         port->mapbase = resource->start;
1816
1817         irq = platform_get_irq(pdev, 0);
1818         if (unlikely(irq < 0))
1819                 return -ENXIO;
1820         port->irq = irq;
1821
1822         platform_set_drvdata(pdev, port);
1823
1824         return uart_add_one_port(&msm_uart_driver, port);
1825 }
1826
1827 static int msm_serial_remove(struct platform_device *pdev)
1828 {
1829         struct uart_port *port = platform_get_drvdata(pdev);
1830
1831         uart_remove_one_port(&msm_uart_driver, port);
1832
1833         return 0;
1834 }
1835
1836 static const struct of_device_id msm_match_table[] = {
1837         { .compatible = "qcom,msm-uart" },
1838         { .compatible = "qcom,msm-uartdm" },
1839         {}
1840 };
1841 MODULE_DEVICE_TABLE(of, msm_match_table);
1842
1843 static int __maybe_unused msm_serial_suspend(struct device *dev)
1844 {
1845         struct msm_port *port = dev_get_drvdata(dev);
1846
1847         uart_suspend_port(&msm_uart_driver, &port->uart);
1848
1849         return 0;
1850 }
1851
1852 static int __maybe_unused msm_serial_resume(struct device *dev)
1853 {
1854         struct msm_port *port = dev_get_drvdata(dev);
1855
1856         uart_resume_port(&msm_uart_driver, &port->uart);
1857
1858         return 0;
1859 }
1860
1861 static const struct dev_pm_ops msm_serial_dev_pm_ops = {
1862         SET_SYSTEM_SLEEP_PM_OPS(msm_serial_suspend, msm_serial_resume)
1863 };
1864
1865 static struct platform_driver msm_platform_driver = {
1866         .remove = msm_serial_remove,
1867         .probe = msm_serial_probe,
1868         .driver = {
1869                 .name = "msm_serial",
1870                 .pm = &msm_serial_dev_pm_ops,
1871                 .of_match_table = msm_match_table,
1872         },
1873 };
1874
1875 static int __init msm_serial_init(void)
1876 {
1877         int ret;
1878
1879         ret = uart_register_driver(&msm_uart_driver);
1880         if (unlikely(ret))
1881                 return ret;
1882
1883         ret = platform_driver_register(&msm_platform_driver);
1884         if (unlikely(ret))
1885                 uart_unregister_driver(&msm_uart_driver);
1886
1887         pr_info("msm_serial: driver initialized\n");
1888
1889         return ret;
1890 }
1891
1892 static void __exit msm_serial_exit(void)
1893 {
1894         platform_driver_unregister(&msm_platform_driver);
1895         uart_unregister_driver(&msm_uart_driver);
1896 }
1897
1898 module_init(msm_serial_init);
1899 module_exit(msm_serial_exit);
1900
1901 MODULE_AUTHOR("Robert Love <rlove@google.com>");
1902 MODULE_DESCRIPTION("Driver for msm7x serial device");
1903 MODULE_LICENSE("GPL");