drivers/gpu/host1x/hw/cdma_hw.c

   1 /*
   2  * Tegra host1x Command DMA
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/slab.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22
  23 #include "../cdma.h"
  24 #include "../channel.h"
  25 #include "../dev.h"
  26 #include "../debug.h"
  27
  28 /*
  29  * Put the restart at the end of pushbuffer memory
  30  */
  31 static void push_buffer_init(struct push_buffer *pb)
  32 {
  33         *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  34 }
  35
  36 /*
  37  * Increment timedout buffer's syncpt via CPU.
  38  */
  39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41 {
  42         struct host1x *host1x = cdma_to_host1x(cdma);
  43         struct push_buffer *pb = &cdma->push_buffer;
  44         unsigned int i;
  45
  46         for (i = 0; i < syncpt_incrs; i++)
  47                 host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49         /* after CPU incr, ensure shadow is up to date */
  50         host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52         /* NOP all the PB slots */
  53         while (nr_slots--) {
  54                 u32 *p = (u32 *)(pb->mapped + getptr);
  55                 *(p++) = HOST1X_OPCODE_NOP;
  56                 *(p++) = HOST1X_OPCODE_NOP;
  57                 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                         &pb->dma, getptr);
  59                 getptr = (getptr + 8) & (pb->size - 1);
  60         }
  61
  62         wmb();
  63 }
  64
  65 /*
  66  * Start channel DMA
  67  */
  68 static void cdma_start(struct host1x_cdma *cdma)
  69 {
  70         struct host1x_channel *ch = cdma_to_channel(cdma);
  71
  72         if (cdma->running)
  73                 return;
  74
  75         cdma->last_pos = cdma->push_buffer.pos;
  76
  77         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  78                          HOST1X_CHANNEL_DMACTRL);
  79
  80         /* set base, put and end pointer */
  81         host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
  82         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  83         host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size + 4,
  84                          HOST1X_CHANNEL_DMAEND);
  85
  86         /* reset GET */
  87         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  88                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  89                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  90                          HOST1X_CHANNEL_DMACTRL);
  91
  92         /* start the command DMA */
  93         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  94
  95         cdma->running = true;
  96 }
  97
  98 /*
  99  * Similar to cdma_start(), but rather than starting from an idle
 100  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 101  * DMA GET from an explicit value (so DMA may again be pending).
 102  */
 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 104 {
 105         struct host1x *host1x = cdma_to_host1x(cdma);
 106         struct host1x_channel *ch = cdma_to_channel(cdma);
 107
 108         if (cdma->running)
 109                 return;
 110
 111         cdma->last_pos = cdma->push_buffer.pos;
 112
 113         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 114                          HOST1X_CHANNEL_DMACTRL);
 115
 116         /* set base, end pointer (all of memory) */
 117         host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
 118         host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size,
 119                          HOST1X_CHANNEL_DMAEND);
 120
 121         /* set GET, by loading the value in PUT (then reset GET) */
 122         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 123         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 124                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 125                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 126                          HOST1X_CHANNEL_DMACTRL);
 127
 128         dev_dbg(host1x->dev,
 129                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 130                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 131                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 132                 cdma->last_pos);
 133
 134         /* deassert GET reset and set PUT */
 135         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 136                          HOST1X_CHANNEL_DMACTRL);
 137         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 138
 139         /* start the command DMA */
 140         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 141
 142         cdma->running = true;
 143 }
 144
 145 /*
 146  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 147  */
 148 static void cdma_flush(struct host1x_cdma *cdma)
 149 {
 150         struct host1x_channel *ch = cdma_to_channel(cdma);
 151
 152         if (cdma->push_buffer.pos != cdma->last_pos) {
 153                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 154                                  HOST1X_CHANNEL_DMAPUT);
 155                 cdma->last_pos = cdma->push_buffer.pos;
 156         }
 157 }
 158
 159 static void cdma_stop(struct host1x_cdma *cdma)
 160 {
 161         struct host1x_channel *ch = cdma_to_channel(cdma);
 162
 163         mutex_lock(&cdma->lock);
 164
 165         if (cdma->running) {
 166                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 167                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 168                                  HOST1X_CHANNEL_DMACTRL);
 169                 cdma->running = false;
 170         }
 171
 172         mutex_unlock(&cdma->lock);
 173 }
 174
 175 static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
 176                                  bool stop)
 177 {
 178 #if HOST1X_HW >= 6
 179         host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
 180 #else
 181         u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 182         if (stop)
 183                 cmdproc_stop |= BIT(ch->id);
 184         else
 185                 cmdproc_stop &= ~BIT(ch->id);
 186         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 187 #endif
 188 }
 189
 190 static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
 191 {
 192 #if HOST1X_HW >= 6
 193         host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
 194 #else
 195         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 196 #endif
 197 }
 198
 199 /*
 200  * Stops both channel's command processor and CDMA immediately.
 201  * Also, tears down the channel and resets corresponding module.
 202  */
 203 static void cdma_freeze(struct host1x_cdma *cdma)
 204 {
 205         struct host1x *host = cdma_to_host1x(cdma);
 206         struct host1x_channel *ch = cdma_to_channel(cdma);
 207
 208         if (cdma->torndown && !cdma->running) {
 209                 dev_warn(host->dev, "Already torn down\n");
 210                 return;
 211         }
 212
 213         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 214
 215         cdma_hw_cmdproc_stop(host, ch, true);
 216
 217         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 218                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 219                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 220                 cdma->last_pos);
 221
 222         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 223                          HOST1X_CHANNEL_DMACTRL);
 224
 225         cdma_hw_teardown(host, ch);
 226
 227         cdma->running = false;
 228         cdma->torndown = true;
 229 }
 230
 231 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 232 {
 233         struct host1x *host1x = cdma_to_host1x(cdma);
 234         struct host1x_channel *ch = cdma_to_channel(cdma);
 235
 236         dev_dbg(host1x->dev,
 237                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 238                 ch->id, getptr);
 239
 240         cdma_hw_cmdproc_stop(host1x, ch, false);
 241
 242         cdma->torndown = false;
 243         cdma_timeout_restart(cdma, getptr);
 244 }
 245
 246 /*
 247  * If this timeout fires, it indicates the current sync_queue entry has
 248  * exceeded its TTL and the userctx should be timed out and remaining
 249  * submits already issued cleaned up (future submits return an error).
 250  */
 251 static void cdma_timeout_handler(struct work_struct *work)
 252 {
 253         u32 syncpt_val;
 254         struct host1x_cdma *cdma;
 255         struct host1x *host1x;
 256         struct host1x_channel *ch;
 257
 258         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 259                             timeout.wq);
 260         host1x = cdma_to_host1x(cdma);
 261         ch = cdma_to_channel(cdma);
 262
 263         host1x_debug_dump(cdma_to_host1x(cdma));
 264
 265         mutex_lock(&cdma->lock);
 266
 267         if (!cdma->timeout.client) {
 268                 dev_dbg(host1x->dev,
 269                         "cdma_timeout: expired, but has no clientid\n");
 270                 mutex_unlock(&cdma->lock);
 271                 return;
 272         }
 273
 274         /* stop processing to get a clean snapshot */
 275         cdma_hw_cmdproc_stop(host1x, ch, true);
 276
 277         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 278
 279         /* has buffer actually completed? */
 280         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 281                 dev_dbg(host1x->dev,
 282                         "cdma_timeout: expired, but buffer had completed\n");
 283                 /* restore */
 284                 cdma_hw_cmdproc_stop(host1x, ch, false);
 285                 mutex_unlock(&cdma->lock);
 286                 return;
 287         }
 288
 289         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 290                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 291                  syncpt_val, cdma->timeout.syncpt_val);
 292
 293         /* stop HW, resetting channel/module */
 294         host1x_hw_cdma_freeze(host1x, cdma);
 295
 296         host1x_cdma_update_sync_queue(cdma, ch->dev);
 297         mutex_unlock(&cdma->lock);
 298 }
 299
 300 /*
 301  * Init timeout resources
 302  */
 303 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 304 {
 305         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 306         cdma->timeout.initialized = true;
 307
 308         return 0;
 309 }
 310
 311 /*
 312  * Clean up timeout resources
 313  */
 314 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 315 {
 316         if (cdma->timeout.initialized)
 317                 cancel_delayed_work(&cdma->timeout.wq);
 318
 319         cdma->timeout.initialized = false;
 320 }
 321
 322 static const struct host1x_cdma_ops host1x_cdma_ops = {
 323         .start = cdma_start,
 324         .stop = cdma_stop,
 325         .flush = cdma_flush,
 326
 327         .timeout_init = cdma_timeout_init,
 328         .timeout_destroy = cdma_timeout_destroy,
 329         .freeze = cdma_freeze,
 330         .resume = cdma_resume,
 331         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 332 };
 333
 334 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 335         .init = push_buffer_init,
 336 };