drivers/gpu/host1x/hw/cdma_hw.c

   1 /*
   2  * Tegra host1x Command DMA
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/slab.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22
  23 #include "../cdma.h"
  24 #include "../channel.h"
  25 #include "../dev.h"
  26 #include "../debug.h"
  27
  28 /*
  29  * Put the restart at the end of pushbuffer memory
  30  */
  31 static void push_buffer_init(struct push_buffer *pb)
  32 {
  33         *(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0);
  34 }
  35
  36 /*
  37  * Increment timedout buffer's syncpt via CPU.
  38  */
  39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41 {
  42         struct host1x *host1x = cdma_to_host1x(cdma);
  43         struct push_buffer *pb = &cdma->push_buffer;
  44         unsigned int i;
  45
  46         for (i = 0; i < syncpt_incrs; i++)
  47                 host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49         /* after CPU incr, ensure shadow is up to date */
  50         host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52         /* NOP all the PB slots */
  53         while (nr_slots--) {
  54                 u32 *p = (u32 *)(pb->mapped + getptr);
  55                 *(p++) = HOST1X_OPCODE_NOP;
  56                 *(p++) = HOST1X_OPCODE_NOP;
  57                 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                         &pb->phys, getptr);
  59                 getptr = (getptr + 8) & (pb->size_bytes - 1);
  60         }
  61
  62         wmb();
  63 }
  64
  65 /*
  66  * Start channel DMA
  67  */
  68 static void cdma_start(struct host1x_cdma *cdma)
  69 {
  70         struct host1x_channel *ch = cdma_to_channel(cdma);
  71
  72         if (cdma->running)
  73                 return;
  74
  75         cdma->last_pos = cdma->push_buffer.pos;
  76
  77         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  78                          HOST1X_CHANNEL_DMACTRL);
  79
  80         /* set base, put and end pointer */
  81         host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
  82         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  83         host1x_ch_writel(ch, cdma->push_buffer.phys +
  84                          cdma->push_buffer.size_bytes + 4,
  85                          HOST1X_CHANNEL_DMAEND);
  86
  87         /* reset GET */
  88         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  89                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  90                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  91                          HOST1X_CHANNEL_DMACTRL);
  92
  93         /* start the command DMA */
  94         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  95
  96         cdma->running = true;
  97 }
  98
  99 /*
 100  * Similar to cdma_start(), but rather than starting from an idle
 101  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 102  * DMA GET from an explicit value (so DMA may again be pending).
 103  */
 104 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 105 {
 106         struct host1x *host1x = cdma_to_host1x(cdma);
 107         struct host1x_channel *ch = cdma_to_channel(cdma);
 108
 109         if (cdma->running)
 110                 return;
 111
 112         cdma->last_pos = cdma->push_buffer.pos;
 113
 114         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 115                          HOST1X_CHANNEL_DMACTRL);
 116
 117         /* set base, end pointer (all of memory) */
 118         host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
 119         host1x_ch_writel(ch, cdma->push_buffer.phys +
 120                          cdma->push_buffer.size_bytes,
 121                          HOST1X_CHANNEL_DMAEND);
 122
 123         /* set GET, by loading the value in PUT (then reset GET) */
 124         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 125         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 126                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 127                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 128                          HOST1X_CHANNEL_DMACTRL);
 129
 130         dev_dbg(host1x->dev,
 131                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 132                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 133                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 134                 cdma->last_pos);
 135
 136         /* deassert GET reset and set PUT */
 137         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 138                          HOST1X_CHANNEL_DMACTRL);
 139         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 140
 141         /* start the command DMA */
 142         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 143
 144         cdma->running = true;
 145 }
 146
 147 /*
 148  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 149  */
 150 static void cdma_flush(struct host1x_cdma *cdma)
 151 {
 152         struct host1x_channel *ch = cdma_to_channel(cdma);
 153
 154         if (cdma->push_buffer.pos != cdma->last_pos) {
 155                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 156                                  HOST1X_CHANNEL_DMAPUT);
 157                 cdma->last_pos = cdma->push_buffer.pos;
 158         }
 159 }
 160
 161 static void cdma_stop(struct host1x_cdma *cdma)
 162 {
 163         struct host1x_channel *ch = cdma_to_channel(cdma);
 164
 165         mutex_lock(&cdma->lock);
 166
 167         if (cdma->running) {
 168                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 169                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 170                                  HOST1X_CHANNEL_DMACTRL);
 171                 cdma->running = false;
 172         }
 173
 174         mutex_unlock(&cdma->lock);
 175 }
 176
 177 /*
 178  * Stops both channel's command processor and CDMA immediately.
 179  * Also, tears down the channel and resets corresponding module.
 180  */
 181 static void cdma_freeze(struct host1x_cdma *cdma)
 182 {
 183         struct host1x *host = cdma_to_host1x(cdma);
 184         struct host1x_channel *ch = cdma_to_channel(cdma);
 185         u32 cmdproc_stop;
 186
 187         if (cdma->torndown && !cdma->running) {
 188                 dev_warn(host->dev, "Already torn down\n");
 189                 return;
 190         }
 191
 192         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 193
 194         cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 195         cmdproc_stop |= BIT(ch->id);
 196         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 197
 198         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 199                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 200                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 201                 cdma->last_pos);
 202
 203         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 204                          HOST1X_CHANNEL_DMACTRL);
 205
 206         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 207
 208         cdma->running = false;
 209         cdma->torndown = true;
 210 }
 211
 212 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 213 {
 214         struct host1x *host1x = cdma_to_host1x(cdma);
 215         struct host1x_channel *ch = cdma_to_channel(cdma);
 216         u32 cmdproc_stop;
 217
 218         dev_dbg(host1x->dev,
 219                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 220                 ch->id, getptr);
 221
 222         cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 223         cmdproc_stop &= ~BIT(ch->id);
 224         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 225
 226         cdma->torndown = false;
 227         cdma_timeout_restart(cdma, getptr);
 228 }
 229
 230 /*
 231  * If this timeout fires, it indicates the current sync_queue entry has
 232  * exceeded its TTL and the userctx should be timed out and remaining
 233  * submits already issued cleaned up (future submits return an error).
 234  */
 235 static void cdma_timeout_handler(struct work_struct *work)
 236 {
 237         u32 prev_cmdproc, cmdproc_stop, syncpt_val;
 238         struct host1x_cdma *cdma;
 239         struct host1x *host1x;
 240         struct host1x_channel *ch;
 241
 242         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 243                             timeout.wq);
 244         host1x = cdma_to_host1x(cdma);
 245         ch = cdma_to_channel(cdma);
 246
 247         host1x_debug_dump(cdma_to_host1x(cdma));
 248
 249         mutex_lock(&cdma->lock);
 250
 251         if (!cdma->timeout.client) {
 252                 dev_dbg(host1x->dev,
 253                         "cdma_timeout: expired, but has no clientid\n");
 254                 mutex_unlock(&cdma->lock);
 255                 return;
 256         }
 257
 258         /* stop processing to get a clean snapshot */
 259         prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 260         cmdproc_stop = prev_cmdproc | BIT(ch->id);
 261         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 262
 263         dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
 264                 prev_cmdproc, cmdproc_stop);
 265
 266         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 267
 268         /* has buffer actually completed? */
 269         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 270                 dev_dbg(host1x->dev,
 271                         "cdma_timeout: expired, but buffer had completed\n");
 272                 /* restore */
 273                 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id));
 274                 host1x_sync_writel(host1x, cmdproc_stop,
 275                                    HOST1X_SYNC_CMDPROC_STOP);
 276                 mutex_unlock(&cdma->lock);
 277                 return;
 278         }
 279
 280         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 281                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 282                  syncpt_val, cdma->timeout.syncpt_val);
 283
 284         /* stop HW, resetting channel/module */
 285         host1x_hw_cdma_freeze(host1x, cdma);
 286
 287         host1x_cdma_update_sync_queue(cdma, ch->dev);
 288         mutex_unlock(&cdma->lock);
 289 }
 290
 291 /*
 292  * Init timeout resources
 293  */
 294 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 295 {
 296         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 297         cdma->timeout.initialized = true;
 298
 299         return 0;
 300 }
 301
 302 /*
 303  * Clean up timeout resources
 304  */
 305 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 306 {
 307         if (cdma->timeout.initialized)
 308                 cancel_delayed_work(&cdma->timeout.wq);
 309
 310         cdma->timeout.initialized = false;
 311 }
 312
 313 static const struct host1x_cdma_ops host1x_cdma_ops = {
 314         .start = cdma_start,
 315         .stop = cdma_stop,
 316         .flush = cdma_flush,
 317
 318         .timeout_init = cdma_timeout_init,
 319         .timeout_destroy = cdma_timeout_destroy,
 320         .freeze = cdma_freeze,
 321         .resume = cdma_resume,
 322         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 323 };
 324
 325 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 326         .init = push_buffer_init,
 327 };