drivers/gpu/drm/msm/msm_gem_submit.c

   1 /*
   2  * Copyright (C) 2013 Red Hat
   3  * Author: Rob Clark <robdclark@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/sync_file.h>
  19
  20 #include "msm_drv.h"
  21 #include "msm_gpu.h"
  22 #include "msm_gem.h"
  23
  24 /*
  25  * Cmdstream submission:
  26  */
  27
  28 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
  29 #define BO_VALID    0x8000   /* is current addr in cmdstream correct/valid? */
  30 #define BO_LOCKED   0x4000
  31 #define BO_PINNED   0x2000
  32
  33 static struct msm_gem_submit *submit_create(struct drm_device *dev,
  34                 struct msm_gpu *gpu, int nr_bos, int nr_cmds)
  35 {
  36         struct msm_gem_submit *submit;
  37         int sz = sizeof(*submit) + (nr_bos * sizeof(submit->bos[0])) +
  38                         (nr_cmds * sizeof(*submit->cmd));
  39
  40         submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
  41         if (!submit)
  42                 return NULL;
  43
  44         submit->dev = dev;
  45         submit->gpu = gpu;
  46         submit->fence = NULL;
  47         submit->pid = get_pid(task_pid(current));
  48         submit->cmd = (void *)&submit->bos[nr_bos];
  49
  50         /* initially, until copy_from_user() and bo lookup succeeds: */
  51         submit->nr_bos = 0;
  52         submit->nr_cmds = 0;
  53
  54         INIT_LIST_HEAD(&submit->node);
  55         INIT_LIST_HEAD(&submit->bo_list);
  56         ww_acquire_init(&submit->ticket, &reservation_ww_class);
  57
  58         return submit;
  59 }
  60
  61 void msm_gem_submit_free(struct msm_gem_submit *submit)
  62 {
  63         fence_put(submit->fence);
  64         list_del(&submit->node);
  65         put_pid(submit->pid);
  66         kfree(submit);
  67 }
  68
  69 static inline unsigned long __must_check
  70 copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
  71 {
  72         if (access_ok(VERIFY_READ, from, n))
  73                 return __copy_from_user_inatomic(to, from, n);
  74         return -EFAULT;
  75 }
  76
  77 static int submit_lookup_objects(struct msm_gem_submit *submit,
  78                 struct drm_msm_gem_submit *args, struct drm_file *file)
  79 {
  80         unsigned i;
  81         int ret = 0;
  82
  83         spin_lock(&file->table_lock);
  84         pagefault_disable();
  85
  86         for (i = 0; i < args->nr_bos; i++) {
  87                 struct drm_msm_gem_submit_bo submit_bo;
  88                 struct drm_gem_object *obj;
  89                 struct msm_gem_object *msm_obj;
  90                 void __user *userptr =
  91                         u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
  92
  93                 /* make sure we don't have garbage flags, in case we hit
  94                  * error path before flags is initialized:
  95                  */
  96                 submit->bos[i].flags = 0;
  97
  98                 ret = copy_from_user_inatomic(&submit_bo, userptr, sizeof(submit_bo));
  99                 if (unlikely(ret)) {
 100                         pagefault_enable();
 101                         spin_unlock(&file->table_lock);
 102                         ret = copy_from_user(&submit_bo, userptr, sizeof(submit_bo));
 103                         if (ret)
 104                                 goto out;
 105                         spin_lock(&file->table_lock);
 106                         pagefault_disable();
 107                 }
 108
 109                 if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
 110                         DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
 111                         ret = -EINVAL;
 112                         goto out_unlock;
 113                 }
 114
 115                 submit->bos[i].flags = submit_bo.flags;
 116                 /* in validate_objects() we figure out if this is true: */
 117                 submit->bos[i].iova  = submit_bo.presumed;
 118
 119                 /* normally use drm_gem_object_lookup(), but for bulk lookup
 120                  * all under single table_lock just hit object_idr directly:
 121                  */
 122                 obj = idr_find(&file->object_idr, submit_bo.handle);
 123                 if (!obj) {
 124                         DRM_ERROR("invalid handle %u at index %u\n", submit_bo.handle, i);
 125                         ret = -EINVAL;
 126                         goto out_unlock;
 127                 }
 128
 129                 msm_obj = to_msm_bo(obj);
 130
 131                 if (!list_empty(&msm_obj->submit_entry)) {
 132                         DRM_ERROR("handle %u at index %u already on submit list\n",
 133                                         submit_bo.handle, i);
 134                         ret = -EINVAL;
 135                         goto out_unlock;
 136                 }
 137
 138                 drm_gem_object_reference(obj);
 139
 140                 submit->bos[i].obj = msm_obj;
 141
 142                 list_add_tail(&msm_obj->submit_entry, &submit->bo_list);
 143         }
 144
 145 out_unlock:
 146         pagefault_enable();
 147         spin_unlock(&file->table_lock);
 148
 149 out:
 150         submit->nr_bos = i;
 151
 152         return ret;
 153 }
 154
 155 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
 156 {
 157         struct msm_gem_object *msm_obj = submit->bos[i].obj;
 158
 159         if (submit->bos[i].flags & BO_PINNED)
 160                 msm_gem_put_iova(&msm_obj->base, submit->gpu->id);
 161
 162         if (submit->bos[i].flags & BO_LOCKED)
 163                 ww_mutex_unlock(&msm_obj->resv->lock);
 164
 165         if (!(submit->bos[i].flags & BO_VALID))
 166                 submit->bos[i].iova = 0;
 167
 168         submit->bos[i].flags &= ~(BO_LOCKED | BO_PINNED);
 169 }
 170
 171 /* This is where we make sure all the bo's are reserved and pin'd: */
 172 static int submit_lock_objects(struct msm_gem_submit *submit)
 173 {
 174         int contended, slow_locked = -1, i, ret = 0;
 175
 176 retry:
 177         for (i = 0; i < submit->nr_bos; i++) {
 178                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 179
 180                 if (slow_locked == i)
 181                         slow_locked = -1;
 182
 183                 contended = i;
 184
 185                 if (!(submit->bos[i].flags & BO_LOCKED)) {
 186                         ret = ww_mutex_lock_interruptible(&msm_obj->resv->lock,
 187                                         &submit->ticket);
 188                         if (ret)
 189                                 goto fail;
 190                         submit->bos[i].flags |= BO_LOCKED;
 191                 }
 192         }
 193
 194         ww_acquire_done(&submit->ticket);
 195
 196         return 0;
 197
 198 fail:
 199         for (; i >= 0; i--)
 200                 submit_unlock_unpin_bo(submit, i);
 201
 202         if (slow_locked > 0)
 203                 submit_unlock_unpin_bo(submit, slow_locked);
 204
 205         if (ret == -EDEADLK) {
 206                 struct msm_gem_object *msm_obj = submit->bos[contended].obj;
 207                 /* we lost out in a seqno race, lock and retry.. */
 208                 ret = ww_mutex_lock_slow_interruptible(&msm_obj->resv->lock,
 209                                 &submit->ticket);
 210                 if (!ret) {
 211                         submit->bos[contended].flags |= BO_LOCKED;
 212                         slow_locked = contended;
 213                         goto retry;
 214                 }
 215         }
 216
 217         return ret;
 218 }
 219
 220 static int submit_fence_sync(struct msm_gem_submit *submit)
 221 {
 222         int i, ret = 0;
 223
 224         for (i = 0; i < submit->nr_bos; i++) {
 225                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 226                 bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
 227
 228                 ret = msm_gem_sync_object(&msm_obj->base, submit->gpu->fctx, write);
 229                 if (ret)
 230                         break;
 231         }
 232
 233         return ret;
 234 }
 235
 236 static int submit_pin_objects(struct msm_gem_submit *submit)
 237 {
 238         int i, ret = 0;
 239
 240         submit->valid = true;
 241
 242         for (i = 0; i < submit->nr_bos; i++) {
 243                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 244                 uint32_t iova;
 245
 246                 /* if locking succeeded, pin bo: */
 247                 ret = msm_gem_get_iova_locked(&msm_obj->base,
 248                                 submit->gpu->id, &iova);
 249
 250                 if (ret)
 251                         break;
 252
 253                 submit->bos[i].flags |= BO_PINNED;
 254
 255                 if (iova == submit->bos[i].iova) {
 256                         submit->bos[i].flags |= BO_VALID;
 257                 } else {
 258                         submit->bos[i].iova = iova;
 259                         /* iova changed, so address in cmdstream is not valid: */
 260                         submit->bos[i].flags &= ~BO_VALID;
 261                         submit->valid = false;
 262                 }
 263         }
 264
 265         return ret;
 266 }
 267
 268 static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
 269                 struct msm_gem_object **obj, uint32_t *iova, bool *valid)
 270 {
 271         if (idx >= submit->nr_bos) {
 272                 DRM_ERROR("invalid buffer index: %u (out of %u)\n",
 273                                 idx, submit->nr_bos);
 274                 return -EINVAL;
 275         }
 276
 277         if (obj)
 278                 *obj = submit->bos[idx].obj;
 279         if (iova)
 280                 *iova = submit->bos[idx].iova;
 281         if (valid)
 282                 *valid = !!(submit->bos[idx].flags & BO_VALID);
 283
 284         return 0;
 285 }
 286
 287 /* process the reloc's and patch up the cmdstream as needed: */
 288 static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *obj,
 289                 uint32_t offset, uint32_t nr_relocs, uint64_t relocs)
 290 {
 291         uint32_t i, last_offset = 0;
 292         uint32_t *ptr;
 293         int ret;
 294
 295         if (offset % 4) {
 296                 DRM_ERROR("non-aligned cmdstream buffer: %u\n", offset);
 297                 return -EINVAL;
 298         }
 299
 300         /* For now, just map the entire thing.  Eventually we probably
 301          * to do it page-by-page, w/ kmap() if not vmap()d..
 302          */
 303         ptr = msm_gem_get_vaddr_locked(&obj->base);
 304
 305         if (IS_ERR(ptr)) {
 306                 ret = PTR_ERR(ptr);
 307                 DBG("failed to map: %d", ret);
 308                 return ret;
 309         }
 310
 311         for (i = 0; i < nr_relocs; i++) {
 312                 struct drm_msm_gem_submit_reloc submit_reloc;
 313                 void __user *userptr =
 314                         u64_to_user_ptr(relocs + (i * sizeof(submit_reloc)));
 315                 uint32_t iova, off;
 316                 bool valid;
 317
 318                 ret = copy_from_user(&submit_reloc, userptr, sizeof(submit_reloc));
 319                 if (ret)
 320                         return -EFAULT;
 321
 322                 if (submit_reloc.submit_offset % 4) {
 323                         DRM_ERROR("non-aligned reloc offset: %u\n",
 324                                         submit_reloc.submit_offset);
 325                         return -EINVAL;
 326                 }
 327
 328                 /* offset in dwords: */
 329                 off = submit_reloc.submit_offset / 4;
 330
 331                 if ((off >= (obj->base.size / 4)) ||
 332                                 (off < last_offset)) {
 333                         DRM_ERROR("invalid offset %u at reloc %u\n", off, i);
 334                         return -EINVAL;
 335                 }
 336
 337                 ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova, &valid);
 338                 if (ret)
 339                         return ret;
 340
 341                 if (valid)
 342                         continue;
 343
 344                 iova += submit_reloc.reloc_offset;
 345
 346                 if (submit_reloc.shift < 0)
 347                         iova >>= -submit_reloc.shift;
 348                 else
 349                         iova <<= submit_reloc.shift;
 350
 351                 ptr[off] = iova | submit_reloc.or;
 352
 353                 last_offset = off;
 354         }
 355
 356         msm_gem_put_vaddr_locked(&obj->base);
 357
 358         return 0;
 359 }
 360
 361 static void submit_cleanup(struct msm_gem_submit *submit)
 362 {
 363         unsigned i;
 364
 365         for (i = 0; i < submit->nr_bos; i++) {
 366                 struct msm_gem_object *msm_obj = submit->bos[i].obj;
 367                 submit_unlock_unpin_bo(submit, i);
 368                 list_del_init(&msm_obj->submit_entry);
 369                 drm_gem_object_unreference(&msm_obj->base);
 370         }
 371
 372         ww_acquire_fini(&submit->ticket);
 373 }
 374
 375 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 376                 struct drm_file *file)
 377 {
 378         struct msm_drm_private *priv = dev->dev_private;
 379         struct drm_msm_gem_submit *args = data;
 380         struct msm_file_private *ctx = file->driver_priv;
 381         struct msm_gem_submit *submit;
 382         struct msm_gpu *gpu = priv->gpu;
 383         struct fence *in_fence = NULL;
 384         struct sync_file *sync_file = NULL;
 385         int out_fence_fd = -1;
 386         unsigned i;
 387         int ret;
 388
 389         if (!gpu)
 390                 return -ENXIO;
 391
 392         /* for now, we just have 3d pipe.. eventually this would need to
 393          * be more clever to dispatch to appropriate gpu module:
 394          */
 395         if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
 396                 return -EINVAL;
 397
 398         if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
 399                 return -EINVAL;
 400
 401         ret = mutex_lock_interruptible(&dev->struct_mutex);
 402         if (ret)
 403                 return ret;
 404
 405         if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
 406                 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
 407                 if (out_fence_fd < 0) {
 408                         ret = out_fence_fd;
 409                         goto out_unlock;
 410                 }
 411         }
 412         priv->struct_mutex_task = current;
 413
 414         submit = submit_create(dev, gpu, args->nr_bos, args->nr_cmds);
 415         if (!submit) {
 416                 ret = -ENOMEM;
 417                 goto out_unlock;
 418         }
 419
 420         ret = submit_lookup_objects(submit, args, file);
 421         if (ret)
 422                 goto out;
 423
 424         ret = submit_lock_objects(submit);
 425         if (ret)
 426                 goto out;
 427
 428         if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
 429                 in_fence = sync_file_get_fence(args->fence_fd);
 430
 431                 if (!in_fence) {
 432                         ret = -EINVAL;
 433                         goto out;
 434                 }
 435
 436                 /* TODO if we get an array-fence due to userspace merging multiple
 437                  * fences, we need a way to determine if all the backing fences
 438                  * are from our own context..
 439                  */
 440
 441                 if (in_fence->context != gpu->fctx->context) {
 442                         ret = fence_wait(in_fence, true);
 443                         if (ret)
 444                                 goto out;
 445                 }
 446
 447         }
 448
 449         if (!(args->fence & MSM_SUBMIT_NO_IMPLICIT)) {
 450                 ret = submit_fence_sync(submit);
 451                 if (ret)
 452                         goto out;
 453         }
 454
 455         ret = submit_pin_objects(submit);
 456         if (ret)
 457                 goto out;
 458
 459         for (i = 0; i < args->nr_cmds; i++) {
 460                 struct drm_msm_gem_submit_cmd submit_cmd;
 461                 void __user *userptr =
 462                         u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
 463                 struct msm_gem_object *msm_obj;
 464                 uint32_t iova;
 465
 466                 ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
 467                 if (ret) {
 468                         ret = -EFAULT;
 469                         goto out;
 470                 }
 471
 472                 /* validate input from userspace: */
 473                 switch (submit_cmd.type) {
 474                 case MSM_SUBMIT_CMD_BUF:
 475                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
 476                 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
 477                         break;
 478                 default:
 479                         DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
 480                         ret = -EINVAL;
 481                         goto out;
 482                 }
 483
 484                 ret = submit_bo(submit, submit_cmd.submit_idx,
 485                                 &msm_obj, &iova, NULL);
 486                 if (ret)
 487                         goto out;
 488
 489                 if (submit_cmd.size % 4) {
 490                         DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
 491                                         submit_cmd.size);
 492                         ret = -EINVAL;
 493                         goto out;
 494                 }
 495
 496                 if ((submit_cmd.size + submit_cmd.submit_offset) >=
 497                                 msm_obj->base.size) {
 498                         DRM_ERROR("invalid cmdstream size: %u\n", submit_cmd.size);
 499                         ret = -EINVAL;
 500                         goto out;
 501                 }
 502
 503                 submit->cmd[i].type = submit_cmd.type;
 504                 submit->cmd[i].size = submit_cmd.size / 4;
 505                 submit->cmd[i].iova = iova + submit_cmd.submit_offset;
 506                 submit->cmd[i].idx  = submit_cmd.submit_idx;
 507
 508                 if (submit->valid)
 509                         continue;
 510
 511                 ret = submit_reloc(submit, msm_obj, submit_cmd.submit_offset,
 512                                 submit_cmd.nr_relocs, submit_cmd.relocs);
 513                 if (ret)
 514                         goto out;
 515         }
 516
 517         submit->nr_cmds = i;
 518
 519         submit->fence = msm_fence_alloc(gpu->fctx);
 520         if (IS_ERR(submit->fence)) {
 521                 ret = PTR_ERR(submit->fence);
 522                 submit->fence = NULL;
 523                 goto out;
 524         }
 525
 526         if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
 527                 sync_file = sync_file_create(submit->fence);
 528                 if (!sync_file) {
 529                         ret = -ENOMEM;
 530                         goto out;
 531                 }
 532         }
 533
 534         msm_gpu_submit(gpu, submit, ctx);
 535
 536         args->fence = submit->fence->seqno;
 537
 538         if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
 539                 fd_install(out_fence_fd, sync_file->file);
 540                 args->fence_fd = out_fence_fd;
 541         }
 542
 543 out:
 544         if (in_fence)
 545                 fence_put(in_fence);
 546         submit_cleanup(submit);
 547         if (ret)
 548                 msm_gem_submit_free(submit);
 549 out_unlock:
 550         if (ret && (out_fence_fd >= 0))
 551                 put_unused_fd(out_fence_fd);
 552         priv->struct_mutex_task = NULL;
 553         mutex_unlock(&dev->struct_mutex);
 554         return ret;
 555 }