drivers/gpu/drm/drm_syncobj.c

   1 /*
   2  * Copyright 2017 Red Hat
   3  * Parts ported from amdgpu (fence wait code).
   4  * Copyright 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  23  * IN THE SOFTWARE.
  24  *
  25  * Authors:
  26  *
  27  */
  28
  29 /**
  30  * DOC: Overview
  31  *
  32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) are
  33  * persistent objects that contain an optional fence. The fence can be updated
  34  * with a new fence, or be NULL.
  35  *
  36  * syncobj's can be waited upon, where it will wait for the underlying
  37  * fence.
  38  *
  39  * syncobj's can be export to fd's and back, these fd's are opaque and
  40  * have no other use case, except passing the syncobj between processes.
  41  *
  42  * Their primary use-case is to implement Vulkan fences and semaphores.
  43  *
  44  * syncobj have a kref reference count, but also have an optional file.
  45  * The file is only created once the syncobj is exported.
  46  * The file takes a reference on the kref.
  47  */
  48
  49 #include <drm/drmP.h>
  50 #include <linux/file.h>
  51 #include <linux/fs.h>
  52 #include <linux/anon_inodes.h>
  53 #include <linux/sync_file.h>
  54 #include <linux/sched/signal.h>
  55
  56 #include "drm_internal.h"
  57 #include <drm/drm_syncobj.h>
  58
  59 /**
  60  * drm_syncobj_find - lookup and reference a sync object.
  61  * @file_private: drm file private pointer
  62  * @handle: sync object handle to lookup.
  63  *
  64  * Returns a reference to the syncobj pointed to by handle or NULL. The
  65  * reference must be released by calling drm_syncobj_put().
  66  */
  67 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
  68                                      u32 handle)
  69 {
  70         struct drm_syncobj *syncobj;
  71
  72         spin_lock(&file_private->syncobj_table_lock);
  73
  74         /* Check if we currently have a reference on the object */
  75         syncobj = idr_find(&file_private->syncobj_idr, handle);
  76         if (syncobj)
  77                 drm_syncobj_get(syncobj);
  78
  79         spin_unlock(&file_private->syncobj_table_lock);
  80
  81         return syncobj;
  82 }
  83 EXPORT_SYMBOL(drm_syncobj_find);
  84
  85 static void drm_syncobj_add_callback_locked(struct drm_syncobj *syncobj,
  86                                             struct drm_syncobj_cb *cb,
  87                                             drm_syncobj_func_t func)
  88 {
  89         cb->func = func;
  90         list_add_tail(&cb->node, &syncobj->cb_list);
  91 }
  92
  93 static int drm_syncobj_fence_get_or_add_callback(struct drm_syncobj *syncobj,
  94                                                  struct dma_fence **fence,
  95                                                  struct drm_syncobj_cb *cb,
  96                                                  drm_syncobj_func_t func)
  97 {
  98         int ret;
  99
 100         WARN_ON(*fence);
 101
 102         *fence = drm_syncobj_fence_get(syncobj);
 103         if (*fence)
 104                 return 1;
 105
 106         spin_lock(&syncobj->lock);
 107         /* We've already tried once to get a fence and failed.  Now that we
 108          * have the lock, try one more time just to be sure we don't add a
 109          * callback when a fence has already been set.
 110          */
 111         if (syncobj->fence) {
 112                 *fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 113                                                                  lockdep_is_held(&syncobj->lock)));
 114                 ret = 1;
 115         } else {
 116                 *fence = NULL;
 117                 drm_syncobj_add_callback_locked(syncobj, cb, func);
 118                 ret = 0;
 119         }
 120         spin_unlock(&syncobj->lock);
 121
 122         return ret;
 123 }
 124
 125 /**
 126  * drm_syncobj_add_callback - adds a callback to syncobj::cb_list
 127  * @syncobj: Sync object to which to add the callback
 128  * @cb: Callback to add
 129  * @func: Func to use when initializing the drm_syncobj_cb struct
 130  *
 131  * This adds a callback to be called next time the fence is replaced
 132  */
 133 void drm_syncobj_add_callback(struct drm_syncobj *syncobj,
 134                               struct drm_syncobj_cb *cb,
 135                               drm_syncobj_func_t func)
 136 {
 137         spin_lock(&syncobj->lock);
 138         drm_syncobj_add_callback_locked(syncobj, cb, func);
 139         spin_unlock(&syncobj->lock);
 140 }
 141 EXPORT_SYMBOL(drm_syncobj_add_callback);
 142
 143 /**
 144  * drm_syncobj_add_callback - removes a callback to syncobj::cb_list
 145  * @syncobj: Sync object from which to remove the callback
 146  * @cb: Callback to remove
 147  */
 148 void drm_syncobj_remove_callback(struct drm_syncobj *syncobj,
 149                                  struct drm_syncobj_cb *cb)
 150 {
 151         spin_lock(&syncobj->lock);
 152         list_del_init(&cb->node);
 153         spin_unlock(&syncobj->lock);
 154 }
 155 EXPORT_SYMBOL(drm_syncobj_remove_callback);
 156
 157 /**
 158  * drm_syncobj_replace_fence - replace fence in a sync object.
 159  * @syncobj: Sync object to replace fence in
 160  * @fence: fence to install in sync file.
 161  *
 162  * This replaces the fence on a sync object.
 163  */
 164 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
 165                                struct dma_fence *fence)
 166 {
 167         struct dma_fence *old_fence;
 168         struct drm_syncobj_cb *cur, *tmp;
 169
 170         if (fence)
 171                 dma_fence_get(fence);
 172
 173         spin_lock(&syncobj->lock);
 174
 175         old_fence = rcu_dereference_protected(syncobj->fence,
 176                                               lockdep_is_held(&syncobj->lock));
 177         rcu_assign_pointer(syncobj->fence, fence);
 178
 179         if (fence != old_fence) {
 180                 list_for_each_entry_safe(cur, tmp, &syncobj->cb_list, node) {
 181                         list_del_init(&cur->node);
 182                         cur->func(syncobj, cur);
 183                 }
 184         }
 185
 186         spin_unlock(&syncobj->lock);
 187
 188         dma_fence_put(old_fence);
 189 }
 190 EXPORT_SYMBOL(drm_syncobj_replace_fence);
 191
 192 struct drm_syncobj_null_fence {
 193         struct dma_fence base;
 194         spinlock_t lock;
 195 };
 196
 197 static const char *drm_syncobj_null_fence_get_name(struct dma_fence *fence)
 198 {
 199         return "syncobjnull";
 200 }
 201
 202 static bool drm_syncobj_null_fence_enable_signaling(struct dma_fence *fence)
 203 {
 204     dma_fence_enable_sw_signaling(fence);
 205     return !dma_fence_is_signaled(fence);
 206 }
 207
 208 static const struct dma_fence_ops drm_syncobj_null_fence_ops = {
 209         .get_driver_name = drm_syncobj_null_fence_get_name,
 210         .get_timeline_name = drm_syncobj_null_fence_get_name,
 211         .enable_signaling = drm_syncobj_null_fence_enable_signaling,
 212         .release = NULL,
 213 };
 214
 215 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
 216 {
 217         struct drm_syncobj_null_fence *fence;
 218         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 219         if (fence == NULL)
 220                 return -ENOMEM;
 221
 222         spin_lock_init(&fence->lock);
 223         dma_fence_init(&fence->base, &drm_syncobj_null_fence_ops,
 224                        &fence->lock, 0, 0);
 225         dma_fence_signal(&fence->base);
 226
 227         drm_syncobj_replace_fence(syncobj, &fence->base);
 228
 229         dma_fence_put(&fence->base);
 230
 231         return 0;
 232 }
 233
 234 /**
 235  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
 236  * @file_private: drm file private pointer
 237  * @handle: sync object handle to lookup.
 238  * @fence: out parameter for the fence
 239  *
 240  * This is just a convenience function that combines drm_syncobj_find() and
 241  * drm_syncobj_fence_get().
 242  *
 243  * Returns 0 on success or a negative error value on failure. On success @fence
 244  * contains a reference to the fence, which must be released by calling
 245  * dma_fence_put().
 246  */
 247 int drm_syncobj_find_fence(struct drm_file *file_private,
 248                            u32 handle,
 249                            struct dma_fence **fence)
 250 {
 251         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 252         int ret = 0;
 253
 254         if (!syncobj)
 255                 return -ENOENT;
 256
 257         *fence = drm_syncobj_fence_get(syncobj);
 258         if (!*fence) {
 259                 ret = -EINVAL;
 260         }
 261         drm_syncobj_put(syncobj);
 262         return ret;
 263 }
 264 EXPORT_SYMBOL(drm_syncobj_find_fence);
 265
 266 /**
 267  * drm_syncobj_free - free a sync object.
 268  * @kref: kref to free.
 269  *
 270  * Only to be called from kref_put in drm_syncobj_put.
 271  */
 272 void drm_syncobj_free(struct kref *kref)
 273 {
 274         struct drm_syncobj *syncobj = container_of(kref,
 275                                                    struct drm_syncobj,
 276                                                    refcount);
 277         drm_syncobj_replace_fence(syncobj, NULL);
 278         kfree(syncobj);
 279 }
 280 EXPORT_SYMBOL(drm_syncobj_free);
 281
 282 /**
 283  * drm_syncobj_create - create a new syncobj
 284  * @out_syncobj: returned syncobj
 285  * @flags: DRM_SYNCOBJ_* flags
 286  * @fence: if non-NULL, the syncobj will represent this fence
 287  *
 288  * This is the first function to create a sync object. After creating, drivers
 289  * probably want to make it available to userspace, either through
 290  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
 291  *
 292  * Returns 0 on success or a negative error value on failure.
 293  */
 294 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
 295                        struct dma_fence *fence)
 296 {
 297         int ret;
 298         struct drm_syncobj *syncobj;
 299
 300         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
 301         if (!syncobj)
 302                 return -ENOMEM;
 303
 304         kref_init(&syncobj->refcount);
 305         INIT_LIST_HEAD(&syncobj->cb_list);
 306         spin_lock_init(&syncobj->lock);
 307
 308         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
 309                 ret = drm_syncobj_assign_null_handle(syncobj);
 310                 if (ret < 0) {
 311                         drm_syncobj_put(syncobj);
 312                         return ret;
 313                 }
 314         }
 315
 316         if (fence)
 317                 drm_syncobj_replace_fence(syncobj, fence);
 318
 319         *out_syncobj = syncobj;
 320         return 0;
 321 }
 322 EXPORT_SYMBOL(drm_syncobj_create);
 323
 324 /**
 325  * drm_syncobj_get_handle - get a handle from a syncobj
 326  * @file_private: drm file private pointer
 327  * @syncobj: Sync object to export
 328  * @handle: out parameter with the new handle
 329  *
 330  * Exports a sync object created with drm_syncobj_create() as a handle on
 331  * @file_private to userspace.
 332  *
 333  * Returns 0 on success or a negative error value on failure.
 334  */
 335 int drm_syncobj_get_handle(struct drm_file *file_private,
 336                            struct drm_syncobj *syncobj, u32 *handle)
 337 {
 338         int ret;
 339
 340         /* take a reference to put in the idr */
 341         drm_syncobj_get(syncobj);
 342
 343         idr_preload(GFP_KERNEL);
 344         spin_lock(&file_private->syncobj_table_lock);
 345         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 346         spin_unlock(&file_private->syncobj_table_lock);
 347
 348         idr_preload_end();
 349
 350         if (ret < 0) {
 351                 drm_syncobj_put(syncobj);
 352                 return ret;
 353         }
 354
 355         *handle = ret;
 356         return 0;
 357 }
 358 EXPORT_SYMBOL(drm_syncobj_get_handle);
 359
 360 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
 361                                         u32 *handle, uint32_t flags)
 362 {
 363         int ret;
 364         struct drm_syncobj *syncobj;
 365
 366         ret = drm_syncobj_create(&syncobj, flags, NULL);
 367         if (ret)
 368                 return ret;
 369
 370         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
 371         drm_syncobj_put(syncobj);
 372         return ret;
 373 }
 374
 375 static int drm_syncobj_destroy(struct drm_file *file_private,
 376                                u32 handle)
 377 {
 378         struct drm_syncobj *syncobj;
 379
 380         spin_lock(&file_private->syncobj_table_lock);
 381         syncobj = idr_remove(&file_private->syncobj_idr, handle);
 382         spin_unlock(&file_private->syncobj_table_lock);
 383
 384         if (!syncobj)
 385                 return -EINVAL;
 386
 387         drm_syncobj_put(syncobj);
 388         return 0;
 389 }
 390
 391 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
 392 {
 393         struct drm_syncobj *syncobj = file->private_data;
 394
 395         drm_syncobj_put(syncobj);
 396         return 0;
 397 }
 398
 399 static const struct file_operations drm_syncobj_file_fops = {
 400         .release = drm_syncobj_file_release,
 401 };
 402
 403 /**
 404  * drm_syncobj_get_fd - get a file descriptor from a syncobj
 405  * @syncobj: Sync object to export
 406  * @p_fd: out parameter with the new file descriptor
 407  *
 408  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
 409  *
 410  * Returns 0 on success or a negative error value on failure.
 411  */
 412 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
 413 {
 414         struct file *file;
 415         int fd;
 416
 417         fd = get_unused_fd_flags(O_CLOEXEC);
 418         if (fd < 0)
 419                 return fd;
 420
 421         file = anon_inode_getfile("syncobj_file",
 422                                   &drm_syncobj_file_fops,
 423                                   syncobj, 0);
 424         if (IS_ERR(file)) {
 425                 put_unused_fd(fd);
 426                 return PTR_ERR(file);
 427         }
 428
 429         drm_syncobj_get(syncobj);
 430         fd_install(fd, file);
 431
 432         *p_fd = fd;
 433         return 0;
 434 }
 435 EXPORT_SYMBOL(drm_syncobj_get_fd);
 436
 437 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
 438                                     u32 handle, int *p_fd)
 439 {
 440         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
 441         int ret;
 442
 443         if (!syncobj)
 444                 return -EINVAL;
 445
 446         ret = drm_syncobj_get_fd(syncobj, p_fd);
 447         drm_syncobj_put(syncobj);
 448         return ret;
 449 }
 450
 451 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
 452                                     int fd, u32 *handle)
 453 {
 454         struct drm_syncobj *syncobj;
 455         struct file *file;
 456         int ret;
 457
 458         file = fget(fd);
 459         if (!file)
 460                 return -EINVAL;
 461
 462         if (file->f_op != &drm_syncobj_file_fops) {
 463                 fput(file);
 464                 return -EINVAL;
 465         }
 466
 467         /* take a reference to put in the idr */
 468         syncobj = file->private_data;
 469         drm_syncobj_get(syncobj);
 470
 471         idr_preload(GFP_KERNEL);
 472         spin_lock(&file_private->syncobj_table_lock);
 473         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
 474         spin_unlock(&file_private->syncobj_table_lock);
 475         idr_preload_end();
 476
 477         if (ret > 0) {
 478                 *handle = ret;
 479                 ret = 0;
 480         } else
 481                 drm_syncobj_put(syncobj);
 482
 483         fput(file);
 484         return ret;
 485 }
 486
 487 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
 488                                               int fd, int handle)
 489 {
 490         struct dma_fence *fence = sync_file_get_fence(fd);
 491         struct drm_syncobj *syncobj;
 492
 493         if (!fence)
 494                 return -EINVAL;
 495
 496         syncobj = drm_syncobj_find(file_private, handle);
 497         if (!syncobj) {
 498                 dma_fence_put(fence);
 499                 return -ENOENT;
 500         }
 501
 502         drm_syncobj_replace_fence(syncobj, fence);
 503         dma_fence_put(fence);
 504         drm_syncobj_put(syncobj);
 505         return 0;
 506 }
 507
 508 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
 509                                         int handle, int *p_fd)
 510 {
 511         int ret;
 512         struct dma_fence *fence;
 513         struct sync_file *sync_file;
 514         int fd = get_unused_fd_flags(O_CLOEXEC);
 515
 516         if (fd < 0)
 517                 return fd;
 518
 519         ret = drm_syncobj_find_fence(file_private, handle, &fence);
 520         if (ret)
 521                 goto err_put_fd;
 522
 523         sync_file = sync_file_create(fence);
 524
 525         dma_fence_put(fence);
 526
 527         if (!sync_file) {
 528                 ret = -EINVAL;
 529                 goto err_put_fd;
 530         }
 531
 532         fd_install(fd, sync_file->file);
 533
 534         *p_fd = fd;
 535         return 0;
 536 err_put_fd:
 537         put_unused_fd(fd);
 538         return ret;
 539 }
 540 /**
 541  * drm_syncobj_open - initalizes syncobj file-private structures at devnode open time
 542  * @file_private: drm file-private structure to set up
 543  *
 544  * Called at device open time, sets up the structure for handling refcounting
 545  * of sync objects.
 546  */
 547 void
 548 drm_syncobj_open(struct drm_file *file_private)
 549 {
 550         idr_init_base(&file_private->syncobj_idr, 1);
 551         spin_lock_init(&file_private->syncobj_table_lock);
 552 }
 553
 554 static int
 555 drm_syncobj_release_handle(int id, void *ptr, void *data)
 556 {
 557         struct drm_syncobj *syncobj = ptr;
 558
 559         drm_syncobj_put(syncobj);
 560         return 0;
 561 }
 562
 563 /**
 564  * drm_syncobj_release - release file-private sync object resources
 565  * @file_private: drm file-private structure to clean up
 566  *
 567  * Called at close time when the filp is going away.
 568  *
 569  * Releases any remaining references on objects by this filp.
 570  */
 571 void
 572 drm_syncobj_release(struct drm_file *file_private)
 573 {
 574         idr_for_each(&file_private->syncobj_idr,
 575                      &drm_syncobj_release_handle, file_private);
 576         idr_destroy(&file_private->syncobj_idr);
 577 }
 578
 579 int
 580 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
 581                          struct drm_file *file_private)
 582 {
 583         struct drm_syncobj_create *args = data;
 584
 585         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 586                 return -ENODEV;
 587
 588         /* no valid flags yet */
 589         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
 590                 return -EINVAL;
 591
 592         return drm_syncobj_create_as_handle(file_private,
 593                                             &args->handle, args->flags);
 594 }
 595
 596 int
 597 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
 598                           struct drm_file *file_private)
 599 {
 600         struct drm_syncobj_destroy *args = data;
 601
 602         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 603                 return -ENODEV;
 604
 605         /* make sure padding is empty */
 606         if (args->pad)
 607                 return -EINVAL;
 608         return drm_syncobj_destroy(file_private, args->handle);
 609 }
 610
 611 int
 612 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
 613                                    struct drm_file *file_private)
 614 {
 615         struct drm_syncobj_handle *args = data;
 616
 617         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 618                 return -ENODEV;
 619
 620         if (args->pad)
 621                 return -EINVAL;
 622
 623         if (args->flags != 0 &&
 624             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 625                 return -EINVAL;
 626
 627         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
 628                 return drm_syncobj_export_sync_file(file_private, args->handle,
 629                                                     &args->fd);
 630
 631         return drm_syncobj_handle_to_fd(file_private, args->handle,
 632                                         &args->fd);
 633 }
 634
 635 int
 636 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
 637                                    struct drm_file *file_private)
 638 {
 639         struct drm_syncobj_handle *args = data;
 640
 641         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 642                 return -ENODEV;
 643
 644         if (args->pad)
 645                 return -EINVAL;
 646
 647         if (args->flags != 0 &&
 648             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 649                 return -EINVAL;
 650
 651         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
 652                 return drm_syncobj_import_sync_file_fence(file_private,
 653                                                           args->fd,
 654                                                           args->handle);
 655
 656         return drm_syncobj_fd_to_handle(file_private, args->fd,
 657                                         &args->handle);
 658 }
 659
 660 struct syncobj_wait_entry {
 661         struct task_struct *task;
 662         struct dma_fence *fence;
 663         struct dma_fence_cb fence_cb;
 664         struct drm_syncobj_cb syncobj_cb;
 665 };
 666
 667 static void syncobj_wait_fence_func(struct dma_fence *fence,
 668                                     struct dma_fence_cb *cb)
 669 {
 670         struct syncobj_wait_entry *wait =
 671                 container_of(cb, struct syncobj_wait_entry, fence_cb);
 672
 673         wake_up_process(wait->task);
 674 }
 675
 676 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
 677                                       struct drm_syncobj_cb *cb)
 678 {
 679         struct syncobj_wait_entry *wait =
 680                 container_of(cb, struct syncobj_wait_entry, syncobj_cb);
 681
 682         /* This happens inside the syncobj lock */
 683         wait->fence = dma_fence_get(rcu_dereference_protected(syncobj->fence,
 684                                                               lockdep_is_held(&syncobj->lock)));
 685         wake_up_process(wait->task);
 686 }
 687
 688 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
 689                                                   uint32_t count,
 690                                                   uint32_t flags,
 691                                                   signed long timeout,
 692                                                   uint32_t *idx)
 693 {
 694         struct syncobj_wait_entry *entries;
 695         struct dma_fence *fence;
 696         signed long ret;
 697         uint32_t signaled_count, i;
 698
 699         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
 700         if (!entries)
 701                 return -ENOMEM;
 702
 703         /* Walk the list of sync objects and initialize entries.  We do
 704          * this up-front so that we can properly return -EINVAL if there is
 705          * a syncobj with a missing fence and then never have the chance of
 706          * returning -EINVAL again.
 707          */
 708         signaled_count = 0;
 709         for (i = 0; i < count; ++i) {
 710                 entries[i].task = current;
 711                 entries[i].fence = drm_syncobj_fence_get(syncobjs[i]);
 712                 if (!entries[i].fence) {
 713                         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 714                                 continue;
 715                         } else {
 716                                 ret = -EINVAL;
 717                                 goto cleanup_entries;
 718                         }
 719                 }
 720
 721                 if (dma_fence_is_signaled(entries[i].fence)) {
 722                         if (signaled_count == 0 && idx)
 723                                 *idx = i;
 724                         signaled_count++;
 725                 }
 726         }
 727
 728         /* Initialize ret to the max of timeout and 1.  That way, the
 729          * default return value indicates a successful wait and not a
 730          * timeout.
 731          */
 732         ret = max_t(signed long, timeout, 1);
 733
 734         if (signaled_count == count ||
 735             (signaled_count > 0 &&
 736              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
 737                 goto cleanup_entries;
 738
 739         /* There's a very annoying laxness in the dma_fence API here, in
 740          * that backends are not required to automatically report when a
 741          * fence is signaled prior to fence->ops->enable_signaling() being
 742          * called.  So here if we fail to match signaled_count, we need to
 743          * fallthough and try a 0 timeout wait!
 744          */
 745
 746         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
 747                 for (i = 0; i < count; ++i) {
 748                         if (entries[i].fence)
 749                                 continue;
 750
 751                         drm_syncobj_fence_get_or_add_callback(syncobjs[i],
 752                                                               &entries[i].fence,
 753                                                               &entries[i].syncobj_cb,
 754                                                               syncobj_wait_syncobj_func);
 755                 }
 756         }
 757
 758         do {
 759                 set_current_state(TASK_INTERRUPTIBLE);
 760
 761                 signaled_count = 0;
 762                 for (i = 0; i < count; ++i) {
 763                         fence = entries[i].fence;
 764                         if (!fence)
 765                                 continue;
 766
 767                         if (dma_fence_is_signaled(fence) ||
 768                             (!entries[i].fence_cb.func &&
 769                              dma_fence_add_callback(fence,
 770                                                     &entries[i].fence_cb,
 771                                                     syncobj_wait_fence_func))) {
 772                                 /* The fence has been signaled */
 773                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
 774                                         signaled_count++;
 775                                 } else {
 776                                         if (idx)
 777                                                 *idx = i;
 778                                         goto done_waiting;
 779                                 }
 780                         }
 781                 }
 782
 783                 if (signaled_count == count)
 784                         goto done_waiting;
 785
 786                 if (timeout == 0) {
 787                         /* If we are doing a 0 timeout wait and we got
 788                          * here, then we just timed out.
 789                          */
 790                         ret = 0;
 791                         goto done_waiting;
 792                 }
 793
 794                 ret = schedule_timeout(ret);
 795
 796                 if (ret > 0 && signal_pending(current))
 797                         ret = -ERESTARTSYS;
 798         } while (ret > 0);
 799
 800 done_waiting:
 801         __set_current_state(TASK_RUNNING);
 802
 803 cleanup_entries:
 804         for (i = 0; i < count; ++i) {
 805                 if (entries[i].syncobj_cb.func)
 806                         drm_syncobj_remove_callback(syncobjs[i],
 807                                                     &entries[i].syncobj_cb);
 808                 if (entries[i].fence_cb.func)
 809                         dma_fence_remove_callback(entries[i].fence,
 810                                                   &entries[i].fence_cb);
 811                 dma_fence_put(entries[i].fence);
 812         }
 813         kfree(entries);
 814
 815         return ret;
 816 }
 817
 818 /**
 819  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
 820  *
 821  * @timeout_nsec: timeout nsec component in ns, 0 for poll
 822  *
 823  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
 824  */
 825 static signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
 826 {
 827         ktime_t abs_timeout, now;
 828         u64 timeout_ns, timeout_jiffies64;
 829
 830         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
 831         if (timeout_nsec == 0)
 832                 return 0;
 833
 834         abs_timeout = ns_to_ktime(timeout_nsec);
 835         now = ktime_get();
 836
 837         if (!ktime_after(abs_timeout, now))
 838                 return 0;
 839
 840         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
 841
 842         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
 843         /*  clamp timeout to avoid infinite timeout */
 844         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
 845                 return MAX_SCHEDULE_TIMEOUT - 1;
 846
 847         return timeout_jiffies64 + 1;
 848 }
 849
 850 static int drm_syncobj_array_wait(struct drm_device *dev,
 851                                   struct drm_file *file_private,
 852                                   struct drm_syncobj_wait *wait,
 853                                   struct drm_syncobj **syncobjs)
 854 {
 855         signed long timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
 856         signed long ret = 0;
 857         uint32_t first = ~0;
 858
 859         ret = drm_syncobj_array_wait_timeout(syncobjs,
 860                                              wait->count_handles,
 861                                              wait->flags,
 862                                              timeout, &first);
 863         if (ret < 0)
 864                 return ret;
 865
 866         wait->first_signaled = first;
 867         if (ret == 0)
 868                 return -ETIME;
 869         return 0;
 870 }
 871
 872 static int drm_syncobj_array_find(struct drm_file *file_private,
 873                                   void __user *user_handles,
 874                                   uint32_t count_handles,
 875                                   struct drm_syncobj ***syncobjs_out)
 876 {
 877         uint32_t i, *handles;
 878         struct drm_syncobj **syncobjs;
 879         int ret;
 880
 881         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
 882         if (handles == NULL)
 883                 return -ENOMEM;
 884
 885         if (copy_from_user(handles, user_handles,
 886                            sizeof(uint32_t) * count_handles)) {
 887                 ret = -EFAULT;
 888                 goto err_free_handles;
 889         }
 890
 891         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
 892         if (syncobjs == NULL) {
 893                 ret = -ENOMEM;
 894                 goto err_free_handles;
 895         }
 896
 897         for (i = 0; i < count_handles; i++) {
 898                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
 899                 if (!syncobjs[i]) {
 900                         ret = -ENOENT;
 901                         goto err_put_syncobjs;
 902                 }
 903         }
 904
 905         kfree(handles);
 906         *syncobjs_out = syncobjs;
 907         return 0;
 908
 909 err_put_syncobjs:
 910         while (i-- > 0)
 911                 drm_syncobj_put(syncobjs[i]);
 912         kfree(syncobjs);
 913 err_free_handles:
 914         kfree(handles);
 915
 916         return ret;
 917 }
 918
 919 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
 920                                    uint32_t count)
 921 {
 922         uint32_t i;
 923         for (i = 0; i < count; i++)
 924                 drm_syncobj_put(syncobjs[i]);
 925         kfree(syncobjs);
 926 }
 927
 928 int
 929 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
 930                        struct drm_file *file_private)
 931 {
 932         struct drm_syncobj_wait *args = data;
 933         struct drm_syncobj **syncobjs;
 934         int ret = 0;
 935
 936         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 937                 return -ENODEV;
 938
 939         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
 940                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
 941                 return -EINVAL;
 942
 943         if (args->count_handles == 0)
 944                 return -EINVAL;
 945
 946         ret = drm_syncobj_array_find(file_private,
 947                                      u64_to_user_ptr(args->handles),
 948                                      args->count_handles,
 949                                      &syncobjs);
 950         if (ret < 0)
 951                 return ret;
 952
 953         ret = drm_syncobj_array_wait(dev, file_private,
 954                                      args, syncobjs);
 955
 956         drm_syncobj_array_free(syncobjs, args->count_handles);
 957
 958         return ret;
 959 }
 960
 961 int
 962 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
 963                         struct drm_file *file_private)
 964 {
 965         struct drm_syncobj_array *args = data;
 966         struct drm_syncobj **syncobjs;
 967         uint32_t i;
 968         int ret;
 969
 970         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
 971                 return -ENODEV;
 972
 973         if (args->pad != 0)
 974                 return -EINVAL;
 975
 976         if (args->count_handles == 0)
 977                 return -EINVAL;
 978
 979         ret = drm_syncobj_array_find(file_private,
 980                                      u64_to_user_ptr(args->handles),
 981                                      args->count_handles,
 982                                      &syncobjs);
 983         if (ret < 0)
 984                 return ret;
 985
 986         for (i = 0; i < args->count_handles; i++)
 987                 drm_syncobj_replace_fence(syncobjs[i], NULL);
 988
 989         drm_syncobj_array_free(syncobjs, args->count_handles);
 990
 991         return 0;
 992 }
 993
 994 int
 995 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
 996                          struct drm_file *file_private)
 997 {
 998         struct drm_syncobj_array *args = data;
 999         struct drm_syncobj **syncobjs;
1000         uint32_t i;
1001         int ret;
1002
1003         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1004                 return -ENODEV;
1005
1006         if (args->pad != 0)
1007                 return -EINVAL;
1008
1009         if (args->count_handles == 0)
1010                 return -EINVAL;
1011
1012         ret = drm_syncobj_array_find(file_private,
1013                                      u64_to_user_ptr(args->handles),
1014                                      args->count_handles,
1015                                      &syncobjs);
1016         if (ret < 0)
1017                 return ret;
1018
1019         for (i = 0; i < args->count_handles; i++) {
1020                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1021                 if (ret < 0)
1022                         break;
1023         }
1024
1025         drm_syncobj_array_free(syncobjs, args->count_handles);
1026
1027         return ret;
1028 }