drivers/gpu/drm/amd/amdgpu/amdgpu_object.c

   1 /*
   2  * Copyright 2009 Jerome Glisse.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  20  *
  21  * The above copyright notice and this permission notice (including the
  22  * next paragraph) shall be included in all copies or substantial portions
  23  * of the Software.
  24  *
  25  */
  26 /*
  27  * Authors:
  28  *    Jerome Glisse <glisse@freedesktop.org>
  29  *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  30  *    Dave Airlie
  31  */
  32 #include <linux/list.h>
  33 #include <linux/slab.h>
  34 #include <drm/drmP.h>
  35 #include <drm/amdgpu_drm.h>
  36 #include <drm/drm_cache.h>
  37 #include "amdgpu.h"
  38 #include "amdgpu_trace.h"
  39
  40
  41
  42 static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
  43                                                 struct ttm_mem_reg *mem)
  44 {
  45         if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
  46                 return 0;
  47
  48         return ((mem->start << PAGE_SHIFT) + mem->size) >
  49                 adev->mc.visible_vram_size ?
  50                 adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
  51                 mem->size;
  52 }
  53
  54 static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
  55                        struct ttm_mem_reg *old_mem,
  56                        struct ttm_mem_reg *new_mem)
  57 {
  58         u64 vis_size;
  59         if (!adev)
  60                 return;
  61
  62         if (new_mem) {
  63                 switch (new_mem->mem_type) {
  64                 case TTM_PL_TT:
  65                         atomic64_add(new_mem->size, &adev->gtt_usage);
  66                         break;
  67                 case TTM_PL_VRAM:
  68                         atomic64_add(new_mem->size, &adev->vram_usage);
  69                         vis_size = amdgpu_get_vis_part_size(adev, new_mem);
  70                         atomic64_add(vis_size, &adev->vram_vis_usage);
  71                         break;
  72                 }
  73         }
  74
  75         if (old_mem) {
  76                 switch (old_mem->mem_type) {
  77                 case TTM_PL_TT:
  78                         atomic64_sub(old_mem->size, &adev->gtt_usage);
  79                         break;
  80                 case TTM_PL_VRAM:
  81                         atomic64_sub(old_mem->size, &adev->vram_usage);
  82                         vis_size = amdgpu_get_vis_part_size(adev, old_mem);
  83                         atomic64_sub(vis_size, &adev->vram_vis_usage);
  84                         break;
  85                 }
  86         }
  87 }
  88
  89 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
  90 {
  91         struct amdgpu_bo *bo;
  92
  93         bo = container_of(tbo, struct amdgpu_bo, tbo);
  94
  95         amdgpu_update_memory_usage(bo->adev, &bo->tbo.mem, NULL);
  96
  97         if (bo->gem_base.import_attach)
  98                 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg);
  99         drm_gem_object_release(&bo->gem_base);
 100         amdgpu_bo_unref(&bo->parent);
 101         if (!list_empty(&bo->shadow_list)) {
 102                 mutex_lock(&bo->adev->shadow_list_lock);
 103                 list_del_init(&bo->shadow_list);
 104                 mutex_unlock(&bo->adev->shadow_list_lock);
 105         }
 106         kfree(bo->metadata);
 107         kfree(bo);
 108 }
 109
 110 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
 111 {
 112         if (bo->destroy == &amdgpu_ttm_bo_destroy)
 113                 return true;
 114         return false;
 115 }
 116
 117 static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
 118                                       struct ttm_placement *placement,
 119                                       struct ttm_place *places,
 120                                       u32 domain, u64 flags)
 121 {
 122         u32 c = 0;
 123
 124         if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
 125                 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
 126
 127                 if (flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS &&
 128                     !(flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
 129                     adev->mc.visible_vram_size < adev->mc.real_vram_size) {
 130                         places[c].fpfn = visible_pfn;
 131                         places[c].lpfn = 0;
 132                         places[c].flags = TTM_PL_FLAG_WC |
 133                                 TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_VRAM |
 134                                 TTM_PL_FLAG_TOPDOWN;
 135                         c++;
 136                 }
 137
 138                 places[c].fpfn = 0;
 139                 places[c].lpfn = 0;
 140                 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
 141                         TTM_PL_FLAG_VRAM;
 142                 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
 143                         places[c].lpfn = visible_pfn;
 144                 else
 145                         places[c].flags |= TTM_PL_FLAG_TOPDOWN;
 146                 c++;
 147         }
 148
 149         if (domain & AMDGPU_GEM_DOMAIN_GTT) {
 150                 places[c].fpfn = 0;
 151                 places[c].lpfn = 0;
 152                 places[c].flags = TTM_PL_FLAG_TT;
 153                 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
 154                         places[c].flags |= TTM_PL_FLAG_WC |
 155                                 TTM_PL_FLAG_UNCACHED;
 156                 else
 157                         places[c].flags |= TTM_PL_FLAG_CACHED;
 158                 c++;
 159         }
 160
 161         if (domain & AMDGPU_GEM_DOMAIN_CPU) {
 162                 places[c].fpfn = 0;
 163                 places[c].lpfn = 0;
 164                 places[c].flags = TTM_PL_FLAG_SYSTEM;
 165                 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
 166                         places[c].flags |= TTM_PL_FLAG_WC |
 167                                 TTM_PL_FLAG_UNCACHED;
 168                 else
 169                         places[c].flags |= TTM_PL_FLAG_CACHED;
 170                 c++;
 171         }
 172
 173         if (domain & AMDGPU_GEM_DOMAIN_GDS) {
 174                 places[c].fpfn = 0;
 175                 places[c].lpfn = 0;
 176                 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
 177                 c++;
 178         }
 179
 180         if (domain & AMDGPU_GEM_DOMAIN_GWS) {
 181                 places[c].fpfn = 0;
 182                 places[c].lpfn = 0;
 183                 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
 184                 c++;
 185         }
 186
 187         if (domain & AMDGPU_GEM_DOMAIN_OA) {
 188                 places[c].fpfn = 0;
 189                 places[c].lpfn = 0;
 190                 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
 191                 c++;
 192         }
 193
 194         if (!c) {
 195                 places[c].fpfn = 0;
 196                 places[c].lpfn = 0;
 197                 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
 198                 c++;
 199         }
 200
 201         placement->num_placement = c;
 202         placement->placement = places;
 203
 204         placement->num_busy_placement = c;
 205         placement->busy_placement = places;
 206 }
 207
 208 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
 209 {
 210         amdgpu_ttm_placement_init(abo->adev, &abo->placement,
 211                                   abo->placements, domain, abo->flags);
 212 }
 213
 214 static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
 215                                         struct ttm_placement *placement)
 216 {
 217         BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));
 218
 219         memcpy(bo->placements, placement->placement,
 220                placement->num_placement * sizeof(struct ttm_place));
 221         bo->placement.num_placement = placement->num_placement;
 222         bo->placement.num_busy_placement = placement->num_busy_placement;
 223         bo->placement.placement = bo->placements;
 224         bo->placement.busy_placement = bo->placements;
 225 }
 226
 227 /**
 228  * amdgpu_bo_create_kernel - create BO for kernel use
 229  *
 230  * @adev: amdgpu device object
 231  * @size: size for the new BO
 232  * @align: alignment for the new BO
 233  * @domain: where to place it
 234  * @bo_ptr: resulting BO
 235  * @gpu_addr: GPU addr of the pinned BO
 236  * @cpu_addr: optional CPU address mapping
 237  *
 238  * Allocates and pins a BO for kernel internal use.
 239  *
 240  * Returns 0 on success, negative error code otherwise.
 241  */
 242 int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
 243                             unsigned long size, int align,
 244                             u32 domain, struct amdgpu_bo **bo_ptr,
 245                             u64 *gpu_addr, void **cpu_addr)
 246 {
 247         int r;
 248
 249         r = amdgpu_bo_create(adev, size, align, true, domain,
 250                              AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
 251                              NULL, NULL, bo_ptr);
 252         if (r) {
 253                 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
 254                 return r;
 255         }
 256
 257         r = amdgpu_bo_reserve(*bo_ptr, false);
 258         if (r) {
 259                 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
 260                 goto error_free;
 261         }
 262
 263         r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
 264         if (r) {
 265                 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
 266                 goto error_unreserve;
 267         }
 268
 269         if (cpu_addr) {
 270                 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
 271                 if (r) {
 272                         dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
 273                         goto error_unreserve;
 274                 }
 275         }
 276
 277         amdgpu_bo_unreserve(*bo_ptr);
 278
 279         return 0;
 280
 281 error_unreserve:
 282         amdgpu_bo_unreserve(*bo_ptr);
 283
 284 error_free:
 285         amdgpu_bo_unref(bo_ptr);
 286
 287         return r;
 288 }
 289
 290 /**
 291  * amdgpu_bo_free_kernel - free BO for kernel use
 292  *
 293  * @bo: amdgpu BO to free
 294  *
 295  * unmaps and unpin a BO for kernel internal use.
 296  */
 297 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 298                            void **cpu_addr)
 299 {
 300         if (*bo == NULL)
 301                 return;
 302
 303         if (likely(amdgpu_bo_reserve(*bo, false) == 0)) {
 304                 if (cpu_addr)
 305                         amdgpu_bo_kunmap(*bo);
 306
 307                 amdgpu_bo_unpin(*bo);
 308                 amdgpu_bo_unreserve(*bo);
 309         }
 310         amdgpu_bo_unref(bo);
 311
 312         if (gpu_addr)
 313                 *gpu_addr = 0;
 314
 315         if (cpu_addr)
 316                 *cpu_addr = NULL;
 317 }
 318
 319 int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
 320                                 unsigned long size, int byte_align,
 321                                 bool kernel, u32 domain, u64 flags,
 322                                 struct sg_table *sg,
 323                                 struct ttm_placement *placement,
 324                                 struct reservation_object *resv,
 325                                 struct amdgpu_bo **bo_ptr)
 326 {
 327         struct amdgpu_bo *bo;
 328         enum ttm_bo_type type;
 329         unsigned long page_align;
 330         size_t acc_size;
 331         int r;
 332
 333         page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
 334         size = ALIGN(size, PAGE_SIZE);
 335
 336         if (kernel) {
 337                 type = ttm_bo_type_kernel;
 338         } else if (sg) {
 339                 type = ttm_bo_type_sg;
 340         } else {
 341                 type = ttm_bo_type_device;
 342         }
 343         *bo_ptr = NULL;
 344
 345         acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
 346                                        sizeof(struct amdgpu_bo));
 347
 348         bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
 349         if (bo == NULL)
 350                 return -ENOMEM;
 351         r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
 352         if (unlikely(r)) {
 353                 kfree(bo);
 354                 return r;
 355         }
 356         bo->adev = adev;
 357         INIT_LIST_HEAD(&bo->shadow_list);
 358         INIT_LIST_HEAD(&bo->va);
 359         bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
 360                                          AMDGPU_GEM_DOMAIN_GTT |
 361                                          AMDGPU_GEM_DOMAIN_CPU |
 362                                          AMDGPU_GEM_DOMAIN_GDS |
 363                                          AMDGPU_GEM_DOMAIN_GWS |
 364                                          AMDGPU_GEM_DOMAIN_OA);
 365         bo->allowed_domains = bo->prefered_domains;
 366         if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
 367                 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
 368
 369         bo->flags = flags;
 370
 371         /* For architectures that don't support WC memory,
 372          * mask out the WC flag from the BO
 373          */
 374         if (!drm_arch_can_wc_memory())
 375                 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 376
 377         amdgpu_fill_placement_to_bo(bo, placement);
 378         /* Kernel allocation are uninterruptible */
 379         r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
 380                         &bo->placement, page_align, !kernel, NULL,
 381                         acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
 382         if (unlikely(r != 0)) {
 383                 return r;
 384         }
 385
 386         if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
 387             bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
 388                 struct fence *fence;
 389
 390                 if (adev->mman.buffer_funcs_ring == NULL ||
 391                    !adev->mman.buffer_funcs_ring->ready) {
 392                         r = -EBUSY;
 393                         goto fail_free;
 394                 }
 395
 396                 r = amdgpu_bo_reserve(bo, false);
 397                 if (unlikely(r != 0))
 398                         goto fail_free;
 399
 400                 amdgpu_ttm_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
 401                 r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 402                 if (unlikely(r != 0))
 403                         goto fail_unreserve;
 404
 405                 amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
 406                 amdgpu_bo_fence(bo, fence, false);
 407                 amdgpu_bo_unreserve(bo);
 408                 fence_put(bo->tbo.moving);
 409                 bo->tbo.moving = fence_get(fence);
 410                 fence_put(fence);
 411         }
 412         *bo_ptr = bo;
 413
 414         trace_amdgpu_bo_create(bo);
 415
 416         return 0;
 417
 418 fail_unreserve:
 419         amdgpu_bo_unreserve(bo);
 420 fail_free:
 421         amdgpu_bo_unref(&bo);
 422         return r;
 423 }
 424
 425 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
 426                                    unsigned long size, int byte_align,
 427                                    struct amdgpu_bo *bo)
 428 {
 429         struct ttm_placement placement = {0};
 430         struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
 431         int r;
 432
 433         if (bo->shadow)
 434                 return 0;
 435
 436         bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
 437         memset(&placements, 0,
 438                (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
 439
 440         amdgpu_ttm_placement_init(adev, &placement,
 441                                   placements, AMDGPU_GEM_DOMAIN_GTT,
 442                                   AMDGPU_GEM_CREATE_CPU_GTT_USWC);
 443
 444         r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
 445                                         AMDGPU_GEM_DOMAIN_GTT,
 446                                         AMDGPU_GEM_CREATE_CPU_GTT_USWC,
 447                                         NULL, &placement,
 448                                         bo->tbo.resv,
 449                                         &bo->shadow);
 450         if (!r) {
 451                 bo->shadow->parent = amdgpu_bo_ref(bo);
 452                 mutex_lock(&adev->shadow_list_lock);
 453                 list_add_tail(&bo->shadow_list, &adev->shadow_list);
 454                 mutex_unlock(&adev->shadow_list_lock);
 455         }
 456
 457         return r;
 458 }
 459
 460 int amdgpu_bo_create(struct amdgpu_device *adev,
 461                      unsigned long size, int byte_align,
 462                      bool kernel, u32 domain, u64 flags,
 463                      struct sg_table *sg,
 464                      struct reservation_object *resv,
 465                      struct amdgpu_bo **bo_ptr)
 466 {
 467         struct ttm_placement placement = {0};
 468         struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
 469         int r;
 470
 471         memset(&placements, 0,
 472                (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));
 473
 474         amdgpu_ttm_placement_init(adev, &placement,
 475                                   placements, domain, flags);
 476
 477         r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
 478                                         domain, flags, sg, &placement,
 479                                         resv, bo_ptr);
 480         if (r)
 481                 return r;
 482
 483         if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
 484                 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
 485                 if (r)
 486                         amdgpu_bo_unref(bo_ptr);
 487         }
 488
 489         return r;
 490 }
 491
 492 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
 493                                struct amdgpu_ring *ring,
 494                                struct amdgpu_bo *bo,
 495                                struct reservation_object *resv,
 496                                struct fence **fence,
 497                                bool direct)
 498
 499 {
 500         struct amdgpu_bo *shadow = bo->shadow;
 501         uint64_t bo_addr, shadow_addr;
 502         int r;
 503
 504         if (!shadow)
 505                 return -EINVAL;
 506
 507         bo_addr = amdgpu_bo_gpu_offset(bo);
 508         shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
 509
 510         r = reservation_object_reserve_shared(bo->tbo.resv);
 511         if (r)
 512                 goto err;
 513
 514         r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
 515                                amdgpu_bo_size(bo), resv, fence,
 516                                direct);
 517         if (!r)
 518                 amdgpu_bo_fence(bo, *fence, true);
 519
 520 err:
 521         return r;
 522 }
 523
 524 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
 525                                   struct amdgpu_ring *ring,
 526                                   struct amdgpu_bo *bo,
 527                                   struct reservation_object *resv,
 528                                   struct fence **fence,
 529                                   bool direct)
 530
 531 {
 532         struct amdgpu_bo *shadow = bo->shadow;
 533         uint64_t bo_addr, shadow_addr;
 534         int r;
 535
 536         if (!shadow)
 537                 return -EINVAL;
 538
 539         bo_addr = amdgpu_bo_gpu_offset(bo);
 540         shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);
 541
 542         r = reservation_object_reserve_shared(bo->tbo.resv);
 543         if (r)
 544                 goto err;
 545
 546         r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
 547                                amdgpu_bo_size(bo), resv, fence,
 548                                direct);
 549         if (!r)
 550                 amdgpu_bo_fence(bo, *fence, true);
 551
 552 err:
 553         return r;
 554 }
 555
 556 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
 557 {
 558         bool is_iomem;
 559         long r;
 560
 561         if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
 562                 return -EPERM;
 563
 564         if (bo->kptr) {
 565                 if (ptr) {
 566                         *ptr = bo->kptr;
 567                 }
 568                 return 0;
 569         }
 570
 571         r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
 572                                                 MAX_SCHEDULE_TIMEOUT);
 573         if (r < 0)
 574                 return r;
 575
 576         r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
 577         if (r)
 578                 return r;
 579
 580         bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
 581         if (ptr)
 582                 *ptr = bo->kptr;
 583
 584         return 0;
 585 }
 586
 587 void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
 588 {
 589         if (bo->kptr == NULL)
 590                 return;
 591         bo->kptr = NULL;
 592         ttm_bo_kunmap(&bo->kmap);
 593 }
 594
 595 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
 596 {
 597         if (bo == NULL)
 598                 return NULL;
 599
 600         ttm_bo_reference(&bo->tbo);
 601         return bo;
 602 }
 603
 604 void amdgpu_bo_unref(struct amdgpu_bo **bo)
 605 {
 606         struct ttm_buffer_object *tbo;
 607
 608         if ((*bo) == NULL)
 609                 return;
 610
 611         tbo = &((*bo)->tbo);
 612         ttm_bo_unref(&tbo);
 613         if (tbo == NULL)
 614                 *bo = NULL;
 615 }
 616
 617 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
 618                              u64 min_offset, u64 max_offset,
 619                              u64 *gpu_addr)
 620 {
 621         int r, i;
 622         unsigned fpfn, lpfn;
 623
 624         if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
 625                 return -EPERM;
 626
 627         if (WARN_ON_ONCE(min_offset > max_offset))
 628                 return -EINVAL;
 629
 630         if (bo->pin_count) {
 631                 uint32_t mem_type = bo->tbo.mem.mem_type;
 632
 633                 if (domain != amdgpu_mem_type_to_domain(mem_type))
 634                         return -EINVAL;
 635
 636                 bo->pin_count++;
 637                 if (gpu_addr)
 638                         *gpu_addr = amdgpu_bo_gpu_offset(bo);
 639
 640                 if (max_offset != 0) {
 641                         u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
 642                         WARN_ON_ONCE(max_offset <
 643                                      (amdgpu_bo_gpu_offset(bo) - domain_start));
 644                 }
 645
 646                 return 0;
 647         }
 648         amdgpu_ttm_placement_from_domain(bo, domain);
 649         for (i = 0; i < bo->placement.num_placement; i++) {
 650                 /* force to pin into visible video ram */
 651                 if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 652                     !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
 653                     (!max_offset || max_offset >
 654                      bo->adev->mc.visible_vram_size)) {
 655                         if (WARN_ON_ONCE(min_offset >
 656                                          bo->adev->mc.visible_vram_size))
 657                                 return -EINVAL;
 658                         fpfn = min_offset >> PAGE_SHIFT;
 659                         lpfn = bo->adev->mc.visible_vram_size >> PAGE_SHIFT;
 660                 } else {
 661                         fpfn = min_offset >> PAGE_SHIFT;
 662                         lpfn = max_offset >> PAGE_SHIFT;
 663                 }
 664                 if (fpfn > bo->placements[i].fpfn)
 665                         bo->placements[i].fpfn = fpfn;
 666                 if (!bo->placements[i].lpfn ||
 667                     (lpfn && lpfn < bo->placements[i].lpfn))
 668                         bo->placements[i].lpfn = lpfn;
 669                 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
 670         }
 671
 672         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 673         if (unlikely(r)) {
 674                 dev_err(bo->adev->dev, "%p pin failed\n", bo);
 675                 goto error;
 676         }
 677         r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
 678         if (unlikely(r)) {
 679                 dev_err(bo->adev->dev, "%p bind failed\n", bo);
 680                 goto error;
 681         }
 682
 683         bo->pin_count = 1;
 684         if (gpu_addr != NULL)
 685                 *gpu_addr = amdgpu_bo_gpu_offset(bo);
 686         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
 687                 bo->adev->vram_pin_size += amdgpu_bo_size(bo);
 688                 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
 689                         bo->adev->invisible_pin_size += amdgpu_bo_size(bo);
 690         } else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
 691                 bo->adev->gart_pin_size += amdgpu_bo_size(bo);
 692         }
 693
 694 error:
 695         return r;
 696 }
 697
 698 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
 699 {
 700         return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
 701 }
 702
 703 int amdgpu_bo_unpin(struct amdgpu_bo *bo)
 704 {
 705         int r, i;
 706
 707         if (!bo->pin_count) {
 708                 dev_warn(bo->adev->dev, "%p unpin not necessary\n", bo);
 709                 return 0;
 710         }
 711         bo->pin_count--;
 712         if (bo->pin_count)
 713                 return 0;
 714         for (i = 0; i < bo->placement.num_placement; i++) {
 715                 bo->placements[i].lpfn = 0;
 716                 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
 717         }
 718         r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
 719         if (unlikely(r)) {
 720                 dev_err(bo->adev->dev, "%p validate failed for unpin\n", bo);
 721                 goto error;
 722         }
 723
 724         if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
 725                 bo->adev->vram_pin_size -= amdgpu_bo_size(bo);
 726                 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
 727                         bo->adev->invisible_pin_size -= amdgpu_bo_size(bo);
 728         } else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
 729                 bo->adev->gart_pin_size -= amdgpu_bo_size(bo);
 730         }
 731
 732 error:
 733         return r;
 734 }
 735
 736 int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
 737 {
 738         /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
 739         if (0 && (adev->flags & AMD_IS_APU)) {
 740                 /* Useless to evict on IGP chips */
 741                 return 0;
 742         }
 743         return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
 744 }
 745
 746 static const char *amdgpu_vram_names[] = {
 747         "UNKNOWN",
 748         "GDDR1",
 749         "DDR2",
 750         "GDDR3",
 751         "GDDR4",
 752         "GDDR5",
 753         "HBM",
 754         "DDR3"
 755 };
 756
 757 int amdgpu_bo_init(struct amdgpu_device *adev)
 758 {
 759         /* reserve PAT memory space to WC for VRAM */
 760         arch_io_reserve_memtype_wc(adev->mc.aper_base,
 761                                    adev->mc.aper_size);
 762
 763         /* Add an MTRR for the VRAM */
 764         adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
 765                                               adev->mc.aper_size);
 766         DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
 767                 adev->mc.mc_vram_size >> 20,
 768                 (unsigned long long)adev->mc.aper_size >> 20);
 769         DRM_INFO("RAM width %dbits %s\n",
 770                  adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
 771         return amdgpu_ttm_init(adev);
 772 }
 773
 774 void amdgpu_bo_fini(struct amdgpu_device *adev)
 775 {
 776         amdgpu_ttm_fini(adev);
 777         arch_phys_wc_del(adev->mc.vram_mtrr);
 778         arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
 779 }
 780
 781 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
 782                              struct vm_area_struct *vma)
 783 {
 784         return ttm_fbdev_mmap(vma, &bo->tbo);
 785 }
 786
 787 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
 788 {
 789         if (AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
 790                 return -EINVAL;
 791
 792         bo->tiling_flags = tiling_flags;
 793         return 0;
 794 }
 795
 796 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
 797 {
 798         lockdep_assert_held(&bo->tbo.resv->lock.base);
 799
 800         if (tiling_flags)
 801                 *tiling_flags = bo->tiling_flags;
 802 }
 803
 804 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
 805                             uint32_t metadata_size, uint64_t flags)
 806 {
 807         void *buffer;
 808
 809         if (!metadata_size) {
 810                 if (bo->metadata_size) {
 811                         kfree(bo->metadata);
 812                         bo->metadata = NULL;
 813                         bo->metadata_size = 0;
 814                 }
 815                 return 0;
 816         }
 817
 818         if (metadata == NULL)
 819                 return -EINVAL;
 820
 821         buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
 822         if (buffer == NULL)
 823                 return -ENOMEM;
 824
 825         kfree(bo->metadata);
 826         bo->metadata_flags = flags;
 827         bo->metadata = buffer;
 828         bo->metadata_size = metadata_size;
 829
 830         return 0;
 831 }
 832
 833 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
 834                            size_t buffer_size, uint32_t *metadata_size,
 835                            uint64_t *flags)
 836 {
 837         if (!buffer && !metadata_size)
 838                 return -EINVAL;
 839
 840         if (buffer) {
 841                 if (buffer_size < bo->metadata_size)
 842                         return -EINVAL;
 843
 844                 if (bo->metadata_size)
 845                         memcpy(buffer, bo->metadata, bo->metadata_size);
 846         }
 847
 848         if (metadata_size)
 849                 *metadata_size = bo->metadata_size;
 850         if (flags)
 851                 *flags = bo->metadata_flags;
 852
 853         return 0;
 854 }
 855
 856 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 857                            struct ttm_mem_reg *new_mem)
 858 {
 859         struct amdgpu_bo *abo;
 860         struct ttm_mem_reg *old_mem = &bo->mem;
 861
 862         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
 863                 return;
 864
 865         abo = container_of(bo, struct amdgpu_bo, tbo);
 866         amdgpu_vm_bo_invalidate(abo->adev, abo);
 867
 868         /* update statistics */
 869         if (!new_mem)
 870                 return;
 871
 872         /* move_notify is called before move happens */
 873         amdgpu_update_memory_usage(abo->adev, &bo->mem, new_mem);
 874
 875         trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
 876 }
 877
 878 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
 879 {
 880         struct amdgpu_device *adev;
 881         struct amdgpu_bo *abo;
 882         unsigned long offset, size, lpfn;
 883         int i, r;
 884
 885         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
 886                 return 0;
 887
 888         abo = container_of(bo, struct amdgpu_bo, tbo);
 889         adev = abo->adev;
 890         if (bo->mem.mem_type != TTM_PL_VRAM)
 891                 return 0;
 892
 893         size = bo->mem.num_pages << PAGE_SHIFT;
 894         offset = bo->mem.start << PAGE_SHIFT;
 895         if ((offset + size) <= adev->mc.visible_vram_size)
 896                 return 0;
 897
 898         /* Can't move a pinned BO to visible VRAM */
 899         if (abo->pin_count > 0)
 900                 return -EINVAL;
 901
 902         /* hurrah the memory is not visible ! */
 903         amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
 904         lpfn =  adev->mc.visible_vram_size >> PAGE_SHIFT;
 905         for (i = 0; i < abo->placement.num_placement; i++) {
 906                 /* Force into visible VRAM */
 907                 if ((abo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
 908                     (!abo->placements[i].lpfn ||
 909                      abo->placements[i].lpfn > lpfn))
 910                         abo->placements[i].lpfn = lpfn;
 911         }
 912         r = ttm_bo_validate(bo, &abo->placement, false, false);
 913         if (unlikely(r == -ENOMEM)) {
 914                 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
 915                 return ttm_bo_validate(bo, &abo->placement, false, false);
 916         } else if (unlikely(r != 0)) {
 917                 return r;
 918         }
 919
 920         offset = bo->mem.start << PAGE_SHIFT;
 921         /* this should never happen */
 922         if ((offset + size) > adev->mc.visible_vram_size)
 923                 return -EINVAL;
 924
 925         return 0;
 926 }
 927
 928 /**
 929  * amdgpu_bo_fence - add fence to buffer object
 930  *
 931  * @bo: buffer object in question
 932  * @fence: fence to add
 933  * @shared: true if fence should be added shared
 934  *
 935  */
 936 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence,
 937                      bool shared)
 938 {
 939         struct reservation_object *resv = bo->tbo.resv;
 940
 941         if (shared)
 942                 reservation_object_add_shared_fence(resv, fence);
 943         else
 944                 reservation_object_add_excl_fence(resv, fence);
 945 }
 946
 947 /**
 948  * amdgpu_bo_gpu_offset - return GPU offset of bo
 949  * @bo: amdgpu object for which we query the offset
 950  *
 951  * Returns current GPU offset of the object.
 952  *
 953  * Note: object should either be pinned or reserved when calling this
 954  * function, it might be useful to add check for this for debugging.
 955  */
 956 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
 957 {
 958         WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
 959         WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
 960                      !amdgpu_ttm_is_bound(bo->tbo.ttm));
 961         WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
 962                      !bo->pin_count);
 963         WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
 964
 965         return bo->tbo.offset;
 966 }