drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

   1 /*
   2  * Copyright 2017 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "vmm.h"
  23
  24 #include <subdev/fb.h>
  25 #include <subdev/ltc.h>
  26 #include <subdev/timer.h>
  27
  28 #include <nvif/if900d.h>
  29 #include <nvif/unpack.h>
  30
  31 static inline void
  32 gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  33                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
  34 {
  35         u64 base = (addr >> 8) | map->type;
  36         u64 data = base;
  37
  38         if (map->ctag && !(map->next & (1ULL << 44))) {
  39                 while (ptes--) {
  40                         data = base | ((map->ctag >> 1) << 44);
  41                         if (!(map->ctag++ & 1))
  42                                 data |= BIT_ULL(60);
  43
  44                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  45                         base += map->next;
  46                 }
  47         } else {
  48                 map->type += ptes * map->ctag;
  49
  50                 while (ptes--) {
  51                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  52                         data += map->next;
  53                 }
  54         }
  55 }
  56
  57 void
  58 gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  59                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  60 {
  61         VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  62 }
  63
  64 void
  65 gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  66                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  67 {
  68         if (map->page->shift == PAGE_SHIFT) {
  69                 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
  70                 nvkm_kmap(pt->memory);
  71                 while (ptes--) {
  72                         const u64 data = (*map->dma++ >> 8) | map->type;
  73                         VMM_WO064(pt, vmm, ptei++ * 8, data);
  74                         map->type += map->ctag;
  75                 }
  76                 nvkm_done(pt->memory);
  77                 return;
  78         }
  79
  80         VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  81 }
  82
  83 void
  84 gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  85                   u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  86 {
  87         VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  88 }
  89
  90 void
  91 gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
  92                     struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
  93 {
  94         VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
  95 }
  96
  97 const struct nvkm_vmm_desc_func
  98 gf100_vmm_pgt = {
  99         .unmap = gf100_vmm_pgt_unmap,
 100         .mem = gf100_vmm_pgt_mem,
 101         .dma = gf100_vmm_pgt_dma,
 102         .sgl = gf100_vmm_pgt_sgl,
 103 };
 104
 105 void
 106 gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
 107 {
 108         struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 109         struct nvkm_mmu_pt *pd = pgd->pt[0];
 110         struct nvkm_mmu_pt *pt;
 111         u64 data = 0;
 112
 113         if ((pt = pgt->pt[0])) {
 114                 switch (nvkm_memory_target(pt->memory)) {
 115                 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
 116                 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
 117                         data |= BIT_ULL(35); /* VOL */
 118                         break;
 119                 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
 120                 default:
 121                         WARN_ON(1);
 122                         return;
 123                 }
 124                 data |= pt->addr >> 8;
 125         }
 126
 127         if ((pt = pgt->pt[1])) {
 128                 switch (nvkm_memory_target(pt->memory)) {
 129                 case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
 130                 case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
 131                         data |= BIT_ULL(34); /* VOL */
 132                         break;
 133                 case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
 134                 default:
 135                         WARN_ON(1);
 136                         return;
 137                 }
 138                 data |= pt->addr << 24;
 139         }
 140
 141         nvkm_kmap(pd->memory);
 142         VMM_WO064(pd, vmm, pdei * 8, data);
 143         nvkm_done(pd->memory);
 144 }
 145
 146 const struct nvkm_vmm_desc_func
 147 gf100_vmm_pgd = {
 148         .unmap = gf100_vmm_pgt_unmap,
 149         .pde = gf100_vmm_pgd_pde,
 150 };
 151
 152 static const struct nvkm_vmm_desc
 153 gf100_vmm_desc_17_12[] = {
 154         { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
 155         { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 156         {}
 157 };
 158
 159 static const struct nvkm_vmm_desc
 160 gf100_vmm_desc_17_17[] = {
 161         { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 162         { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 163         {}
 164 };
 165
 166 static const struct nvkm_vmm_desc
 167 gf100_vmm_desc_16_12[] = {
 168         { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
 169         { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 170         {}
 171 };
 172
 173 static const struct nvkm_vmm_desc
 174 gf100_vmm_desc_16_16[] = {
 175         { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 176         { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 177         {}
 178 };
 179
 180 void
 181 gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth)
 182 {
 183         struct nvkm_subdev *subdev = &vmm->mmu->subdev;
 184         struct nvkm_device *device = subdev->device;
 185         u32 type = depth << 24;
 186
 187         type = 0x00000001; /* PAGE_ALL */
 188         if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
 189                 type |= 0x00000004; /* HUB_ONLY */
 190
 191         mutex_lock(&subdev->mutex);
 192         /* Looks like maybe a "free flush slots" counter, the
 193          * faster you write to 0x100cbc to more it decreases.
 194          */
 195         nvkm_msec(device, 2000,
 196                 if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
 197                         break;
 198         );
 199
 200         nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8);
 201         nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
 202
 203         /* Wait for flush to be queued? */
 204         nvkm_msec(device, 2000,
 205                 if (nvkm_rd32(device, 0x100c80) & 0x00008000)
 206                         break;
 207         );
 208         mutex_unlock(&subdev->mutex);
 209 }
 210
 211 void
 212 gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
 213 {
 214         gf100_vmm_flush_(vmm, 0);
 215 }
 216
 217 int
 218 gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
 219                 struct nvkm_vmm_map *map)
 220 {
 221         const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
 222         const struct nvkm_vmm_page *page = map->page;
 223         const bool gm20x = page->desc->func->sparse != NULL;
 224         union {
 225                 struct gf100_vmm_map_vn vn;
 226                 struct gf100_vmm_map_v0 v0;
 227         } *args = argv;
 228         struct nvkm_device *device = vmm->mmu->subdev.device;
 229         struct nvkm_memory *memory = map->memory;
 230         u8  kind, priv, ro, vol;
 231         int kindn, aper, ret = -ENOSYS;
 232         const u8 *kindm;
 233
 234         map->next = (1 << page->shift) >> 8;
 235         map->type = map->ctag = 0;
 236
 237         if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 238                 vol  = !!args->v0.vol;
 239                 ro   = !!args->v0.ro;
 240                 priv = !!args->v0.priv;
 241                 kind =   args->v0.kind;
 242         } else
 243         if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
 244                 vol  = target == NVKM_MEM_TARGET_HOST;
 245                 ro   = 0;
 246                 priv = 0;
 247                 kind = 0x00;
 248         } else {
 249                 VMM_DEBUG(vmm, "args");
 250                 return ret;
 251         }
 252
 253         aper = vmm->func->aper(target);
 254         if (WARN_ON(aper < 0))
 255                 return aper;
 256
 257         kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
 258         if (kind >= kindn || kindm[kind] == 0xff) {
 259                 VMM_DEBUG(vmm, "kind %02x", kind);
 260                 return -EINVAL;
 261         }
 262
 263         if (kindm[kind] != kind) {
 264                 u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
 265                 u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
 266                 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
 267                         VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
 268                         return -EINVAL;
 269                 }
 270
 271                 ret = nvkm_memory_tags_get(memory, device, tags,
 272                                            nvkm_ltc_tags_clear,
 273                                            &map->tags);
 274                 if (ret) {
 275                         VMM_DEBUG(vmm, "comp %d", ret);
 276                         return ret;
 277                 }
 278
 279                 if (map->tags->mn) {
 280                         u64 tags = map->tags->mn->offset + (map->offset >> 17);
 281                         if (page->shift == 17 || !gm20x) {
 282                                 map->type |= tags << 44;
 283                                 map->ctag |= 1ULL << 44;
 284                                 map->next |= 1ULL << 44;
 285                         } else {
 286                                 map->ctag |= tags << 1 | 1;
 287                         }
 288                 } else {
 289                         kind = kindm[kind];
 290                 }
 291         }
 292
 293         map->type |= BIT(0);
 294         map->type |= (u64)priv << 1;
 295         map->type |= (u64)  ro << 2;
 296         map->type |= (u64) vol << 32;
 297         map->type |= (u64)aper << 33;
 298         map->type |= (u64)kind << 36;
 299         return 0;
 300 }
 301
 302 int
 303 gf100_vmm_aper(enum nvkm_memory_target target)
 304 {
 305         switch (target) {
 306         case NVKM_MEM_TARGET_VRAM: return 0;
 307         case NVKM_MEM_TARGET_HOST: return 2;
 308         case NVKM_MEM_TARGET_NCOH: return 3;
 309         default:
 310                 return -EINVAL;
 311         }
 312 }
 313
 314 void
 315 gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 316 {
 317         nvkm_fo64(inst, 0x0200, 0x00000000, 2);
 318 }
 319
 320 int
 321 gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
 322 {
 323         struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
 324
 325         switch (nvkm_memory_target(pd->memory)) {
 326         case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
 327         case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
 328                 base |= BIT_ULL(2) /* VOL. */;
 329                 break;
 330         case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
 331         default:
 332                 WARN_ON(1);
 333                 return -EINVAL;
 334         }
 335         base |= pd->addr;
 336
 337         nvkm_kmap(inst);
 338         nvkm_wo64(inst, 0x0200, base);
 339         nvkm_wo64(inst, 0x0208, vmm->limit - 1);
 340         nvkm_done(inst);
 341         return 0;
 342 }
 343
 344 int
 345 gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 346 {
 347         return gf100_vmm_join_(vmm, inst, 0);
 348 }
 349
 350 static const struct nvkm_vmm_func
 351 gf100_vmm_17 = {
 352         .join = gf100_vmm_join,
 353         .part = gf100_vmm_part,
 354         .aper = gf100_vmm_aper,
 355         .valid = gf100_vmm_valid,
 356         .flush = gf100_vmm_flush,
 357         .page = {
 358                 { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
 359                 { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
 360                 {}
 361         }
 362 };
 363
 364 static const struct nvkm_vmm_func
 365 gf100_vmm_16 = {
 366         .join = gf100_vmm_join,
 367         .part = gf100_vmm_part,
 368         .aper = gf100_vmm_aper,
 369         .valid = gf100_vmm_valid,
 370         .flush = gf100_vmm_flush,
 371         .page = {
 372                 { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
 373                 { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
 374                 {}
 375         }
 376 };
 377
 378 int
 379 gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
 380                const struct nvkm_vmm_func *func_17,
 381                struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
 382                struct lock_class_key *key, const char *name,
 383                struct nvkm_vmm **pvmm)
 384 {
 385         switch (mmu->subdev.device->fb->page) {
 386         case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size,
 387                                       argv, argc, key, name, pvmm);
 388         case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size,
 389                                       argv, argc, key, name, pvmm);
 390         default:
 391                 WARN_ON(1);
 392                 return -EINVAL;
 393         }
 394 }
 395
 396 int
 397 gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
 398               struct lock_class_key *key, const char *name,
 399               struct nvkm_vmm **pvmm)
 400 {
 401         return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr,
 402                               size, argv, argc, key, name, pvmm);
 403 }