drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.h

   1 #ifndef __NVKM_VMM_H__
   2 #define __NVKM_VMM_H__
   3 #include "priv.h"
   4 #include <core/memory.h>
   5 enum nvkm_memory_target;
   6
   7 struct nvkm_vmm_pt {
   8         /* Some GPUs have a mapping level with a dual page tables to
   9          * support large and small pages in the same address-range.
  10          *
  11          * We track the state of both page tables in one place, which
  12          * is why there's multiple PT pointers/refcounts here.
  13          */
  14         struct nvkm_mmu_pt *pt[2];
  15         u32 refs[2];
  16
  17         /* Page size handled by this PT.
  18          *
  19          * Tesla backend needs to know this when writinge PDEs,
  20          * otherwise unnecessary.
  21          */
  22         u8 page;
  23
  24         /* Entire page table sparse.
  25          *
  26          * Used to propagate sparseness to child page tables.
  27          */
  28         bool sparse:1;
  29
  30         /* Tracking for page directories.
  31          *
  32          * The array is indexed by PDE, and will either point to the
  33          * child page table, or indicate the PDE is marked as sparse.
  34          **/
  35 #define NVKM_VMM_PDE_INVALID(pde) IS_ERR_OR_NULL(pde)
  36 #define NVKM_VMM_PDE_SPARSED(pde) IS_ERR(pde)
  37 #define NVKM_VMM_PDE_SPARSE       ERR_PTR(-EBUSY)
  38         struct nvkm_vmm_pt **pde;
  39
  40         /* Tracking for dual page tables.
  41          *
  42          * There's one entry for each LPTE, keeping track of whether
  43          * there are valid SPTEs in the same address-range.
  44          *
  45          * This information is used to manage LPTE state transitions.
  46          */
  47 #define NVKM_VMM_PTE_SPARSE 0x80
  48 #define NVKM_VMM_PTE_VALID  0x40
  49 #define NVKM_VMM_PTE_SPTES  0x3f
  50         u8 pte[];
  51 };
  52
  53 typedef void (*nvkm_vmm_pxe_func)(struct nvkm_vmm *,
  54                                   struct nvkm_mmu_pt *, u32 ptei, u32 ptes);
  55 typedef void (*nvkm_vmm_pde_func)(struct nvkm_vmm *,
  56                                   struct nvkm_vmm_pt *, u32 pdei);
  57 typedef void (*nvkm_vmm_pte_func)(struct nvkm_vmm *, struct nvkm_mmu_pt *,
  58                                   u32 ptei, u32 ptes, struct nvkm_vmm_map *);
  59
  60 struct nvkm_vmm_desc_func {
  61         nvkm_vmm_pxe_func invalid;
  62         nvkm_vmm_pxe_func unmap;
  63         nvkm_vmm_pxe_func sparse;
  64
  65         nvkm_vmm_pde_func pde;
  66
  67         nvkm_vmm_pte_func mem;
  68         nvkm_vmm_pte_func dma;
  69         nvkm_vmm_pte_func sgl;
  70 };
  71
  72 extern const struct nvkm_vmm_desc_func gf100_vmm_pgd;
  73 void gf100_vmm_pgd_pde(struct nvkm_vmm *, struct nvkm_vmm_pt *, u32);
  74 extern const struct nvkm_vmm_desc_func gf100_vmm_pgt;
  75 void gf100_vmm_pgt_unmap(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
  76 void gf100_vmm_pgt_mem(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
  77                        struct nvkm_vmm_map *);
  78 void gf100_vmm_pgt_dma(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
  79                        struct nvkm_vmm_map *);
  80 void gf100_vmm_pgt_sgl(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32,
  81                        struct nvkm_vmm_map *);
  82
  83 void gk104_vmm_lpt_invalid(struct nvkm_vmm *, struct nvkm_mmu_pt *, u32, u32);
  84
  85 struct nvkm_vmm_desc {
  86         enum {
  87                 PGD,
  88                 PGT,
  89                 SPT,
  90                 LPT,
  91         } type;
  92         u8 bits;        /* VMA bits covered by PT. */
  93         u8 size;        /* Bytes-per-PTE. */
  94         u32 align;      /* PT address alignment. */
  95         const struct nvkm_vmm_desc_func *func;
  96 };
  97
  98 extern const struct nvkm_vmm_desc nv50_vmm_desc_12[];
  99 extern const struct nvkm_vmm_desc nv50_vmm_desc_16[];
 100
 101 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_12[];
 102 extern const struct nvkm_vmm_desc gk104_vmm_desc_16_16[];
 103 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_12[];
 104 extern const struct nvkm_vmm_desc gk104_vmm_desc_17_17[];
 105
 106 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_12[];
 107 extern const struct nvkm_vmm_desc gm200_vmm_desc_16_16[];
 108 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_12[];
 109 extern const struct nvkm_vmm_desc gm200_vmm_desc_17_17[];
 110
 111 extern const struct nvkm_vmm_desc gp100_vmm_desc_12[];
 112 extern const struct nvkm_vmm_desc gp100_vmm_desc_16[];
 113
 114 struct nvkm_vmm_page {
 115         u8 shift;
 116         const struct nvkm_vmm_desc *desc;
 117 #define NVKM_VMM_PAGE_SPARSE                                               0x01
 118 #define NVKM_VMM_PAGE_VRAM                                                 0x02
 119 #define NVKM_VMM_PAGE_HOST                                                 0x04
 120 #define NVKM_VMM_PAGE_COMP                                                 0x08
 121 #define NVKM_VMM_PAGE_Sxxx                                (NVKM_VMM_PAGE_SPARSE)
 122 #define NVKM_VMM_PAGE_xVxx                                  (NVKM_VMM_PAGE_VRAM)
 123 #define NVKM_VMM_PAGE_SVxx             (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_VRAM)
 124 #define NVKM_VMM_PAGE_xxHx                                  (NVKM_VMM_PAGE_HOST)
 125 #define NVKM_VMM_PAGE_SxHx             (NVKM_VMM_PAGE_Sxxx | NVKM_VMM_PAGE_HOST)
 126 #define NVKM_VMM_PAGE_xVHx             (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_HOST)
 127 #define NVKM_VMM_PAGE_SVHx             (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_HOST)
 128 #define NVKM_VMM_PAGE_xVxC             (NVKM_VMM_PAGE_xVxx | NVKM_VMM_PAGE_COMP)
 129 #define NVKM_VMM_PAGE_SVxC             (NVKM_VMM_PAGE_SVxx | NVKM_VMM_PAGE_COMP)
 130 #define NVKM_VMM_PAGE_xxHC             (NVKM_VMM_PAGE_xxHx | NVKM_VMM_PAGE_COMP)
 131 #define NVKM_VMM_PAGE_SxHC             (NVKM_VMM_PAGE_SxHx | NVKM_VMM_PAGE_COMP)
 132         u8 type;
 133 };
 134
 135 struct nvkm_vmm_func {
 136         int (*join)(struct nvkm_vmm *, struct nvkm_memory *inst);
 137         void (*part)(struct nvkm_vmm *, struct nvkm_memory *inst);
 138
 139         int (*aper)(enum nvkm_memory_target);
 140         int (*valid)(struct nvkm_vmm *, void *argv, u32 argc,
 141                      struct nvkm_vmm_map *);
 142         void (*flush)(struct nvkm_vmm *, int depth);
 143
 144         u64 page_block;
 145         const struct nvkm_vmm_page page[];
 146 };
 147
 148 struct nvkm_vmm_join {
 149         struct nvkm_memory *inst;
 150         struct list_head head;
 151 };
 152
 153 int nvkm_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *,
 154                   u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
 155                   const char *name, struct nvkm_vmm **);
 156 int nvkm_vmm_ctor(const struct nvkm_vmm_func *, struct nvkm_mmu *,
 157                   u32 pd_header, u64 addr, u64 size, struct lock_class_key *,
 158                   const char *name, struct nvkm_vmm *);
 159 struct nvkm_vma *nvkm_vmm_node_search(struct nvkm_vmm *, u64 addr);
 160 int nvkm_vmm_get_locked(struct nvkm_vmm *, bool getref, bool mapref,
 161                         bool sparse, u8 page, u8 align, u64 size,
 162                         struct nvkm_vma **pvma);
 163 void nvkm_vmm_put_locked(struct nvkm_vmm *, struct nvkm_vma *);
 164 void nvkm_vmm_unmap_locked(struct nvkm_vmm *, struct nvkm_vma *);
 165 void nvkm_vmm_unmap_region(struct nvkm_vmm *vmm, struct nvkm_vma *vma);
 166
 167 struct nvkm_vma *nvkm_vma_tail(struct nvkm_vma *, u64 tail);
 168 void nvkm_vmm_node_insert(struct nvkm_vmm *, struct nvkm_vma *);
 169
 170 int nv04_vmm_new_(const struct nvkm_vmm_func *, struct nvkm_mmu *, u32,
 171                   u64, u64, void *, u32, struct lock_class_key *,
 172                   const char *, struct nvkm_vmm **);
 173 int nv04_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
 174
 175 int nv50_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
 176 void nv50_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
 177 int nv50_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
 178 void nv50_vmm_flush(struct nvkm_vmm *, int);
 179
 180 int gf100_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
 181                    struct nvkm_mmu *, u64, u64, void *, u32,
 182                    struct lock_class_key *, const char *, struct nvkm_vmm **);
 183 int gf100_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
 184 int gf100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
 185 void gf100_vmm_part(struct nvkm_vmm *, struct nvkm_memory *);
 186 int gf100_vmm_aper(enum nvkm_memory_target);
 187 int gf100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
 188 void gf100_vmm_flush_(struct nvkm_vmm *, int);
 189 void gf100_vmm_flush(struct nvkm_vmm *, int);
 190
 191 int gk20a_vmm_aper(enum nvkm_memory_target);
 192
 193 int gm200_vmm_new_(const struct nvkm_vmm_func *, const struct nvkm_vmm_func *,
 194                    struct nvkm_mmu *, u64, u64, void *, u32,
 195                    struct lock_class_key *, const char *, struct nvkm_vmm **);
 196 int gm200_vmm_join_(struct nvkm_vmm *, struct nvkm_memory *, u64 base);
 197 int gm200_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
 198
 199 int gp100_vmm_join(struct nvkm_vmm *, struct nvkm_memory *);
 200 int gp100_vmm_valid(struct nvkm_vmm *, void *, u32, struct nvkm_vmm_map *);
 201 void gp100_vmm_flush(struct nvkm_vmm *, int);
 202
 203 int nv04_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 204                  struct lock_class_key *, const char *, struct nvkm_vmm **);
 205 int nv41_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 206                  struct lock_class_key *, const char *, struct nvkm_vmm **);
 207 int nv44_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 208                  struct lock_class_key *, const char *, struct nvkm_vmm **);
 209 int nv50_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 210                  struct lock_class_key *, const char *, struct nvkm_vmm **);
 211 int mcp77_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 212                   struct lock_class_key *, const char *, struct nvkm_vmm **);
 213 int g84_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 214                 struct lock_class_key *, const char *, struct nvkm_vmm **);
 215 int gf100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 216                   struct lock_class_key *, const char *, struct nvkm_vmm **);
 217 int gk104_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 218                   struct lock_class_key *, const char *, struct nvkm_vmm **);
 219 int gk20a_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 220                   struct lock_class_key *, const char *, struct nvkm_vmm **);
 221 int gm200_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
 222                         struct lock_class_key *, const char *,
 223                         struct nvkm_vmm **);
 224 int gm200_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 225                   struct lock_class_key *, const char *,
 226                   struct nvkm_vmm **);
 227 int gm20b_vmm_new_fixed(struct nvkm_mmu *, u64, u64, void *, u32,
 228                         struct lock_class_key *, const char *,
 229                         struct nvkm_vmm **);
 230 int gm20b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 231                   struct lock_class_key *, const char *,
 232                   struct nvkm_vmm **);
 233 int gp100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 234                   struct lock_class_key *, const char *,
 235                   struct nvkm_vmm **);
 236 int gp10b_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 237                   struct lock_class_key *, const char *,
 238                   struct nvkm_vmm **);
 239 int gv100_vmm_new(struct nvkm_mmu *, u64, u64, void *, u32,
 240                   struct lock_class_key *, const char *,
 241                   struct nvkm_vmm **);
 242
 243 #define VMM_PRINT(l,v,p,f,a...) do {                                           \
 244         struct nvkm_vmm *_vmm = (v);                                           \
 245         if (CONFIG_NOUVEAU_DEBUG >= (l) && _vmm->debug >= (l)) {               \
 246                 nvkm_printk_(&_vmm->mmu->subdev, 0, p, "%s: "f"\n",            \
 247                              _vmm->name, ##a);                                 \
 248         }                                                                      \
 249 } while(0)
 250 #define VMM_DEBUG(v,f,a...) VMM_PRINT(NV_DBG_DEBUG, (v), info, f, ##a)
 251 #define VMM_TRACE(v,f,a...) VMM_PRINT(NV_DBG_TRACE, (v), info, f, ##a)
 252 #define VMM_SPAM(v,f,a...)  VMM_PRINT(NV_DBG_SPAM , (v),  dbg, f, ##a)
 253
 254 #define VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,BASE,SIZE,NEXT) do {            \
 255         nvkm_kmap((PT)->memory);                                               \
 256         while (PTEN) {                                                         \
 257                 u64 _ptes = ((SIZE) - MAP->off) >> MAP->page->shift;           \
 258                 u64 _addr = ((BASE) + MAP->off);                               \
 259                                                                                \
 260                 if (_ptes > PTEN) {                                            \
 261                         MAP->off += PTEN << MAP->page->shift;                  \
 262                         _ptes = PTEN;                                          \
 263                 } else {                                                       \
 264                         MAP->off = 0;                                          \
 265                         NEXT;                                                  \
 266                 }                                                              \
 267                                                                                \
 268                 VMM_SPAM(VMM, "ITER %08x %08x PTE(s)", PTEI, (u32)_ptes);      \
 269                                                                                \
 270                 FILL(VMM, PT, PTEI, _ptes, MAP, _addr);                        \
 271                 PTEI += _ptes;                                                 \
 272                 PTEN -= _ptes;                                                 \
 273         };                                                                     \
 274         nvkm_done((PT)->memory);                                               \
 275 } while(0)
 276
 277 #define VMM_MAP_ITER_MEM(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
 278         VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
 279                      ((u64)MAP->mem->offset << NVKM_RAM_MM_SHIFT),             \
 280                      ((u64)MAP->mem->length << NVKM_RAM_MM_SHIFT),             \
 281                      (MAP->mem = MAP->mem->next))
 282 #define VMM_MAP_ITER_DMA(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
 283         VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
 284                      *MAP->dma, PAGE_SIZE, MAP->dma++)
 285 #define VMM_MAP_ITER_SGL(VMM,PT,PTEI,PTEN,MAP,FILL)                            \
 286         VMM_MAP_ITER(VMM,PT,PTEI,PTEN,MAP,FILL,                                \
 287                      sg_dma_address(MAP->sgl), sg_dma_len(MAP->sgl),           \
 288                      (MAP->sgl = sg_next(MAP->sgl)))
 289
 290 #define VMM_FO(m,o,d,c,b) nvkm_fo##b((m)->memory, (o), (d), (c))
 291 #define VMM_WO(m,o,d,c,b) nvkm_wo##b((m)->memory, (o), (d))
 292 #define VMM_XO(m,v,o,d,c,b,fn,f,a...) do {                                     \
 293         const u32 _pteo = (o); u##b _data = (d);                               \
 294         VMM_SPAM((v), "   %010llx "f, (m)->addr + _pteo, _data, ##a);          \
 295         VMM_##fn((m), (m)->base + _pteo, _data, (c), b);                       \
 296 } while(0)
 297
 298 #define VMM_WO032(m,v,o,d) VMM_XO((m),(v),(o),(d),  1, 32, WO, "%08x")
 299 #define VMM_FO032(m,v,o,d,c)                                                   \
 300         VMM_XO((m),(v),(o),(d),(c), 32, FO, "%08x %08x", (c))
 301
 302 #define VMM_WO064(m,v,o,d) VMM_XO((m),(v),(o),(d),  1, 64, WO, "%016llx")
 303 #define VMM_FO064(m,v,o,d,c)                                                   \
 304         VMM_XO((m),(v),(o),(d),(c), 64, FO, "%016llx %08x", (c))
 305
 306 #define VMM_XO128(m,v,o,lo,hi,c,f,a...) do {                                   \
 307         u32 _pteo = (o), _ptes = (c);                                          \
 308         const u64 _addr = (m)->addr + _pteo;                                   \
 309         VMM_SPAM((v), "   %010llx %016llx%016llx"f, _addr, (hi), (lo), ##a);   \
 310         while (_ptes--) {                                                      \
 311                 nvkm_wo64((m)->memory, (m)->base + _pteo + 0, (lo));           \
 312                 nvkm_wo64((m)->memory, (m)->base + _pteo + 8, (hi));           \
 313                 _pteo += 0x10;                                                 \
 314         }                                                                      \
 315 } while(0)
 316
 317 #define VMM_WO128(m,v,o,lo,hi) VMM_XO128((m),(v),(o),(lo),(hi), 1, "")
 318 #define VMM_FO128(m,v,o,lo,hi,c) do {                                          \
 319         nvkm_kmap((m)->memory);                                                \
 320         VMM_XO128((m),(v),(o),(lo),(hi),(c), " %08x", (c));                    \
 321         nvkm_done((m)->memory);                                                \
 322 } while(0)
 323 #endif