drivers/staging/gasket/gasket_page_table.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Implementation of Gasket page table support.
   4  *
   5  * Copyright (C) 2018 Google, Inc.
   6  */
   7
   8 /*
   9  * Implementation of Gasket page table support.
  10  *
  11  * This file assumes 4kB pages throughout; can be factored out when necessary.
  12  *
  13  * Address format is as follows:
  14  * Simple addresses - those whose containing pages are directly placed in the
  15  * device's address translation registers - are laid out as:
  16  * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ]
  17  * page index:  The index of the containing page in the device's address
  18  *              translation registers.
  19  * page offset: The index of the address into the containing page.
  20  *
  21  * Extended address - those whose containing pages are contained in a second-
  22  * level page table whose address is present in the device's address translation
  23  * registers - are laid out as:
  24  * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index |
  25  *   20 - 12: host/level 1 index | 11 - 0: page offset ]
  26  * flag:        Marker indicating that this is an extended address. Always 1.
  27  * dev index:   The index of the first-level page in the device's extended
  28  *              address translation registers.
  29  * host index:  The index of the containing page in the [host-resident] second-
  30  *              level page table.
  31  * page offset: The index of the address into the containing [second-level]
  32  *              page.
  33  */
  34 #include "gasket_page_table.h"
  35
  36 #include <linux/device.h>
  37 #include <linux/file.h>
  38 #include <linux/init.h>
  39 #include <linux/kernel.h>
  40 #include <linux/module.h>
  41 #include <linux/moduleparam.h>
  42 #include <linux/pagemap.h>
  43 #include <linux/vmalloc.h>
  44
  45 #include "gasket_constants.h"
  46 #include "gasket_core.h"
  47
  48 /* Constants & utility macros */
  49 /* The number of pages that can be mapped into each second-level page table. */
  50 #define GASKET_PAGES_PER_SUBTABLE 512
  51
  52 /* The starting position of the page index in a simple virtual address. */
  53 #define GASKET_SIMPLE_PAGE_SHIFT 12
  54
  55 /* Flag indicating that a [device] slot is valid for use. */
  56 #define GASKET_VALID_SLOT_FLAG 1
  57
  58 /*
  59  * The starting position of the level 0 page index (i.e., the entry in the
  60  * device's extended address registers) in an extended address.
  61  * Also can be thought of as (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)),
  62  * or (12 + 9).
  63  */
  64 #define GASKET_EXTENDED_LVL0_SHIFT 21
  65
  66 /*
  67  * Number of first level pages that Gasket chips support. Equivalent to
  68  * log2(NUM_LVL0_PAGE_TABLES)
  69  *
  70  * At a maximum, allowing for a 34 bits address space (or 16GB)
  71  *   = GASKET_EXTENDED_LVL0_WIDTH + (log2(PAGE_SIZE) + log2(PAGES_PER_SUBTABLE)
  72  * or, = 13 + 9 + 12
  73  */
  74 #define GASKET_EXTENDED_LVL0_WIDTH 13
  75
  76 /*
  77  * The starting position of the level 1 page index (i.e., the entry in the
  78  * host second-level/sub- table) in an extended address.
  79  */
  80 #define GASKET_EXTENDED_LVL1_SHIFT 12
  81
  82 /* Type declarations */
  83 /* Valid states for a struct gasket_page_table_entry. */
  84 enum pte_status {
  85         PTE_FREE,
  86         PTE_INUSE,
  87 };
  88
  89 /*
  90  * Mapping metadata for a single page.
  91  *
  92  * In this file, host-side page table entries are referred to as that (or PTEs).
  93  * Where device vs. host entries are differentiated, device-side or -visible
  94  * entries are called "slots". A slot may be either an entry in the device's
  95  * address translation table registers or an entry in a second-level page
  96  * table ("subtable").
  97  *
  98  * The full data in this structure is visible on the host [of course]. Only
  99  * the address contained in dma_addr is communicated to the device; that points
 100  * to the actual page mapped and described by this structure.
 101  */
 102 struct gasket_page_table_entry {
 103         /* The status of this entry/slot: free or in use. */
 104         enum pte_status status;
 105
 106         /* Address of the page in DMA space. */
 107         dma_addr_t dma_addr;
 108
 109         /* Linux page descriptor for the page described by this structure. */
 110         struct page *page;
 111
 112         /*
 113          * Index for alignment into host vaddrs.
 114          * When a user specifies a host address for a mapping, that address may
 115          * not be page-aligned. Offset is the index into the containing page of
 116          * the host address (i.e., host_vaddr & (PAGE_SIZE - 1)).
 117          * This is necessary for translating between user-specified addresses
 118          * and page-aligned addresses.
 119          */
 120         int offset;
 121
 122         /*
 123          * If this is an extended and first-level entry, sublevel points
 124          * to the second-level entries underneath this entry.
 125          */
 126         struct gasket_page_table_entry *sublevel;
 127 };
 128
 129 /*
 130  * Maintains virtual to physical address mapping for a coherent page that is
 131  * allocated by this module for a given device.
 132  * Note that coherent pages mappings virt mapping cannot be tracked by the
 133  * Linux kernel, and coherent pages don't have a struct page associated,
 134  * hence Linux kernel cannot perform a get_user_page_xx() on a phys address
 135  * that was allocated coherent.
 136  * This structure trivially implements this mechanism.
 137  */
 138 struct gasket_coherent_page_entry {
 139         /* Phys address, dma'able by the owner device */
 140         dma_addr_t paddr;
 141
 142         /* Kernel virtual address */
 143         u64 user_virt;
 144
 145         /* User virtual address that was mapped by the mmap kernel subsystem */
 146         u64 kernel_virt;
 147
 148         /*
 149          * Whether this page has been mapped into a user land process virtual
 150          * space
 151          */
 152         u32 in_use;
 153 };
 154
 155 /*
 156  * [Host-side] page table descriptor.
 157  *
 158  * This structure tracks the metadata necessary to manage both simple and
 159  * extended page tables.
 160  */
 161 struct gasket_page_table {
 162         /* The config used to create this page table. */
 163         struct gasket_page_table_config config;
 164
 165         /* The number of simple (single-level) entries in the page table. */
 166         uint num_simple_entries;
 167
 168         /* The number of extended (two-level) entries in the page table. */
 169         uint num_extended_entries;
 170
 171         /* Array of [host-side] page table entries. */
 172         struct gasket_page_table_entry *entries;
 173
 174         /* Number of actively mapped kernel pages in this table. */
 175         uint num_active_pages;
 176
 177         /* Device register: base of/first slot in the page table. */
 178         u64 __iomem *base_slot;
 179
 180         /* Device register: holds the offset indicating the start of the
 181          * extended address region of the device's address translation table.
 182          */
 183         u64 __iomem *extended_offset_reg;
 184
 185         /* Device structure for the underlying device. Only used for logging. */
 186         struct device *device;
 187
 188         /* PCI system descriptor for the underlying device. */
 189         struct pci_dev *pci_dev;
 190
 191         /* Location of the extended address bit for this Gasket device. */
 192         u64 extended_flag;
 193
 194         /* Mutex to protect page table internals. */
 195         struct mutex mutex;
 196
 197         /* Number of coherent pages accessible thru by this page table */
 198         int num_coherent_pages;
 199
 200         /*
 201          * List of coherent memory (physical) allocated for a device.
 202          *
 203          * This structure also remembers the user virtual mapping, this is
 204          * hacky, but we need to do this because the kernel doesn't keep track
 205          * of the user coherent pages (pfn pages), and virt to coherent page
 206          * mapping.
 207          * TODO: use find_vma() APIs to convert host address to vm_area, to
 208          * dma_addr_t instead of storing user virtu address in
 209          * gasket_coherent_page_entry
 210          *
 211          * Note that the user virtual mapping is created by the driver, in
 212          * gasket_mmap function, so user_virt belongs in the driver anyhow.
 213          */
 214         struct gasket_coherent_page_entry *coherent_pages;
 215 };
 216
 217 /* See gasket_page_table.h for description. */
 218 int gasket_page_table_init(struct gasket_page_table **ppg_tbl,
 219                            const struct gasket_bar_data *bar_data,
 220                            const struct gasket_page_table_config *page_table_config,
 221                            struct device *device, struct pci_dev *pci_dev)
 222 {
 223         ulong bytes;
 224         struct gasket_page_table *pg_tbl;
 225         ulong total_entries = page_table_config->total_entries;
 226
 227         /*
 228          * TODO: Verify config->total_entries against value read from the
 229          * hardware register that contains the page table size.
 230          */
 231         if (total_entries == ULONG_MAX) {
 232                 dev_dbg(device, "Error reading page table size. "
 233                         "Initializing page table with size 0\n");
 234                 total_entries = 0;
 235         }
 236
 237         dev_dbg(device,
 238                 "Attempting to initialize page table of size 0x%lx\n",
 239                 total_entries);
 240
 241         dev_dbg(device,
 242                 "Table has base reg 0x%x, extended offset reg 0x%x\n",
 243                 page_table_config->base_reg,
 244                 page_table_config->extended_reg);
 245
 246         *ppg_tbl = kzalloc(sizeof(**ppg_tbl), GFP_KERNEL);
 247         if (!*ppg_tbl) {
 248                 dev_dbg(device, "No memory for page table\n");
 249                 return -ENOMEM;
 250         }
 251
 252         pg_tbl = *ppg_tbl;
 253         bytes = total_entries * sizeof(struct gasket_page_table_entry);
 254         if (bytes != 0) {
 255                 pg_tbl->entries = vzalloc(bytes);
 256                 if (!pg_tbl->entries) {
 257                         dev_dbg(device,
 258                                 "No memory for address translation metadata\n");
 259                         kfree(pg_tbl);
 260                         *ppg_tbl = NULL;
 261                         return -ENOMEM;
 262                 }
 263         }
 264
 265         mutex_init(&pg_tbl->mutex);
 266         memcpy(&pg_tbl->config, page_table_config, sizeof(*page_table_config));
 267         if (pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_NORMAL ||
 268             pg_tbl->config.mode == GASKET_PAGE_TABLE_MODE_SIMPLE) {
 269                 pg_tbl->num_simple_entries = total_entries;
 270                 pg_tbl->num_extended_entries = 0;
 271                 pg_tbl->extended_flag = 1ull << page_table_config->extended_bit;
 272         } else {
 273                 pg_tbl->num_simple_entries = 0;
 274                 pg_tbl->num_extended_entries = total_entries;
 275                 pg_tbl->extended_flag = 0;
 276         }
 277         pg_tbl->num_active_pages = 0;
 278         pg_tbl->base_slot =
 279                 (u64 __iomem *)&bar_data->virt_base[page_table_config->base_reg];
 280         pg_tbl->extended_offset_reg =
 281                 (u64 __iomem *)&bar_data->virt_base[page_table_config->extended_reg];
 282         pg_tbl->device = get_device(device);
 283         pg_tbl->pci_dev = pci_dev;
 284
 285         dev_dbg(device, "Page table initialized successfully\n");
 286
 287         return 0;
 288 }
 289
 290 /*
 291  * Check if a range of PTEs is free.
 292  * The page table mutex must be held by the caller.
 293  */
 294 static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes,
 295                                      uint num_entries)
 296 {
 297         int i;
 298
 299         for (i = 0; i < num_entries; i++) {
 300                 if (ptes[i].status != PTE_FREE)
 301                         return false;
 302         }
 303
 304         return true;
 305 }
 306
 307 /*
 308  * Free a second level page [sub]table.
 309  * The page table mutex must be held before this call.
 310  */
 311 static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl,
 312                                           struct gasket_page_table_entry *pte,
 313                                           u64 __iomem *slot)
 314 {
 315         /* Release the page table from the driver */
 316         pte->status = PTE_FREE;
 317
 318         /* Release the page table from the device */
 319         writeq(0, slot);
 320         /* Force sync around the address release. */
 321         mb();
 322
 323         if (pte->dma_addr)
 324                 dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE,
 325                                DMA_BIDIRECTIONAL);
 326
 327         vfree(pte->sublevel);
 328
 329         if (pte->page)
 330                 free_page((ulong)page_address(pte->page));
 331
 332         memset(pte, 0, sizeof(struct gasket_page_table_entry));
 333 }
 334
 335 /*
 336  * Actually perform collection.
 337  * The page table mutex must be held by the caller.
 338  */
 339 static void
 340 gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl)
 341 {
 342         struct gasket_page_table_entry *pte;
 343         u64 __iomem *slot;
 344
 345         /* XXX FIX ME XXX -- more efficient to keep a usage count */
 346         /* rather than scanning the second level page tables */
 347
 348         for (pte = pg_tbl->entries + pg_tbl->num_simple_entries,
 349              slot = pg_tbl->base_slot + pg_tbl->num_simple_entries;
 350              pte < pg_tbl->entries + pg_tbl->config.total_entries;
 351              pte++, slot++) {
 352                 if (pte->status == PTE_INUSE) {
 353                         if (gasket_is_pte_range_free(pte->sublevel,
 354                                                      GASKET_PAGES_PER_SUBTABLE))
 355                                 gasket_free_extended_subtable(pg_tbl, pte,
 356                                                               slot);
 357                 }
 358         }
 359 }
 360
 361 /* See gasket_page_table.h for description. */
 362 void gasket_page_table_garbage_collect(struct gasket_page_table *pg_tbl)
 363 {
 364         mutex_lock(&pg_tbl->mutex);
 365         gasket_page_table_garbage_collect_nolock(pg_tbl);
 366         mutex_unlock(&pg_tbl->mutex);
 367 }
 368
 369 /* See gasket_page_table.h for description. */
 370 void gasket_page_table_cleanup(struct gasket_page_table *pg_tbl)
 371 {
 372         /* Deallocate free second-level tables. */
 373         gasket_page_table_garbage_collect(pg_tbl);
 374
 375         /* TODO: Check that all PTEs have been freed? */
 376
 377         vfree(pg_tbl->entries);
 378         pg_tbl->entries = NULL;
 379
 380         put_device(pg_tbl->device);
 381         kfree(pg_tbl);
 382 }
 383
 384 /* See gasket_page_table.h for description. */
 385 int gasket_page_table_partition(struct gasket_page_table *pg_tbl,
 386                                 uint num_simple_entries)
 387 {
 388         int i, start;
 389
 390         mutex_lock(&pg_tbl->mutex);
 391         if (num_simple_entries > pg_tbl->config.total_entries) {
 392                 mutex_unlock(&pg_tbl->mutex);
 393                 return -EINVAL;
 394         }
 395
 396         gasket_page_table_garbage_collect_nolock(pg_tbl);
 397
 398         start = min(pg_tbl->num_simple_entries, num_simple_entries);
 399
 400         for (i = start; i < pg_tbl->config.total_entries; i++) {
 401                 if (pg_tbl->entries[i].status != PTE_FREE) {
 402                         dev_err(pg_tbl->device, "entry %d is not free\n", i);
 403                         mutex_unlock(&pg_tbl->mutex);
 404                         return -EBUSY;
 405                 }
 406         }
 407
 408         pg_tbl->num_simple_entries = num_simple_entries;
 409         pg_tbl->num_extended_entries =
 410                 pg_tbl->config.total_entries - num_simple_entries;
 411         writeq(num_simple_entries, pg_tbl->extended_offset_reg);
 412
 413         mutex_unlock(&pg_tbl->mutex);
 414         return 0;
 415 }
 416 EXPORT_SYMBOL(gasket_page_table_partition);
 417
 418 /*
 419  * Return whether a host buffer was mapped as coherent memory.
 420  *
 421  * A Gasket page_table currently support one contiguous dma range, mapped to one
 422  * contiguous virtual memory range. Check if the host_addr is within that range.
 423  */
 424 static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr)
 425 {
 426         u64 min, max;
 427
 428         /* whether the host address is within user virt range */
 429         if (!pg_tbl->coherent_pages)
 430                 return 0;
 431
 432         min = (u64)pg_tbl->coherent_pages[0].user_virt;
 433         max = min + PAGE_SIZE * pg_tbl->num_coherent_pages;
 434
 435         return min <= host_addr && host_addr < max;
 436 }
 437
 438 /*
 439  * Get and map last level page table buffers.
 440  *
 441  * slots is the location(s) to write device-mapped page address. If this is a
 442  * simple mapping, these will be address translation registers. If this is
 443  * an extended mapping, these will be within a second-level page table
 444  * allocated by the host and so must have their __iomem attribute casted away.
 445  */
 446 static int gasket_perform_mapping(struct gasket_page_table *pg_tbl,
 447                                   struct gasket_page_table_entry *ptes,
 448                                   u64 __iomem *slots, ulong host_addr,
 449                                   uint num_pages, int is_simple_mapping)
 450 {
 451         int ret;
 452         ulong offset;
 453         struct page *page;
 454         dma_addr_t dma_addr;
 455         ulong page_addr;
 456         int i;
 457
 458         for (i = 0; i < num_pages; i++) {
 459                 page_addr = host_addr + i * PAGE_SIZE;
 460                 offset = page_addr & (PAGE_SIZE - 1);
 461                 dev_dbg(pg_tbl->device, "%s i %d\n", __func__, i);
 462                 if (is_coherent(pg_tbl, host_addr)) {
 463                         u64 off =
 464                                 (u64)host_addr -
 465                                 (u64)pg_tbl->coherent_pages[0].user_virt;
 466                         ptes[i].page = NULL;
 467                         ptes[i].offset = offset;
 468                         ptes[i].dma_addr = pg_tbl->coherent_pages[0].paddr +
 469                                            off + i * PAGE_SIZE;
 470                 } else {
 471                         ret = get_user_pages_fast(page_addr - offset, 1, 1,
 472                                                   &page);
 473
 474                         if (ret <= 0) {
 475                                 dev_err(pg_tbl->device,
 476                                         "get user pages failed for addr=0x%lx, "
 477                                         "offset=0x%lx [ret=%d]\n",
 478                                         page_addr, offset, ret);
 479                                 return ret ? ret : -ENOMEM;
 480                         }
 481                         ++pg_tbl->num_active_pages;
 482
 483                         ptes[i].page = page;
 484                         ptes[i].offset = offset;
 485
 486                         /* Map the page into DMA space. */
 487                         ptes[i].dma_addr =
 488                                 dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE,
 489                                              DMA_BIDIRECTIONAL);
 490                         dev_dbg(pg_tbl->device,
 491                                 "%s i %d pte %p pfn %p -> mapped %llx\n",
 492                                 __func__, i, &ptes[i],
 493                                 (void *)page_to_pfn(page),
 494                                 (unsigned long long)ptes[i].dma_addr);
 495
 496                         if (ptes[i].dma_addr == -1) {
 497                                 dev_dbg(pg_tbl->device,
 498                                         "%s i %d -> fail to map page %llx "
 499                                         "[pfn %p ohys %p]\n",
 500                                         __func__, i,
 501                                         (unsigned long long)ptes[i].dma_addr,
 502                                         (void *)page_to_pfn(page),
 503                                         (void *)page_to_phys(page));
 504                                 return -1;
 505                         }
 506                         /* Wait until the page is mapped. */
 507                         mb();
 508                 }
 509
 510                 /* Make the DMA-space address available to the device. */
 511                 dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG;
 512
 513                 if (is_simple_mapping) {
 514                         writeq(dma_addr, &slots[i]);
 515                 } else {
 516                         ((u64 __force *)slots)[i] = dma_addr;
 517                         /* Extended page table vectors are in DRAM,
 518                          * and so need to be synced each time they are updated.
 519                          */
 520                         dma_map_single(pg_tbl->device,
 521                                        (void *)&((u64 __force *)slots)[i],
 522                                        sizeof(u64), DMA_TO_DEVICE);
 523                 }
 524                 ptes[i].status = PTE_INUSE;
 525         }
 526         return 0;
 527 }
 528
 529 /*
 530  * Return the index of the page for the address in the simple table.
 531  * Does not perform validity checking.
 532  */
 533 static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl,
 534                                   ulong dev_addr)
 535 {
 536         return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) &
 537                 (pg_tbl->config.total_entries - 1);
 538 }
 539
 540 /*
 541  * Return the level 0 page index for the given address.
 542  * Does not perform validity checking.
 543  */
 544 static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl,
 545                                            ulong dev_addr)
 546 {
 547         return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) &
 548                ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1);
 549 }
 550
 551 /*
 552  * Return the level 1 page index for the given address.
 553  * Does not perform validity checking.
 554  */
 555 static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl,
 556                                            ulong dev_addr)
 557 {
 558         return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) &
 559                (GASKET_PAGES_PER_SUBTABLE - 1);
 560 }
 561
 562 /*
 563  * Allocate page table entries in a simple table.
 564  * The page table mutex must be held by the caller.
 565  */
 566 static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl,
 567                                        ulong dev_addr, uint num_pages)
 568 {
 569         if (!gasket_is_pte_range_free(pg_tbl->entries +
 570                                       gasket_simple_page_idx(pg_tbl, dev_addr),
 571                                       num_pages))
 572                 return -EBUSY;
 573
 574         return 0;
 575 }
 576
 577 /* Safely return a page to the OS. */
 578 static bool gasket_release_page(struct page *page)
 579 {
 580         if (!page)
 581                 return false;
 582
 583         if (!PageReserved(page))
 584                 SetPageDirty(page);
 585         put_page(page);
 586
 587         return true;
 588 }
 589
 590 /*
 591  * Unmap and release mapped pages.
 592  * The page table mutex must be held by the caller.
 593  */
 594 static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl,
 595                                      struct gasket_page_table_entry *ptes,
 596                                      u64 __iomem *slots, uint num_pages,
 597                                      int is_simple_mapping)
 598 {
 599         int i;
 600         /*
 601          * For each page table entry and corresponding entry in the device's
 602          * address translation table:
 603          */
 604         for (i = 0; i < num_pages; i++) {
 605                 /* release the address from the device, */
 606                 if (is_simple_mapping || ptes[i].status == PTE_INUSE)
 607                         writeq(0, &slots[i]);
 608                 else
 609                         ((u64 __force *)slots)[i] = 0;
 610                 /* Force sync around the address release. */
 611                 mb();
 612
 613                 /* release the address from the driver, */
 614                 if (ptes[i].status == PTE_INUSE) {
 615                         if (ptes[i].dma_addr) {
 616                                 dma_unmap_page(pg_tbl->device, ptes[i].dma_addr,
 617                                                PAGE_SIZE, DMA_FROM_DEVICE);
 618                         }
 619                         if (gasket_release_page(ptes[i].page))
 620                                 --pg_tbl->num_active_pages;
 621                 }
 622                 ptes[i].status = PTE_FREE;
 623
 624                 /* and clear the PTE. */
 625                 memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry));
 626         }
 627 }
 628
 629 /*
 630  * Unmap and release pages mapped to simple addresses.
 631  * The page table mutex must be held by the caller.
 632  */
 633 static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl,
 634                                       ulong dev_addr, uint num_pages)
 635 {
 636         uint slot = gasket_simple_page_idx(pg_tbl, dev_addr);
 637
 638         gasket_perform_unmapping(pg_tbl, pg_tbl->entries + slot,
 639                                  pg_tbl->base_slot + slot, num_pages, 1);
 640 }
 641
 642 /*
 643  * Unmap and release buffers to extended addresses.
 644  * The page table mutex must be held by the caller.
 645  */
 646 static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl,
 647                                         ulong dev_addr, uint num_pages)
 648 {
 649         uint slot_idx, remain, len;
 650         struct gasket_page_table_entry *pte;
 651         u64 __iomem *slot_base;
 652
 653         remain = num_pages;
 654         slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
 655         pte = pg_tbl->entries + pg_tbl->num_simple_entries +
 656               gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
 657
 658         while (remain > 0) {
 659                 /* TODO: Add check to ensure pte remains valid? */
 660                 len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
 661
 662                 if (pte->status == PTE_INUSE) {
 663                         slot_base = (u64 __iomem *)(page_address(pte->page) +
 664                                                     pte->offset);
 665                         gasket_perform_unmapping(pg_tbl,
 666                                                  pte->sublevel + slot_idx,
 667                                                  slot_base + slot_idx, len, 0);
 668                 }
 669
 670                 remain -= len;
 671                 slot_idx = 0;
 672                 pte++;
 673         }
 674 }
 675
 676 /* Evaluates to nonzero if the specified virtual address is simple. */
 677 static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl,
 678                                          ulong addr)
 679 {
 680         return !((addr) & (pg_tbl)->extended_flag);
 681 }
 682
 683 /*
 684  * Convert (simple, page, offset) into a device address.
 685  * Examples:
 686  * Simple page 0, offset 32:
 687  *  Input (0, 0, 32), Output 0x20
 688  * Simple page 1000, offset 511:
 689  *  Input (0, 1000, 512), Output 0x3E81FF
 690  * Extended page 0, offset 32:
 691  *  Input (0, 0, 32), Output 0x8000000020
 692  * Extended page 1000, offset 511:
 693  *  Input (1, 1000, 512), Output 0x8003E81FF
 694  */
 695 static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl,
 696                                               int is_simple, uint page_index,
 697                                               uint offset)
 698 {
 699         ulong lvl0_index, lvl1_index;
 700
 701         if (is_simple) {
 702                 /* Return simple addresses directly. */
 703                 lvl0_index = page_index & (pg_tbl->config.total_entries - 1);
 704                 return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset;
 705         }
 706
 707         /*
 708          * This could be compressed into fewer statements, but
 709          * A) the compiler should optimize it
 710          * B) this is not slow
 711          * C) this is an uncommon operation
 712          * D) this is actually readable this way.
 713          */
 714         lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE;
 715         lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1);
 716         return (pg_tbl)->extended_flag |
 717                (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) |
 718                (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset;
 719 }
 720
 721 /*
 722  * Validity checking for simple addresses.
 723  *
 724  * Verify that address translation commutes (from address to/from page + offset)
 725  * and that the requested page range starts and ends within the set of
 726  * currently-partitioned simple pages.
 727  */
 728 static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl,
 729                                           ulong dev_addr, uint num_pages)
 730 {
 731         ulong page_offset = dev_addr & (PAGE_SIZE - 1);
 732         ulong page_index =
 733                 (dev_addr / PAGE_SIZE) & (pg_tbl->config.total_entries - 1);
 734
 735         if (gasket_components_to_dev_address(pg_tbl, 1, page_index,
 736                                              page_offset) != dev_addr) {
 737                 dev_err(pg_tbl->device, "address is invalid, 0x%lX\n",
 738                         dev_addr);
 739                 return true;
 740         }
 741
 742         if (page_index >= pg_tbl->num_simple_entries) {
 743                 dev_err(pg_tbl->device,
 744                         "starting slot at %lu is too large, max is < %u\n",
 745                         page_index, pg_tbl->num_simple_entries);
 746                 return true;
 747         }
 748
 749         if (page_index + num_pages > pg_tbl->num_simple_entries) {
 750                 dev_err(pg_tbl->device,
 751                         "ending slot at %lu is too large, max is <= %u\n",
 752                         page_index + num_pages, pg_tbl->num_simple_entries);
 753                 return true;
 754         }
 755
 756         return false;
 757 }
 758
 759 /*
 760  * Validity checking for extended addresses.
 761  *
 762  * Verify that address translation commutes (from address to/from page +
 763  * offset) and that the requested page range starts and ends within the set of
 764  * currently-partitioned extended pages.
 765  */
 766 static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl,
 767                                             ulong dev_addr, uint num_pages)
 768 {
 769         /* Starting byte index of dev_addr into the first mapped page */
 770         ulong page_offset = dev_addr & (PAGE_SIZE - 1);
 771         ulong page_global_idx, page_lvl0_idx;
 772         ulong num_lvl0_pages;
 773         ulong addr;
 774
 775         /* check if the device address is out of bound */
 776         addr = dev_addr & ~((pg_tbl)->extended_flag);
 777         if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) {
 778                 dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n",
 779                         dev_addr);
 780                 return true;
 781         }
 782
 783         /* Find the starting sub-page index in the space of all sub-pages. */
 784         page_global_idx = (dev_addr / PAGE_SIZE) &
 785                 (pg_tbl->config.total_entries * GASKET_PAGES_PER_SUBTABLE - 1);
 786
 787         /* Find the starting level 0 index. */
 788         page_lvl0_idx = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
 789
 790         /* Get the count of affected level 0 pages. */
 791         num_lvl0_pages = (num_pages + GASKET_PAGES_PER_SUBTABLE - 1) /
 792                 GASKET_PAGES_PER_SUBTABLE;
 793
 794         if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx,
 795                                              page_offset) != dev_addr) {
 796                 dev_err(pg_tbl->device, "address is invalid: 0x%lx\n",
 797                         dev_addr);
 798                 return true;
 799         }
 800
 801         if (page_lvl0_idx >= pg_tbl->num_extended_entries) {
 802                 dev_err(pg_tbl->device,
 803                         "starting level 0 slot at %lu is too large, max is < "
 804                         "%u\n", page_lvl0_idx, pg_tbl->num_extended_entries);
 805                 return true;
 806         }
 807
 808         if (page_lvl0_idx + num_lvl0_pages > pg_tbl->num_extended_entries) {
 809                 dev_err(pg_tbl->device,
 810                         "ending level 0 slot at %lu is too large, max is <= %u\n",
 811                         page_lvl0_idx + num_lvl0_pages,
 812                         pg_tbl->num_extended_entries);
 813                 return true;
 814         }
 815
 816         return false;
 817 }
 818
 819 /*
 820  * Non-locking entry to unmapping routines.
 821  * The page table mutex must be held by the caller.
 822  */
 823 static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl,
 824                                            ulong dev_addr, uint num_pages)
 825 {
 826         if (!num_pages)
 827                 return;
 828
 829         if (gasket_addr_is_simple(pg_tbl, dev_addr))
 830                 gasket_unmap_simple_pages(pg_tbl, dev_addr, num_pages);
 831         else
 832                 gasket_unmap_extended_pages(pg_tbl, dev_addr, num_pages);
 833 }
 834
 835 /*
 836  * Allocate and map pages to simple addresses.
 837  * If there is an error, no pages are mapped.
 838  */
 839 static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl,
 840                                    ulong host_addr, ulong dev_addr,
 841                                    uint num_pages)
 842 {
 843         int ret;
 844         uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr);
 845
 846         ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages);
 847         if (ret) {
 848                 dev_err(pg_tbl->device,
 849                         "page table slots %u (@ 0x%lx) to %u are not available\n",
 850                         slot_idx, dev_addr, slot_idx + num_pages - 1);
 851                 return ret;
 852         }
 853
 854         ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx,
 855                                      pg_tbl->base_slot + slot_idx, host_addr,
 856                                      num_pages, 1);
 857
 858         if (ret) {
 859                 gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
 860                 dev_err(pg_tbl->device, "gasket_perform_mapping %d\n", ret);
 861         }
 862         return ret;
 863 }
 864
 865 /*
 866  * Allocate a second level page table.
 867  * The page table mutex must be held by the caller.
 868  */
 869 static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl,
 870                                           struct gasket_page_table_entry *pte,
 871                                           u64 __iomem *slot)
 872 {
 873         ulong page_addr, subtable_bytes;
 874         dma_addr_t dma_addr;
 875
 876         /* XXX FIX ME XXX this is inefficient for non-4K page sizes */
 877
 878         /* GFP_DMA flag must be passed to architectures for which
 879          * part of the memory range is not considered DMA'able.
 880          * This seems to be the case for Juno board with 4.5.0 Linaro kernel
 881          */
 882         page_addr = get_zeroed_page(GFP_KERNEL | GFP_DMA);
 883         if (!page_addr)
 884                 return -ENOMEM;
 885         pte->page = virt_to_page((void *)page_addr);
 886         pte->offset = 0;
 887
 888         subtable_bytes = sizeof(struct gasket_page_table_entry) *
 889                 GASKET_PAGES_PER_SUBTABLE;
 890         pte->sublevel = vzalloc(subtable_bytes);
 891         if (!pte->sublevel) {
 892                 free_page(page_addr);
 893                 memset(pte, 0, sizeof(struct gasket_page_table_entry));
 894                 return -ENOMEM;
 895         }
 896
 897         /* Map the page into DMA space. */
 898         pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE,
 899                                      DMA_BIDIRECTIONAL);
 900         /* Wait until the page is mapped. */
 901         mb();
 902
 903         /* make the addresses available to the device */
 904         dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG;
 905         writeq(dma_addr, slot);
 906
 907         pte->status = PTE_INUSE;
 908
 909         return 0;
 910 }
 911
 912 /*
 913  * Allocate slots in an extended page table.  Check to see if a range of page
 914  * table slots are available. If necessary, memory is allocated for second level
 915  * page tables.
 916  *
 917  * Note that memory for second level page tables is allocated as needed, but
 918  * that memory is only freed on the final close of the device file, when the
 919  * page tables are repartitioned, or the the device is removed.  If there is an
 920  * error or if the full range of slots is not available, any memory
 921  * allocated for second level page tables remains allocated until final close,
 922  * repartition, or device removal.
 923  *
 924  * The page table mutex must be held by the caller.
 925  */
 926 static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl,
 927                                          ulong dev_addr, uint num_entries)
 928 {
 929         int ret = 0;
 930         uint remain, subtable_slot_idx, len;
 931         struct gasket_page_table_entry *pte;
 932         u64 __iomem *slot;
 933
 934         remain = num_entries;
 935         subtable_slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
 936         pte = pg_tbl->entries + pg_tbl->num_simple_entries +
 937               gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
 938         slot = pg_tbl->base_slot + pg_tbl->num_simple_entries +
 939                gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
 940
 941         while (remain > 0) {
 942                 len = min(remain,
 943                           GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx);
 944
 945                 if (pte->status == PTE_FREE) {
 946                         ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot);
 947                         if (ret) {
 948                                 dev_err(pg_tbl->device,
 949                                         "no memory for extended addr subtable\n");
 950                                 return ret;
 951                         }
 952                 } else {
 953                         if (!gasket_is_pte_range_free(pte->sublevel +
 954                                                       subtable_slot_idx, len))
 955                                 return -EBUSY;
 956                 }
 957
 958                 remain -= len;
 959                 subtable_slot_idx = 0;
 960                 pte++;
 961                 slot++;
 962         }
 963
 964         return 0;
 965 }
 966
 967 /*
 968  * gasket_map_extended_pages - Get and map buffers to extended addresses.
 969  * If there is an error, no pages are mapped.
 970  */
 971 static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl,
 972                                      ulong host_addr, ulong dev_addr,
 973                                      uint num_pages)
 974 {
 975         int ret;
 976         ulong dev_addr_end;
 977         uint slot_idx, remain, len;
 978         struct gasket_page_table_entry *pte;
 979         u64 __iomem *slot_base;
 980
 981         ret = gasket_alloc_extended_entries(pg_tbl, dev_addr, num_pages);
 982         if (ret) {
 983                 dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1;
 984                 dev_err(pg_tbl->device,
 985                         "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are "
 986                         "not available\n",
 987                         gasket_extended_lvl0_page_idx(pg_tbl, dev_addr),
 988                         dev_addr,
 989                         gasket_extended_lvl1_page_idx(pg_tbl, dev_addr),
 990                         gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end),
 991                         gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end));
 992                 return ret;
 993         }
 994
 995         remain = num_pages;
 996         slot_idx = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
 997         pte = pg_tbl->entries + pg_tbl->num_simple_entries +
 998               gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
 999
1000         while (remain > 0) {
1001                 len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx);
1002
1003                 slot_base =
1004                         (u64 __iomem *)(page_address(pte->page) + pte->offset);
1005                 ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx,
1006                                              slot_base + slot_idx, host_addr,
1007                                              len, 0);
1008                 if (ret) {
1009                         gasket_page_table_unmap_nolock(pg_tbl, dev_addr,
1010                                                        num_pages);
1011                         return ret;
1012                 }
1013
1014                 remain -= len;
1015                 slot_idx = 0;
1016                 pte++;
1017                 host_addr += len * PAGE_SIZE;
1018         }
1019
1020         return 0;
1021 }
1022
1023 /*
1024  * See gasket_page_table.h for general description.
1025  *
1026  * gasket_page_table_map calls either gasket_map_simple_pages() or
1027  * gasket_map_extended_pages() to actually perform the mapping.
1028  *
1029  * The page table mutex is held for the entire operation.
1030  */
1031 int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr,
1032                           ulong dev_addr, uint num_pages)
1033 {
1034         int ret;
1035
1036         if (!num_pages)
1037                 return 0;
1038
1039         mutex_lock(&pg_tbl->mutex);
1040
1041         if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
1042                 ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr,
1043                                               num_pages);
1044         } else {
1045                 ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr,
1046                                                 num_pages);
1047         }
1048
1049         mutex_unlock(&pg_tbl->mutex);
1050
1051         dev_dbg(pg_tbl->device,
1052                 "%s done: ha %llx daddr %llx num %d, ret %d\n",
1053                 __func__, (unsigned long long)host_addr,
1054                 (unsigned long long)dev_addr, num_pages, ret);
1055         return ret;
1056 }
1057 EXPORT_SYMBOL(gasket_page_table_map);
1058
1059 /*
1060  * See gasket_page_table.h for general description.
1061  *
1062  * gasket_page_table_unmap takes the page table lock and calls either
1063  * gasket_unmap_simple_pages() or gasket_unmap_extended_pages() to
1064  * actually unmap the pages from device space.
1065  *
1066  * The page table mutex is held for the entire operation.
1067  */
1068 void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr,
1069                              uint num_pages)
1070 {
1071         if (!num_pages)
1072                 return;
1073
1074         mutex_lock(&pg_tbl->mutex);
1075         gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages);
1076         mutex_unlock(&pg_tbl->mutex);
1077 }
1078 EXPORT_SYMBOL(gasket_page_table_unmap);
1079
1080 static void gasket_page_table_unmap_all_nolock(struct gasket_page_table *pg_tbl)
1081 {
1082         gasket_unmap_simple_pages(pg_tbl,
1083                                   gasket_components_to_dev_address(pg_tbl, 1, 0,
1084                                                                    0),
1085                                   pg_tbl->num_simple_entries);
1086         gasket_unmap_extended_pages(pg_tbl,
1087                                     gasket_components_to_dev_address(pg_tbl, 0,
1088                                                                      0, 0),
1089                                     pg_tbl->num_extended_entries *
1090                                     GASKET_PAGES_PER_SUBTABLE);
1091 }
1092
1093 /* See gasket_page_table.h for description. */
1094 void gasket_page_table_unmap_all(struct gasket_page_table *pg_tbl)
1095 {
1096         mutex_lock(&pg_tbl->mutex);
1097         gasket_page_table_unmap_all_nolock(pg_tbl);
1098         mutex_unlock(&pg_tbl->mutex);
1099 }
1100 EXPORT_SYMBOL(gasket_page_table_unmap_all);
1101
1102 /* See gasket_page_table.h for description. */
1103 void gasket_page_table_reset(struct gasket_page_table *pg_tbl)
1104 {
1105         mutex_lock(&pg_tbl->mutex);
1106         gasket_page_table_unmap_all_nolock(pg_tbl);
1107         writeq(pg_tbl->config.total_entries, pg_tbl->extended_offset_reg);
1108         mutex_unlock(&pg_tbl->mutex);
1109 }
1110
1111 /* See gasket_page_table.h for description. */
1112 int gasket_page_table_lookup_page(
1113         struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage,
1114         ulong *poffset)
1115 {
1116         uint page_num;
1117         struct gasket_page_table_entry *pte;
1118
1119         mutex_lock(&pg_tbl->mutex);
1120         if (gasket_addr_is_simple(pg_tbl, dev_addr)) {
1121                 page_num = gasket_simple_page_idx(pg_tbl, dev_addr);
1122                 if (page_num >= pg_tbl->num_simple_entries)
1123                         goto fail;
1124
1125                 pte = pg_tbl->entries + page_num;
1126                 if (pte->status != PTE_INUSE)
1127                         goto fail;
1128         } else {
1129                 /* Find the level 0 entry, */
1130                 page_num = gasket_extended_lvl0_page_idx(pg_tbl, dev_addr);
1131                 if (page_num >= pg_tbl->num_extended_entries)
1132                         goto fail;
1133
1134                 pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num;
1135                 if (pte->status != PTE_INUSE)
1136                         goto fail;
1137
1138                 /* and its contained level 1 entry. */
1139                 page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr);
1140                 pte = pte->sublevel + page_num;
1141                 if (pte->status != PTE_INUSE)
1142                         goto fail;
1143         }
1144
1145         *ppage = pte->page;
1146         *poffset = pte->offset;
1147         mutex_unlock(&pg_tbl->mutex);
1148         return 0;
1149
1150 fail:
1151         *ppage = NULL;
1152         *poffset = 0;
1153         mutex_unlock(&pg_tbl->mutex);
1154         return -1;
1155 }
1156
1157 /* See gasket_page_table.h for description. */
1158 bool gasket_page_table_are_addrs_bad(
1159         struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr,
1160         ulong bytes)
1161 {
1162         if (host_addr & (PAGE_SIZE - 1)) {
1163                 dev_err(pg_tbl->device,
1164                         "host mapping address 0x%lx must be page aligned\n",
1165                         host_addr);
1166                 return true;
1167         }
1168
1169         return gasket_page_table_is_dev_addr_bad(pg_tbl, dev_addr, bytes);
1170 }
1171 EXPORT_SYMBOL(gasket_page_table_are_addrs_bad);
1172
1173 /* See gasket_page_table.h for description. */
1174 bool gasket_page_table_is_dev_addr_bad(
1175         struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes)
1176 {
1177         uint num_pages = bytes / PAGE_SIZE;
1178
1179         if (bytes & (PAGE_SIZE - 1)) {
1180                 dev_err(pg_tbl->device,
1181                         "mapping size 0x%lX must be page aligned\n", bytes);
1182                 return true;
1183         }
1184
1185         if (num_pages == 0) {
1186                 dev_err(pg_tbl->device,
1187                         "requested mapping is less than one page: %lu / %lu\n",
1188                         bytes, PAGE_SIZE);
1189                 return true;
1190         }
1191
1192         if (gasket_addr_is_simple(pg_tbl, dev_addr))
1193                 return gasket_is_simple_dev_addr_bad(pg_tbl, dev_addr,
1194                                                      num_pages);
1195         return gasket_is_extended_dev_addr_bad(pg_tbl, dev_addr, num_pages);
1196 }
1197 EXPORT_SYMBOL(gasket_page_table_is_dev_addr_bad);
1198
1199 /* See gasket_page_table.h for description. */
1200 uint gasket_page_table_max_size(struct gasket_page_table *page_table)
1201 {
1202         if (!page_table)
1203                 return 0;
1204         return page_table->config.total_entries;
1205 }
1206 EXPORT_SYMBOL(gasket_page_table_max_size);
1207
1208 /* See gasket_page_table.h for description. */
1209 uint gasket_page_table_num_entries(struct gasket_page_table *pg_tbl)
1210 {
1211         if (!pg_tbl)
1212                 return 0;
1213         return pg_tbl->num_simple_entries + pg_tbl->num_extended_entries;
1214 }
1215 EXPORT_SYMBOL(gasket_page_table_num_entries);
1216
1217 /* See gasket_page_table.h for description. */
1218 uint gasket_page_table_num_simple_entries(struct gasket_page_table *pg_tbl)
1219 {
1220         if (!pg_tbl)
1221                 return 0;
1222         return pg_tbl->num_simple_entries;
1223 }
1224 EXPORT_SYMBOL(gasket_page_table_num_simple_entries);
1225
1226 /* See gasket_page_table.h for description. */
1227 uint gasket_page_table_num_active_pages(struct gasket_page_table *pg_tbl)
1228 {
1229         if (!pg_tbl)
1230                 return 0;
1231         return pg_tbl->num_active_pages;
1232 }
1233 EXPORT_SYMBOL(gasket_page_table_num_active_pages);
1234
1235 /* See gasket_page_table.h */
1236 int gasket_page_table_system_status(struct gasket_page_table *page_table)
1237 {
1238         if (!page_table)
1239                 return GASKET_STATUS_LAMED;
1240
1241         if (gasket_page_table_num_entries(page_table) == 0) {
1242                 dev_dbg(page_table->device, "Page table size is 0\n");
1243                 return GASKET_STATUS_LAMED;
1244         }
1245
1246         return GASKET_STATUS_ALIVE;
1247 }
1248
1249 /* Record the host_addr to coherent dma memory mapping. */
1250 int gasket_set_user_virt(
1251         struct gasket_dev *gasket_dev, u64 size, dma_addr_t dma_address,
1252         ulong vma)
1253 {
1254         int j;
1255         struct gasket_page_table *pg_tbl;
1256
1257         unsigned int num_pages = size / PAGE_SIZE;
1258
1259         /*
1260          * TODO: for future chipset, better handling of the case where multiple
1261          * page tables are supported on a given device
1262          */
1263         pg_tbl = gasket_dev->page_table[0];
1264         if (!pg_tbl) {
1265                 dev_dbg(gasket_dev->dev, "%s: invalid page table index\n",
1266                         __func__);
1267                 return 0;
1268         }
1269         for (j = 0; j < num_pages; j++) {
1270                 pg_tbl->coherent_pages[j].user_virt =
1271                         (u64)vma + j * PAGE_SIZE;
1272         }
1273         return 0;
1274 }
1275
1276 /* Allocate a block of coherent memory. */
1277 int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
1278                                  dma_addr_t *dma_address, u64 index)
1279 {
1280         dma_addr_t handle;
1281         void *mem;
1282         int j;
1283         unsigned int num_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
1284         const struct gasket_driver_desc *driver_desc =
1285                 gasket_get_driver_desc(gasket_dev);
1286
1287         if (!gasket_dev->page_table[index])
1288                 return -EFAULT;
1289
1290         if (num_pages == 0)
1291                 return -EINVAL;
1292
1293         mem = dma_alloc_coherent(gasket_get_device(gasket_dev),
1294                                  num_pages * PAGE_SIZE, &handle, 0);
1295         if (!mem)
1296                 goto nomem;
1297
1298         gasket_dev->page_table[index]->num_coherent_pages = num_pages;
1299
1300         /* allocate the physical memory block */
1301         gasket_dev->page_table[index]->coherent_pages =
1302                 kcalloc(num_pages, sizeof(struct gasket_coherent_page_entry),
1303                         GFP_KERNEL);
1304         if (!gasket_dev->page_table[index]->coherent_pages)
1305                 goto nomem;
1306         *dma_address = 0;
1307
1308         gasket_dev->coherent_buffer.length_bytes =
1309                 PAGE_SIZE * (num_pages);
1310         gasket_dev->coherent_buffer.phys_base = handle;
1311         gasket_dev->coherent_buffer.virt_base = mem;
1312
1313         *dma_address = driver_desc->coherent_buffer_description.base;
1314         for (j = 0; j < num_pages; j++) {
1315                 gasket_dev->page_table[index]->coherent_pages[j].paddr =
1316                         handle + j * PAGE_SIZE;
1317                 gasket_dev->page_table[index]->coherent_pages[j].kernel_virt =
1318                         (u64)mem + j * PAGE_SIZE;
1319         }
1320
1321         if (*dma_address == 0)
1322                 goto nomem;
1323         return 0;
1324
1325 nomem:
1326         if (mem) {
1327                 dma_free_coherent(gasket_get_device(gasket_dev),
1328                                   num_pages * PAGE_SIZE, mem, handle);
1329         }
1330
1331         if (gasket_dev->page_table[index]->coherent_pages) {
1332                 kfree(gasket_dev->page_table[index]->coherent_pages);
1333                 gasket_dev->page_table[index]->coherent_pages = NULL;
1334         }
1335         gasket_dev->page_table[index]->num_coherent_pages = 0;
1336         return -ENOMEM;
1337 }
1338
1339 /* Free a block of coherent memory. */
1340 int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size,
1341                                 dma_addr_t dma_address, u64 index)
1342 {
1343         const struct gasket_driver_desc *driver_desc;
1344
1345         if (!gasket_dev->page_table[index])
1346                 return -EFAULT;
1347
1348         driver_desc = gasket_get_driver_desc(gasket_dev);
1349
1350         if (driver_desc->coherent_buffer_description.base != dma_address)
1351                 return -EADDRNOTAVAIL;
1352
1353         if (gasket_dev->coherent_buffer.length_bytes) {
1354                 dma_free_coherent(gasket_get_device(gasket_dev),
1355                                   gasket_dev->coherent_buffer.length_bytes,
1356                                   gasket_dev->coherent_buffer.virt_base,
1357                                   gasket_dev->coherent_buffer.phys_base);
1358                 gasket_dev->coherent_buffer.length_bytes = 0;
1359                 gasket_dev->coherent_buffer.virt_base = NULL;
1360                 gasket_dev->coherent_buffer.phys_base = 0;
1361         }
1362         return 0;
1363 }
1364
1365 /* Release all coherent memory. */
1366 void gasket_free_coherent_memory_all(
1367         struct gasket_dev *gasket_dev, u64 index)
1368 {
1369         if (!gasket_dev->page_table[index])
1370                 return;
1371
1372         if (gasket_dev->coherent_buffer.length_bytes) {
1373                 dma_free_coherent(gasket_get_device(gasket_dev),
1374                                   gasket_dev->coherent_buffer.length_bytes,
1375                                   gasket_dev->coherent_buffer.virt_base,
1376                                   gasket_dev->coherent_buffer.phys_base);
1377                 gasket_dev->coherent_buffer.length_bytes = 0;
1378                 gasket_dev->coherent_buffer.virt_base = NULL;
1379                 gasket_dev->coherent_buffer.phys_base = 0;
1380         }
1381 }