drivers/iommu/intel-iommu.c

   1 /*
   2  * Copyright © 2006-2014 Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * Authors: David Woodhouse <dwmw2@infradead.org>,
  14  *          Ashok Raj <ashok.raj@intel.com>,
  15  *          Shaohua Li <shaohua.li@intel.com>,
  16  *          Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
  17  *          Fenghua Yu <fenghua.yu@intel.com>
  18  *          Joerg Roedel <jroedel@suse.de>
  19  */
  20
  21 #define pr_fmt(fmt)     "DMAR: " fmt
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/export.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/memory.h>
  36 #include <linux/cpu.h>
  37 #include <linux/timer.h>
  38 #include <linux/io.h>
  39 #include <linux/iova.h>
  40 #include <linux/iommu.h>
  41 #include <linux/intel-iommu.h>
  42 #include <linux/syscore_ops.h>
  43 #include <linux/tboot.h>
  44 #include <linux/dmi.h>
  45 #include <linux/pci-ats.h>
  46 #include <linux/memblock.h>
  47 #include <linux/dma-contiguous.h>
  48 #include <linux/dma-direct.h>
  49 #include <linux/crash_dump.h>
  50 #include <asm/irq_remapping.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/iommu.h>
  53
  54 #include "irq_remapping.h"
  55 #include "intel-pasid.h"
  56
  57 #define ROOT_SIZE               VTD_PAGE_SIZE
  58 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  59
  60 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  61 #define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
  62 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  63 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
  64
  65 #define IOAPIC_RANGE_START      (0xfee00000)
  66 #define IOAPIC_RANGE_END        (0xfeefffff)
  67 #define IOVA_START_ADDR         (0x1000)
  68
  69 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
  70
  71 #define MAX_AGAW_WIDTH 64
  72 #define MAX_AGAW_PFN_WIDTH      (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
  73
  74 #define __DOMAIN_MAX_PFN(gaw)  ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
  75 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
  76
  77 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
  78    to match. That way, we can use 'unsigned long' for PFNs with impunity. */
  79 #define DOMAIN_MAX_PFN(gaw)     ((unsigned long) min_t(uint64_t, \
  80                                 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
  81 #define DOMAIN_MAX_ADDR(gaw)    (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
  82
  83 /* IO virtual address start page frame number */
  84 #define IOVA_START_PFN          (1)
  85
  86 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  87
  88 /* page table handling */
  89 #define LEVEL_STRIDE            (9)
  90 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
  91
  92 /*
  93  * This bitmap is used to advertise the page sizes our hardware support
  94  * to the IOMMU core, which will then use this information to split
  95  * physically contiguous memory regions it is mapping into page sizes
  96  * that we support.
  97  *
  98  * Traditionally the IOMMU core just handed us the mappings directly,
  99  * after making sure the size is an order of a 4KiB page and that the
 100  * mapping has natural alignment.
 101  *
 102  * To retain this behavior, we currently advertise that we support
 103  * all page sizes that are an order of 4KiB.
 104  *
 105  * If at some point we'd like to utilize the IOMMU core's new behavior,
 106  * we could change this to advertise the real page sizes we support.
 107  */
 108 #define INTEL_IOMMU_PGSIZES     (~0xFFFUL)
 109
 110 static inline int agaw_to_level(int agaw)
 111 {
 112         return agaw + 2;
 113 }
 114
 115 static inline int agaw_to_width(int agaw)
 116 {
 117         return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
 118 }
 119
 120 static inline int width_to_agaw(int width)
 121 {
 122         return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
 123 }
 124
 125 static inline unsigned int level_to_offset_bits(int level)
 126 {
 127         return (level - 1) * LEVEL_STRIDE;
 128 }
 129
 130 static inline int pfn_level_offset(unsigned long pfn, int level)
 131 {
 132         return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
 133 }
 134
 135 static inline unsigned long level_mask(int level)
 136 {
 137         return -1UL << level_to_offset_bits(level);
 138 }
 139
 140 static inline unsigned long level_size(int level)
 141 {
 142         return 1UL << level_to_offset_bits(level);
 143 }
 144
 145 static inline unsigned long align_to_level(unsigned long pfn, int level)
 146 {
 147         return (pfn + level_size(level) - 1) & level_mask(level);
 148 }
 149
 150 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
 151 {
 152         return  1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
 153 }
 154
 155 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
 156    are never going to work. */
 157 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
 158 {
 159         return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
 160 }
 161
 162 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
 163 {
 164         return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
 165 }
 166 static inline unsigned long page_to_dma_pfn(struct page *pg)
 167 {
 168         return mm_to_dma_pfn(page_to_pfn(pg));
 169 }
 170 static inline unsigned long virt_to_dma_pfn(void *p)
 171 {
 172         return page_to_dma_pfn(virt_to_page(p));
 173 }
 174
 175 /* global iommu list, set NULL for ignored DMAR units */
 176 static struct intel_iommu **g_iommus;
 177
 178 static void __init check_tylersburg_isoch(void);
 179 static int rwbf_quirk;
 180
 181 /*
 182  * set to 1 to panic kernel if can't successfully enable VT-d
 183  * (used when kernel is launched w/ TXT)
 184  */
 185 static int force_on = 0;
 186 int intel_iommu_tboot_noforce;
 187
 188 /*
 189  * 0: Present
 190  * 1-11: Reserved
 191  * 12-63: Context Ptr (12 - (haw-1))
 192  * 64-127: Reserved
 193  */
 194 struct root_entry {
 195         u64     lo;
 196         u64     hi;
 197 };
 198 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
 199
 200 /*
 201  * Take a root_entry and return the Lower Context Table Pointer (LCTP)
 202  * if marked present.
 203  */
 204 static phys_addr_t root_entry_lctp(struct root_entry *re)
 205 {
 206         if (!(re->lo & 1))
 207                 return 0;
 208
 209         return re->lo & VTD_PAGE_MASK;
 210 }
 211
 212 /*
 213  * Take a root_entry and return the Upper Context Table Pointer (UCTP)
 214  * if marked present.
 215  */
 216 static phys_addr_t root_entry_uctp(struct root_entry *re)
 217 {
 218         if (!(re->hi & 1))
 219                 return 0;
 220
 221         return re->hi & VTD_PAGE_MASK;
 222 }
 223 /*
 224  * low 64 bits:
 225  * 0: present
 226  * 1: fault processing disable
 227  * 2-3: translation type
 228  * 12-63: address space root
 229  * high 64 bits:
 230  * 0-2: address width
 231  * 3-6: aval
 232  * 8-23: domain id
 233  */
 234 struct context_entry {
 235         u64 lo;
 236         u64 hi;
 237 };
 238
 239 static inline void context_clear_pasid_enable(struct context_entry *context)
 240 {
 241         context->lo &= ~(1ULL << 11);
 242 }
 243
 244 static inline bool context_pasid_enabled(struct context_entry *context)
 245 {
 246         return !!(context->lo & (1ULL << 11));
 247 }
 248
 249 static inline void context_set_copied(struct context_entry *context)
 250 {
 251         context->hi |= (1ull << 3);
 252 }
 253
 254 static inline bool context_copied(struct context_entry *context)
 255 {
 256         return !!(context->hi & (1ULL << 3));
 257 }
 258
 259 static inline bool __context_present(struct context_entry *context)
 260 {
 261         return (context->lo & 1);
 262 }
 263
 264 static inline bool context_present(struct context_entry *context)
 265 {
 266         return context_pasid_enabled(context) ?
 267              __context_present(context) :
 268              __context_present(context) && !context_copied(context);
 269 }
 270
 271 static inline void context_set_present(struct context_entry *context)
 272 {
 273         context->lo |= 1;
 274 }
 275
 276 static inline void context_set_fault_enable(struct context_entry *context)
 277 {
 278         context->lo &= (((u64)-1) << 2) | 1;
 279 }
 280
 281 static inline void context_set_translation_type(struct context_entry *context,
 282                                                 unsigned long value)
 283 {
 284         context->lo &= (((u64)-1) << 4) | 3;
 285         context->lo |= (value & 3) << 2;
 286 }
 287
 288 static inline void context_set_address_root(struct context_entry *context,
 289                                             unsigned long value)
 290 {
 291         context->lo &= ~VTD_PAGE_MASK;
 292         context->lo |= value & VTD_PAGE_MASK;
 293 }
 294
 295 static inline void context_set_address_width(struct context_entry *context,
 296                                              unsigned long value)
 297 {
 298         context->hi |= value & 7;
 299 }
 300
 301 static inline void context_set_domain_id(struct context_entry *context,
 302                                          unsigned long value)
 303 {
 304         context->hi |= (value & ((1 << 16) - 1)) << 8;
 305 }
 306
 307 static inline int context_domain_id(struct context_entry *c)
 308 {
 309         return((c->hi >> 8) & 0xffff);
 310 }
 311
 312 static inline void context_clear_entry(struct context_entry *context)
 313 {
 314         context->lo = 0;
 315         context->hi = 0;
 316 }
 317
 318 /*
 319  * 0: readable
 320  * 1: writable
 321  * 2-6: reserved
 322  * 7: super page
 323  * 8-10: available
 324  * 11: snoop behavior
 325  * 12-63: Host physcial address
 326  */
 327 struct dma_pte {
 328         u64 val;
 329 };
 330
 331 static inline void dma_clear_pte(struct dma_pte *pte)
 332 {
 333         pte->val = 0;
 334 }
 335
 336 static inline u64 dma_pte_addr(struct dma_pte *pte)
 337 {
 338 #ifdef CONFIG_64BIT
 339         return pte->val & VTD_PAGE_MASK;
 340 #else
 341         /* Must have a full atomic 64-bit read */
 342         return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
 343 #endif
 344 }
 345
 346 static inline bool dma_pte_present(struct dma_pte *pte)
 347 {
 348         return (pte->val & 3) != 0;
 349 }
 350
 351 static inline bool dma_pte_superpage(struct dma_pte *pte)
 352 {
 353         return (pte->val & DMA_PTE_LARGE_PAGE);
 354 }
 355
 356 static inline int first_pte_in_page(struct dma_pte *pte)
 357 {
 358         return !((unsigned long)pte & ~VTD_PAGE_MASK);
 359 }
 360
 361 /*
 362  * This domain is a statically identity mapping domain.
 363  *      1. This domain creats a static 1:1 mapping to all usable memory.
 364  *      2. It maps to each iommu if successful.
 365  *      3. Each iommu mapps to this domain if successful.
 366  */
 367 static struct dmar_domain *si_domain;
 368 static int hw_pass_through = 1;
 369
 370 /*
 371  * Domain represents a virtual machine, more than one devices
 372  * across iommus may be owned in one domain, e.g. kvm guest.
 373  */
 374 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 0)
 375
 376 /* si_domain contains mulitple devices */
 377 #define DOMAIN_FLAG_STATIC_IDENTITY     (1 << 1)
 378
 379 #define for_each_domain_iommu(idx, domain)                      \
 380         for (idx = 0; idx < g_num_of_iommus; idx++)             \
 381                 if (domain->iommu_refcnt[idx])
 382
 383 struct dmar_rmrr_unit {
 384         struct list_head list;          /* list of rmrr units   */
 385         struct acpi_dmar_header *hdr;   /* ACPI header          */
 386         u64     base_address;           /* reserved base address*/
 387         u64     end_address;            /* reserved end address */
 388         struct dmar_dev_scope *devices; /* target devices */
 389         int     devices_cnt;            /* target device count */
 390 };
 391
 392 struct dmar_atsr_unit {
 393         struct list_head list;          /* list of ATSR units */
 394         struct acpi_dmar_header *hdr;   /* ACPI header */
 395         struct dmar_dev_scope *devices; /* target devices */
 396         int devices_cnt;                /* target device count */
 397         u8 include_all:1;               /* include all ports */
 398 };
 399
 400 static LIST_HEAD(dmar_atsr_units);
 401 static LIST_HEAD(dmar_rmrr_units);
 402
 403 #define for_each_rmrr_units(rmrr) \
 404         list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 405
 406 /* bitmap for indexing intel_iommus */
 407 static int g_num_of_iommus;
 408
 409 static void domain_exit(struct dmar_domain *domain);
 410 static void domain_remove_dev_info(struct dmar_domain *domain);
 411 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
 412                                      struct device *dev);
 413 static void __dmar_remove_one_dev_info(struct device_domain_info *info);
 414 static void domain_context_clear(struct intel_iommu *iommu,
 415                                  struct device *dev);
 416 static int domain_detach_iommu(struct dmar_domain *domain,
 417                                struct intel_iommu *iommu);
 418
 419 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
 420 int dmar_disabled = 0;
 421 #else
 422 int dmar_disabled = 1;
 423 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
 424
 425 int intel_iommu_enabled = 0;
 426 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
 427
 428 static int dmar_map_gfx = 1;
 429 static int dmar_forcedac;
 430 static int intel_iommu_strict;
 431 static int intel_iommu_superpage = 1;
 432 static int intel_iommu_ecs = 1;
 433 static int intel_iommu_pasid28;
 434 static int iommu_identity_mapping;
 435
 436 #define IDENTMAP_ALL            1
 437 #define IDENTMAP_GFX            2
 438 #define IDENTMAP_AZALIA         4
 439
 440 /* Broadwell and Skylake have broken ECS support — normal so-called "second
 441  * level" translation of DMA requests-without-PASID doesn't actually happen
 442  * unless you also set the NESTE bit in an extended context-entry. Which of
 443  * course means that SVM doesn't work because it's trying to do nested
 444  * translation of the physical addresses it finds in the process page tables,
 445  * through the IOVA->phys mapping found in the "second level" page tables.
 446  *
 447  * The VT-d specification was retroactively changed to change the definition
 448  * of the capability bits and pretend that Broadwell/Skylake never happened...
 449  * but unfortunately the wrong bit was changed. It's ECS which is broken, but
 450  * for some reason it was the PASID capability bit which was redefined (from
 451  * bit 28 on BDW/SKL to bit 40 in future).
 452  *
 453  * So our test for ECS needs to eschew those implementations which set the old
 454  * PASID capabiity bit 28, since those are the ones on which ECS is broken.
 455  * Unless we are working around the 'pasid28' limitations, that is, by putting
 456  * the device into passthrough mode for normal DMA and thus masking the bug.
 457  */
 458 #define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
 459                             (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap)))
 460 /* PASID support is thus enabled if ECS is enabled and *either* of the old
 461  * or new capability bits are set. */
 462 #define pasid_enabled(iommu) (ecs_enabled(iommu) &&                     \
 463                               (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap)))
 464
 465 int intel_iommu_gfx_mapped;
 466 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 467
 468 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 469 static DEFINE_SPINLOCK(device_domain_lock);
 470 static LIST_HEAD(device_domain_list);
 471
 472 /*
 473  * Iterate over elements in device_domain_list and call the specified
 474  * callback @fn against each element. This helper should only be used
 475  * in the context where the device_domain_lock has already been holden.
 476  */
 477 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
 478                                      void *data), void *data)
 479 {
 480         int ret = 0;
 481         struct device_domain_info *info;
 482
 483         assert_spin_locked(&device_domain_lock);
 484         list_for_each_entry(info, &device_domain_list, global) {
 485                 ret = fn(info, data);
 486                 if (ret)
 487                         return ret;
 488         }
 489
 490         return 0;
 491 }
 492
 493 const struct iommu_ops intel_iommu_ops;
 494
 495 static bool translation_pre_enabled(struct intel_iommu *iommu)
 496 {
 497         return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
 498 }
 499
 500 static void clear_translation_pre_enabled(struct intel_iommu *iommu)
 501 {
 502         iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
 503 }
 504
 505 static void init_translation_status(struct intel_iommu *iommu)
 506 {
 507         u32 gsts;
 508
 509         gsts = readl(iommu->reg + DMAR_GSTS_REG);
 510         if (gsts & DMA_GSTS_TES)
 511                 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
 512 }
 513
 514 /* Convert generic 'struct iommu_domain to private struct dmar_domain */
 515 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 516 {
 517         return container_of(dom, struct dmar_domain, domain);
 518 }
 519
 520 static int __init intel_iommu_setup(char *str)
 521 {
 522         if (!str)
 523                 return -EINVAL;
 524         while (*str) {
 525                 if (!strncmp(str, "on", 2)) {
 526                         dmar_disabled = 0;
 527                         pr_info("IOMMU enabled\n");
 528                 } else if (!strncmp(str, "off", 3)) {
 529                         dmar_disabled = 1;
 530                         pr_info("IOMMU disabled\n");
 531                 } else if (!strncmp(str, "igfx_off", 8)) {
 532                         dmar_map_gfx = 0;
 533                         pr_info("Disable GFX device mapping\n");
 534                 } else if (!strncmp(str, "forcedac", 8)) {
 535                         pr_info("Forcing DAC for PCI devices\n");
 536                         dmar_forcedac = 1;
 537                 } else if (!strncmp(str, "strict", 6)) {
 538                         pr_info("Disable batched IOTLB flush\n");
 539                         intel_iommu_strict = 1;
 540                 } else if (!strncmp(str, "sp_off", 6)) {
 541                         pr_info("Disable supported super page\n");
 542                         intel_iommu_superpage = 0;
 543                 } else if (!strncmp(str, "ecs_off", 7)) {
 544                         printk(KERN_INFO
 545                                 "Intel-IOMMU: disable extended context table support\n");
 546                         intel_iommu_ecs = 0;
 547                 } else if (!strncmp(str, "pasid28", 7)) {
 548                         printk(KERN_INFO
 549                                 "Intel-IOMMU: enable pre-production PASID support\n");
 550                         intel_iommu_pasid28 = 1;
 551                         iommu_identity_mapping |= IDENTMAP_GFX;
 552                 } else if (!strncmp(str, "tboot_noforce", 13)) {
 553                         printk(KERN_INFO
 554                                 "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
 555                         intel_iommu_tboot_noforce = 1;
 556                 }
 557
 558                 str += strcspn(str, ",");
 559                 while (*str == ',')
 560                         str++;
 561         }
 562         return 0;
 563 }
 564 __setup("intel_iommu=", intel_iommu_setup);
 565
 566 static struct kmem_cache *iommu_domain_cache;
 567 static struct kmem_cache *iommu_devinfo_cache;
 568
 569 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)
 570 {
 571         struct dmar_domain **domains;
 572         int idx = did >> 8;
 573
 574         domains = iommu->domains[idx];
 575         if (!domains)
 576                 return NULL;
 577
 578         return domains[did & 0xff];
 579 }
 580
 581 static void set_iommu_domain(struct intel_iommu *iommu, u16 did,
 582                              struct dmar_domain *domain)
 583 {
 584         struct dmar_domain **domains;
 585         int idx = did >> 8;
 586
 587         if (!iommu->domains[idx]) {
 588                 size_t size = 256 * sizeof(struct dmar_domain *);
 589                 iommu->domains[idx] = kzalloc(size, GFP_ATOMIC);
 590         }
 591
 592         domains = iommu->domains[idx];
 593         if (WARN_ON(!domains))
 594                 return;
 595         else
 596                 domains[did & 0xff] = domain;
 597 }
 598
 599 void *alloc_pgtable_page(int node)
 600 {
 601         struct page *page;
 602         void *vaddr = NULL;
 603
 604         page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
 605         if (page)
 606                 vaddr = page_address(page);
 607         return vaddr;
 608 }
 609
 610 void free_pgtable_page(void *vaddr)
 611 {
 612         free_page((unsigned long)vaddr);
 613 }
 614
 615 static inline void *alloc_domain_mem(void)
 616 {
 617         return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
 618 }
 619
 620 static void free_domain_mem(void *vaddr)
 621 {
 622         kmem_cache_free(iommu_domain_cache, vaddr);
 623 }
 624
 625 static inline void * alloc_devinfo_mem(void)
 626 {
 627         return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
 628 }
 629
 630 static inline void free_devinfo_mem(void *vaddr)
 631 {
 632         kmem_cache_free(iommu_devinfo_cache, vaddr);
 633 }
 634
 635 static inline int domain_type_is_vm(struct dmar_domain *domain)
 636 {
 637         return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
 638 }
 639
 640 static inline int domain_type_is_si(struct dmar_domain *domain)
 641 {
 642         return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY;
 643 }
 644
 645 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
 646 {
 647         return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
 648                                 DOMAIN_FLAG_STATIC_IDENTITY);
 649 }
 650
 651 static inline int domain_pfn_supported(struct dmar_domain *domain,
 652                                        unsigned long pfn)
 653 {
 654         int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
 655
 656         return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
 657 }
 658
 659 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
 660 {
 661         unsigned long sagaw;
 662         int agaw = -1;
 663
 664         sagaw = cap_sagaw(iommu->cap);
 665         for (agaw = width_to_agaw(max_gaw);
 666              agaw >= 0; agaw--) {
 667                 if (test_bit(agaw, &sagaw))
 668                         break;
 669         }
 670
 671         return agaw;
 672 }
 673
 674 /*
 675  * Calculate max SAGAW for each iommu.
 676  */
 677 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
 678 {
 679         return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
 680 }
 681
 682 /*
 683  * calculate agaw for each iommu.
 684  * "SAGAW" may be different across iommus, use a default agaw, and
 685  * get a supported less agaw for iommus that don't support the default agaw.
 686  */
 687 int iommu_calculate_agaw(struct intel_iommu *iommu)
 688 {
 689         return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 690 }
 691
 692 /* This functionin only returns single iommu in a domain */
 693 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 694 {
 695         int iommu_id;
 696
 697         /* si_domain and vm domain should not get here. */
 698         BUG_ON(domain_type_is_vm_or_si(domain));
 699         for_each_domain_iommu(iommu_id, domain)
 700                 break;
 701
 702         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 703                 return NULL;
 704
 705         return g_iommus[iommu_id];
 706 }
 707
 708 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 709 {
 710         struct dmar_drhd_unit *drhd;
 711         struct intel_iommu *iommu;
 712         bool found = false;
 713         int i;
 714
 715         domain->iommu_coherency = 1;
 716
 717         for_each_domain_iommu(i, domain) {
 718                 found = true;
 719                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 720                         domain->iommu_coherency = 0;
 721                         break;
 722                 }
 723         }
 724         if (found)
 725                 return;
 726
 727         /* No hardware attached; use lowest common denominator */
 728         rcu_read_lock();
 729         for_each_active_iommu(iommu, drhd) {
 730                 if (!ecap_coherent(iommu->ecap)) {
 731                         domain->iommu_coherency = 0;
 732                         break;
 733                 }
 734         }
 735         rcu_read_unlock();
 736 }
 737
 738 static int domain_update_iommu_snooping(struct intel_iommu *skip)
 739 {
 740         struct dmar_drhd_unit *drhd;
 741         struct intel_iommu *iommu;
 742         int ret = 1;
 743
 744         rcu_read_lock();
 745         for_each_active_iommu(iommu, drhd) {
 746                 if (iommu != skip) {
 747                         if (!ecap_sc_support(iommu->ecap)) {
 748                                 ret = 0;
 749                                 break;
 750                         }
 751                 }
 752         }
 753         rcu_read_unlock();
 754
 755         return ret;
 756 }
 757
 758 static int domain_update_iommu_superpage(struct intel_iommu *skip)
 759 {
 760         struct dmar_drhd_unit *drhd;
 761         struct intel_iommu *iommu;
 762         int mask = 0xf;
 763
 764         if (!intel_iommu_superpage) {
 765                 return 0;
 766         }
 767
 768         /* set iommu_superpage to the smallest common denominator */
 769         rcu_read_lock();
 770         for_each_active_iommu(iommu, drhd) {
 771                 if (iommu != skip) {
 772                         mask &= cap_super_page_val(iommu->cap);
 773                         if (!mask)
 774                                 break;
 775                 }
 776         }
 777         rcu_read_unlock();
 778
 779         return fls(mask);
 780 }
 781
 782 /* Some capabilities may be different across iommus */
 783 static void domain_update_iommu_cap(struct dmar_domain *domain)
 784 {
 785         domain_update_iommu_coherency(domain);
 786         domain->iommu_snooping = domain_update_iommu_snooping(NULL);
 787         domain->iommu_superpage = domain_update_iommu_superpage(NULL);
 788 }
 789
 790 static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
 791                                                        u8 bus, u8 devfn, int alloc)
 792 {
 793         struct root_entry *root = &iommu->root_entry[bus];
 794         struct context_entry *context;
 795         u64 *entry;
 796
 797         entry = &root->lo;
 798         if (ecs_enabled(iommu)) {
 799                 if (devfn >= 0x80) {
 800                         devfn -= 0x80;
 801                         entry = &root->hi;
 802                 }
 803                 devfn *= 2;
 804         }
 805         if (*entry & 1)
 806                 context = phys_to_virt(*entry & VTD_PAGE_MASK);
 807         else {
 808                 unsigned long phy_addr;
 809                 if (!alloc)
 810                         return NULL;
 811
 812                 context = alloc_pgtable_page(iommu->node);
 813                 if (!context)
 814                         return NULL;
 815
 816                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 817                 phy_addr = virt_to_phys((void *)context);
 818                 *entry = phy_addr | 1;
 819                 __iommu_flush_cache(iommu, entry, sizeof(*entry));
 820         }
 821         return &context[devfn];
 822 }
 823
 824 static int iommu_dummy(struct device *dev)
 825 {
 826         return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
 827 }
 828
 829 static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
 830 {
 831         struct dmar_drhd_unit *drhd = NULL;
 832         struct intel_iommu *iommu;
 833         struct device *tmp;
 834         struct pci_dev *ptmp, *pdev = NULL;
 835         u16 segment = 0;
 836         int i;
 837
 838         if (iommu_dummy(dev))
 839                 return NULL;
 840
 841         if (dev_is_pci(dev)) {
 842                 struct pci_dev *pf_pdev;
 843
 844                 pdev = to_pci_dev(dev);
 845
 846 #ifdef CONFIG_X86
 847                 /* VMD child devices currently cannot be handled individually */
 848                 if (is_vmd(pdev->bus))
 849                         return NULL;
 850 #endif
 851
 852                 /* VFs aren't listed in scope tables; we need to look up
 853                  * the PF instead to find the IOMMU. */
 854                 pf_pdev = pci_physfn(pdev);
 855                 dev = &pf_pdev->dev;
 856                 segment = pci_domain_nr(pdev->bus);
 857         } else if (has_acpi_companion(dev))
 858                 dev = &ACPI_COMPANION(dev)->dev;
 859
 860         rcu_read_lock();
 861         for_each_active_iommu(iommu, drhd) {
 862                 if (pdev && segment != drhd->segment)
 863                         continue;
 864
 865                 for_each_active_dev_scope(drhd->devices,
 866                                           drhd->devices_cnt, i, tmp) {
 867                         if (tmp == dev) {
 868                                 /* For a VF use its original BDF# not that of the PF
 869                                  * which we used for the IOMMU lookup. Strictly speaking
 870                                  * we could do this for all PCI devices; we only need to
 871                                  * get the BDF# from the scope table for ACPI matches. */
 872                                 if (pdev && pdev->is_virtfn)
 873                                         goto got_pdev;
 874
 875                                 *bus = drhd->devices[i].bus;
 876                                 *devfn = drhd->devices[i].devfn;
 877                                 goto out;
 878                         }
 879
 880                         if (!pdev || !dev_is_pci(tmp))
 881                                 continue;
 882
 883                         ptmp = to_pci_dev(tmp);
 884                         if (ptmp->subordinate &&
 885                             ptmp->subordinate->number <= pdev->bus->number &&
 886                             ptmp->subordinate->busn_res.end >= pdev->bus->number)
 887                                 goto got_pdev;
 888                 }
 889
 890                 if (pdev && drhd->include_all) {
 891                 got_pdev:
 892                         *bus = pdev->bus->number;
 893                         *devfn = pdev->devfn;
 894                         goto out;
 895                 }
 896         }
 897         iommu = NULL;
 898  out:
 899         rcu_read_unlock();
 900
 901         return iommu;
 902 }
 903
 904 static void domain_flush_cache(struct dmar_domain *domain,
 905                                void *addr, int size)
 906 {
 907         if (!domain->iommu_coherency)
 908                 clflush_cache_range(addr, size);
 909 }
 910
 911 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 912 {
 913         struct context_entry *context;
 914         int ret = 0;
 915         unsigned long flags;
 916
 917         spin_lock_irqsave(&iommu->lock, flags);
 918         context = iommu_context_addr(iommu, bus, devfn, 0);
 919         if (context)
 920                 ret = context_present(context);
 921         spin_unlock_irqrestore(&iommu->lock, flags);
 922         return ret;
 923 }
 924
 925 static void free_context_table(struct intel_iommu *iommu)
 926 {
 927         int i;
 928         unsigned long flags;
 929         struct context_entry *context;
 930
 931         spin_lock_irqsave(&iommu->lock, flags);
 932         if (!iommu->root_entry) {
 933                 goto out;
 934         }
 935         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 936                 context = iommu_context_addr(iommu, i, 0, 0);
 937                 if (context)
 938                         free_pgtable_page(context);
 939
 940                 if (!ecs_enabled(iommu))
 941                         continue;
 942
 943                 context = iommu_context_addr(iommu, i, 0x80, 0);
 944                 if (context)
 945                         free_pgtable_page(context);
 946
 947         }
 948         free_pgtable_page(iommu->root_entry);
 949         iommu->root_entry = NULL;
 950 out:
 951         spin_unlock_irqrestore(&iommu->lock, flags);
 952 }
 953
 954 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
 955                                       unsigned long pfn, int *target_level)
 956 {
 957         struct dma_pte *parent, *pte = NULL;
 958         int level = agaw_to_level(domain->agaw);
 959         int offset;
 960
 961         BUG_ON(!domain->pgd);
 962
 963         if (!domain_pfn_supported(domain, pfn))
 964                 /* Address beyond IOMMU's addressing capabilities. */
 965                 return NULL;
 966
 967         parent = domain->pgd;
 968
 969         while (1) {
 970                 void *tmp_page;
 971
 972                 offset = pfn_level_offset(pfn, level);
 973                 pte = &parent[offset];
 974                 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
 975                         break;
 976                 if (level == *target_level)
 977                         break;
 978
 979                 if (!dma_pte_present(pte)) {
 980                         uint64_t pteval;
 981
 982                         tmp_page = alloc_pgtable_page(domain->nid);
 983
 984                         if (!tmp_page)
 985                                 return NULL;
 986
 987                         domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
 988                         pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
 989                         if (cmpxchg64(&pte->val, 0ULL, pteval))
 990                                 /* Someone else set it while we were thinking; use theirs. */
 991                                 free_pgtable_page(tmp_page);
 992                         else
 993                                 domain_flush_cache(domain, pte, sizeof(*pte));
 994                 }
 995                 if (level == 1)
 996                         break;
 997
 998                 parent = phys_to_virt(dma_pte_addr(pte));
 999                 level--;
1000         }
1001
1002         if (!*target_level)
1003                 *target_level = level;
1004
1005         return pte;
1006 }
1007
1008
1009 /* return address's pte at specific level */
1010 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
1011                                          unsigned long pfn,
1012                                          int level, int *large_page)
1013 {
1014         struct dma_pte *parent, *pte = NULL;
1015         int total = agaw_to_level(domain->agaw);
1016         int offset;
1017
1018         parent = domain->pgd;
1019         while (level <= total) {
1020                 offset = pfn_level_offset(pfn, total);
1021                 pte = &parent[offset];
1022                 if (level == total)
1023                         return pte;
1024
1025                 if (!dma_pte_present(pte)) {
1026                         *large_page = total;
1027                         break;
1028                 }
1029
1030                 if (dma_pte_superpage(pte)) {
1031                         *large_page = total;
1032                         return pte;
1033                 }
1034
1035                 parent = phys_to_virt(dma_pte_addr(pte));
1036                 total--;
1037         }
1038         return NULL;
1039 }
1040
1041 /* clear last level pte, a tlb flush should be followed */
1042 static void dma_pte_clear_range(struct dmar_domain *domain,
1043                                 unsigned long start_pfn,
1044                                 unsigned long last_pfn)
1045 {
1046         unsigned int large_page = 1;
1047         struct dma_pte *first_pte, *pte;
1048
1049         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1050         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1051         BUG_ON(start_pfn > last_pfn);
1052
1053         /* we don't need lock here; nobody else touches the iova range */
1054         do {
1055                 large_page = 1;
1056                 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1057                 if (!pte) {
1058                         start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1059                         continue;
1060                 }
1061                 do {
1062                         dma_clear_pte(pte);
1063                         start_pfn += lvl_to_nr_pages(large_page);
1064                         pte++;
1065                 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1066
1067                 domain_flush_cache(domain, first_pte,
1068                                    (void *)pte - (void *)first_pte);
1069
1070         } while (start_pfn && start_pfn <= last_pfn);
1071 }
1072
1073 static void dma_pte_free_level(struct dmar_domain *domain, int level,
1074                                int retain_level, struct dma_pte *pte,
1075                                unsigned long pfn, unsigned long start_pfn,
1076                                unsigned long last_pfn)
1077 {
1078         pfn = max(start_pfn, pfn);
1079         pte = &pte[pfn_level_offset(pfn, level)];
1080
1081         do {
1082                 unsigned long level_pfn;
1083                 struct dma_pte *level_pte;
1084
1085                 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1086                         goto next;
1087
1088                 level_pfn = pfn & level_mask(level);
1089                 level_pte = phys_to_virt(dma_pte_addr(pte));
1090
1091                 if (level > 2) {
1092                         dma_pte_free_level(domain, level - 1, retain_level,
1093                                            level_pte, level_pfn, start_pfn,
1094                                            last_pfn);
1095                 }
1096
1097                 /*
1098                  * Free the page table if we're below the level we want to
1099                  * retain and the range covers the entire table.
1100                  */
1101                 if (level < retain_level && !(start_pfn > level_pfn ||
1102                       last_pfn < level_pfn + level_size(level) - 1)) {
1103                         dma_clear_pte(pte);
1104                         domain_flush_cache(domain, pte, sizeof(*pte));
1105                         free_pgtable_page(level_pte);
1106                 }
1107 next:
1108                 pfn += level_size(level);
1109         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1110 }
1111
1112 /*
1113  * clear last level (leaf) ptes and free page table pages below the
1114  * level we wish to keep intact.
1115  */
1116 static void dma_pte_free_pagetable(struct dmar_domain *domain,
1117                                    unsigned long start_pfn,
1118                                    unsigned long last_pfn,
1119                                    int retain_level)
1120 {
1121         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1122         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1123         BUG_ON(start_pfn > last_pfn);
1124
1125         dma_pte_clear_range(domain, start_pfn, last_pfn);
1126
1127         /* We don't need lock here; nobody else touches the iova range */
1128         dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1129                            domain->pgd, 0, start_pfn, last_pfn);
1130
1131         /* free pgd */
1132         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1133                 free_pgtable_page(domain->pgd);
1134                 domain->pgd = NULL;
1135         }
1136 }
1137
1138 /* When a page at a given level is being unlinked from its parent, we don't
1139    need to *modify* it at all. All we need to do is make a list of all the
1140    pages which can be freed just as soon as we've flushed the IOTLB and we
1141    know the hardware page-walk will no longer touch them.
1142    The 'pte' argument is the *parent* PTE, pointing to the page that is to
1143    be freed. */
1144 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1145                                             int level, struct dma_pte *pte,
1146                                             struct page *freelist)
1147 {
1148         struct page *pg;
1149
1150         pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1151         pg->freelist = freelist;
1152         freelist = pg;
1153
1154         if (level == 1)
1155                 return freelist;
1156
1157         pte = page_address(pg);
1158         do {
1159                 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1160                         freelist = dma_pte_list_pagetables(domain, level - 1,
1161                                                            pte, freelist);
1162                 pte++;
1163         } while (!first_pte_in_page(pte));
1164
1165         return freelist;
1166 }
1167
1168 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1169                                         struct dma_pte *pte, unsigned long pfn,
1170                                         unsigned long start_pfn,
1171                                         unsigned long last_pfn,
1172                                         struct page *freelist)
1173 {
1174         struct dma_pte *first_pte = NULL, *last_pte = NULL;
1175
1176         pfn = max(start_pfn, pfn);
1177         pte = &pte[pfn_level_offset(pfn, level)];
1178
1179         do {
1180                 unsigned long level_pfn;
1181
1182                 if (!dma_pte_present(pte))
1183                         goto next;
1184
1185                 level_pfn = pfn & level_mask(level);
1186
1187                 /* If range covers entire pagetable, free it */
1188                 if (start_pfn <= level_pfn &&
1189                     last_pfn >= level_pfn + level_size(level) - 1) {
1190                         /* These suborbinate page tables are going away entirely. Don't
1191                            bother to clear them; we're just going to *free* them. */
1192                         if (level > 1 && !dma_pte_superpage(pte))
1193                                 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1194
1195                         dma_clear_pte(pte);
1196                         if (!first_pte)
1197                                 first_pte = pte;
1198                         last_pte = pte;
1199                 } else if (level > 1) {
1200                         /* Recurse down into a level that isn't *entirely* obsolete */
1201                         freelist = dma_pte_clear_level(domain, level - 1,
1202                                                        phys_to_virt(dma_pte_addr(pte)),
1203                                                        level_pfn, start_pfn, last_pfn,
1204                                                        freelist);
1205                 }
1206 next:
1207                 pfn += level_size(level);
1208         } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1209
1210         if (first_pte)
1211                 domain_flush_cache(domain, first_pte,
1212                                    (void *)++last_pte - (void *)first_pte);
1213
1214         return freelist;
1215 }
1216
1217 /* We can't just free the pages because the IOMMU may still be walking
1218    the page tables, and may have cached the intermediate levels. The
1219    pages can only be freed after the IOTLB flush has been done. */
1220 static struct page *domain_unmap(struct dmar_domain *domain,
1221                                  unsigned long start_pfn,
1222                                  unsigned long last_pfn)
1223 {
1224         struct page *freelist = NULL;
1225
1226         BUG_ON(!domain_pfn_supported(domain, start_pfn));
1227         BUG_ON(!domain_pfn_supported(domain, last_pfn));
1228         BUG_ON(start_pfn > last_pfn);
1229
1230         /* we don't need lock here; nobody else touches the iova range */
1231         freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1232                                        domain->pgd, 0, start_pfn, last_pfn, NULL);
1233
1234         /* free pgd */
1235         if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1236                 struct page *pgd_page = virt_to_page(domain->pgd);
1237                 pgd_page->freelist = freelist;
1238                 freelist = pgd_page;
1239
1240                 domain->pgd = NULL;
1241         }
1242
1243         return freelist;
1244 }
1245
1246 static void dma_free_pagelist(struct page *freelist)
1247 {
1248         struct page *pg;
1249
1250         while ((pg = freelist)) {
1251                 freelist = pg->freelist;
1252                 free_pgtable_page(page_address(pg));
1253         }
1254 }
1255
1256 static void iova_entry_free(unsigned long data)
1257 {
1258         struct page *freelist = (struct page *)data;
1259
1260         dma_free_pagelist(freelist);
1261 }
1262
1263 /* iommu handling */
1264 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1265 {
1266         struct root_entry *root;
1267         unsigned long flags;
1268
1269         root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1270         if (!root) {
1271                 pr_err("Allocating root entry for %s failed\n",
1272                         iommu->name);
1273                 return -ENOMEM;
1274         }
1275
1276         __iommu_flush_cache(iommu, root, ROOT_SIZE);
1277
1278         spin_lock_irqsave(&iommu->lock, flags);
1279         iommu->root_entry = root;
1280         spin_unlock_irqrestore(&iommu->lock, flags);
1281
1282         return 0;
1283 }
1284
1285 static void iommu_set_root_entry(struct intel_iommu *iommu)
1286 {
1287         u64 addr;
1288         u32 sts;
1289         unsigned long flag;
1290
1291         addr = virt_to_phys(iommu->root_entry);
1292         if (ecs_enabled(iommu))
1293                 addr |= DMA_RTADDR_RTT;
1294
1295         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1296         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1297
1298         writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1299
1300         /* Make sure hardware complete it */
1301         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1302                       readl, (sts & DMA_GSTS_RTPS), sts);
1303
1304         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1305 }
1306
1307 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1308 {
1309         u32 val;
1310         unsigned long flag;
1311
1312         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1313                 return;
1314
1315         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1316         writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1317
1318         /* Make sure hardware complete it */
1319         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1320                       readl, (!(val & DMA_GSTS_WBFS)), val);
1321
1322         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1323 }
1324
1325 /* return value determine if we need a write buffer flush */
1326 static void __iommu_flush_context(struct intel_iommu *iommu,
1327                                   u16 did, u16 source_id, u8 function_mask,
1328                                   u64 type)
1329 {
1330         u64 val = 0;
1331         unsigned long flag;
1332
1333         switch (type) {
1334         case DMA_CCMD_GLOBAL_INVL:
1335                 val = DMA_CCMD_GLOBAL_INVL;
1336                 break;
1337         case DMA_CCMD_DOMAIN_INVL:
1338                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1339                 break;
1340         case DMA_CCMD_DEVICE_INVL:
1341                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1342                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1343                 break;
1344         default:
1345                 BUG();
1346         }
1347         val |= DMA_CCMD_ICC;
1348
1349         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1350         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1351
1352         /* Make sure hardware complete it */
1353         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1354                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1355
1356         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1357 }
1358
1359 /* return value determine if we need a write buffer flush */
1360 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1361                                 u64 addr, unsigned int size_order, u64 type)
1362 {
1363         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1364         u64 val = 0, val_iva = 0;
1365         unsigned long flag;
1366
1367         switch (type) {
1368         case DMA_TLB_GLOBAL_FLUSH:
1369                 /* global flush doesn't need set IVA_REG */
1370                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1371                 break;
1372         case DMA_TLB_DSI_FLUSH:
1373                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1374                 break;
1375         case DMA_TLB_PSI_FLUSH:
1376                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1377                 /* IH bit is passed in as part of address */
1378                 val_iva = size_order | addr;
1379                 break;
1380         default:
1381                 BUG();
1382         }
1383         /* Note: set drain read/write */
1384 #if 0
1385         /*
1386          * This is probably to be super secure.. Looks like we can
1387          * ignore it without any impact.
1388          */
1389         if (cap_read_drain(iommu->cap))
1390                 val |= DMA_TLB_READ_DRAIN;
1391 #endif
1392         if (cap_write_drain(iommu->cap))
1393                 val |= DMA_TLB_WRITE_DRAIN;
1394
1395         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1396         /* Note: Only uses first TLB reg currently */
1397         if (val_iva)
1398                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1399         dmar_writeq(iommu->reg + tlb_offset + 8, val);
1400
1401         /* Make sure hardware complete it */
1402         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1403                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1404
1405         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1406
1407         /* check IOTLB invalidation granularity */
1408         if (DMA_TLB_IAIG(val) == 0)
1409                 pr_err("Flush IOTLB failed\n");
1410         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1411                 pr_debug("TLB flush request %Lx, actual %Lx\n",
1412                         (unsigned long long)DMA_TLB_IIRG(type),
1413                         (unsigned long long)DMA_TLB_IAIG(val));
1414 }
1415
1416 static struct device_domain_info *
1417 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1418                          u8 bus, u8 devfn)
1419 {
1420         struct device_domain_info *info;
1421
1422         assert_spin_locked(&device_domain_lock);
1423
1424         if (!iommu->qi)
1425                 return NULL;
1426
1427         list_for_each_entry(info, &domain->devices, link)
1428                 if (info->iommu == iommu && info->bus == bus &&
1429                     info->devfn == devfn) {
1430                         if (info->ats_supported && info->dev)
1431                                 return info;
1432                         break;
1433                 }
1434
1435         return NULL;
1436 }
1437
1438 static void domain_update_iotlb(struct dmar_domain *domain)
1439 {
1440         struct device_domain_info *info;
1441         bool has_iotlb_device = false;
1442
1443         assert_spin_locked(&device_domain_lock);
1444
1445         list_for_each_entry(info, &domain->devices, link) {
1446                 struct pci_dev *pdev;
1447
1448                 if (!info->dev || !dev_is_pci(info->dev))
1449                         continue;
1450
1451                 pdev = to_pci_dev(info->dev);
1452                 if (pdev->ats_enabled) {
1453                         has_iotlb_device = true;
1454                         break;
1455                 }
1456         }
1457
1458         domain->has_iotlb_device = has_iotlb_device;
1459 }
1460
1461 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1462 {
1463         struct pci_dev *pdev;
1464
1465         assert_spin_locked(&device_domain_lock);
1466
1467         if (!info || !dev_is_pci(info->dev))
1468                 return;
1469
1470         pdev = to_pci_dev(info->dev);
1471         /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1472          * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1473          * queue depth at PF level. If DIT is not set, PFSID will be treated as
1474          * reserved, which should be set to 0.
1475          */
1476         if (!ecap_dit(info->iommu->ecap))
1477                 info->pfsid = 0;
1478         else {
1479                 struct pci_dev *pf_pdev;
1480
1481                 /* pdev will be returned if device is not a vf */
1482                 pf_pdev = pci_physfn(pdev);
1483                 info->pfsid = PCI_DEVID(pf_pdev->bus->number, pf_pdev->devfn);
1484         }
1485
1486 #ifdef CONFIG_INTEL_IOMMU_SVM
1487         /* The PCIe spec, in its wisdom, declares that the behaviour of
1488            the device if you enable PASID support after ATS support is
1489            undefined. So always enable PASID support on devices which
1490            have it, even if we can't yet know if we're ever going to
1491            use it. */
1492         if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1493                 info->pasid_enabled = 1;
1494
1495         if (info->pri_supported && !pci_reset_pri(pdev) && !pci_enable_pri(pdev, 32))
1496                 info->pri_enabled = 1;
1497 #endif
1498         if (info->ats_supported && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1499                 info->ats_enabled = 1;
1500                 domain_update_iotlb(info->domain);
1501                 info->ats_qdep = pci_ats_queue_depth(pdev);
1502         }
1503 }
1504
1505 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1506 {
1507         struct pci_dev *pdev;
1508
1509         assert_spin_locked(&device_domain_lock);
1510
1511         if (!dev_is_pci(info->dev))
1512                 return;
1513
1514         pdev = to_pci_dev(info->dev);
1515
1516         if (info->ats_enabled) {
1517                 pci_disable_ats(pdev);
1518                 info->ats_enabled = 0;
1519                 domain_update_iotlb(info->domain);
1520         }
1521 #ifdef CONFIG_INTEL_IOMMU_SVM
1522         if (info->pri_enabled) {
1523                 pci_disable_pri(pdev);
1524                 info->pri_enabled = 0;
1525         }
1526         if (info->pasid_enabled) {
1527                 pci_disable_pasid(pdev);
1528                 info->pasid_enabled = 0;
1529         }
1530 #endif
1531 }
1532
1533 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1534                                   u64 addr, unsigned mask)
1535 {
1536         u16 sid, qdep;
1537         unsigned long flags;
1538         struct device_domain_info *info;
1539
1540         if (!domain->has_iotlb_device)
1541                 return;
1542
1543         spin_lock_irqsave(&device_domain_lock, flags);
1544         list_for_each_entry(info, &domain->devices, link) {
1545                 if (!info->ats_enabled)
1546                         continue;
1547
1548                 sid = info->bus << 8 | info->devfn;
1549                 qdep = info->ats_qdep;
1550                 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1551                                 qdep, addr, mask);
1552         }
1553         spin_unlock_irqrestore(&device_domain_lock, flags);
1554 }
1555
1556 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1557                                   struct dmar_domain *domain,
1558                                   unsigned long pfn, unsigned int pages,
1559                                   int ih, int map)
1560 {
1561         unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1562         uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1563         u16 did = domain->iommu_did[iommu->seq_id];
1564
1565         BUG_ON(pages == 0);
1566
1567         if (ih)
1568                 ih = 1 << 6;
1569         /*
1570          * Fallback to domain selective flush if no PSI support or the size is
1571          * too big.
1572          * PSI requires page size to be 2 ^ x, and the base address is naturally
1573          * aligned to the size
1574          */
1575         if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1576                 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1577                                                 DMA_TLB_DSI_FLUSH);
1578         else
1579                 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1580                                                 DMA_TLB_PSI_FLUSH);
1581
1582         /*
1583          * In caching mode, changes of pages from non-present to present require
1584          * flush. However, device IOTLB doesn't need to be flushed in this case.
1585          */
1586         if (!cap_caching_mode(iommu->cap) || !map)
1587                 iommu_flush_dev_iotlb(domain, addr, mask);
1588 }
1589
1590 /* Notification for newly created mappings */
1591 static inline void __mapping_notify_one(struct intel_iommu *iommu,
1592                                         struct dmar_domain *domain,
1593                                         unsigned long pfn, unsigned int pages)
1594 {
1595         /* It's a non-present to present mapping. Only flush if caching mode */
1596         if (cap_caching_mode(iommu->cap))
1597                 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1598         else
1599                 iommu_flush_write_buffer(iommu);
1600 }
1601
1602 static void iommu_flush_iova(struct iova_domain *iovad)
1603 {
1604         struct dmar_domain *domain;
1605         int idx;
1606
1607         domain = container_of(iovad, struct dmar_domain, iovad);
1608
1609         for_each_domain_iommu(idx, domain) {
1610                 struct intel_iommu *iommu = g_iommus[idx];
1611                 u16 did = domain->iommu_did[iommu->seq_id];
1612
1613                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1614
1615                 if (!cap_caching_mode(iommu->cap))
1616                         iommu_flush_dev_iotlb(get_iommu_domain(iommu, did),
1617                                               0, MAX_AGAW_PFN_WIDTH);
1618         }
1619 }
1620
1621 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1622 {
1623         u32 pmen;
1624         unsigned long flags;
1625
1626         if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1627                 return;
1628
1629         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1630         pmen = readl(iommu->reg + DMAR_PMEN_REG);
1631         pmen &= ~DMA_PMEN_EPM;
1632         writel(pmen, iommu->reg + DMAR_PMEN_REG);
1633
1634         /* wait for the protected region status bit to clear */
1635         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1636                 readl, !(pmen & DMA_PMEN_PRS), pmen);
1637
1638         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1639 }
1640
1641 static void iommu_enable_translation(struct intel_iommu *iommu)
1642 {
1643         u32 sts;
1644         unsigned long flags;
1645
1646         raw_spin_lock_irqsave(&iommu->register_lock, flags);
1647         iommu->gcmd |= DMA_GCMD_TE;
1648         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1649
1650         /* Make sure hardware complete it */
1651         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1652                       readl, (sts & DMA_GSTS_TES), sts);
1653
1654         raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1655 }
1656
1657 static void iommu_disable_translation(struct intel_iommu *iommu)
1658 {
1659         u32 sts;
1660         unsigned long flag;
1661
1662         raw_spin_lock_irqsave(&iommu->register_lock, flag);
1663         iommu->gcmd &= ~DMA_GCMD_TE;
1664         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1665
1666         /* Make sure hardware complete it */
1667         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1668                       readl, (!(sts & DMA_GSTS_TES)), sts);
1669
1670         raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1671 }
1672
1673
1674 static int iommu_init_domains(struct intel_iommu *iommu)
1675 {
1676         u32 ndomains, nlongs;
1677         size_t size;
1678
1679         ndomains = cap_ndoms(iommu->cap);
1680         pr_debug("%s: Number of Domains supported <%d>\n",
1681                  iommu->name, ndomains);
1682         nlongs = BITS_TO_LONGS(ndomains);
1683
1684         spin_lock_init(&iommu->lock);
1685
1686         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1687         if (!iommu->domain_ids) {
1688                 pr_err("%s: Allocating domain id array failed\n",
1689                        iommu->name);
1690                 return -ENOMEM;
1691         }
1692
1693         size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
1694         iommu->domains = kzalloc(size, GFP_KERNEL);
1695
1696         if (iommu->domains) {
1697                 size = 256 * sizeof(struct dmar_domain *);
1698                 iommu->domains[0] = kzalloc(size, GFP_KERNEL);
1699         }
1700
1701         if (!iommu->domains || !iommu->domains[0]) {
1702                 pr_err("%s: Allocating domain array failed\n",
1703                        iommu->name);
1704                 kfree(iommu->domain_ids);
1705                 kfree(iommu->domains);
1706                 iommu->domain_ids = NULL;
1707                 iommu->domains    = NULL;
1708                 return -ENOMEM;
1709         }
1710
1711
1712
1713         /*
1714          * If Caching mode is set, then invalid translations are tagged
1715          * with domain-id 0, hence we need to pre-allocate it. We also
1716          * use domain-id 0 as a marker for non-allocated domain-id, so
1717          * make sure it is not used for a real domain.
1718          */
1719         set_bit(0, iommu->domain_ids);
1720
1721         return 0;
1722 }
1723
1724 static void disable_dmar_iommu(struct intel_iommu *iommu)
1725 {
1726         struct device_domain_info *info, *tmp;
1727         unsigned long flags;
1728
1729         if (!iommu->domains || !iommu->domain_ids)
1730                 return;
1731
1732 again:
1733         spin_lock_irqsave(&device_domain_lock, flags);
1734         list_for_each_entry_safe(info, tmp, &device_domain_list, global) {
1735                 struct dmar_domain *domain;
1736
1737                 if (info->iommu != iommu)
1738                         continue;
1739
1740                 if (!info->dev || !info->domain)
1741                         continue;
1742
1743                 domain = info->domain;
1744
1745                 __dmar_remove_one_dev_info(info);
1746
1747                 if (!domain_type_is_vm_or_si(domain)) {
1748                         /*
1749                          * The domain_exit() function  can't be called under
1750                          * device_domain_lock, as it takes this lock itself.
1751                          * So release the lock here and re-run the loop
1752                          * afterwards.
1753                          */
1754                         spin_unlock_irqrestore(&device_domain_lock, flags);
1755                         domain_exit(domain);
1756                         goto again;
1757                 }
1758         }
1759         spin_unlock_irqrestore(&device_domain_lock, flags);
1760
1761         if (iommu->gcmd & DMA_GCMD_TE)
1762                 iommu_disable_translation(iommu);
1763 }
1764
1765 static void free_dmar_iommu(struct intel_iommu *iommu)
1766 {
1767         if ((iommu->domains) && (iommu->domain_ids)) {
1768                 int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8;
1769                 int i;
1770
1771                 for (i = 0; i < elems; i++)
1772                         kfree(iommu->domains[i]);
1773                 kfree(iommu->domains);
1774                 kfree(iommu->domain_ids);
1775                 iommu->domains = NULL;
1776                 iommu->domain_ids = NULL;
1777         }
1778
1779         g_iommus[iommu->seq_id] = NULL;
1780
1781         /* free context mapping */
1782         free_context_table(iommu);
1783
1784 #ifdef CONFIG_INTEL_IOMMU_SVM
1785         if (pasid_enabled(iommu)) {
1786                 if (ecap_prs(iommu->ecap))
1787                         intel_svm_finish_prq(iommu);
1788                 intel_svm_exit(iommu);
1789         }
1790 #endif
1791 }
1792
1793 static struct dmar_domain *alloc_domain(int flags)
1794 {
1795         struct dmar_domain *domain;
1796
1797         domain = alloc_domain_mem();
1798         if (!domain)
1799                 return NULL;
1800
1801         memset(domain, 0, sizeof(*domain));
1802         domain->nid = -1;
1803         domain->flags = flags;
1804         domain->has_iotlb_device = false;
1805         INIT_LIST_HEAD(&domain->devices);
1806
1807         return domain;
1808 }
1809
1810 /* Must be called with iommu->lock */
1811 static int domain_attach_iommu(struct dmar_domain *domain,
1812                                struct intel_iommu *iommu)
1813 {
1814         unsigned long ndomains;
1815         int num;
1816
1817         assert_spin_locked(&device_domain_lock);
1818         assert_spin_locked(&iommu->lock);
1819
1820         domain->iommu_refcnt[iommu->seq_id] += 1;
1821         domain->iommu_count += 1;
1822         if (domain->iommu_refcnt[iommu->seq_id] == 1) {
1823                 ndomains = cap_ndoms(iommu->cap);
1824                 num      = find_first_zero_bit(iommu->domain_ids, ndomains);
1825
1826                 if (num >= ndomains) {
1827                         pr_err("%s: No free domain ids\n", iommu->name);
1828                         domain->iommu_refcnt[iommu->seq_id] -= 1;
1829                         domain->iommu_count -= 1;
1830                         return -ENOSPC;
1831                 }
1832
1833                 set_bit(num, iommu->domain_ids);
1834                 set_iommu_domain(iommu, num, domain);
1835
1836                 domain->iommu_did[iommu->seq_id] = num;
1837                 domain->nid                      = iommu->node;
1838
1839                 domain_update_iommu_cap(domain);
1840         }
1841
1842         return 0;
1843 }
1844
1845 static int domain_detach_iommu(struct dmar_domain *domain,
1846                                struct intel_iommu *iommu)
1847 {
1848         int num, count = INT_MAX;
1849
1850         assert_spin_locked(&device_domain_lock);
1851         assert_spin_locked(&iommu->lock);
1852
1853         domain->iommu_refcnt[iommu->seq_id] -= 1;
1854         count = --domain->iommu_count;
1855         if (domain->iommu_refcnt[iommu->seq_id] == 0) {
1856                 num = domain->iommu_did[iommu->seq_id];
1857                 clear_bit(num, iommu->domain_ids);
1858                 set_iommu_domain(iommu, num, NULL);
1859
1860                 domain_update_iommu_cap(domain);
1861                 domain->iommu_did[iommu->seq_id] = 0;
1862         }
1863
1864         return count;
1865 }
1866
1867 static struct iova_domain reserved_iova_list;
1868 static struct lock_class_key reserved_rbtree_key;
1869
1870 static int dmar_init_reserved_ranges(void)
1871 {
1872         struct pci_dev *pdev = NULL;
1873         struct iova *iova;
1874         int i;
1875
1876         init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
1877
1878         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1879                 &reserved_rbtree_key);
1880
1881         /* IOAPIC ranges shouldn't be accessed by DMA */
1882         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1883                 IOVA_PFN(IOAPIC_RANGE_END));
1884         if (!iova) {
1885                 pr_err("Reserve IOAPIC range failed\n");
1886                 return -ENODEV;
1887         }
1888
1889         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1890         for_each_pci_dev(pdev) {
1891                 struct resource *r;
1892
1893                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1894                         r = &pdev->resource[i];
1895                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1896                                 continue;
1897                         iova = reserve_iova(&reserved_iova_list,
1898                                             IOVA_PFN(r->start),
1899                                             IOVA_PFN(r->end));
1900                         if (!iova) {
1901                                 pr_err("Reserve iova failed\n");
1902                                 return -ENODEV;
1903                         }
1904                 }
1905         }
1906         return 0;
1907 }
1908
1909 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1910 {
1911         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1912 }
1913
1914 static inline int guestwidth_to_adjustwidth(int gaw)
1915 {
1916         int agaw;
1917         int r = (gaw - 12) % 9;
1918
1919         if (r == 0)
1920                 agaw = gaw;
1921         else
1922                 agaw = gaw + 9 - r;
1923         if (agaw > 64)
1924                 agaw = 64;
1925         return agaw;
1926 }
1927
1928 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,
1929                        int guest_width)
1930 {
1931         int adjust_width, agaw, cap_width;
1932         unsigned long sagaw;
1933         int err;
1934
1935         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
1936
1937         err = init_iova_flush_queue(&domain->iovad,
1938                                     iommu_flush_iova, iova_entry_free);
1939         if (err)
1940                 return err;
1941
1942         domain_reserve_special_ranges(domain);
1943
1944         /* calculate AGAW */
1945         cap_width = min_t(int, cap_mgaw(iommu->cap), agaw_to_width(iommu->agaw));
1946         if (guest_width > cap_width)
1947                 guest_width = cap_width;
1948         domain->gaw = guest_width;
1949         adjust_width = guestwidth_to_adjustwidth(guest_width);
1950         agaw = width_to_agaw(adjust_width);
1951         sagaw = cap_sagaw(iommu->cap);
1952         if (!test_bit(agaw, &sagaw)) {
1953                 /* hardware doesn't support it, choose a bigger one */
1954                 pr_debug("Hardware doesn't support agaw %d\n", agaw);
1955                 agaw = find_next_bit(&sagaw, 5, agaw);
1956                 if (agaw >= 5)
1957                         return -ENODEV;
1958         }
1959         domain->agaw = agaw;
1960
1961         if (ecap_coherent(iommu->ecap))
1962                 domain->iommu_coherency = 1;
1963         else
1964                 domain->iommu_coherency = 0;
1965
1966         if (ecap_sc_support(iommu->ecap))
1967                 domain->iommu_snooping = 1;
1968         else
1969                 domain->iommu_snooping = 0;
1970
1971         if (intel_iommu_superpage)
1972                 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1973         else
1974                 domain->iommu_superpage = 0;
1975
1976         domain->nid = iommu->node;
1977
1978         /* always allocate the top pgd */
1979         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1980         if (!domain->pgd)
1981                 return -ENOMEM;
1982         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1983         return 0;
1984 }
1985
1986 static void domain_exit(struct dmar_domain *domain)
1987 {
1988         struct page *freelist = NULL;
1989
1990         /* Domain 0 is reserved, so dont process it */
1991         if (!domain)
1992                 return;
1993
1994         /* Remove associated devices and clear attached or cached domains */
1995         rcu_read_lock();
1996         domain_remove_dev_info(domain);
1997         rcu_read_unlock();
1998
1999         /* destroy iovas */
2000         put_iova_domain(&domain->iovad);
2001
2002         freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
2003
2004         dma_free_pagelist(freelist);
2005
2006         free_domain_mem(domain);
2007 }
2008
2009 static int domain_context_mapping_one(struct dmar_domain *domain,
2010                                       struct intel_iommu *iommu,
2011                                       u8 bus, u8 devfn)
2012 {
2013         u16 did = domain->iommu_did[iommu->seq_id];
2014         int translation = CONTEXT_TT_MULTI_LEVEL;
2015         struct device_domain_info *info = NULL;
2016         struct context_entry *context;
2017         unsigned long flags;
2018         struct dma_pte *pgd;
2019         int ret, agaw;
2020
2021         WARN_ON(did == 0);
2022
2023         if (hw_pass_through && domain_type_is_si(domain))
2024                 translation = CONTEXT_TT_PASS_THROUGH;
2025
2026         pr_debug("Set context mapping for %02x:%02x.%d\n",
2027                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
2028
2029         BUG_ON(!domain->pgd);
2030
2031         spin_lock_irqsave(&device_domain_lock, flags);
2032         spin_lock(&iommu->lock);
2033
2034         ret = -ENOMEM;
2035         context = iommu_context_addr(iommu, bus, devfn, 1);
2036         if (!context)
2037                 goto out_unlock;
2038
2039         ret = 0;
2040         if (context_present(context))
2041                 goto out_unlock;
2042
2043         /*
2044          * For kdump cases, old valid entries may be cached due to the
2045          * in-flight DMA and copied pgtable, but there is no unmapping
2046          * behaviour for them, thus we need an explicit cache flush for
2047          * the newly-mapped device. For kdump, at this point, the device
2048          * is supposed to finish reset at its driver probe stage, so no
2049          * in-flight DMA will exist, and we don't need to worry anymore
2050          * hereafter.
2051          */
2052         if (context_copied(context)) {
2053                 u16 did_old = context_domain_id(context);
2054
2055                 if (did_old < cap_ndoms(iommu->cap)) {
2056                         iommu->flush.flush_context(iommu, did_old,
2057                                                    (((u16)bus) << 8) | devfn,
2058                                                    DMA_CCMD_MASK_NOBIT,
2059                                                    DMA_CCMD_DEVICE_INVL);
2060                         iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
2061                                                  DMA_TLB_DSI_FLUSH);
2062                 }
2063         }
2064
2065         pgd = domain->pgd;
2066
2067         context_clear_entry(context);
2068         context_set_domain_id(context, did);
2069
2070         /*
2071          * Skip top levels of page tables for iommu which has less agaw
2072          * than default.  Unnecessary for PT mode.
2073          */
2074         if (translation != CONTEXT_TT_PASS_THROUGH) {
2075                 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2076                         ret = -ENOMEM;
2077                         pgd = phys_to_virt(dma_pte_addr(pgd));
2078                         if (!dma_pte_present(pgd))
2079                                 goto out_unlock;
2080                 }
2081
2082                 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
2083                 if (info && info->ats_supported)
2084                         translation = CONTEXT_TT_DEV_IOTLB;
2085                 else
2086                         translation = CONTEXT_TT_MULTI_LEVEL;
2087
2088                 context_set_address_root(context, virt_to_phys(pgd));
2089                 context_set_address_width(context, agaw);
2090         } else {
2091                 /*
2092                  * In pass through mode, AW must be programmed to
2093                  * indicate the largest AGAW value supported by
2094                  * hardware. And ASR is ignored by hardware.
2095                  */
2096                 context_set_address_width(context, iommu->msagaw);
2097         }
2098
2099         context_set_translation_type(context, translation);
2100         context_set_fault_enable(context);
2101         context_set_present(context);
2102         domain_flush_cache(domain, context, sizeof(*context));
2103
2104         /*
2105          * It's a non-present to present mapping. If hardware doesn't cache
2106          * non-present entry we only need to flush the write-buffer. If the
2107          * _does_ cache non-present entries, then it does so in the special
2108          * domain #0, which we have to flush:
2109          */
2110         if (cap_caching_mode(iommu->cap)) {
2111                 iommu->flush.flush_context(iommu, 0,
2112                                            (((u16)bus) << 8) | devfn,
2113                                            DMA_CCMD_MASK_NOBIT,
2114                                            DMA_CCMD_DEVICE_INVL);
2115                 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2116         } else {
2117                 iommu_flush_write_buffer(iommu);
2118         }
2119         iommu_enable_dev_iotlb(info);
2120
2121         ret = 0;
2122
2123 out_unlock:
2124         spin_unlock(&iommu->lock);
2125         spin_unlock_irqrestore(&device_domain_lock, flags);
2126
2127         return ret;
2128 }
2129
2130 struct domain_context_mapping_data {
2131         struct dmar_domain *domain;
2132         struct intel_iommu *iommu;
2133 };
2134
2135 static int domain_context_mapping_cb(struct pci_dev *pdev,
2136                                      u16 alias, void *opaque)
2137 {
2138         struct domain_context_mapping_data *data = opaque;
2139
2140         return domain_context_mapping_one(data->domain, data->iommu,
2141                                           PCI_BUS_NUM(alias), alias & 0xff);
2142 }
2143
2144 static int
2145 domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2146 {
2147         struct intel_iommu *iommu;
2148         u8 bus, devfn;
2149         struct domain_context_mapping_data data;
2150
2151         iommu = device_to_iommu(dev, &bus, &devfn);
2152         if (!iommu)
2153                 return -ENODEV;
2154
2155         if (!dev_is_pci(dev))
2156                 return domain_context_mapping_one(domain, iommu, bus, devfn);
2157
2158         data.domain = domain;
2159         data.iommu = iommu;
2160
2161         return pci_for_each_dma_alias(to_pci_dev(dev),
2162                                       &domain_context_mapping_cb, &data);
2163 }
2164
2165 static int domain_context_mapped_cb(struct pci_dev *pdev,
2166                                     u16 alias, void *opaque)
2167 {
2168         struct intel_iommu *iommu = opaque;
2169
2170         return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
2171 }
2172
2173 static int domain_context_mapped(struct device *dev)
2174 {
2175         struct intel_iommu *iommu;
2176         u8 bus, devfn;
2177
2178         iommu = device_to_iommu(dev, &bus, &devfn);
2179         if (!iommu)
2180                 return -ENODEV;
2181
2182         if (!dev_is_pci(dev))
2183                 return device_context_mapped(iommu, bus, devfn);
2184
2185         return !pci_for_each_dma_alias(to_pci_dev(dev),
2186                                        domain_context_mapped_cb, iommu);
2187 }
2188
2189 /* Returns a number of VTD pages, but aligned to MM page size */
2190 static inline unsigned long aligned_nrpages(unsigned long host_addr,
2191                                             size_t size)
2192 {
2193         host_addr &= ~PAGE_MASK;
2194         return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2195 }
2196
2197 /* Return largest possible superpage level for a given mapping */
2198 static inline int hardware_largepage_caps(struct dmar_domain *domain,
2199                                           unsigned long iov_pfn,
2200                                           unsigned long phy_pfn,
2201                                           unsigned long pages)
2202 {
2203         int support, level = 1;
2204         unsigned long pfnmerge;
2205
2206         support = domain->iommu_superpage;
2207
2208         /* To use a large page, the virtual *and* physical addresses
2209            must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2210            of them will mean we have to use smaller pages. So just
2211            merge them and check both at once. */
2212         pfnmerge = iov_pfn | phy_pfn;
2213
2214         while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2215                 pages >>= VTD_STRIDE_SHIFT;
2216                 if (!pages)
2217                         break;
2218                 pfnmerge >>= VTD_STRIDE_SHIFT;
2219                 level++;
2220                 support--;
2221         }
2222         return level;
2223 }
2224
2225 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2226                             struct scatterlist *sg, unsigned long phys_pfn,
2227                             unsigned long nr_pages, int prot)
2228 {
2229         struct dma_pte *first_pte = NULL, *pte = NULL;
2230         phys_addr_t uninitialized_var(pteval);
2231         unsigned long sg_res = 0;
2232         unsigned int largepage_lvl = 0;
2233         unsigned long lvl_pages = 0;
2234
2235         BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2236
2237         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2238                 return -EINVAL;
2239
2240         prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2241
2242         if (!sg) {
2243                 sg_res = nr_pages;
2244                 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2245         }
2246
2247         while (nr_pages > 0) {
2248                 uint64_t tmp;
2249
2250                 if (!sg_res) {
2251                         unsigned int pgoff = sg->offset & ~PAGE_MASK;
2252
2253                         sg_res = aligned_nrpages(sg->offset, sg->length);
2254                         sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + pgoff;
2255                         sg->dma_length = sg->length;
2256                         pteval = (sg_phys(sg) - pgoff) | prot;
2257                         phys_pfn = pteval >> VTD_PAGE_SHIFT;
2258                 }
2259
2260                 if (!pte) {
2261                         largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2262
2263                         first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2264                         if (!pte)
2265                                 return -ENOMEM;
2266                         /* It is large page*/
2267                         if (largepage_lvl > 1) {
2268                                 unsigned long nr_superpages, end_pfn;
2269
2270                                 pteval |= DMA_PTE_LARGE_PAGE;
2271                                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2272
2273                                 nr_superpages = sg_res / lvl_pages;
2274                                 end_pfn = iov_pfn + nr_superpages * lvl_pages - 1;
2275
2276                                 /*
2277                                  * Ensure that old small page tables are
2278                                  * removed to make room for superpage(s).
2279                                  * We're adding new large pages, so make sure
2280                                  * we don't remove their parent tables.
2281                                  */
2282                                 dma_pte_free_pagetable(domain, iov_pfn, end_pfn,
2283                                                        largepage_lvl + 1);
2284                         } else {
2285                                 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2286                         }
2287
2288                 }
2289                 /* We don't need lock here, nobody else
2290                  * touches the iova range
2291                  */
2292                 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2293                 if (tmp) {
2294                         static int dumps = 5;
2295                         pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2296                                 iov_pfn, tmp, (unsigned long long)pteval);
2297                         if (dumps) {
2298                                 dumps--;
2299                                 debug_dma_dump_mappings(NULL);
2300                         }
2301                         WARN_ON(1);
2302                 }
2303
2304                 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2305
2306                 BUG_ON(nr_pages < lvl_pages);
2307                 BUG_ON(sg_res < lvl_pages);
2308
2309                 nr_pages -= lvl_pages;
2310                 iov_pfn += lvl_pages;
2311                 phys_pfn += lvl_pages;
2312                 pteval += lvl_pages * VTD_PAGE_SIZE;
2313                 sg_res -= lvl_pages;
2314
2315                 /* If the next PTE would be the first in a new page, then we
2316                    need to flush the cache on the entries we've just written.
2317                    And then we'll need to recalculate 'pte', so clear it and
2318                    let it get set again in the if (!pte) block above.
2319
2320                    If we're done (!nr_pages) we need to flush the cache too.
2321
2322                    Also if we've been setting superpages, we may need to
2323                    recalculate 'pte' and switch back to smaller pages for the
2324                    end of the mapping, if the trailing size is not enough to
2325                    use another superpage (i.e. sg_res < lvl_pages). */
2326                 pte++;
2327                 if (!nr_pages || first_pte_in_page(pte) ||
2328                     (largepage_lvl > 1 && sg_res < lvl_pages)) {
2329                         domain_flush_cache(domain, first_pte,
2330                                            (void *)pte - (void *)first_pte);
2331                         pte = NULL;
2332                 }
2333
2334                 if (!sg_res && nr_pages)
2335                         sg = sg_next(sg);
2336         }
2337         return 0;
2338 }
2339
2340 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2341                          struct scatterlist *sg, unsigned long phys_pfn,
2342                          unsigned long nr_pages, int prot)
2343 {
2344        int ret;
2345        struct intel_iommu *iommu;
2346
2347        /* Do the real mapping first */
2348        ret = __domain_mapping(domain, iov_pfn, sg, phys_pfn, nr_pages, prot);
2349        if (ret)
2350                return ret;
2351
2352        /* Notify about the new mapping */
2353        if (domain_type_is_vm(domain)) {
2354                /* VM typed domains can have more than one IOMMUs */
2355                int iommu_id;
2356                for_each_domain_iommu(iommu_id, domain) {
2357                        iommu = g_iommus[iommu_id];
2358                        __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2359                }
2360        } else {
2361                /* General domains only have one IOMMU */
2362                iommu = domain_get_iommu(domain);
2363                __mapping_notify_one(iommu, domain, iov_pfn, nr_pages);
2364        }
2365
2366        return 0;
2367 }
2368
2369 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2370                                     struct scatterlist *sg, unsigned long nr_pages,
2371                                     int prot)
2372 {
2373         return domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2374 }
2375
2376 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2377                                      unsigned long phys_pfn, unsigned long nr_pages,
2378                                      int prot)
2379 {
2380         return domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
2381 }
2382
2383 static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn)
2384 {
2385         unsigned long flags;
2386         struct context_entry *context;
2387         u16 did_old;
2388
2389         if (!iommu)
2390                 return;
2391
2392         spin_lock_irqsave(&iommu->lock, flags);
2393         context = iommu_context_addr(iommu, bus, devfn, 0);
2394         if (!context) {
2395                 spin_unlock_irqrestore(&iommu->lock, flags);
2396                 return;
2397         }
2398         did_old = context_domain_id(context);
2399         context_clear_entry(context);
2400         __iommu_flush_cache(iommu, context, sizeof(*context));
2401         spin_unlock_irqrestore(&iommu->lock, flags);
2402         iommu->flush.flush_context(iommu,
2403                                    did_old,
2404                                    (((u16)bus) << 8) | devfn,
2405                                    DMA_CCMD_MASK_NOBIT,
2406                                    DMA_CCMD_DEVICE_INVL);
2407         iommu->flush.flush_iotlb(iommu,
2408                                  did_old,
2409                                  0,
2410                                  0,
2411                                  DMA_TLB_DSI_FLUSH);
2412 }
2413
2414 static inline void unlink_domain_info(struct device_domain_info *info)
2415 {
2416         assert_spin_locked(&device_domain_lock);
2417         list_del(&info->link);
2418         list_del(&info->global);
2419         if (info->dev)
2420                 info->dev->archdata.iommu = NULL;
2421 }
2422
2423 static void domain_remove_dev_info(struct dmar_domain *domain)
2424 {
2425         struct device_domain_info *info, *tmp;
2426         unsigned long flags;
2427
2428         spin_lock_irqsave(&device_domain_lock, flags);
2429         list_for_each_entry_safe(info, tmp, &domain->devices, link)
2430                 __dmar_remove_one_dev_info(info);
2431         spin_unlock_irqrestore(&device_domain_lock, flags);
2432 }
2433
2434 /*
2435  * find_domain
2436  * Note: we use struct device->archdata.iommu stores the info
2437  */
2438 static struct dmar_domain *find_domain(struct device *dev)
2439 {
2440         struct device_domain_info *info;
2441
2442         /* No lock here, assumes no domain exit in normal case */
2443         info = dev->archdata.iommu;
2444         if (likely(info))
2445                 return info->domain;
2446         return NULL;
2447 }
2448
2449 static inline struct device_domain_info *
2450 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2451 {
2452         struct device_domain_info *info;
2453
2454         list_for_each_entry(info, &device_domain_list, global)
2455                 if (info->iommu->segment == segment && info->bus == bus &&
2456                     info->devfn == devfn)
2457                         return info;
2458
2459         return NULL;
2460 }
2461
2462 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
2463                                                     int bus, int devfn,
2464                                                     struct device *dev,
2465                                                     struct dmar_domain *domain)
2466 {
2467         struct dmar_domain *found = NULL;
2468         struct device_domain_info *info;
2469         unsigned long flags;
2470         int ret;
2471
2472         info = alloc_devinfo_mem();
2473         if (!info)
2474                 return NULL;
2475
2476         info->bus = bus;
2477         info->devfn = devfn;
2478         info->ats_supported = info->pasid_supported = info->pri_supported = 0;
2479         info->ats_enabled = info->pasid_enabled = info->pri_enabled = 0;
2480         info->ats_qdep = 0;
2481         info->dev = dev;
2482         info->domain = domain;
2483         info->iommu = iommu;
2484         info->pasid_table = NULL;
2485
2486         if (dev && dev_is_pci(dev)) {
2487                 struct pci_dev *pdev = to_pci_dev(info->dev);
2488
2489                 if (!pci_ats_disabled() &&
2490                     ecap_dev_iotlb_support(iommu->ecap) &&
2491                     pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS) &&
2492                     dmar_find_matched_atsr_unit(pdev))
2493                         info->ats_supported = 1;
2494
2495                 if (ecs_enabled(iommu)) {
2496                         if (pasid_enabled(iommu)) {
2497                                 int features = pci_pasid_features(pdev);
2498                                 if (features >= 0)
2499                                         info->pasid_supported = features | 1;
2500                         }
2501
2502                         if (info->ats_supported && ecap_prs(iommu->ecap) &&
2503                             pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI))
2504                                 info->pri_supported = 1;
2505                 }
2506         }
2507
2508         spin_lock_irqsave(&device_domain_lock, flags);
2509         if (dev)
2510                 found = find_domain(dev);
2511
2512         if (!found) {
2513                 struct device_domain_info *info2;
2514                 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
2515                 if (info2) {
2516                         found      = info2->domain;
2517                         info2->dev = dev;
2518                 }
2519         }
2520
2521         if (found) {
2522                 spin_unlock_irqrestore(&device_domain_lock, flags);
2523                 free_devinfo_mem(info);
2524                 /* Caller must free the original domain */
2525                 return found;
2526         }
2527
2528         spin_lock(&iommu->lock);
2529         ret = domain_attach_iommu(domain, iommu);
2530         spin_unlock(&iommu->lock);
2531
2532         if (ret) {
2533                 spin_unlock_irqrestore(&device_domain_lock, flags);
2534                 free_devinfo_mem(info);
2535                 return NULL;
2536         }
2537
2538         list_add(&info->link, &domain->devices);
2539         list_add(&info->global, &device_domain_list);
2540         if (dev)
2541                 dev->archdata.iommu = info;
2542
2543         if (dev && dev_is_pci(dev) && info->pasid_supported) {
2544                 ret = intel_pasid_alloc_table(dev);
2545                 if (ret) {
2546                         pr_warn("No pasid table for %s, pasid disabled\n",
2547                                 dev_name(dev));
2548                         info->pasid_supported = 0;
2549                 }
2550         }
2551         spin_unlock_irqrestore(&device_domain_lock, flags);
2552
2553         if (dev && domain_context_mapping(domain, dev)) {
2554                 pr_err("Domain context map for %s failed\n", dev_name(dev));
2555                 dmar_remove_one_dev_info(domain, dev);
2556                 return NULL;
2557         }
2558
2559         return domain;
2560 }
2561
2562 static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2563 {
2564         *(u16 *)opaque = alias;
2565         return 0;
2566 }
2567
2568 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)
2569 {
2570         struct device_domain_info *info = NULL;
2571         struct dmar_domain *domain = NULL;
2572         struct intel_iommu *iommu;
2573         u16 dma_alias;
2574         unsigned long flags;
2575         u8 bus, devfn;
2576
2577         iommu = device_to_iommu(dev, &bus, &devfn);
2578         if (!iommu)
2579                 return NULL;
2580
2581         if (dev_is_pci(dev)) {
2582                 struct pci_dev *pdev = to_pci_dev(dev);
2583
2584                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2585
2586                 spin_lock_irqsave(&device_domain_lock, flags);
2587                 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2588                                                       PCI_BUS_NUM(dma_alias),
2589                                                       dma_alias & 0xff);
2590                 if (info) {
2591                         iommu = info->iommu;
2592                         domain = info->domain;
2593                 }
2594                 spin_unlock_irqrestore(&device_domain_lock, flags);
2595
2596                 /* DMA alias already has a domain, use it */
2597                 if (info)
2598                         goto out;
2599         }
2600
2601         /* Allocate and initialize new domain for the device */
2602         domain = alloc_domain(0);
2603         if (!domain)
2604                 return NULL;
2605         if (domain_init(domain, iommu, gaw)) {
2606                 domain_exit(domain);
2607                 return NULL;
2608         }
2609
2610 out:
2611
2612         return domain;
2613 }
2614
2615 static struct dmar_domain *set_domain_for_dev(struct device *dev,
2616                                               struct dmar_domain *domain)
2617 {
2618         struct intel_iommu *iommu;
2619         struct dmar_domain *tmp;
2620         u16 req_id, dma_alias;
2621         u8 bus, devfn;
2622
2623         iommu = device_to_iommu(dev, &bus, &devfn);
2624         if (!iommu)
2625                 return NULL;
2626
2627         req_id = ((u16)bus << 8) | devfn;
2628
2629         if (dev_is_pci(dev)) {
2630                 struct pci_dev *pdev = to_pci_dev(dev);
2631
2632                 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2633
2634                 /* register PCI DMA alias device */
2635                 if (req_id != dma_alias) {
2636                         tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2637                                         dma_alias & 0xff, NULL, domain);
2638
2639                         if (!tmp || tmp != domain)
2640                                 return tmp;
2641                 }
2642         }
2643
2644         tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2645         if (!tmp || tmp != domain)
2646                 return tmp;
2647
2648         return domain;
2649 }
2650
2651 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
2652 {
2653         struct dmar_domain *domain, *tmp;
2654
2655         domain = find_domain(dev);
2656         if (domain)
2657                 goto out;
2658
2659         domain = find_or_alloc_domain(dev, gaw);
2660         if (!domain)
2661                 goto out;
2662
2663         tmp = set_domain_for_dev(dev, domain);
2664         if (!tmp || domain != tmp) {
2665                 domain_exit(domain);
2666                 domain = tmp;
2667         }
2668
2669 out:
2670
2671         return domain;
2672 }
2673
2674 static int iommu_domain_identity_map(struct dmar_domain *domain,
2675                                      unsigned long long start,
2676                                      unsigned long long end)
2677 {
2678         unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2679         unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2680
2681         if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2682                           dma_to_mm_pfn(last_vpfn))) {
2683                 pr_err("Reserving iova failed\n");
2684                 return -ENOMEM;
2685         }
2686
2687         pr_debug("Mapping reserved region %llx-%llx\n", start, end);
2688         /*
2689          * RMRR range might have overlap with physical memory range,
2690          * clear it first
2691          */
2692         dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2693
2694         return __domain_mapping(domain, first_vpfn, NULL,
2695                                 first_vpfn, last_vpfn - first_vpfn + 1,
2696                                 DMA_PTE_READ|DMA_PTE_WRITE);
2697 }
2698
2699 static int domain_prepare_identity_map(struct device *dev,
2700                                        struct dmar_domain *domain,
2701                                        unsigned long long start,
2702                                        unsigned long long end)
2703 {
2704         /* For _hardware_ passthrough, don't bother. But for software
2705            passthrough, we do it anyway -- it may indicate a memory
2706            range which is reserved in E820, so which didn't get set
2707            up to start with in si_domain */
2708         if (domain == si_domain && hw_pass_through) {
2709                 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2710                         dev_name(dev), start, end);
2711                 return 0;
2712         }
2713
2714         pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2715                 dev_name(dev), start, end);
2716
2717         if (end < start) {
2718                 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2719                         "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2720                         dmi_get_system_info(DMI_BIOS_VENDOR),
2721                         dmi_get_system_info(DMI_BIOS_VERSION),
2722                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2723                 return -EIO;
2724         }
2725
2726         if (end >> agaw_to_width(domain->agaw)) {
2727                 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2728                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2729                      agaw_to_width(domain->agaw),
2730                      dmi_get_system_info(DMI_BIOS_VENDOR),
2731                      dmi_get_system_info(DMI_BIOS_VERSION),
2732                      dmi_get_system_info(DMI_PRODUCT_VERSION));
2733                 return -EIO;
2734         }
2735
2736         return iommu_domain_identity_map(domain, start, end);
2737 }
2738
2739 static int iommu_prepare_identity_map(struct device *dev,
2740                                       unsigned long long start,
2741                                       unsigned long long end)
2742 {
2743         struct dmar_domain *domain;
2744         int ret;
2745
2746         domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2747         if (!domain)
2748                 return -ENOMEM;
2749
2750         ret = domain_prepare_identity_map(dev, domain, start, end);
2751         if (ret)
2752                 domain_exit(domain);
2753
2754         return ret;
2755 }
2756
2757 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2758                                          struct device *dev)
2759 {
2760         if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2761                 return 0;
2762         return iommu_prepare_identity_map(dev, rmrr->base_address,
2763                                           rmrr->end_address);
2764 }
2765
2766 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2767 static inline void iommu_prepare_isa(void)
2768 {
2769         struct pci_dev *pdev;
2770         int ret;
2771
2772         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2773         if (!pdev)
2774                 return;
2775
2776         pr_info("Prepare 0-16MiB unity mapping for LPC\n");
2777         ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
2778
2779         if (ret)
2780                 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
2781
2782         pci_dev_put(pdev);
2783 }
2784 #else
2785 static inline void iommu_prepare_isa(void)
2786 {
2787         return;
2788 }
2789 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2790
2791 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2792
2793 static int __init si_domain_init(int hw)
2794 {
2795         int nid, ret = 0;
2796
2797         si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2798         if (!si_domain)
2799                 return -EFAULT;
2800
2801         if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2802                 domain_exit(si_domain);
2803                 si_domain = NULL;
2804                 return -EFAULT;
2805         }
2806
2807         pr_debug("Identity mapping domain allocated\n");
2808
2809         if (hw)
2810                 return 0;
2811
2812         for_each_online_node(nid) {
2813                 unsigned long start_pfn, end_pfn;
2814                 int i;
2815
2816                 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2817                         ret = iommu_domain_identity_map(si_domain,
2818                                         PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2819                         if (ret)
2820                                 return ret;
2821                 }
2822         }
2823
2824         return 0;
2825 }
2826
2827 static int identity_mapping(struct device *dev)
2828 {
2829         struct device_domain_info *info;
2830
2831         if (likely(!iommu_identity_mapping))
2832                 return 0;
2833
2834         info = dev->archdata.iommu;
2835         if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2836                 return (info->domain == si_domain);
2837
2838         return 0;
2839 }
2840
2841 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)
2842 {
2843         struct dmar_domain *ndomain;
2844         struct intel_iommu *iommu;
2845         u8 bus, devfn;
2846
2847         iommu = device_to_iommu(dev, &bus, &devfn);
2848         if (!iommu)
2849                 return -ENODEV;
2850
2851         ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain);
2852         if (ndomain != domain)
2853                 return -EBUSY;
2854
2855         return 0;
2856 }
2857
2858 static bool device_has_rmrr(struct device *dev)
2859 {
2860         struct dmar_rmrr_unit *rmrr;
2861         struct device *tmp;
2862         int i;
2863
2864         rcu_read_lock();
2865         for_each_rmrr_units(rmrr) {
2866                 /*
2867                  * Return TRUE if this RMRR contains the device that
2868                  * is passed in.
2869                  */
2870                 for_each_active_dev_scope(rmrr->devices,
2871                                           rmrr->devices_cnt, i, tmp)
2872                         if (tmp == dev) {
2873                                 rcu_read_unlock();
2874                                 return true;
2875                         }
2876         }
2877         rcu_read_unlock();
2878         return false;
2879 }
2880
2881 /*
2882  * There are a couple cases where we need to restrict the functionality of
2883  * devices associated with RMRRs.  The first is when evaluating a device for
2884  * identity mapping because problems exist when devices are moved in and out
2885  * of domains and their respective RMRR information is lost.  This means that
2886  * a device with associated RMRRs will never be in a "passthrough" domain.
2887  * The second is use of the device through the IOMMU API.  This interface
2888  * expects to have full control of the IOVA space for the device.  We cannot
2889  * satisfy both the requirement that RMRR access is maintained and have an
2890  * unencumbered IOVA space.  We also have no ability to quiesce the device's
2891  * use of the RMRR space or even inform the IOMMU API user of the restriction.
2892  * We therefore prevent devices associated with an RMRR from participating in
2893  * the IOMMU API, which eliminates them from device assignment.
2894  *
2895  * In both cases we assume that PCI USB devices with RMRRs have them largely
2896  * for historical reasons and that the RMRR space is not actively used post
2897  * boot.  This exclusion may change if vendors begin to abuse it.
2898  *
2899  * The same exception is made for graphics devices, with the requirement that
2900  * any use of the RMRR regions will be torn down before assigning the device
2901  * to a guest.
2902  */
2903 static bool device_is_rmrr_locked(struct device *dev)
2904 {
2905         if (!device_has_rmrr(dev))
2906                 return false;
2907
2908         if (dev_is_pci(dev)) {
2909                 struct pci_dev *pdev = to_pci_dev(dev);
2910
2911                 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2912                         return false;
2913         }
2914
2915         return true;
2916 }
2917
2918 static int iommu_should_identity_map(struct device *dev, int startup)
2919 {
2920
2921         if (dev_is_pci(dev)) {
2922                 struct pci_dev *pdev = to_pci_dev(dev);
2923
2924                 if (device_is_rmrr_locked(dev))
2925                         return 0;
2926
2927                 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2928                         return 1;
2929
2930                 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2931                         return 1;
2932
2933                 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2934                         return 0;
2935
2936                 /*
2937                  * We want to start off with all devices in the 1:1 domain, and
2938                  * take them out later if we find they can't access all of memory.
2939                  *
2940                  * However, we can't do this for PCI devices behind bridges,
2941                  * because all PCI devices behind the same bridge will end up
2942                  * with the same source-id on their transactions.
2943                  *
2944                  * Practically speaking, we can't change things around for these
2945                  * devices at run-time, because we can't be sure there'll be no
2946                  * DMA transactions in flight for any of their siblings.
2947                  *
2948                  * So PCI devices (unless they're on the root bus) as well as
2949                  * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2950                  * the 1:1 domain, just in _case_ one of their siblings turns out
2951                  * not to be able to map all of memory.
2952                  */
2953                 if (!pci_is_pcie(pdev)) {
2954                         if (!pci_is_root_bus(pdev->bus))
2955                                 return 0;
2956                         if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2957                                 return 0;
2958                 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2959                         return 0;
2960         } else {
2961                 if (device_has_rmrr(dev))
2962                         return 0;
2963         }
2964
2965         /*
2966          * At boot time, we don't yet know if devices will be 64-bit capable.
2967          * Assume that they will — if they turn out not to be, then we can
2968          * take them out of the 1:1 domain later.
2969          */
2970         if (!startup) {
2971                 /*
2972                  * If the device's dma_mask is less than the system's memory
2973                  * size then this is not a candidate for identity mapping.
2974                  */
2975                 u64 dma_mask = *dev->dma_mask;
2976
2977                 if (dev->coherent_dma_mask &&
2978                     dev->coherent_dma_mask < dma_mask)
2979                         dma_mask = dev->coherent_dma_mask;
2980
2981                 return dma_mask >= dma_get_required_mask(dev);
2982         }
2983
2984         return 1;
2985 }
2986
2987 static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2988 {
2989         int ret;
2990
2991         if (!iommu_should_identity_map(dev, 1))
2992                 return 0;
2993
2994         ret = domain_add_dev_info(si_domain, dev);
2995         if (!ret)
2996                 pr_info("%s identity mapping for device %s\n",
2997                         hw ? "Hardware" : "Software", dev_name(dev));
2998         else if (ret == -ENODEV)
2999                 /* device not associated with an iommu */
3000                 ret = 0;
3001
3002         return ret;
3003 }
3004
3005
3006 static int __init iommu_prepare_static_identity_mapping(int hw)
3007 {
3008         struct pci_dev *pdev = NULL;
3009         struct dmar_drhd_unit *drhd;
3010         struct intel_iommu *iommu;
3011         struct device *dev;
3012         int i;
3013         int ret = 0;
3014
3015         for_each_pci_dev(pdev) {
3016                 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
3017                 if (ret)
3018                         return ret;
3019         }
3020
3021         for_each_active_iommu(iommu, drhd)
3022                 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
3023                         struct acpi_device_physical_node *pn;
3024                         struct acpi_device *adev;
3025
3026                         if (dev->bus != &acpi_bus_type)
3027                                 continue;
3028
3029                         adev= to_acpi_device(dev);
3030                         mutex_lock(&adev->physical_node_lock);
3031                         list_for_each_entry(pn, &adev->physical_node_list, node) {
3032                                 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
3033                                 if (ret)
3034                                         break;
3035                         }
3036                         mutex_unlock(&adev->physical_node_lock);
3037                         if (ret)
3038                                 return ret;
3039                 }
3040
3041         return 0;
3042 }
3043
3044 static void intel_iommu_init_qi(struct intel_iommu *iommu)
3045 {
3046         /*
3047          * Start from the sane iommu hardware state.
3048          * If the queued invalidation is already initialized by us
3049          * (for example, while enabling interrupt-remapping) then
3050          * we got the things already rolling from a sane state.
3051          */
3052         if (!iommu->qi) {
3053                 /*
3054                  * Clear any previous faults.
3055                  */
3056                 dmar_fault(-1, iommu);
3057                 /*
3058                  * Disable queued invalidation if supported and already enabled
3059                  * before OS handover.
3060                  */
3061                 dmar_disable_qi(iommu);
3062         }
3063
3064         if (dmar_enable_qi(iommu)) {
3065                 /*
3066                  * Queued Invalidate not enabled, use Register Based Invalidate
3067                  */
3068                 iommu->flush.flush_context = __iommu_flush_context;
3069                 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
3070                 pr_info("%s: Using Register based invalidation\n",
3071                         iommu->name);
3072         } else {
3073                 iommu->flush.flush_context = qi_flush_context;
3074                 iommu->flush.flush_iotlb = qi_flush_iotlb;
3075                 pr_info("%s: Using Queued invalidation\n", iommu->name);
3076         }
3077 }
3078
3079 static int copy_context_table(struct intel_iommu *iommu,
3080                               struct root_entry *old_re,
3081                               struct context_entry **tbl,
3082                               int bus, bool ext)
3083 {
3084         int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
3085         struct context_entry *new_ce = NULL, ce;
3086         struct context_entry *old_ce = NULL;
3087         struct root_entry re;
3088         phys_addr_t old_ce_phys;
3089
3090         tbl_idx = ext ? bus * 2 : bus;
3091         memcpy(&re, old_re, sizeof(re));
3092
3093         for (devfn = 0; devfn < 256; devfn++) {
3094                 /* First calculate the correct index */
3095                 idx = (ext ? devfn * 2 : devfn) % 256;
3096
3097                 if (idx == 0) {
3098                         /* First save what we may have and clean up */
3099                         if (new_ce) {
3100                                 tbl[tbl_idx] = new_ce;
3101                                 __iommu_flush_cache(iommu, new_ce,
3102                                                     VTD_PAGE_SIZE);
3103                                 pos = 1;
3104                         }
3105
3106                         if (old_ce)
3107                                 memunmap(old_ce);
3108
3109                         ret = 0;
3110                         if (devfn < 0x80)
3111                                 old_ce_phys = root_entry_lctp(&re);
3112                         else
3113                                 old_ce_phys = root_entry_uctp(&re);
3114
3115                         if (!old_ce_phys) {
3116                                 if (ext && devfn == 0) {
3117                                         /* No LCTP, try UCTP */
3118                                         devfn = 0x7f;
3119                                         continue;
3120                                 } else {
3121                                         goto out;
3122                                 }
3123                         }
3124
3125                         ret = -ENOMEM;
3126                         old_ce = memremap(old_ce_phys, PAGE_SIZE,
3127                                         MEMREMAP_WB);
3128                         if (!old_ce)
3129                                 goto out;
3130
3131                         new_ce = alloc_pgtable_page(iommu->node);
3132                         if (!new_ce)
3133                                 goto out_unmap;
3134
3135                         ret = 0;
3136                 }
3137
3138                 /* Now copy the context entry */
3139                 memcpy(&ce, old_ce + idx, sizeof(ce));
3140
3141                 if (!__context_present(&ce))
3142                         continue;
3143
3144                 did = context_domain_id(&ce);
3145                 if (did >= 0 && did < cap_ndoms(iommu->cap))
3146                         set_bit(did, iommu->domain_ids);
3147
3148                 /*
3149                  * We need a marker for copied context entries. This
3150                  * marker needs to work for the old format as well as
3151                  * for extended context entries.
3152                  *
3153                  * Bit 67 of the context entry is used. In the old
3154                  * format this bit is available to software, in the
3155                  * extended format it is the PGE bit, but PGE is ignored
3156                  * by HW if PASIDs are disabled (and thus still
3157                  * available).
3158                  *
3159                  * So disable PASIDs first and then mark the entry
3160                  * copied. This means that we don't copy PASID
3161                  * translations from the old kernel, but this is fine as
3162                  * faults there are not fatal.
3163                  */
3164                 context_clear_pasid_enable(&ce);
3165                 context_set_copied(&ce);
3166
3167                 new_ce[idx] = ce;
3168         }
3169
3170         tbl[tbl_idx + pos] = new_ce;
3171
3172         __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
3173
3174 out_unmap:
3175         memunmap(old_ce);
3176
3177 out:
3178         return ret;
3179 }
3180
3181 static int copy_translation_tables(struct intel_iommu *iommu)
3182 {
3183         struct context_entry **ctxt_tbls;
3184         struct root_entry *old_rt;
3185         phys_addr_t old_rt_phys;
3186         int ctxt_table_entries;
3187         unsigned long flags;
3188         u64 rtaddr_reg;
3189         int bus, ret;
3190         bool new_ext, ext;
3191
3192         rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
3193         ext        = !!(rtaddr_reg & DMA_RTADDR_RTT);
3194         new_ext    = !!ecap_ecs(iommu->ecap);
3195
3196         /*
3197          * The RTT bit can only be changed when translation is disabled,
3198          * but disabling translation means to open a window for data
3199          * corruption. So bail out and don't copy anything if we would
3200          * have to change the bit.
3201          */
3202         if (new_ext != ext)
3203                 return -EINVAL;
3204
3205         old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
3206         if (!old_rt_phys)
3207                 return -EINVAL;
3208
3209         old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
3210         if (!old_rt)
3211                 return -ENOMEM;
3212
3213         /* This is too big for the stack - allocate it from slab */
3214         ctxt_table_entries = ext ? 512 : 256;
3215         ret = -ENOMEM;
3216         ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
3217         if (!ctxt_tbls)
3218                 goto out_unmap;
3219
3220         for (bus = 0; bus < 256; bus++) {
3221                 ret = copy_context_table(iommu, &old_rt[bus],
3222                                          ctxt_tbls, bus, ext);
3223                 if (ret) {
3224                         pr_err("%s: Failed to copy context table for bus %d\n",
3225                                 iommu->name, bus);
3226                         continue;
3227                 }
3228         }
3229
3230         spin_lock_irqsave(&iommu->lock, flags);
3231
3232         /* Context tables are copied, now write them to the root_entry table */
3233         for (bus = 0; bus < 256; bus++) {
3234                 int idx = ext ? bus * 2 : bus;
3235                 u64 val;
3236
3237                 if (ctxt_tbls[idx]) {
3238                         val = virt_to_phys(ctxt_tbls[idx]) | 1;
3239                         iommu->root_entry[bus].lo = val;
3240                 }
3241
3242                 if (!ext || !ctxt_tbls[idx + 1])
3243                         continue;
3244
3245                 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
3246                 iommu->root_entry[bus].hi = val;
3247         }
3248
3249         spin_unlock_irqrestore(&iommu->lock, flags);
3250
3251         kfree(ctxt_tbls);
3252
3253         __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
3254
3255         ret = 0;
3256
3257 out_unmap:
3258         memunmap(old_rt);
3259
3260         return ret;
3261 }
3262
3263 static int __init init_dmars(void)
3264 {
3265         struct dmar_drhd_unit *drhd;
3266         struct dmar_rmrr_unit *rmrr;
3267         bool copied_tables = false;
3268         struct device *dev;
3269         struct intel_iommu *iommu;
3270         int i, ret;
3271
3272         /*
3273          * for each drhd
3274          *    allocate root
3275          *    initialize and program root entry to not present
3276          * endfor
3277          */
3278         for_each_drhd_unit(drhd) {
3279                 /*
3280                  * lock not needed as this is only incremented in the single
3281                  * threaded kernel __init code path all other access are read
3282                  * only
3283                  */
3284                 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
3285                         g_num_of_iommus++;
3286                         continue;
3287                 }
3288                 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
3289         }
3290
3291         /* Preallocate enough resources for IOMMU hot-addition */
3292         if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
3293                 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
3294
3295         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
3296                         GFP_KERNEL);
3297         if (!g_iommus) {
3298                 pr_err("Allocating global iommu array failed\n");
3299                 ret = -ENOMEM;
3300                 goto error;
3301         }
3302
3303         for_each_active_iommu(iommu, drhd) {
3304                 /*
3305                  * Find the max pasid size of all IOMMU's in the system.
3306                  * We need to ensure the system pasid table is no bigger
3307                  * than the smallest supported.
3308                  */
3309                 if (pasid_enabled(iommu)) {
3310                         u32 temp = 2 << ecap_pss(iommu->ecap);
3311
3312                         intel_pasid_max_id = min_t(u32, temp,
3313                                                    intel_pasid_max_id);
3314                 }
3315
3316                 g_iommus[iommu->seq_id] = iommu;
3317
3318                 intel_iommu_init_qi(iommu);
3319
3320                 ret = iommu_init_domains(iommu);
3321                 if (ret)
3322                         goto free_iommu;
3323
3324                 init_translation_status(iommu);
3325
3326                 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
3327                         iommu_disable_translation(iommu);
3328                         clear_translation_pre_enabled(iommu);
3329                         pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
3330                                 iommu->name);
3331                 }
3332
3333                 /*
3334                  * TBD:
3335                  * we could share the same root & context tables
3336                  * among all IOMMU's. Need to Split it later.
3337                  */
3338                 ret = iommu_alloc_root_entry(iommu);
3339                 if (ret)
3340                         goto free_iommu;
3341
3342                 if (translation_pre_enabled(iommu)) {
3343                         pr_info("Translation already enabled - trying to copy translation structures\n");
3344
3345                         ret = copy_translation_tables(iommu);
3346                         if (ret) {
3347                                 /*
3348                                  * We found the IOMMU with translation
3349                                  * enabled - but failed to copy over the
3350                                  * old root-entry table. Try to proceed
3351                                  * by disabling translation now and
3352                                  * allocating a clean root-entry table.
3353                                  * This might cause DMAR faults, but
3354                                  * probably the dump will still succeed.
3355                                  */
3356                                 pr_err("Failed to copy translation tables from previous kernel for %s\n",
3357                                        iommu->name);
3358                                 iommu_disable_translation(iommu);
3359                                 clear_translation_pre_enabled(iommu);
3360                         } else {
3361                                 pr_info("Copied translation tables from previous kernel for %s\n",
3362                                         iommu->name);
3363                                 copied_tables = true;
3364                         }
3365                 }
3366
3367                 if (!ecap_pass_through(iommu->ecap))
3368                         hw_pass_through = 0;
3369
3370                 if (!intel_iommu_strict && cap_caching_mode(iommu->cap)) {
3371                         pr_info("Disable batched IOTLB flush due to virtualization");
3372                         intel_iommu_strict = 1;
3373                 }
3374
3375 #ifdef CONFIG_INTEL_IOMMU_SVM
3376                 if (pasid_enabled(iommu))
3377                         intel_svm_init(iommu);
3378 #endif
3379         }
3380
3381         /*
3382          * Now that qi is enabled on all iommus, set the root entry and flush
3383          * caches. This is required on some Intel X58 chipsets, otherwise the
3384          * flush_context function will loop forever and the boot hangs.
3385          */
3386         for_each_active_iommu(iommu, drhd) {
3387                 iommu_flush_write_buffer(iommu);
3388                 iommu_set_root_entry(iommu);
3389                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3390                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3391         }
3392
3393         if (iommu_pass_through)
3394                 iommu_identity_mapping |= IDENTMAP_ALL;
3395
3396 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
3397         dmar_map_gfx = 0;
3398 #endif
3399
3400         if (!dmar_map_gfx)
3401                 iommu_identity_mapping |= IDENTMAP_GFX;
3402
3403         check_tylersburg_isoch();
3404
3405         if (iommu_identity_mapping) {
3406                 ret = si_domain_init(hw_pass_through);
3407                 if (ret)
3408                         goto free_iommu;
3409         }
3410
3411
3412         /*
3413          * If we copied translations from a previous kernel in the kdump
3414          * case, we can not assign the devices to domains now, as that
3415          * would eliminate the old mappings. So skip this part and defer
3416          * the assignment to device driver initialization time.
3417          */
3418         if (copied_tables)
3419                 goto domains_done;
3420
3421         /*
3422          * If pass through is not set or not enabled, setup context entries for
3423          * identity mappings for rmrr, gfx, and isa and may fall back to static
3424          * identity mapping if iommu_identity_mapping is set.
3425          */
3426         if (iommu_identity_mapping) {
3427                 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
3428                 if (ret) {
3429                         pr_crit("Failed to setup IOMMU pass-through\n");
3430                         goto free_iommu;
3431                 }
3432         }
3433         /*
3434          * For each rmrr
3435          *   for each dev attached to rmrr
3436          *   do
3437          *     locate drhd for dev, alloc domain for dev
3438          *     allocate free domain
3439          *     allocate page table entries for rmrr
3440          *     if context not allocated for bus
3441          *           allocate and init context
3442          *           set present in root table for this bus
3443          *     init context with domain, translation etc
3444          *    endfor
3445          * endfor
3446          */
3447         pr_info("Setting RMRR:\n");
3448         for_each_rmrr_units(rmrr) {
3449                 /* some BIOS lists non-exist devices in DMAR table. */
3450                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3451                                           i, dev) {
3452                         ret = iommu_prepare_rmrr_dev(rmrr, dev);
3453                         if (ret)
3454                                 pr_err("Mapping reserved region failed\n");
3455                 }
3456         }
3457
3458         iommu_prepare_isa();
3459
3460 domains_done:
3461
3462         /*
3463          * for each drhd
3464          *   enable fault log
3465          *   global invalidate context cache
3466          *   global invalidate iotlb
3467          *   enable translation
3468          */
3469         for_each_iommu(iommu, drhd) {
3470                 if (drhd->ignored) {
3471                         /*
3472                          * we always have to disable PMRs or DMA may fail on
3473                          * this device
3474                          */
3475                         if (force_on)
3476                                 iommu_disable_protect_mem_regions(iommu);
3477                         continue;
3478                 }
3479
3480                 iommu_flush_write_buffer(iommu);
3481
3482 #ifdef CONFIG_INTEL_IOMMU_SVM
3483                 if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
3484                         ret = intel_svm_enable_prq(iommu);
3485                         if (ret)
3486                                 goto free_iommu;
3487                 }
3488 #endif
3489                 ret = dmar_set_interrupt(iommu);
3490                 if (ret)
3491                         goto free_iommu;
3492
3493                 if (!translation_pre_enabled(iommu))
3494                         iommu_enable_translation(iommu);
3495
3496                 iommu_disable_protect_mem_regions(iommu);
3497         }
3498
3499         return 0;
3500
3501 free_iommu:
3502         for_each_active_iommu(iommu, drhd) {
3503                 disable_dmar_iommu(iommu);
3504                 free_dmar_iommu(iommu);
3505         }
3506         if (si_domain) {
3507                 domain_exit(si_domain);
3508                 si_domain = NULL;
3509         }
3510
3511         kfree(g_iommus);
3512
3513 error:
3514         return ret;
3515 }
3516
3517 /* This takes a number of _MM_ pages, not VTD pages */
3518 static unsigned long intel_alloc_iova(struct device *dev,
3519                                      struct dmar_domain *domain,
3520                                      unsigned long nrpages, uint64_t dma_mask)
3521 {
3522         unsigned long iova_pfn = 0;
3523
3524         /* Restrict dma_mask to the width that the iommu can handle */
3525         dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
3526         /* Ensure we reserve the whole size-aligned region */
3527         nrpages = __roundup_pow_of_two(nrpages);
3528
3529         if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
3530                 /*
3531                  * First try to allocate an io virtual address in
3532                  * DMA_BIT_MASK(32) and if that fails then try allocating
3533                  * from higher range
3534                  */
3535                 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3536                                            IOVA_PFN(DMA_BIT_MASK(32)), false);
3537                 if (iova_pfn)
3538                         return iova_pfn;
3539         }
3540         iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
3541                                    IOVA_PFN(dma_mask), true);
3542         if (unlikely(!iova_pfn)) {
3543                 pr_err("Allocating %ld-page iova for %s failed",
3544                        nrpages, dev_name(dev));
3545                 return 0;
3546         }
3547
3548         return iova_pfn;
3549 }
3550
3551 struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
3552 {
3553         struct dmar_domain *domain, *tmp;
3554         struct dmar_rmrr_unit *rmrr;
3555         struct device *i_dev;
3556         int i, ret;
3557
3558         domain = find_domain(dev);
3559         if (domain)
3560                 goto out;
3561
3562         domain = find_or_alloc_domain(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
3563         if (!domain)
3564                 goto out;
3565
3566         /* We have a new domain - setup possible RMRRs for the device */
3567         rcu_read_lock();
3568         for_each_rmrr_units(rmrr) {
3569                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3570                                           i, i_dev) {
3571                         if (i_dev != dev)
3572                                 continue;
3573
3574                         ret = domain_prepare_identity_map(dev, domain,
3575                                                           rmrr->base_address,
3576                                                           rmrr->end_address);
3577                         if (ret)
3578                                 dev_err(dev, "Mapping reserved region failed\n");
3579                 }
3580         }
3581         rcu_read_unlock();
3582
3583         tmp = set_domain_for_dev(dev, domain);
3584         if (!tmp || domain != tmp) {
3585                 domain_exit(domain);
3586                 domain = tmp;
3587         }
3588
3589 out:
3590
3591         if (!domain)
3592                 pr_err("Allocating domain for %s failed\n", dev_name(dev));
3593
3594
3595         return domain;
3596 }
3597
3598 /* Check if the dev needs to go through non-identity map and unmap process.*/
3599 static int iommu_no_mapping(struct device *dev)
3600 {
3601         int found;
3602
3603         if (iommu_dummy(dev))
3604                 return 1;
3605
3606         if (!iommu_identity_mapping)
3607                 return 0;
3608
3609         found = identity_mapping(dev);
3610         if (found) {
3611                 if (iommu_should_identity_map(dev, 0))
3612                         return 1;
3613                 else {
3614                         /*
3615                          * 32 bit DMA is removed from si_domain and fall back
3616                          * to non-identity mapping.
3617                          */
3618                         dmar_remove_one_dev_info(si_domain, dev);
3619                         pr_info("32bit %s uses non-identity mapping\n",
3620                                 dev_name(dev));
3621                         return 0;
3622                 }
3623         } else {
3624                 /*
3625                  * In case of a detached 64 bit DMA device from vm, the device
3626                  * is put into si_domain for identity mapping.
3627                  */
3628                 if (iommu_should_identity_map(dev, 0)) {
3629                         int ret;
3630                         ret = domain_add_dev_info(si_domain, dev);
3631                         if (!ret) {
3632                                 pr_info("64bit %s uses identity mapping\n",
3633                                         dev_name(dev));
3634                                 return 1;
3635                         }
3636                 }
3637         }
3638
3639         return 0;
3640 }
3641
3642 static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
3643                                      size_t size, int dir, u64 dma_mask)
3644 {
3645         struct dmar_domain *domain;
3646         phys_addr_t start_paddr;
3647         unsigned long iova_pfn;
3648         int prot = 0;
3649         int ret;
3650         struct intel_iommu *iommu;
3651         unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
3652
3653         BUG_ON(dir == DMA_NONE);
3654
3655         if (iommu_no_mapping(dev))
3656                 return paddr;
3657
3658         domain = get_valid_domain_for_dev(dev);
3659         if (!domain)
3660                 return 0;
3661
3662         iommu = domain_get_iommu(domain);
3663         size = aligned_nrpages(paddr, size);
3664
3665         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
3666         if (!iova_pfn)
3667                 goto error;
3668
3669         /*
3670          * Check if DMAR supports zero-length reads on write only
3671          * mappings..
3672          */
3673         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3674                         !cap_zlr(iommu->cap))
3675                 prot |= DMA_PTE_READ;
3676         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3677                 prot |= DMA_PTE_WRITE;
3678         /*
3679          * paddr - (paddr + size) might be partial page, we should map the whole
3680          * page.  Note: if two part of one page are separately mapped, we
3681          * might have two guest_addr mapping to the same host paddr, but this
3682          * is not a big problem
3683          */
3684         ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
3685                                  mm_to_dma_pfn(paddr_pfn), size, prot);
3686         if (ret)
3687                 goto error;
3688
3689         start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
3690         start_paddr += paddr & ~PAGE_MASK;
3691         return start_paddr;
3692
3693 error:
3694         if (iova_pfn)
3695                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3696         pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
3697                 dev_name(dev), size, (unsigned long long)paddr, dir);
3698         return 0;
3699 }
3700
3701 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3702                                  unsigned long offset, size_t size,
3703                                  enum dma_data_direction dir,
3704                                  unsigned long attrs)
3705 {
3706         return __intel_map_single(dev, page_to_phys(page) + offset, size,
3707                                   dir, *dev->dma_mask);
3708 }
3709
3710 static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
3711 {
3712         struct dmar_domain *domain;
3713         unsigned long start_pfn, last_pfn;
3714         unsigned long nrpages;
3715         unsigned long iova_pfn;
3716         struct intel_iommu *iommu;
3717         struct page *freelist;
3718
3719         if (iommu_no_mapping(dev))
3720                 return;
3721
3722         domain = find_domain(dev);
3723         BUG_ON(!domain);
3724
3725         iommu = domain_get_iommu(domain);
3726
3727         iova_pfn = IOVA_PFN(dev_addr);
3728
3729         nrpages = aligned_nrpages(dev_addr, size);
3730         start_pfn = mm_to_dma_pfn(iova_pfn);
3731         last_pfn = start_pfn + nrpages - 1;
3732
3733         pr_debug("Device %s unmapping: pfn %lx-%lx\n",
3734                  dev_name(dev), start_pfn, last_pfn);
3735
3736         freelist = domain_unmap(domain, start_pfn, last_pfn);
3737
3738         if (intel_iommu_strict || !has_iova_flush_queue(&domain->iovad)) {
3739                 iommu_flush_iotlb_psi(iommu, domain, start_pfn,
3740                                       nrpages, !freelist, 0);
3741                 /* free iova */
3742                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
3743                 dma_free_pagelist(freelist);
3744         } else {
3745                 queue_iova(&domain->iovad, iova_pfn, nrpages,
3746                            (unsigned long)freelist);
3747                 /*
3748                  * queue up the release of the unmap to save the 1/6th of the
3749                  * cpu used up by the iotlb flush operation...
3750                  */
3751         }
3752 }
3753
3754 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3755                              size_t size, enum dma_data_direction dir,
3756                              unsigned long attrs)
3757 {
3758         intel_unmap(dev, dev_addr, size);
3759 }
3760
3761 static void *intel_alloc_coherent(struct device *dev, size_t size,
3762                                   dma_addr_t *dma_handle, gfp_t flags,
3763                                   unsigned long attrs)
3764 {
3765         struct page *page = NULL;
3766         int order;
3767
3768         size = PAGE_ALIGN(size);
3769         order = get_order(size);
3770
3771         if (!iommu_no_mapping(dev))
3772                 flags &= ~(GFP_DMA | GFP_DMA32);
3773         else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3774                 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
3775                         flags |= GFP_DMA;
3776                 else
3777                         flags |= GFP_DMA32;
3778         }
3779
3780         if (gfpflags_allow_blocking(flags)) {
3781                 unsigned int count = size >> PAGE_SHIFT;
3782
3783                 page = dma_alloc_from_contiguous(dev, count, order,
3784                                                  flags & __GFP_NOWARN);
3785                 if (page && iommu_no_mapping(dev) &&
3786                     page_to_phys(page) + size > dev->coherent_dma_mask) {
3787                         dma_release_from_contiguous(dev, page, count);
3788                         page = NULL;
3789                 }
3790         }
3791
3792         if (!page)
3793                 page = alloc_pages(flags, order);
3794         if (!page)
3795                 return NULL;
3796         memset(page_address(page), 0, size);
3797
3798         *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
3799                                          DMA_BIDIRECTIONAL,
3800                                          dev->coherent_dma_mask);
3801         if (*dma_handle)
3802                 return page_address(page);
3803         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3804                 __free_pages(page, order);
3805
3806         return NULL;
3807 }
3808
3809 static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
3810                                 dma_addr_t dma_handle, unsigned long attrs)
3811 {
3812         int order;
3813         struct page *page = virt_to_page(vaddr);
3814
3815         size = PAGE_ALIGN(size);
3816         order = get_order(size);
3817
3818         intel_unmap(dev, dma_handle, size);
3819         if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3820                 __free_pages(page, order);
3821 }
3822
3823 static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
3824                            int nelems, enum dma_data_direction dir,
3825                            unsigned long attrs)
3826 {
3827         dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
3828         unsigned long nrpages = 0;
3829         struct scatterlist *sg;
3830         int i;
3831
3832         for_each_sg(sglist, sg, nelems, i) {
3833                 nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
3834         }
3835
3836         intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
3837 }
3838
3839 static int intel_nontranslate_map_sg(struct device *hddev,
3840         struct scatterlist *sglist, int nelems, int dir)
3841 {
3842         int i;
3843         struct scatterlist *sg;
3844
3845         for_each_sg(sglist, sg, nelems, i) {
3846                 BUG_ON(!sg_page(sg));
3847                 sg->dma_address = sg_phys(sg);
3848                 sg->dma_length = sg->length;
3849         }
3850         return nelems;
3851 }
3852
3853 static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
3854                         enum dma_data_direction dir, unsigned long attrs)
3855 {
3856         int i;
3857         struct dmar_domain *domain;
3858         size_t size = 0;
3859         int prot = 0;
3860         unsigned long iova_pfn;
3861         int ret;
3862         struct scatterlist *sg;
3863         unsigned long start_vpfn;
3864         struct intel_iommu *iommu;
3865
3866         BUG_ON(dir == DMA_NONE);
3867         if (iommu_no_mapping(dev))
3868                 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
3869
3870         domain = get_valid_domain_for_dev(dev);
3871         if (!domain)
3872                 return 0;
3873
3874         iommu = domain_get_iommu(domain);
3875
3876         for_each_sg(sglist, sg, nelems, i)
3877                 size += aligned_nrpages(sg->offset, sg->length);
3878
3879         iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3880                                 *dev->dma_mask);
3881         if (!iova_pfn) {
3882                 sglist->dma_length = 0;
3883                 return 0;
3884         }
3885
3886         /*
3887          * Check if DMAR supports zero-length reads on write only
3888          * mappings..
3889          */
3890         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3891                         !cap_zlr(iommu->cap))
3892                 prot |= DMA_PTE_READ;
3893         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3894                 prot |= DMA_PTE_WRITE;
3895
3896         start_vpfn = mm_to_dma_pfn(iova_pfn);
3897
3898         ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3899         if (unlikely(ret)) {
3900                 dma_pte_free_pagetable(domain, start_vpfn,
3901                                        start_vpfn + size - 1,
3902                                        agaw_to_level(domain->agaw) + 1);
3903                 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
3904                 return 0;
3905         }
3906
3907         return nelems;
3908 }
3909
3910 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3911 {
3912         return !dma_addr;
3913 }
3914
3915 const struct dma_map_ops intel_dma_ops = {
3916         .alloc = intel_alloc_coherent,
3917         .free = intel_free_coherent,
3918         .map_sg = intel_map_sg,
3919         .unmap_sg = intel_unmap_sg,
3920         .map_page = intel_map_page,
3921         .unmap_page = intel_unmap_page,
3922         .mapping_error = intel_mapping_error,
3923 #ifdef CONFIG_X86
3924         .dma_supported = dma_direct_supported,
3925 #endif
3926 };
3927
3928 static inline int iommu_domain_cache_init(void)
3929 {
3930         int ret = 0;
3931
3932         iommu_domain_cache = kmem_cache_create("iommu_domain",
3933                                          sizeof(struct dmar_domain),
3934                                          0,
3935                                          SLAB_HWCACHE_ALIGN,
3936
3937                                          NULL);
3938         if (!iommu_domain_cache) {
3939                 pr_err("Couldn't create iommu_domain cache\n");
3940                 ret = -ENOMEM;
3941         }
3942
3943         return ret;
3944 }
3945
3946 static inline int iommu_devinfo_cache_init(void)
3947 {
3948         int ret = 0;
3949
3950         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3951                                          sizeof(struct device_domain_info),
3952                                          0,
3953                                          SLAB_HWCACHE_ALIGN,
3954                                          NULL);
3955         if (!iommu_devinfo_cache) {
3956                 pr_err("Couldn't create devinfo cache\n");
3957                 ret = -ENOMEM;
3958         }
3959
3960         return ret;
3961 }
3962
3963 static int __init iommu_init_mempool(void)
3964 {
3965         int ret;
3966         ret = iova_cache_get();
3967         if (ret)
3968                 return ret;
3969
3970         ret = iommu_domain_cache_init();
3971         if (ret)
3972                 goto domain_error;
3973
3974         ret = iommu_devinfo_cache_init();
3975         if (!ret)
3976                 return ret;
3977
3978         kmem_cache_destroy(iommu_domain_cache);
3979 domain_error:
3980         iova_cache_put();
3981
3982         return -ENOMEM;
3983 }
3984
3985 static void __init iommu_exit_mempool(void)
3986 {
3987         kmem_cache_destroy(iommu_devinfo_cache);
3988         kmem_cache_destroy(iommu_domain_cache);
3989         iova_cache_put();
3990 }
3991
3992 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3993 {
3994         struct dmar_drhd_unit *drhd;
3995         u32 vtbar;
3996         int rc;
3997
3998         /* We know that this device on this chipset has its own IOMMU.
3999          * If we find it under a different IOMMU, then the BIOS is lying
4000          * to us. Hope that the IOMMU for this device is actually
4001          * disabled, and it needs no translation...
4002          */
4003         rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
4004         if (rc) {
4005                 /* "can't" happen */
4006                 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
4007                 return;
4008         }
4009         vtbar &= 0xffff0000;
4010
4011         /* we know that the this iommu should be at offset 0xa000 from vtbar */
4012         drhd = dmar_find_matched_drhd_unit(pdev);
4013         if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
4014                 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
4015                 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
4016                 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4017         }
4018 }
4019 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
4020
4021 static void __init init_no_remapping_devices(void)
4022 {
4023         struct dmar_drhd_unit *drhd;
4024         struct device *dev;
4025         int i;
4026
4027         for_each_drhd_unit(drhd) {
4028                 if (!drhd->include_all) {
4029                         for_each_active_dev_scope(drhd->devices,
4030                                                   drhd->devices_cnt, i, dev)
4031                                 break;
4032                         /* ignore DMAR unit if no devices exist */
4033                         if (i == drhd->devices_cnt)
4034                                 drhd->ignored = 1;
4035                 }
4036         }
4037
4038         for_each_active_drhd_unit(drhd) {
4039                 if (drhd->include_all)
4040                         continue;
4041
4042                 for_each_active_dev_scope(drhd->devices,
4043                                           drhd->devices_cnt, i, dev)
4044                         if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
4045                                 break;
4046                 if (i < drhd->devices_cnt)
4047                         continue;
4048
4049                 /* This IOMMU has *only* gfx devices. Either bypass it or
4050                    set the gfx_mapped flag, as appropriate */
4051                 if (!dmar_map_gfx) {
4052                         drhd->ignored = 1;
4053                         for_each_active_dev_scope(drhd->devices,
4054                                                   drhd->devices_cnt, i, dev)
4055                                 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
4056                 }
4057         }
4058 }
4059
4060 #ifdef CONFIG_SUSPEND
4061 static int init_iommu_hw(void)
4062 {
4063         struct dmar_drhd_unit *drhd;
4064         struct intel_iommu *iommu = NULL;
4065
4066         for_each_active_iommu(iommu, drhd)
4067                 if (iommu->qi)
4068                         dmar_reenable_qi(iommu);
4069
4070         for_each_iommu(iommu, drhd) {
4071                 if (drhd->ignored) {
4072                         /*
4073                          * we always have to disable PMRs or DMA may fail on
4074                          * this device
4075                          */
4076                         if (force_on)
4077                                 iommu_disable_protect_mem_regions(iommu);
4078                         continue;
4079                 }
4080
4081                 iommu_flush_write_buffer(iommu);
4082
4083                 iommu_set_root_entry(iommu);
4084
4085                 iommu->flush.flush_context(iommu, 0, 0, 0,
4086                                            DMA_CCMD_GLOBAL_INVL);
4087                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4088                 iommu_enable_translation(iommu);
4089                 iommu_disable_protect_mem_regions(iommu);
4090         }
4091
4092         return 0;
4093 }
4094
4095 static void iommu_flush_all(void)
4096 {
4097         struct dmar_drhd_unit *drhd;
4098         struct intel_iommu *iommu;
4099
4100         for_each_active_iommu(iommu, drhd) {
4101                 iommu->flush.flush_context(iommu, 0, 0, 0,
4102                                            DMA_CCMD_GLOBAL_INVL);
4103                 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
4104                                          DMA_TLB_GLOBAL_FLUSH);
4105         }
4106 }
4107
4108 static int iommu_suspend(void)
4109 {
4110         struct dmar_drhd_unit *drhd;
4111         struct intel_iommu *iommu = NULL;
4112         unsigned long flag;
4113
4114         for_each_active_iommu(iommu, drhd) {
4115                 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
4116                                                  GFP_ATOMIC);
4117                 if (!iommu->iommu_state)
4118                         goto nomem;
4119         }
4120
4121         iommu_flush_all();
4122
4123         for_each_active_iommu(iommu, drhd) {
4124                 iommu_disable_translation(iommu);
4125
4126                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4127
4128                 iommu->iommu_state[SR_DMAR_FECTL_REG] =
4129                         readl(iommu->reg + DMAR_FECTL_REG);
4130                 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
4131                         readl(iommu->reg + DMAR_FEDATA_REG);
4132                 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
4133                         readl(iommu->reg + DMAR_FEADDR_REG);
4134                 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
4135                         readl(iommu->reg + DMAR_FEUADDR_REG);
4136
4137                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4138         }
4139         return 0;
4140
4141 nomem:
4142         for_each_active_iommu(iommu, drhd)
4143                 kfree(iommu->iommu_state);
4144
4145         return -ENOMEM;
4146 }
4147
4148 static void iommu_resume(void)
4149 {
4150         struct dmar_drhd_unit *drhd;
4151         struct intel_iommu *iommu = NULL;
4152         unsigned long flag;
4153
4154         if (init_iommu_hw()) {
4155                 if (force_on)
4156                         panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
4157                 else
4158                         WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
4159                 return;
4160         }
4161
4162         for_each_active_iommu(iommu, drhd) {
4163
4164                 raw_spin_lock_irqsave(&iommu->register_lock, flag);
4165
4166                 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
4167                         iommu->reg + DMAR_FECTL_REG);
4168                 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
4169                         iommu->reg + DMAR_FEDATA_REG);
4170                 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
4171                         iommu->reg + DMAR_FEADDR_REG);
4172                 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
4173                         iommu->reg + DMAR_FEUADDR_REG);
4174
4175                 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
4176         }
4177
4178         for_each_active_iommu(iommu, drhd)
4179                 kfree(iommu->iommu_state);
4180 }
4181
4182 static struct syscore_ops iommu_syscore_ops = {
4183         .resume         = iommu_resume,
4184         .suspend        = iommu_suspend,
4185 };
4186
4187 static void __init init_iommu_pm_ops(void)
4188 {
4189         register_syscore_ops(&iommu_syscore_ops);
4190 }
4191
4192 #else
4193 static inline void init_iommu_pm_ops(void) {}
4194 #endif  /* CONFIG_PM */
4195
4196
4197 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
4198 {
4199         struct acpi_dmar_reserved_memory *rmrr;
4200         struct dmar_rmrr_unit *rmrru;
4201         size_t length;
4202
4203         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
4204         if (!rmrru)
4205                 goto out;
4206
4207         rmrru->hdr = header;
4208         rmrr = (struct acpi_dmar_reserved_memory *)header;
4209         rmrru->base_address = rmrr->base_address;
4210         rmrru->end_address = rmrr->end_address;
4211
4212         length = rmrr->end_address - rmrr->base_address + 1;
4213
4214         rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
4215                                 ((void *)rmrr) + rmrr->header.length,
4216                                 &rmrru->devices_cnt);
4217         if (rmrru->devices_cnt && rmrru->devices == NULL)
4218                 goto free_rmrru;
4219
4220         list_add(&rmrru->list, &dmar_rmrr_units);
4221
4222         return 0;
4223 free_rmrru:
4224         kfree(rmrru);
4225 out:
4226         return -ENOMEM;
4227 }
4228
4229 static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
4230 {
4231         struct dmar_atsr_unit *atsru;
4232         struct acpi_dmar_atsr *tmp;
4233
4234         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4235                 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
4236                 if (atsr->segment != tmp->segment)
4237                         continue;
4238                 if (atsr->header.length != tmp->header.length)
4239                         continue;
4240                 if (memcmp(atsr, tmp, atsr->header.length) == 0)
4241                         return atsru;
4242         }
4243
4244         return NULL;
4245 }
4246
4247 int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4248 {
4249         struct acpi_dmar_atsr *atsr;
4250         struct dmar_atsr_unit *atsru;
4251
4252         if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
4253                 return 0;
4254
4255         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4256         atsru = dmar_find_atsr(atsr);
4257         if (atsru)
4258                 return 0;
4259
4260         atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
4261         if (!atsru)
4262                 return -ENOMEM;
4263
4264         /*
4265          * If memory is allocated from slab by ACPI _DSM method, we need to
4266          * copy the memory content because the memory buffer will be freed
4267          * on return.
4268          */
4269         atsru->hdr = (void *)(atsru + 1);
4270         memcpy(atsru->hdr, hdr, hdr->length);
4271         atsru->include_all = atsr->flags & 0x1;
4272         if (!atsru->include_all) {
4273                 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
4274                                 (void *)atsr + atsr->header.length,
4275                                 &atsru->devices_cnt);
4276                 if (atsru->devices_cnt && atsru->devices == NULL) {
4277                         kfree(atsru);
4278                         return -ENOMEM;
4279                 }
4280         }
4281
4282         list_add_rcu(&atsru->list, &dmar_atsr_units);
4283
4284         return 0;
4285 }
4286
4287 static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
4288 {
4289         dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
4290         kfree(atsru);
4291 }
4292
4293 int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4294 {
4295         struct acpi_dmar_atsr *atsr;
4296         struct dmar_atsr_unit *atsru;
4297
4298         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4299         atsru = dmar_find_atsr(atsr);
4300         if (atsru) {
4301                 list_del_rcu(&atsru->list);
4302                 synchronize_rcu();
4303                 intel_iommu_free_atsr(atsru);
4304         }
4305
4306         return 0;
4307 }
4308
4309 int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
4310 {
4311         int i;
4312         struct device *dev;
4313         struct acpi_dmar_atsr *atsr;
4314         struct dmar_atsr_unit *atsru;
4315
4316         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
4317         atsru = dmar_find_atsr(atsr);
4318         if (!atsru)
4319                 return 0;
4320
4321         if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
4322                 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
4323                                           i, dev)
4324                         return -EBUSY;
4325         }
4326
4327         return 0;
4328 }
4329
4330 static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
4331 {
4332         int sp, ret = 0;
4333         struct intel_iommu *iommu = dmaru->iommu;
4334
4335         if (g_iommus[iommu->seq_id])
4336                 return 0;
4337
4338         if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
4339                 pr_warn("%s: Doesn't support hardware pass through.\n",
4340                         iommu->name);
4341                 return -ENXIO;
4342         }
4343         if (!ecap_sc_support(iommu->ecap) &&
4344             domain_update_iommu_snooping(iommu)) {
4345                 pr_warn("%s: Doesn't support snooping.\n",
4346                         iommu->name);
4347                 return -ENXIO;
4348         }
4349         sp = domain_update_iommu_superpage(iommu) - 1;
4350         if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
4351                 pr_warn("%s: Doesn't support large page.\n",
4352                         iommu->name);
4353                 return -ENXIO;
4354         }
4355
4356         /*
4357          * Disable translation if already enabled prior to OS handover.
4358          */
4359         if (iommu->gcmd & DMA_GCMD_TE)
4360                 iommu_disable_translation(iommu);
4361
4362         g_iommus[iommu->seq_id] = iommu;
4363         ret = iommu_init_domains(iommu);
4364         if (ret == 0)
4365                 ret = iommu_alloc_root_entry(iommu);
4366         if (ret)
4367                 goto out;
4368
4369 #ifdef CONFIG_INTEL_IOMMU_SVM
4370         if (pasid_enabled(iommu))
4371                 intel_svm_init(iommu);
4372 #endif
4373
4374         if (dmaru->ignored) {
4375                 /*
4376                  * we always have to disable PMRs or DMA may fail on this device
4377                  */
4378                 if (force_on)
4379                         iommu_disable_protect_mem_regions(iommu);
4380                 return 0;
4381         }
4382
4383         intel_iommu_init_qi(iommu);
4384         iommu_flush_write_buffer(iommu);
4385
4386 #ifdef CONFIG_INTEL_IOMMU_SVM
4387         if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) {
4388                 ret = intel_svm_enable_prq(iommu);
4389                 if (ret)
4390                         goto disable_iommu;
4391         }
4392 #endif
4393         ret = dmar_set_interrupt(iommu);
4394         if (ret)
4395                 goto disable_iommu;
4396
4397         iommu_set_root_entry(iommu);
4398         iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
4399         iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
4400         iommu_enable_translation(iommu);
4401
4402         iommu_disable_protect_mem_regions(iommu);
4403         return 0;
4404
4405 disable_iommu:
4406         disable_dmar_iommu(iommu);
4407 out:
4408         free_dmar_iommu(iommu);
4409         return ret;
4410 }
4411
4412 int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
4413 {
4414         int ret = 0;
4415         struct intel_iommu *iommu = dmaru->iommu;
4416
4417         if (!intel_iommu_enabled)
4418                 return 0;
4419         if (iommu == NULL)
4420                 return -EINVAL;
4421
4422         if (insert) {
4423                 ret = intel_iommu_add(dmaru);
4424         } else {
4425                 disable_dmar_iommu(iommu);
4426                 free_dmar_iommu(iommu);
4427         }
4428
4429         return ret;
4430 }
4431
4432 static void intel_iommu_free_dmars(void)
4433 {
4434         struct dmar_rmrr_unit *rmrru, *rmrr_n;
4435         struct dmar_atsr_unit *atsru, *atsr_n;
4436
4437         list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
4438                 list_del(&rmrru->list);
4439                 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
4440                 kfree(rmrru);
4441         }
4442
4443         list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
4444                 list_del(&atsru->list);
4445                 intel_iommu_free_atsr(atsru);
4446         }
4447 }
4448
4449 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
4450 {
4451         int i, ret = 1;
4452         struct pci_bus *bus;
4453         struct pci_dev *bridge = NULL;
4454         struct device *tmp;
4455         struct acpi_dmar_atsr *atsr;
4456         struct dmar_atsr_unit *atsru;
4457
4458         dev = pci_physfn(dev);
4459         for (bus = dev->bus; bus; bus = bus->parent) {
4460                 bridge = bus->self;
4461                 /* If it's an integrated device, allow ATS */
4462                 if (!bridge)
4463                         return 1;
4464                 /* Connected via non-PCIe: no ATS */
4465                 if (!pci_is_pcie(bridge) ||
4466                     pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
4467                         return 0;
4468                 /* If we found the root port, look it up in the ATSR */
4469                 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
4470                         break;
4471         }
4472
4473         rcu_read_lock();
4474         list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
4475                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4476                 if (atsr->segment != pci_domain_nr(dev->bus))
4477                         continue;
4478
4479                 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
4480                         if (tmp == &bridge->dev)
4481                                 goto out;
4482
4483                 if (atsru->include_all)
4484                         goto out;
4485         }
4486         ret = 0;
4487 out:
4488         rcu_read_unlock();
4489
4490         return ret;
4491 }
4492
4493 int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
4494 {
4495         int ret = 0;
4496         struct dmar_rmrr_unit *rmrru;
4497         struct dmar_atsr_unit *atsru;
4498         struct acpi_dmar_atsr *atsr;
4499         struct acpi_dmar_reserved_memory *rmrr;
4500
4501         if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
4502                 return 0;
4503
4504         list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
4505                 rmrr = container_of(rmrru->hdr,
4506                                     struct acpi_dmar_reserved_memory, header);
4507                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4508                         ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
4509                                 ((void *)rmrr) + rmrr->header.length,
4510                                 rmrr->segment, rmrru->devices,
4511                                 rmrru->devices_cnt);
4512                         if(ret < 0)
4513                                 return ret;
4514                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4515                         dmar_remove_dev_scope(info, rmrr->segment,
4516                                 rmrru->devices, rmrru->devices_cnt);
4517                 }
4518         }
4519
4520         list_for_each_entry(atsru, &dmar_atsr_units, list) {
4521                 if (atsru->include_all)
4522                         continue;
4523
4524                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
4525                 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
4526                         ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
4527                                         (void *)atsr + atsr->header.length,
4528                                         atsr->segment, atsru->devices,
4529                                         atsru->devices_cnt);
4530                         if (ret > 0)
4531                                 break;
4532                         else if(ret < 0)
4533                                 return ret;
4534                 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
4535                         if (dmar_remove_dev_scope(info, atsr->segment,
4536                                         atsru->devices, atsru->devices_cnt))
4537                                 break;
4538                 }
4539         }
4540
4541         return 0;
4542 }
4543
4544 /*
4545  * Here we only respond to action of unbound device from driver.
4546  *
4547  * Added device is not attached to its DMAR domain here yet. That will happen
4548  * when mapping the device to iova.
4549  */
4550 static int device_notifier(struct notifier_block *nb,
4551                                   unsigned long action, void *data)
4552 {
4553         struct device *dev = data;
4554         struct dmar_domain *domain;
4555
4556         if (iommu_dummy(dev))
4557                 return 0;
4558
4559         if (action != BUS_NOTIFY_REMOVED_DEVICE)
4560                 return 0;
4561
4562         domain = find_domain(dev);
4563         if (!domain)
4564                 return 0;
4565
4566         dmar_remove_one_dev_info(domain, dev);
4567         if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
4568                 domain_exit(domain);
4569
4570         return 0;
4571 }
4572
4573 static struct notifier_block device_nb = {
4574         .notifier_call = device_notifier,
4575 };
4576
4577 static int intel_iommu_memory_notifier(struct notifier_block *nb,
4578                                        unsigned long val, void *v)
4579 {
4580         struct memory_notify *mhp = v;
4581         unsigned long long start, end;
4582         unsigned long start_vpfn, last_vpfn;
4583
4584         switch (val) {
4585         case MEM_GOING_ONLINE:
4586                 start = mhp->start_pfn << PAGE_SHIFT;
4587                 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4588                 if (iommu_domain_identity_map(si_domain, start, end)) {
4589                         pr_warn("Failed to build identity map for [%llx-%llx]\n",
4590                                 start, end);
4591                         return NOTIFY_BAD;
4592                 }
4593                 break;
4594
4595         case MEM_OFFLINE:
4596         case MEM_CANCEL_ONLINE:
4597                 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4598                 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4599                 while (start_vpfn <= last_vpfn) {
4600                         struct iova *iova;
4601                         struct dmar_drhd_unit *drhd;
4602                         struct intel_iommu *iommu;
4603                         struct page *freelist;
4604
4605                         iova = find_iova(&si_domain->iovad, start_vpfn);
4606                         if (iova == NULL) {
4607                                 pr_debug("Failed get IOVA for PFN %lx\n",
4608                                          start_vpfn);
4609                                 break;
4610                         }
4611
4612                         iova = split_and_remove_iova(&si_domain->iovad, iova,
4613                                                      start_vpfn, last_vpfn);
4614                         if (iova == NULL) {
4615                                 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
4616                                         start_vpfn, last_vpfn);
4617                                 return NOTIFY_BAD;
4618                         }
4619
4620                         freelist = domain_unmap(si_domain, iova->pfn_lo,
4621                                                iova->pfn_hi);
4622
4623                         rcu_read_lock();
4624                         for_each_active_iommu(iommu, drhd)
4625                                 iommu_flush_iotlb_psi(iommu, si_domain,
4626                                         iova->pfn_lo, iova_size(iova),
4627                                         !freelist, 0);
4628                         rcu_read_unlock();
4629                         dma_free_pagelist(freelist);
4630
4631                         start_vpfn = iova->pfn_hi + 1;
4632                         free_iova_mem(iova);
4633                 }
4634                 break;
4635         }
4636
4637         return NOTIFY_OK;
4638 }
4639
4640 static struct notifier_block intel_iommu_memory_nb = {
4641         .notifier_call = intel_iommu_memory_notifier,
4642         .priority = 0
4643 };
4644
4645 static void free_all_cpu_cached_iovas(unsigned int cpu)
4646 {
4647         int i;
4648
4649         for (i = 0; i < g_num_of_iommus; i++) {
4650                 struct intel_iommu *iommu = g_iommus[i];
4651                 struct dmar_domain *domain;
4652                 int did;
4653
4654                 if (!iommu)
4655                         continue;
4656
4657                 for (did = 0; did < cap_ndoms(iommu->cap); did++) {
4658                         domain = get_iommu_domain(iommu, (u16)did);
4659
4660                         if (!domain)
4661                                 continue;
4662                         free_cpu_cached_iovas(cpu, &domain->iovad);
4663                 }
4664         }
4665 }
4666
4667 static int intel_iommu_cpu_dead(unsigned int cpu)
4668 {
4669         free_all_cpu_cached_iovas(cpu);
4670         return 0;
4671 }
4672
4673 static void intel_disable_iommus(void)
4674 {
4675         struct intel_iommu *iommu = NULL;
4676         struct dmar_drhd_unit *drhd;
4677
4678         for_each_iommu(iommu, drhd)
4679                 iommu_disable_translation(iommu);
4680 }
4681
4682 static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
4683 {
4684         struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
4685
4686         return container_of(iommu_dev, struct intel_iommu, iommu);
4687 }
4688
4689 static ssize_t intel_iommu_show_version(struct device *dev,
4690                                         struct device_attribute *attr,
4691                                         char *buf)
4692 {
4693         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4694         u32 ver = readl(iommu->reg + DMAR_VER_REG);
4695         return sprintf(buf, "%d:%d\n",
4696                        DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4697 }
4698 static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4699
4700 static ssize_t intel_iommu_show_address(struct device *dev,
4701                                         struct device_attribute *attr,
4702                                         char *buf)
4703 {
4704         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4705         return sprintf(buf, "%llx\n", iommu->reg_phys);
4706 }
4707 static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4708
4709 static ssize_t intel_iommu_show_cap(struct device *dev,
4710                                     struct device_attribute *attr,
4711                                     char *buf)
4712 {
4713         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4714         return sprintf(buf, "%llx\n", iommu->cap);
4715 }
4716 static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4717
4718 static ssize_t intel_iommu_show_ecap(struct device *dev,
4719                                     struct device_attribute *attr,
4720                                     char *buf)
4721 {
4722         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4723         return sprintf(buf, "%llx\n", iommu->ecap);
4724 }
4725 static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4726
4727 static ssize_t intel_iommu_show_ndoms(struct device *dev,
4728                                       struct device_attribute *attr,
4729                                       char *buf)
4730 {
4731         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4732         return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
4733 }
4734 static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL);
4735
4736 static ssize_t intel_iommu_show_ndoms_used(struct device *dev,
4737                                            struct device_attribute *attr,
4738                                            char *buf)
4739 {
4740         struct intel_iommu *iommu = dev_to_intel_iommu(dev);
4741         return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
4742                                                   cap_ndoms(iommu->cap)));
4743 }
4744 static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL);
4745
4746 static struct attribute *intel_iommu_attrs[] = {
4747         &dev_attr_version.attr,
4748         &dev_attr_address.attr,
4749         &dev_attr_cap.attr,
4750         &dev_attr_ecap.attr,
4751         &dev_attr_domains_supported.attr,
4752         &dev_attr_domains_used.attr,
4753         NULL,
4754 };
4755
4756 static struct attribute_group intel_iommu_group = {
4757         .name = "intel-iommu",
4758         .attrs = intel_iommu_attrs,
4759 };
4760
4761 const struct attribute_group *intel_iommu_groups[] = {
4762         &intel_iommu_group,
4763         NULL,
4764 };
4765
4766 int __init intel_iommu_init(void)
4767 {
4768         int ret = -ENODEV;
4769         struct dmar_drhd_unit *drhd;
4770         struct intel_iommu *iommu;
4771
4772         /* VT-d is required for a TXT/tboot launch, so enforce that */
4773         force_on = tboot_force_iommu();
4774
4775         if (iommu_init_mempool()) {
4776                 if (force_on)
4777                         panic("tboot: Failed to initialize iommu memory\n");
4778                 return -ENOMEM;
4779         }
4780
4781         down_write(&dmar_global_lock);
4782         if (dmar_table_init()) {
4783                 if (force_on)
4784                         panic("tboot: Failed to initialize DMAR table\n");
4785                 goto out_free_dmar;
4786         }
4787
4788         if (dmar_dev_scope_init() < 0) {
4789                 if (force_on)
4790                         panic("tboot: Failed to initialize DMAR device scope\n");
4791                 goto out_free_dmar;
4792         }
4793
4794         up_write(&dmar_global_lock);
4795
4796         /*
4797          * The bus notifier takes the dmar_global_lock, so lockdep will
4798          * complain later when we register it under the lock.
4799          */
4800         dmar_register_bus_notifier();
4801
4802         down_write(&dmar_global_lock);
4803
4804         if (no_iommu || dmar_disabled) {
4805                 /*
4806                  * We exit the function here to ensure IOMMU's remapping and
4807                  * mempool aren't setup, which means that the IOMMU's PMRs
4808                  * won't be disabled via the call to init_dmars(). So disable
4809                  * it explicitly here. The PMRs were setup by tboot prior to
4810                  * calling SENTER, but the kernel is expected to reset/tear
4811                  * down the PMRs.
4812                  */
4813                 if (intel_iommu_tboot_noforce) {
4814                         for_each_iommu(iommu, drhd)
4815                                 iommu_disable_protect_mem_regions(iommu);
4816                 }
4817
4818                 /*
4819                  * Make sure the IOMMUs are switched off, even when we
4820                  * boot into a kexec kernel and the previous kernel left
4821                  * them enabled
4822                  */
4823                 intel_disable_iommus();
4824                 goto out_free_dmar;
4825         }
4826
4827         if (list_empty(&dmar_rmrr_units))
4828                 pr_info("No RMRR found\n");
4829
4830         if (list_empty(&dmar_atsr_units))
4831                 pr_info("No ATSR found\n");
4832
4833         if (dmar_init_reserved_ranges()) {
4834                 if (force_on)
4835                         panic("tboot: Failed to reserve iommu ranges\n");
4836                 goto out_free_reserved_range;
4837         }
4838
4839         if (dmar_map_gfx)
4840                 intel_iommu_gfx_mapped = 1;
4841
4842         init_no_remapping_devices();
4843
4844         ret = init_dmars();
4845         if (ret) {
4846                 if (force_on)
4847                         panic("tboot: Failed to initialize DMARs\n");
4848                 pr_err("Initialization failed\n");
4849                 goto out_free_reserved_range;
4850         }
4851         up_write(&dmar_global_lock);
4852         pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4853
4854 #if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
4855         swiotlb = 0;
4856 #endif
4857         dma_ops = &intel_dma_ops;
4858
4859         init_iommu_pm_ops();
4860
4861         for_each_active_iommu(iommu, drhd) {
4862                 iommu_device_sysfs_add(&iommu->iommu, NULL,
4863                                        intel_iommu_groups,
4864                                        "%s", iommu->name);
4865                 iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);
4866                 iommu_device_register(&iommu->iommu);
4867         }
4868
4869         bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
4870         bus_register_notifier(&pci_bus_type, &device_nb);
4871         if (si_domain && !hw_pass_through)
4872                 register_memory_notifier(&intel_iommu_memory_nb);
4873         cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL,
4874                           intel_iommu_cpu_dead);
4875         intel_iommu_enabled = 1;
4876
4877         return 0;
4878
4879 out_free_reserved_range:
4880         put_iova_domain(&reserved_iova_list);
4881 out_free_dmar:
4882         intel_iommu_free_dmars();
4883         up_write(&dmar_global_lock);
4884         iommu_exit_mempool();
4885         return ret;
4886 }
4887
4888 static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4889 {
4890         struct intel_iommu *iommu = opaque;
4891
4892         domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4893         return 0;
4894 }
4895
4896 /*
4897  * NB - intel-iommu lacks any sort of reference counting for the users of
4898  * dependent devices.  If multiple endpoints have intersecting dependent
4899  * devices, unbinding the driver from any one of them will possibly leave
4900  * the others unable to operate.
4901  */
4902 static void domain_context_clear(struct intel_iommu *iommu, struct device *dev)
4903 {
4904         if (!iommu || !dev || !dev_is_pci(dev))
4905                 return;
4906
4907         pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu);
4908 }
4909
4910 static void __dmar_remove_one_dev_info(struct device_domain_info *info)
4911 {
4912         struct intel_iommu *iommu;
4913         unsigned long flags;
4914
4915         assert_spin_locked(&device_domain_lock);
4916
4917         if (WARN_ON(!info))
4918                 return;
4919
4920         iommu = info->iommu;
4921
4922         if (info->dev) {
4923                 iommu_disable_dev_iotlb(info);
4924                 domain_context_clear(iommu, info->dev);
4925                 intel_pasid_free_table(info->dev);
4926         }
4927
4928         unlink_domain_info(info);
4929
4930         spin_lock_irqsave(&iommu->lock, flags);
4931         domain_detach_iommu(info->domain, iommu);
4932         spin_unlock_irqrestore(&iommu->lock, flags);
4933
4934         free_devinfo_mem(info);
4935 }
4936
4937 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
4938                                      struct device *dev)
4939 {
4940         struct device_domain_info *info;
4941         unsigned long flags;
4942
4943         spin_lock_irqsave(&device_domain_lock, flags);
4944         info = dev->archdata.iommu;
4945         __dmar_remove_one_dev_info(info);
4946         spin_unlock_irqrestore(&device_domain_lock, flags);
4947 }
4948
4949 static int md_domain_init(struct dmar_domain *domain, int guest_width)
4950 {
4951         int adjust_width;
4952
4953         init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
4954         domain_reserve_special_ranges(domain);
4955
4956         /* calculate AGAW */
4957         domain->gaw = guest_width;
4958         adjust_width = guestwidth_to_adjustwidth(guest_width);
4959         domain->agaw = width_to_agaw(adjust_width);
4960
4961         domain->iommu_coherency = 0;
4962         domain->iommu_snooping = 0;
4963         domain->iommu_superpage = 0;
4964         domain->max_addr = 0;
4965
4966         /* always allocate the top pgd */
4967         domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
4968         if (!domain->pgd)
4969                 return -ENOMEM;
4970         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4971         return 0;
4972 }
4973
4974 static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4975 {
4976         struct dmar_domain *dmar_domain;
4977         struct iommu_domain *domain;
4978
4979         if (type != IOMMU_DOMAIN_UNMANAGED)
4980                 return NULL;
4981
4982         dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
4983         if (!dmar_domain) {
4984                 pr_err("Can't allocate dmar_domain\n");
4985                 return NULL;
4986         }
4987         if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4988                 pr_err("Domain initialization failed\n");
4989                 domain_exit(dmar_domain);
4990                 return NULL;
4991         }
4992         domain_update_iommu_cap(dmar_domain);
4993
4994         domain = &dmar_domain->domain;
4995         domain->geometry.aperture_start = 0;
4996         domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4997         domain->geometry.force_aperture = true;
4998
4999         return domain;
5000 }
5001
5002 static void intel_iommu_domain_free(struct iommu_domain *domain)
5003 {
5004         domain_exit(to_dmar_domain(domain));
5005 }
5006
5007 static int intel_iommu_attach_device(struct iommu_domain *domain,
5008                                      struct device *dev)
5009 {
5010         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5011         struct intel_iommu *iommu;
5012         int addr_width;
5013         u8 bus, devfn;
5014
5015         if (device_is_rmrr_locked(dev)) {
5016                 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement.  Contact your platform vendor.\n");
5017                 return -EPERM;
5018         }
5019
5020         /* normally dev is not mapped */
5021         if (unlikely(domain_context_mapped(dev))) {
5022                 struct dmar_domain *old_domain;
5023
5024                 old_domain = find_domain(dev);
5025                 if (old_domain) {
5026                         rcu_read_lock();
5027                         dmar_remove_one_dev_info(old_domain, dev);
5028                         rcu_read_unlock();
5029
5030                         if (!domain_type_is_vm_or_si(old_domain) &&
5031                              list_empty(&old_domain->devices))
5032                                 domain_exit(old_domain);
5033                 }
5034         }
5035
5036         iommu = device_to_iommu(dev, &bus, &devfn);
5037         if (!iommu)
5038                 return -ENODEV;
5039
5040         /* check if this iommu agaw is sufficient for max mapped address */
5041         addr_width = agaw_to_width(iommu->agaw);
5042         if (addr_width > cap_mgaw(iommu->cap))
5043                 addr_width = cap_mgaw(iommu->cap);
5044
5045         if (dmar_domain->max_addr > (1LL << addr_width)) {
5046                 pr_err("%s: iommu width (%d) is not "
5047                        "sufficient for the mapped address (%llx)\n",
5048                        __func__, addr_width, dmar_domain->max_addr);
5049                 return -EFAULT;
5050         }
5051         dmar_domain->gaw = addr_width;
5052
5053         /*
5054          * Knock out extra levels of page tables if necessary
5055          */
5056         while (iommu->agaw < dmar_domain->agaw) {
5057                 struct dma_pte *pte;
5058
5059                 pte = dmar_domain->pgd;
5060                 if (dma_pte_present(pte)) {
5061                         dmar_domain->pgd = (struct dma_pte *)
5062                                 phys_to_virt(dma_pte_addr(pte));
5063                         free_pgtable_page(pte);
5064                 }
5065                 dmar_domain->agaw--;
5066         }
5067
5068         return domain_add_dev_info(dmar_domain, dev);
5069 }
5070
5071 static void intel_iommu_detach_device(struct iommu_domain *domain,
5072                                       struct device *dev)
5073 {
5074         dmar_remove_one_dev_info(to_dmar_domain(domain), dev);
5075 }
5076
5077 static int intel_iommu_map(struct iommu_domain *domain,
5078                            unsigned long iova, phys_addr_t hpa,
5079                            size_t size, int iommu_prot)
5080 {
5081         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5082         u64 max_addr;
5083         int prot = 0;
5084         int ret;
5085
5086         if (iommu_prot & IOMMU_READ)
5087                 prot |= DMA_PTE_READ;
5088         if (iommu_prot & IOMMU_WRITE)
5089                 prot |= DMA_PTE_WRITE;
5090         if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
5091                 prot |= DMA_PTE_SNP;
5092
5093         max_addr = iova + size;
5094         if (dmar_domain->max_addr < max_addr) {
5095                 u64 end;
5096
5097                 /* check if minimum agaw is sufficient for mapped address */
5098                 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
5099                 if (end < max_addr) {
5100                         pr_err("%s: iommu width (%d) is not "
5101                                "sufficient for the mapped address (%llx)\n",
5102                                __func__, dmar_domain->gaw, max_addr);
5103                         return -EFAULT;
5104                 }
5105                 dmar_domain->max_addr = max_addr;
5106         }
5107         /* Round up size to next multiple of PAGE_SIZE, if it and
5108            the low bits of hpa would take us onto the next page */
5109         size = aligned_nrpages(hpa, size);
5110         ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
5111                                  hpa >> VTD_PAGE_SHIFT, size, prot);
5112         return ret;
5113 }
5114
5115 static size_t intel_iommu_unmap(struct iommu_domain *domain,
5116                                 unsigned long iova, size_t size)
5117 {
5118         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5119         struct page *freelist = NULL;
5120         unsigned long start_pfn, last_pfn;
5121         unsigned int npages;
5122         int iommu_id, level = 0;
5123
5124         /* Cope with horrid API which requires us to unmap more than the
5125            size argument if it happens to be a large-page mapping. */
5126         BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
5127
5128         if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
5129                 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
5130
5131         start_pfn = iova >> VTD_PAGE_SHIFT;
5132         last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
5133
5134         freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
5135
5136         npages = last_pfn - start_pfn + 1;
5137
5138         for_each_domain_iommu(iommu_id, dmar_domain)
5139                 iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,
5140                                       start_pfn, npages, !freelist, 0);
5141
5142         dma_free_pagelist(freelist);
5143
5144         if (dmar_domain->max_addr == iova + size)
5145                 dmar_domain->max_addr = iova;
5146
5147         return size;
5148 }
5149
5150 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
5151                                             dma_addr_t iova)
5152 {
5153         struct dmar_domain *dmar_domain = to_dmar_domain(domain);
5154         struct dma_pte *pte;
5155         int level = 0;
5156         u64 phys = 0;
5157
5158         pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
5159         if (pte && dma_pte_present(pte))
5160                 phys = dma_pte_addr(pte) +
5161                         (iova & (BIT_MASK(level_to_offset_bits(level) +
5162                                                 VTD_PAGE_SHIFT) - 1));
5163
5164         return phys;
5165 }
5166
5167 static bool intel_iommu_capable(enum iommu_cap cap)
5168 {
5169         if (cap == IOMMU_CAP_CACHE_COHERENCY)
5170                 return domain_update_iommu_snooping(NULL) == 1;
5171         if (cap == IOMMU_CAP_INTR_REMAP)
5172                 return irq_remapping_enabled == 1;
5173
5174         return false;
5175 }
5176
5177 static int intel_iommu_add_device(struct device *dev)
5178 {
5179         struct intel_iommu *iommu;
5180         struct iommu_group *group;
5181         u8 bus, devfn;
5182
5183         iommu = device_to_iommu(dev, &bus, &devfn);
5184         if (!iommu)
5185                 return -ENODEV;
5186
5187         iommu_device_link(&iommu->iommu, dev);
5188
5189         group = iommu_group_get_for_dev(dev);
5190
5191         if (IS_ERR(group))
5192                 return PTR_ERR(group);
5193
5194         iommu_group_put(group);
5195         return 0;
5196 }
5197
5198 static void intel_iommu_remove_device(struct device *dev)
5199 {
5200         struct intel_iommu *iommu;
5201         u8 bus, devfn;
5202
5203         iommu = device_to_iommu(dev, &bus, &devfn);
5204         if (!iommu)
5205                 return;
5206
5207         iommu_group_remove_device(dev);
5208
5209         iommu_device_unlink(&iommu->iommu, dev);
5210 }
5211
5212 static void intel_iommu_get_resv_regions(struct device *device,
5213                                          struct list_head *head)
5214 {
5215         int prot = DMA_PTE_READ | DMA_PTE_WRITE;
5216         struct iommu_resv_region *reg;
5217         struct dmar_rmrr_unit *rmrr;
5218         struct device *i_dev;
5219         int i;
5220
5221         down_read(&dmar_global_lock);
5222         for_each_rmrr_units(rmrr) {
5223                 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
5224                                           i, i_dev) {
5225                         struct iommu_resv_region *resv;
5226                         size_t length;
5227
5228                         if (i_dev != device)
5229                                 continue;
5230
5231                         length = rmrr->end_address - rmrr->base_address + 1;
5232                         resv = iommu_alloc_resv_region(rmrr->base_address,
5233                                                        length, prot,
5234                                                        IOMMU_RESV_DIRECT);
5235                         if (!resv)
5236                                 break;
5237
5238                         list_add_tail(&resv->list, head);
5239                 }
5240         }
5241         up_read(&dmar_global_lock);
5242
5243         reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
5244                                       IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
5245                                       0, IOMMU_RESV_MSI);
5246         if (!reg)
5247                 return;
5248         list_add_tail(&reg->list, head);
5249 }
5250
5251 static void intel_iommu_put_resv_regions(struct device *dev,
5252                                          struct list_head *head)
5253 {
5254         struct iommu_resv_region *entry, *next;
5255
5256         list_for_each_entry_safe(entry, next, head, list)
5257                 kfree(entry);
5258 }
5259
5260 #ifdef CONFIG_INTEL_IOMMU_SVM
5261 #define MAX_NR_PASID_BITS (20)
5262 static inline unsigned long intel_iommu_get_pts(struct device *dev)
5263 {
5264         int pts, max_pasid;
5265
5266         max_pasid = intel_pasid_get_dev_max_id(dev);
5267         pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS);
5268         if (pts < 5)
5269                 return 0;
5270
5271         return pts - 5;
5272 }
5273
5274 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev)
5275 {
5276         struct device_domain_info *info;
5277         struct context_entry *context;
5278         struct dmar_domain *domain;
5279         unsigned long flags;
5280         u64 ctx_lo;
5281         int ret;
5282
5283         domain = get_valid_domain_for_dev(sdev->dev);
5284         if (!domain)
5285                 return -EINVAL;
5286
5287         spin_lock_irqsave(&device_domain_lock, flags);
5288         spin_lock(&iommu->lock);
5289
5290         ret = -EINVAL;
5291         info = sdev->dev->archdata.iommu;
5292         if (!info || !info->pasid_supported)
5293                 goto out;
5294
5295         context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
5296         if (WARN_ON(!context))
5297                 goto out;
5298
5299         ctx_lo = context[0].lo;
5300
5301         sdev->did = domain->iommu_did[iommu->seq_id];
5302         sdev->sid = PCI_DEVID(info->bus, info->devfn);
5303
5304         if (!(ctx_lo & CONTEXT_PASIDE)) {
5305                 if (iommu->pasid_state_table)
5306                         context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table);
5307                 context[1].lo = (u64)virt_to_phys(info->pasid_table->table) |
5308                         intel_iommu_get_pts(sdev->dev);
5309
5310                 wmb();
5311                 /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both
5312                  * extended to permit requests-with-PASID if the PASIDE bit
5313                  * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH,
5314                  * however, the PASIDE bit is ignored and requests-with-PASID
5315                  * are unconditionally blocked. Which makes less sense.
5316                  * So convert from CONTEXT_TT_PASS_THROUGH to one of the new
5317                  * "guest mode" translation types depending on whether ATS
5318                  * is available or not. Annoyingly, we can't use the new
5319                  * modes *unless* PASIDE is set. */
5320                 if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) {
5321                         ctx_lo &= ~CONTEXT_TT_MASK;
5322                         if (info->ats_supported)
5323                                 ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2;
5324                         else
5325                                 ctx_lo |= CONTEXT_TT_PT_PASID << 2;
5326                 }
5327                 ctx_lo |= CONTEXT_PASIDE;
5328                 if (iommu->pasid_state_table)
5329                         ctx_lo |= CONTEXT_DINVE;
5330                 if (info->pri_supported)
5331                         ctx_lo |= CONTEXT_PRS;
5332                 context[0].lo = ctx_lo;
5333                 wmb();
5334                 iommu->flush.flush_context(iommu, sdev->did, sdev->sid,
5335                                            DMA_CCMD_MASK_NOBIT,
5336                                            DMA_CCMD_DEVICE_INVL);
5337         }
5338
5339         /* Enable PASID support in the device, if it wasn't already */
5340         if (!info->pasid_enabled)
5341                 iommu_enable_dev_iotlb(info);
5342
5343         if (info->ats_enabled) {
5344                 sdev->dev_iotlb = 1;
5345                 sdev->qdep = info->ats_qdep;
5346                 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS)
5347                         sdev->qdep = 0;
5348         }
5349         ret = 0;
5350
5351  out:
5352         spin_unlock(&iommu->lock);
5353         spin_unlock_irqrestore(&device_domain_lock, flags);
5354
5355         return ret;
5356 }
5357
5358 struct intel_iommu *intel_svm_device_to_iommu(struct device *dev)
5359 {
5360         struct intel_iommu *iommu;
5361         u8 bus, devfn;
5362
5363         if (iommu_dummy(dev)) {
5364                 dev_warn(dev,
5365                          "No IOMMU translation for device; cannot enable SVM\n");
5366                 return NULL;
5367         }
5368
5369         iommu = device_to_iommu(dev, &bus, &devfn);
5370         if ((!iommu)) {
5371                 dev_err(dev, "No IOMMU for device; cannot enable SVM\n");
5372                 return NULL;
5373         }
5374
5375         return iommu;
5376 }
5377 #endif /* CONFIG_INTEL_IOMMU_SVM */
5378
5379 const struct iommu_ops intel_iommu_ops = {
5380         .capable                = intel_iommu_capable,
5381         .domain_alloc           = intel_iommu_domain_alloc,
5382         .domain_free            = intel_iommu_domain_free,
5383         .attach_dev             = intel_iommu_attach_device,
5384         .detach_dev             = intel_iommu_detach_device,
5385         .map                    = intel_iommu_map,
5386         .unmap                  = intel_iommu_unmap,
5387         .iova_to_phys           = intel_iommu_iova_to_phys,
5388         .add_device             = intel_iommu_add_device,
5389         .remove_device          = intel_iommu_remove_device,
5390         .get_resv_regions       = intel_iommu_get_resv_regions,
5391         .put_resv_regions       = intel_iommu_put_resv_regions,
5392         .device_group           = pci_device_group,
5393         .pgsize_bitmap          = INTEL_IOMMU_PGSIZES,
5394 };
5395
5396 static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
5397 {
5398         /* G4x/GM45 integrated gfx dmar support is totally busted. */
5399         pr_info("Disabling IOMMU for graphics on this chipset\n");
5400         dmar_map_gfx = 0;
5401 }
5402
5403 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
5404 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
5405 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
5406 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
5407 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
5408 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
5409 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
5410
5411 static void quirk_iommu_rwbf(struct pci_dev *dev)
5412 {
5413         /*
5414          * Mobile 4 Series Chipset neglects to set RWBF capability,
5415          * but needs it. Same seems to hold for the desktop versions.
5416          */
5417         pr_info("Forcing write-buffer flush capability\n");
5418         rwbf_quirk = 1;
5419 }
5420
5421 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
5422 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
5423 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
5424 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
5425 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
5426 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
5427 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
5428
5429 #define GGC 0x52
5430 #define GGC_MEMORY_SIZE_MASK    (0xf << 8)
5431 #define GGC_MEMORY_SIZE_NONE    (0x0 << 8)
5432 #define GGC_MEMORY_SIZE_1M      (0x1 << 8)
5433 #define GGC_MEMORY_SIZE_2M      (0x3 << 8)
5434 #define GGC_MEMORY_VT_ENABLED   (0x8 << 8)
5435 #define GGC_MEMORY_SIZE_2M_VT   (0x9 << 8)
5436 #define GGC_MEMORY_SIZE_3M_VT   (0xa << 8)
5437 #define GGC_MEMORY_SIZE_4M_VT   (0xb << 8)
5438
5439 static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
5440 {
5441         unsigned short ggc;
5442
5443         if (pci_read_config_word(dev, GGC, &ggc))
5444                 return;
5445
5446         if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
5447                 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
5448                 dmar_map_gfx = 0;
5449         } else if (dmar_map_gfx) {
5450                 /* we have to ensure the gfx device is idle before we flush */
5451                 pr_info("Disabling batched IOTLB flush on Ironlake\n");
5452                 intel_iommu_strict = 1;
5453        }
5454 }
5455 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
5456 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
5457 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
5458 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
5459
5460 /* On Tylersburg chipsets, some BIOSes have been known to enable the
5461    ISOCH DMAR unit for the Azalia sound device, but not give it any
5462    TLB entries, which causes it to deadlock. Check for that.  We do
5463    this in a function called from init_dmars(), instead of in a PCI
5464    quirk, because we don't want to print the obnoxious "BIOS broken"
5465    message if VT-d is actually disabled.
5466 */
5467 static void __init check_tylersburg_isoch(void)
5468 {
5469         struct pci_dev *pdev;
5470         uint32_t vtisochctrl;
5471
5472         /* If there's no Azalia in the system anyway, forget it. */
5473         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
5474         if (!pdev)
5475                 return;
5476         pci_dev_put(pdev);
5477
5478         /* System Management Registers. Might be hidden, in which case
5479            we can't do the sanity check. But that's OK, because the
5480            known-broken BIOSes _don't_ actually hide it, so far. */
5481         pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
5482         if (!pdev)
5483                 return;
5484
5485         if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
5486                 pci_dev_put(pdev);
5487                 return;
5488         }
5489
5490         pci_dev_put(pdev);
5491
5492         /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
5493         if (vtisochctrl & 1)
5494                 return;
5495
5496         /* Drop all bits other than the number of TLB entries */
5497         vtisochctrl &= 0x1c;
5498
5499         /* If we have the recommended number of TLB entries (16), fine. */
5500         if (vtisochctrl == 0x10)
5501                 return;
5502
5503         /* Zero TLB entries? You get to ride the short bus to school. */
5504         if (!vtisochctrl) {
5505                 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
5506                      "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
5507                      dmi_get_system_info(DMI_BIOS_VENDOR),
5508                      dmi_get_system_info(DMI_BIOS_VERSION),
5509                      dmi_get_system_info(DMI_PRODUCT_VERSION));
5510                 iommu_identity_mapping |= IDENTMAP_AZALIA;
5511                 return;
5512         }
5513
5514         pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
5515                vtisochctrl);
5516 }