drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/atomic.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-iommu.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/io.h>
  38 #include <linux/io-64-nonatomic-hi-lo.h>
  39 #include <linux/iommu.h>
  40 #include <linux/iopoll.h>
  41 #include <linux/module.h>
  42 #include <linux/of.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_device.h>
  45 #include <linux/of_iommu.h>
  46 #include <linux/pci.h>
  47 #include <linux/platform_device.h>
  48 #include <linux/slab.h>
  49 #include <linux/spinlock.h>
  50
  51 #include <linux/amba/bus.h>
  52
  53 #include "io-pgtable.h"
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* SMMU global address space */
  59 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  60 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  61
  62 /*
  63  * SMMU global address space with conditional offset to access secure
  64  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  65  * nsGFSYNR0: 0x450)
  66  */
  67 #define ARM_SMMU_GR0_NS(smmu)                                           \
  68         ((smmu)->base +                                                 \
  69                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  70                         ? 0x400 : 0))
  71
  72 /*
  73  * Some 64-bit registers only make sense to write atomically, but in such
  74  * cases all the data relevant to AArch32 formats lies within the lower word,
  75  * therefore this actually makes more sense than it might first appear.
  76  */
  77 #ifdef CONFIG_64BIT
  78 #define smmu_write_atomic_lq            writeq_relaxed
  79 #else
  80 #define smmu_write_atomic_lq            writel_relaxed
  81 #endif
  82
  83 /* Configuration registers */
  84 #define ARM_SMMU_GR0_sCR0               0x0
  85 #define sCR0_CLIENTPD                   (1 << 0)
  86 #define sCR0_GFRE                       (1 << 1)
  87 #define sCR0_GFIE                       (1 << 2)
  88 #define sCR0_GCFGFRE                    (1 << 4)
  89 #define sCR0_GCFGFIE                    (1 << 5)
  90 #define sCR0_USFCFG                     (1 << 10)
  91 #define sCR0_VMIDPNE                    (1 << 11)
  92 #define sCR0_PTM                        (1 << 12)
  93 #define sCR0_FB                         (1 << 13)
  94 #define sCR0_VMID16EN                   (1 << 31)
  95 #define sCR0_BSU_SHIFT                  14
  96 #define sCR0_BSU_MASK                   0x3
  97
  98 /* Auxiliary Configuration register */
  99 #define ARM_SMMU_GR0_sACR               0x10
 100
 101 /* Identification registers */
 102 #define ARM_SMMU_GR0_ID0                0x20
 103 #define ARM_SMMU_GR0_ID1                0x24
 104 #define ARM_SMMU_GR0_ID2                0x28
 105 #define ARM_SMMU_GR0_ID3                0x2c
 106 #define ARM_SMMU_GR0_ID4                0x30
 107 #define ARM_SMMU_GR0_ID5                0x34
 108 #define ARM_SMMU_GR0_ID6                0x38
 109 #define ARM_SMMU_GR0_ID7                0x3c
 110 #define ARM_SMMU_GR0_sGFSR              0x48
 111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 114
 115 #define ID0_S1TS                        (1 << 30)
 116 #define ID0_S2TS                        (1 << 29)
 117 #define ID0_NTS                         (1 << 28)
 118 #define ID0_SMS                         (1 << 27)
 119 #define ID0_ATOSNS                      (1 << 26)
 120 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 121 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 122 #define ID0_CTTW                        (1 << 14)
 123 #define ID0_NUMIRPT_SHIFT               16
 124 #define ID0_NUMIRPT_MASK                0xff
 125 #define ID0_NUMSIDB_SHIFT               9
 126 #define ID0_NUMSIDB_MASK                0xf
 127 #define ID0_NUMSMRG_SHIFT               0
 128 #define ID0_NUMSMRG_MASK                0xff
 129
 130 #define ID1_PAGESIZE                    (1 << 31)
 131 #define ID1_NUMPAGENDXB_SHIFT           28
 132 #define ID1_NUMPAGENDXB_MASK            7
 133 #define ID1_NUMS2CB_SHIFT               16
 134 #define ID1_NUMS2CB_MASK                0xff
 135 #define ID1_NUMCB_SHIFT                 0
 136 #define ID1_NUMCB_MASK                  0xff
 137
 138 #define ID2_OAS_SHIFT                   4
 139 #define ID2_OAS_MASK                    0xf
 140 #define ID2_IAS_SHIFT                   0
 141 #define ID2_IAS_MASK                    0xf
 142 #define ID2_UBS_SHIFT                   8
 143 #define ID2_UBS_MASK                    0xf
 144 #define ID2_PTFS_4K                     (1 << 12)
 145 #define ID2_PTFS_16K                    (1 << 13)
 146 #define ID2_PTFS_64K                    (1 << 14)
 147 #define ID2_VMID16                      (1 << 15)
 148
 149 #define ID7_MAJOR_SHIFT                 4
 150 #define ID7_MAJOR_MASK                  0xf
 151
 152 /* Global TLB invalidation */
 153 #define ARM_SMMU_GR0_TLBIVMID           0x64
 154 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 155 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 156 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 157 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 158 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 159 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 160
 161 /* Stream mapping registers */
 162 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 163 #define SMR_VALID                       (1 << 31)
 164 #define SMR_MASK_SHIFT                  16
 165 #define SMR_ID_SHIFT                    0
 166
 167 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 168 #define S2CR_CBNDX_SHIFT                0
 169 #define S2CR_CBNDX_MASK                 0xff
 170 #define S2CR_TYPE_SHIFT                 16
 171 #define S2CR_TYPE_MASK                  0x3
 172 enum arm_smmu_s2cr_type {
 173         S2CR_TYPE_TRANS,
 174         S2CR_TYPE_BYPASS,
 175         S2CR_TYPE_FAULT,
 176 };
 177
 178 #define S2CR_PRIVCFG_SHIFT              24
 179 #define S2CR_PRIVCFG_MASK               0x3
 180 enum arm_smmu_s2cr_privcfg {
 181         S2CR_PRIVCFG_DEFAULT,
 182         S2CR_PRIVCFG_DIPAN,
 183         S2CR_PRIVCFG_UNPRIV,
 184         S2CR_PRIVCFG_PRIV,
 185 };
 186
 187 /* Context bank attribute registers */
 188 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 189 #define CBAR_VMID_SHIFT                 0
 190 #define CBAR_VMID_MASK                  0xff
 191 #define CBAR_S1_BPSHCFG_SHIFT           8
 192 #define CBAR_S1_BPSHCFG_MASK            3
 193 #define CBAR_S1_BPSHCFG_NSH             3
 194 #define CBAR_S1_MEMATTR_SHIFT           12
 195 #define CBAR_S1_MEMATTR_MASK            0xf
 196 #define CBAR_S1_MEMATTR_WB              0xf
 197 #define CBAR_TYPE_SHIFT                 16
 198 #define CBAR_TYPE_MASK                  0x3
 199 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 201 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 203 #define CBAR_IRPTNDX_SHIFT              24
 204 #define CBAR_IRPTNDX_MASK               0xff
 205
 206 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 207 #define CBA2R_RW64_32BIT                (0 << 0)
 208 #define CBA2R_RW64_64BIT                (1 << 0)
 209 #define CBA2R_VMID_SHIFT                16
 210 #define CBA2R_VMID_MASK                 0xffff
 211
 212 /* Translation context bank */
 213 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 214 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 215
 216 #define ARM_SMMU_CB_SCTLR               0x0
 217 #define ARM_SMMU_CB_ACTLR               0x4
 218 #define ARM_SMMU_CB_RESUME              0x8
 219 #define ARM_SMMU_CB_TTBCR2              0x10
 220 #define ARM_SMMU_CB_TTBR0               0x20
 221 #define ARM_SMMU_CB_TTBR1               0x28
 222 #define ARM_SMMU_CB_TTBCR               0x30
 223 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 224 #define ARM_SMMU_CB_S1_MAIR0            0x38
 225 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 226 #define ARM_SMMU_CB_PAR                 0x50
 227 #define ARM_SMMU_CB_FSR                 0x58
 228 #define ARM_SMMU_CB_FAR                 0x60
 229 #define ARM_SMMU_CB_FSYNR0              0x68
 230 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 231 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 232 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 233 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 234 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 235 #define ARM_SMMU_CB_ATS1PR              0x800
 236 #define ARM_SMMU_CB_ATSR                0x8f0
 237
 238 #define SCTLR_S1_ASIDPNE                (1 << 12)
 239 #define SCTLR_CFCFG                     (1 << 7)
 240 #define SCTLR_CFIE                      (1 << 6)
 241 #define SCTLR_CFRE                      (1 << 5)
 242 #define SCTLR_E                         (1 << 4)
 243 #define SCTLR_AFE                       (1 << 2)
 244 #define SCTLR_TRE                       (1 << 1)
 245 #define SCTLR_M                         (1 << 0)
 246
 247 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 248
 249 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 250
 251 #define CB_PAR_F                        (1 << 0)
 252
 253 #define ATSR_ACTIVE                     (1 << 0)
 254
 255 #define RESUME_RETRY                    (0 << 0)
 256 #define RESUME_TERMINATE                (1 << 0)
 257
 258 #define TTBCR2_SEP_SHIFT                15
 259 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 260
 261 #define TTBRn_ASID_SHIFT                48
 262
 263 #define FSR_MULTI                       (1 << 31)
 264 #define FSR_SS                          (1 << 30)
 265 #define FSR_UUT                         (1 << 8)
 266 #define FSR_ASF                         (1 << 7)
 267 #define FSR_TLBLKF                      (1 << 6)
 268 #define FSR_TLBMCF                      (1 << 5)
 269 #define FSR_EF                          (1 << 4)
 270 #define FSR_PF                          (1 << 3)
 271 #define FSR_AFF                         (1 << 2)
 272 #define FSR_TF                          (1 << 1)
 273
 274 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 275                                          FSR_TLBMCF | FSR_TLBLKF)
 276 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 277                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 278
 279 #define FSYNR0_WNR                      (1 << 4)
 280
 281 static int force_stage;
 282 module_param(force_stage, int, S_IRUGO);
 283 MODULE_PARM_DESC(force_stage,
 284         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 285 static bool disable_bypass;
 286 module_param(disable_bypass, bool, S_IRUGO);
 287 MODULE_PARM_DESC(disable_bypass,
 288         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 289
 290 enum arm_smmu_arch_version {
 291         ARM_SMMU_V1,
 292         ARM_SMMU_V1_64K,
 293         ARM_SMMU_V2,
 294 };
 295
 296 enum arm_smmu_implementation {
 297         GENERIC_SMMU,
 298         ARM_MMU500,
 299         CAVIUM_SMMUV2,
 300         QCOM_SMMUV2,
 301 };
 302
 303 struct arm_smmu_s2cr {
 304         struct iommu_group              *group;
 305         int                             count;
 306         enum arm_smmu_s2cr_type         type;
 307         enum arm_smmu_s2cr_privcfg      privcfg;
 308         u8                              cbndx;
 309 };
 310
 311 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 312         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 313 }
 314
 315 struct arm_smmu_smr {
 316         u16                             mask;
 317         u16                             id;
 318         bool                            valid;
 319 };
 320
 321 struct arm_smmu_master_cfg {
 322         struct arm_smmu_device          *smmu;
 323         s16                             smendx[];
 324 };
 325 #define INVALID_SMENDX                  -1
 326 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 327 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 328 #define fwspec_smendx(fw, i) \
 329         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 330 #define for_each_cfg_sme(fw, i, idx) \
 331         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 332
 333 struct arm_smmu_device {
 334         struct device                   *dev;
 335
 336         void __iomem                    *base;
 337         unsigned long                   size;
 338         unsigned long                   pgshift;
 339
 340 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 341 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 342 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 343 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 344 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 345 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 346 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 347 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 348 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 349 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 350 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 351 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 352         u32                             features;
 353
 354 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 355         u32                             options;
 356         enum arm_smmu_arch_version      version;
 357         enum arm_smmu_implementation    model;
 358
 359         u32                             num_context_banks;
 360         u32                             num_s2_context_banks;
 361         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 362         atomic_t                        irptndx;
 363
 364         u32                             num_mapping_groups;
 365         u16                             streamid_mask;
 366         u16                             smr_mask_mask;
 367         struct arm_smmu_smr             *smrs;
 368         struct arm_smmu_s2cr            *s2crs;
 369         struct mutex                    stream_map_mutex;
 370
 371         unsigned long                   va_size;
 372         unsigned long                   ipa_size;
 373         unsigned long                   pa_size;
 374         unsigned long                   pgsize_bitmap;
 375
 376         u32                             num_global_irqs;
 377         u32                             num_context_irqs;
 378         unsigned int                    *irqs;
 379
 380         u32                             cavium_id_base; /* Specific to Cavium */
 381 };
 382
 383 enum arm_smmu_context_fmt {
 384         ARM_SMMU_CTX_FMT_NONE,
 385         ARM_SMMU_CTX_FMT_AARCH64,
 386         ARM_SMMU_CTX_FMT_AARCH32_L,
 387         ARM_SMMU_CTX_FMT_AARCH32_S,
 388 };
 389
 390 struct arm_smmu_cfg {
 391         u8                              cbndx;
 392         u8                              irptndx;
 393         u32                             cbar;
 394         enum arm_smmu_context_fmt       fmt;
 395 };
 396 #define INVALID_IRPTNDX                 0xff
 397
 398 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 399 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 400
 401 enum arm_smmu_domain_stage {
 402         ARM_SMMU_DOMAIN_S1 = 0,
 403         ARM_SMMU_DOMAIN_S2,
 404         ARM_SMMU_DOMAIN_NESTED,
 405 };
 406
 407 struct arm_smmu_domain {
 408         struct arm_smmu_device          *smmu;
 409         struct io_pgtable_ops           *pgtbl_ops;
 410         spinlock_t                      pgtbl_lock;
 411         struct arm_smmu_cfg             cfg;
 412         enum arm_smmu_domain_stage      stage;
 413         struct mutex                    init_mutex; /* Protects smmu pointer */
 414         struct iommu_domain             domain;
 415 };
 416
 417 struct arm_smmu_option_prop {
 418         u32 opt;
 419         const char *prop;
 420 };
 421
 422 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 423
 424 static bool using_legacy_binding, using_generic_binding;
 425
 426 static struct arm_smmu_option_prop arm_smmu_options[] = {
 427         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 428         { 0, NULL},
 429 };
 430
 431 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 432 {
 433         return container_of(dom, struct arm_smmu_domain, domain);
 434 }
 435
 436 static void parse_driver_options(struct arm_smmu_device *smmu)
 437 {
 438         int i = 0;
 439
 440         do {
 441                 if (of_property_read_bool(smmu->dev->of_node,
 442                                                 arm_smmu_options[i].prop)) {
 443                         smmu->options |= arm_smmu_options[i].opt;
 444                         dev_notice(smmu->dev, "option %s\n",
 445                                 arm_smmu_options[i].prop);
 446                 }
 447         } while (arm_smmu_options[++i].opt);
 448 }
 449
 450 static struct device_node *dev_get_dev_node(struct device *dev)
 451 {
 452         if (dev_is_pci(dev)) {
 453                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 454
 455                 while (!pci_is_root_bus(bus))
 456                         bus = bus->parent;
 457                 return of_node_get(bus->bridge->parent->of_node);
 458         }
 459
 460         return of_node_get(dev->of_node);
 461 }
 462
 463 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 464 {
 465         *((__be32 *)data) = cpu_to_be32(alias);
 466         return 0; /* Continue walking */
 467 }
 468
 469 static int __find_legacy_master_phandle(struct device *dev, void *data)
 470 {
 471         struct of_phandle_iterator *it = *(void **)data;
 472         struct device_node *np = it->node;
 473         int err;
 474
 475         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 476                             "#stream-id-cells", 0)
 477                 if (it->node == np) {
 478                         *(void **)data = dev;
 479                         return 1;
 480                 }
 481         it->node = np;
 482         return err == -ENOENT ? 0 : err;
 483 }
 484
 485 static struct platform_driver arm_smmu_driver;
 486 static struct iommu_ops arm_smmu_ops;
 487
 488 static int arm_smmu_register_legacy_master(struct device *dev,
 489                                            struct arm_smmu_device **smmu)
 490 {
 491         struct device *smmu_dev;
 492         struct device_node *np;
 493         struct of_phandle_iterator it;
 494         void *data = &it;
 495         u32 *sids;
 496         __be32 pci_sid;
 497         int err;
 498
 499         np = dev_get_dev_node(dev);
 500         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 501                 of_node_put(np);
 502                 return -ENODEV;
 503         }
 504
 505         it.node = np;
 506         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 507                                      __find_legacy_master_phandle);
 508         smmu_dev = data;
 509         of_node_put(np);
 510         if (err == 0)
 511                 return -ENODEV;
 512         if (err < 0)
 513                 return err;
 514
 515         if (dev_is_pci(dev)) {
 516                 /* "mmu-masters" assumes Stream ID == Requester ID */
 517                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 518                                        &pci_sid);
 519                 it.cur = &pci_sid;
 520                 it.cur_count = 1;
 521         }
 522
 523         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 524                                 &arm_smmu_ops);
 525         if (err)
 526                 return err;
 527
 528         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 529         if (!sids)
 530                 return -ENOMEM;
 531
 532         *smmu = dev_get_drvdata(smmu_dev);
 533         of_phandle_iterator_args(&it, sids, it.cur_count);
 534         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 535         kfree(sids);
 536         return err;
 537 }
 538
 539 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 540 {
 541         int idx;
 542
 543         do {
 544                 idx = find_next_zero_bit(map, end, start);
 545                 if (idx == end)
 546                         return -ENOSPC;
 547         } while (test_and_set_bit(idx, map));
 548
 549         return idx;
 550 }
 551
 552 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 553 {
 554         clear_bit(idx, map);
 555 }
 556
 557 /* Wait for any pending TLB invalidations to complete */
 558 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 559 {
 560         int count = 0;
 561         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 562
 563         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 564         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 565                & sTLBGSTATUS_GSACTIVE) {
 566                 cpu_relax();
 567                 if (++count == TLB_LOOP_TIMEOUT) {
 568                         dev_err_ratelimited(smmu->dev,
 569                         "TLB sync timed out -- SMMU may be deadlocked\n");
 570                         return;
 571                 }
 572                 udelay(1);
 573         }
 574 }
 575
 576 static void arm_smmu_tlb_sync(void *cookie)
 577 {
 578         struct arm_smmu_domain *smmu_domain = cookie;
 579         __arm_smmu_tlb_sync(smmu_domain->smmu);
 580 }
 581
 582 static void arm_smmu_tlb_inv_context(void *cookie)
 583 {
 584         struct arm_smmu_domain *smmu_domain = cookie;
 585         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 586         struct arm_smmu_device *smmu = smmu_domain->smmu;
 587         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 588         void __iomem *base;
 589
 590         if (stage1) {
 591                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 592                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 593                                base + ARM_SMMU_CB_S1_TLBIASID);
 594         } else {
 595                 base = ARM_SMMU_GR0(smmu);
 596                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 597                                base + ARM_SMMU_GR0_TLBIVMID);
 598         }
 599
 600         __arm_smmu_tlb_sync(smmu);
 601 }
 602
 603 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 604                                           size_t granule, bool leaf, void *cookie)
 605 {
 606         struct arm_smmu_domain *smmu_domain = cookie;
 607         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 608         struct arm_smmu_device *smmu = smmu_domain->smmu;
 609         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 610         void __iomem *reg;
 611
 612         if (stage1) {
 613                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 614                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 615
 616                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 617                         iova &= ~12UL;
 618                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 619                         do {
 620                                 writel_relaxed(iova, reg);
 621                                 iova += granule;
 622                         } while (size -= granule);
 623                 } else {
 624                         iova >>= 12;
 625                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 626                         do {
 627                                 writeq_relaxed(iova, reg);
 628                                 iova += granule >> 12;
 629                         } while (size -= granule);
 630                 }
 631         } else if (smmu->version == ARM_SMMU_V2) {
 632                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 633                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 634                               ARM_SMMU_CB_S2_TLBIIPAS2;
 635                 iova >>= 12;
 636                 do {
 637                         smmu_write_atomic_lq(iova, reg);
 638                         iova += granule >> 12;
 639                 } while (size -= granule);
 640         } else {
 641                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 642                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 643         }
 644 }
 645
 646 static struct iommu_gather_ops arm_smmu_gather_ops = {
 647         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 648         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 649         .tlb_sync       = arm_smmu_tlb_sync,
 650 };
 651
 652 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 653 {
 654         u32 fsr, fsynr;
 655         unsigned long iova;
 656         struct iommu_domain *domain = dev;
 657         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 658         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 659         struct arm_smmu_device *smmu = smmu_domain->smmu;
 660         void __iomem *cb_base;
 661
 662         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 663         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 664
 665         if (!(fsr & FSR_FAULT))
 666                 return IRQ_NONE;
 667
 668         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 669         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 670
 671         dev_err_ratelimited(smmu->dev,
 672         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 673                             fsr, iova, fsynr, cfg->cbndx);
 674
 675         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 676         return IRQ_HANDLED;
 677 }
 678
 679 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 680 {
 681         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 682         struct arm_smmu_device *smmu = dev;
 683         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 684
 685         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 686         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 687         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 688         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 689
 690         if (!gfsr)
 691                 return IRQ_NONE;
 692
 693         dev_err_ratelimited(smmu->dev,
 694                 "Unexpected global fault, this could be serious\n");
 695         dev_err_ratelimited(smmu->dev,
 696                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 697                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 698
 699         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 700         return IRQ_HANDLED;
 701 }
 702
 703 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 704                                        struct io_pgtable_cfg *pgtbl_cfg)
 705 {
 706         u32 reg, reg2;
 707         u64 reg64;
 708         bool stage1;
 709         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 710         struct arm_smmu_device *smmu = smmu_domain->smmu;
 711         void __iomem *cb_base, *gr1_base;
 712
 713         gr1_base = ARM_SMMU_GR1(smmu);
 714         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 715         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 716
 717         if (smmu->version > ARM_SMMU_V1) {
 718                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 719                         reg = CBA2R_RW64_64BIT;
 720                 else
 721                         reg = CBA2R_RW64_32BIT;
 722                 /* 16-bit VMIDs live in CBA2R */
 723                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 724                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 725
 726                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 727         }
 728
 729         /* CBAR */
 730         reg = cfg->cbar;
 731         if (smmu->version < ARM_SMMU_V2)
 732                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 733
 734         /*
 735          * Use the weakest shareability/memory types, so they are
 736          * overridden by the ttbcr/pte.
 737          */
 738         if (stage1) {
 739                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 740                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 741         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 742                 /* 8-bit VMIDs live in CBAR */
 743                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 744         }
 745         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 746
 747         /* TTBRs */
 748         if (stage1) {
 749                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 750
 751                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 752                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 753                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 754                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 755                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 756                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 757                 } else {
 758                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 759                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 760                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 761                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 762                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 763                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 764                 }
 765         } else {
 766                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 767                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 768         }
 769
 770         /* TTBCR */
 771         if (stage1) {
 772                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 773                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 774                         reg2 = 0;
 775                 } else {
 776                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 777                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 778                         reg2 |= TTBCR2_SEP_UPSTREAM;
 779                 }
 780                 if (smmu->version > ARM_SMMU_V1)
 781                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 782         } else {
 783                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 784         }
 785         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 786
 787         /* MAIRs (stage-1 only) */
 788         if (stage1) {
 789                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 790                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 791                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 792                 } else {
 793                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 794                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 795                 }
 796                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 797                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 798         }
 799
 800         /* SCTLR */
 801         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 802         if (stage1)
 803                 reg |= SCTLR_S1_ASIDPNE;
 804 #ifdef __BIG_ENDIAN
 805         reg |= SCTLR_E;
 806 #endif
 807         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 808 }
 809
 810 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 811                                         struct arm_smmu_device *smmu)
 812 {
 813         int irq, start, ret = 0;
 814         unsigned long ias, oas;
 815         struct io_pgtable_ops *pgtbl_ops;
 816         struct io_pgtable_cfg pgtbl_cfg;
 817         enum io_pgtable_fmt fmt;
 818         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 819         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 820
 821         mutex_lock(&smmu_domain->init_mutex);
 822         if (smmu_domain->smmu)
 823                 goto out_unlock;
 824
 825         /*
 826          * Mapping the requested stage onto what we support is surprisingly
 827          * complicated, mainly because the spec allows S1+S2 SMMUs without
 828          * support for nested translation. That means we end up with the
 829          * following table:
 830          *
 831          * Requested        Supported        Actual
 832          *     S1               N              S1
 833          *     S1             S1+S2            S1
 834          *     S1               S2             S2
 835          *     S1               S1             S1
 836          *     N                N              N
 837          *     N              S1+S2            S2
 838          *     N                S2             S2
 839          *     N                S1             S1
 840          *
 841          * Note that you can't actually request stage-2 mappings.
 842          */
 843         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 844                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 845         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 846                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 847
 848         /*
 849          * Choosing a suitable context format is even more fiddly. Until we
 850          * grow some way for the caller to express a preference, and/or move
 851          * the decision into the io-pgtable code where it arguably belongs,
 852          * just aim for the closest thing to the rest of the system, and hope
 853          * that the hardware isn't esoteric enough that we can't assume AArch64
 854          * support to be a superset of AArch32 support...
 855          */
 856         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 857                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 858         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 859             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 860             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 861             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 862                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 863         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 864             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 865                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 866                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 867                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 868
 869         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 870                 ret = -EINVAL;
 871                 goto out_unlock;
 872         }
 873
 874         switch (smmu_domain->stage) {
 875         case ARM_SMMU_DOMAIN_S1:
 876                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 877                 start = smmu->num_s2_context_banks;
 878                 ias = smmu->va_size;
 879                 oas = smmu->ipa_size;
 880                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 881                         fmt = ARM_64_LPAE_S1;
 882                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 883                         fmt = ARM_32_LPAE_S1;
 884                         ias = min(ias, 32UL);
 885                         oas = min(oas, 40UL);
 886                 } else {
 887                         fmt = ARM_V7S;
 888                         ias = min(ias, 32UL);
 889                         oas = min(oas, 32UL);
 890                 }
 891                 break;
 892         case ARM_SMMU_DOMAIN_NESTED:
 893                 /*
 894                  * We will likely want to change this if/when KVM gets
 895                  * involved.
 896                  */
 897         case ARM_SMMU_DOMAIN_S2:
 898                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 899                 start = 0;
 900                 ias = smmu->ipa_size;
 901                 oas = smmu->pa_size;
 902                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 903                         fmt = ARM_64_LPAE_S2;
 904                 } else {
 905                         fmt = ARM_32_LPAE_S2;
 906                         ias = min(ias, 40UL);
 907                         oas = min(oas, 40UL);
 908                 }
 909                 break;
 910         default:
 911                 ret = -EINVAL;
 912                 goto out_unlock;
 913         }
 914
 915         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 916                                       smmu->num_context_banks);
 917         if (ret < 0)
 918                 goto out_unlock;
 919
 920         cfg->cbndx = ret;
 921         if (smmu->version < ARM_SMMU_V2) {
 922                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 923                 cfg->irptndx %= smmu->num_context_irqs;
 924         } else {
 925                 cfg->irptndx = cfg->cbndx;
 926         }
 927
 928         pgtbl_cfg = (struct io_pgtable_cfg) {
 929                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 930                 .ias            = ias,
 931                 .oas            = oas,
 932                 .tlb            = &arm_smmu_gather_ops,
 933                 .iommu_dev      = smmu->dev,
 934         };
 935
 936         smmu_domain->smmu = smmu;
 937         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 938         if (!pgtbl_ops) {
 939                 ret = -ENOMEM;
 940                 goto out_clear_smmu;
 941         }
 942
 943         /* Update the domain's page sizes to reflect the page table format */
 944         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 945         domain->geometry.aperture_end = (1UL << ias) - 1;
 946         domain->geometry.force_aperture = true;
 947
 948         /* Initialise the context bank with our page table cfg */
 949         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 950
 951         /*
 952          * Request context fault interrupt. Do this last to avoid the
 953          * handler seeing a half-initialised domain state.
 954          */
 955         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 956         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 957                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 958         if (ret < 0) {
 959                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 960                         cfg->irptndx, irq);
 961                 cfg->irptndx = INVALID_IRPTNDX;
 962         }
 963
 964         mutex_unlock(&smmu_domain->init_mutex);
 965
 966         /* Publish page table ops for map/unmap */
 967         smmu_domain->pgtbl_ops = pgtbl_ops;
 968         return 0;
 969
 970 out_clear_smmu:
 971         smmu_domain->smmu = NULL;
 972 out_unlock:
 973         mutex_unlock(&smmu_domain->init_mutex);
 974         return ret;
 975 }
 976
 977 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 978 {
 979         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 980         struct arm_smmu_device *smmu = smmu_domain->smmu;
 981         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 982         void __iomem *cb_base;
 983         int irq;
 984
 985         if (!smmu)
 986                 return;
 987
 988         /*
 989          * Disable the context bank and free the page tables before freeing
 990          * it.
 991          */
 992         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 993         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 994
 995         if (cfg->irptndx != INVALID_IRPTNDX) {
 996                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 997                 devm_free_irq(smmu->dev, irq, domain);
 998         }
 999
1000         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1001         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1002 }
1003
1004 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1005 {
1006         struct arm_smmu_domain *smmu_domain;
1007
1008         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1009                 return NULL;
1010         /*
1011          * Allocate the domain and initialise some of its data structures.
1012          * We can't really do anything meaningful until we've added a
1013          * master.
1014          */
1015         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1016         if (!smmu_domain)
1017                 return NULL;
1018
1019         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1020             iommu_get_dma_cookie(&smmu_domain->domain))) {
1021                 kfree(smmu_domain);
1022                 return NULL;
1023         }
1024
1025         mutex_init(&smmu_domain->init_mutex);
1026         spin_lock_init(&smmu_domain->pgtbl_lock);
1027
1028         return &smmu_domain->domain;
1029 }
1030
1031 static void arm_smmu_domain_free(struct iommu_domain *domain)
1032 {
1033         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1034
1035         /*
1036          * Free the domain resources. We assume that all devices have
1037          * already been detached.
1038          */
1039         iommu_put_dma_cookie(domain);
1040         arm_smmu_destroy_domain_context(domain);
1041         kfree(smmu_domain);
1042 }
1043
1044 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1045 {
1046         struct arm_smmu_smr *smr = smmu->smrs + idx;
1047         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1048
1049         if (smr->valid)
1050                 reg |= SMR_VALID;
1051         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1052 }
1053
1054 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1055 {
1056         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1057         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1058                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1059                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1060
1061         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1062 }
1063
1064 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1065 {
1066         arm_smmu_write_s2cr(smmu, idx);
1067         if (smmu->smrs)
1068                 arm_smmu_write_smr(smmu, idx);
1069 }
1070
1071 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1072 {
1073         struct arm_smmu_smr *smrs = smmu->smrs;
1074         int i, free_idx = -ENOSPC;
1075
1076         /* Stream indexing is blissfully easy */
1077         if (!smrs)
1078                 return id;
1079
1080         /* Validating SMRs is... less so */
1081         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1082                 if (!smrs[i].valid) {
1083                         /*
1084                          * Note the first free entry we come across, which
1085                          * we'll claim in the end if nothing else matches.
1086                          */
1087                         if (free_idx < 0)
1088                                 free_idx = i;
1089                         continue;
1090                 }
1091                 /*
1092                  * If the new entry is _entirely_ matched by an existing entry,
1093                  * then reuse that, with the guarantee that there also cannot
1094                  * be any subsequent conflicting entries. In normal use we'd
1095                  * expect simply identical entries for this case, but there's
1096                  * no harm in accommodating the generalisation.
1097                  */
1098                 if ((mask & smrs[i].mask) == mask &&
1099                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1100                         return i;
1101                 /*
1102                  * If the new entry has any other overlap with an existing one,
1103                  * though, then there always exists at least one stream ID
1104                  * which would cause a conflict, and we can't allow that risk.
1105                  */
1106                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1107                         return -EINVAL;
1108         }
1109
1110         return free_idx;
1111 }
1112
1113 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1114 {
1115         if (--smmu->s2crs[idx].count)
1116                 return false;
1117
1118         smmu->s2crs[idx] = s2cr_init_val;
1119         if (smmu->smrs)
1120                 smmu->smrs[idx].valid = false;
1121
1122         return true;
1123 }
1124
1125 static int arm_smmu_master_alloc_smes(struct device *dev)
1126 {
1127         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1128         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1129         struct arm_smmu_device *smmu = cfg->smmu;
1130         struct arm_smmu_smr *smrs = smmu->smrs;
1131         struct iommu_group *group;
1132         int i, idx, ret;
1133
1134         mutex_lock(&smmu->stream_map_mutex);
1135         /* Figure out a viable stream map entry allocation */
1136         for_each_cfg_sme(fwspec, i, idx) {
1137                 u16 sid = fwspec->ids[i];
1138                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1139
1140                 if (idx != INVALID_SMENDX) {
1141                         ret = -EEXIST;
1142                         goto out_err;
1143                 }
1144
1145                 ret = arm_smmu_find_sme(smmu, sid, mask);
1146                 if (ret < 0)
1147                         goto out_err;
1148
1149                 idx = ret;
1150                 if (smrs && smmu->s2crs[idx].count == 0) {
1151                         smrs[idx].id = sid;
1152                         smrs[idx].mask = mask;
1153                         smrs[idx].valid = true;
1154                 }
1155                 smmu->s2crs[idx].count++;
1156                 cfg->smendx[i] = (s16)idx;
1157         }
1158
1159         group = iommu_group_get_for_dev(dev);
1160         if (!group)
1161                 group = ERR_PTR(-ENOMEM);
1162         if (IS_ERR(group)) {
1163                 ret = PTR_ERR(group);
1164                 goto out_err;
1165         }
1166         iommu_group_put(group);
1167
1168         /* It worked! Now, poke the actual hardware */
1169         for_each_cfg_sme(fwspec, i, idx) {
1170                 arm_smmu_write_sme(smmu, idx);
1171                 smmu->s2crs[idx].group = group;
1172         }
1173
1174         mutex_unlock(&smmu->stream_map_mutex);
1175         return 0;
1176
1177 out_err:
1178         while (i--) {
1179                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1180                 cfg->smendx[i] = INVALID_SMENDX;
1181         }
1182         mutex_unlock(&smmu->stream_map_mutex);
1183         return ret;
1184 }
1185
1186 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1187 {
1188         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1189         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1190         int i, idx;
1191
1192         mutex_lock(&smmu->stream_map_mutex);
1193         for_each_cfg_sme(fwspec, i, idx) {
1194                 if (arm_smmu_free_sme(smmu, idx))
1195                         arm_smmu_write_sme(smmu, idx);
1196                 cfg->smendx[i] = INVALID_SMENDX;
1197         }
1198         mutex_unlock(&smmu->stream_map_mutex);
1199 }
1200
1201 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1202                                       struct iommu_fwspec *fwspec)
1203 {
1204         struct arm_smmu_device *smmu = smmu_domain->smmu;
1205         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1206         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1207         u8 cbndx = smmu_domain->cfg.cbndx;
1208         int i, idx;
1209
1210         for_each_cfg_sme(fwspec, i, idx) {
1211                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1212                         continue;
1213
1214                 s2cr[idx].type = type;
1215                 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1216                 s2cr[idx].cbndx = cbndx;
1217                 arm_smmu_write_s2cr(smmu, idx);
1218         }
1219         return 0;
1220 }
1221
1222 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1223 {
1224         int ret;
1225         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1226         struct arm_smmu_device *smmu;
1227         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1228
1229         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1230                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1231                 return -ENXIO;
1232         }
1233
1234         /*
1235          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1236          * domains between of_xlate() and add_device() - we have no way to cope
1237          * with that, so until ARM gets converted to rely on groups and default
1238          * domains, just say no (but more politely than by dereferencing NULL).
1239          * This should be at least a WARN_ON once that's sorted.
1240          */
1241         if (!fwspec->iommu_priv)
1242                 return -ENODEV;
1243
1244         smmu = fwspec_smmu(fwspec);
1245         /* Ensure that the domain is finalised */
1246         ret = arm_smmu_init_domain_context(domain, smmu);
1247         if (ret < 0)
1248                 return ret;
1249
1250         /*
1251          * Sanity check the domain. We don't support domains across
1252          * different SMMUs.
1253          */
1254         if (smmu_domain->smmu != smmu) {
1255                 dev_err(dev,
1256                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1257                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1258                 return -EINVAL;
1259         }
1260
1261         /* Looks ok, so add the device to the domain */
1262         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1263 }
1264
1265 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1266                         phys_addr_t paddr, size_t size, int prot)
1267 {
1268         int ret;
1269         unsigned long flags;
1270         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1271         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1272
1273         if (!ops)
1274                 return -ENODEV;
1275
1276         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1277         ret = ops->map(ops, iova, paddr, size, prot);
1278         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1279         return ret;
1280 }
1281
1282 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1283                              size_t size)
1284 {
1285         size_t ret;
1286         unsigned long flags;
1287         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1288         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1289
1290         if (!ops)
1291                 return 0;
1292
1293         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1294         ret = ops->unmap(ops, iova, size);
1295         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1296         return ret;
1297 }
1298
1299 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1300                                               dma_addr_t iova)
1301 {
1302         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1303         struct arm_smmu_device *smmu = smmu_domain->smmu;
1304         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1305         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1306         struct device *dev = smmu->dev;
1307         void __iomem *cb_base;
1308         u32 tmp;
1309         u64 phys;
1310         unsigned long va;
1311
1312         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1313
1314         /* ATS1 registers can only be written atomically */
1315         va = iova & ~0xfffUL;
1316         if (smmu->version == ARM_SMMU_V2)
1317                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1318         else /* Register is only 32-bit in v1 */
1319                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1320
1321         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1322                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1323                 dev_err(dev,
1324                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1325                         &iova);
1326                 return ops->iova_to_phys(ops, iova);
1327         }
1328
1329         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1330         if (phys & CB_PAR_F) {
1331                 dev_err(dev, "translation fault!\n");
1332                 dev_err(dev, "PAR = 0x%llx\n", phys);
1333                 return 0;
1334         }
1335
1336         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1337 }
1338
1339 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1340                                         dma_addr_t iova)
1341 {
1342         phys_addr_t ret;
1343         unsigned long flags;
1344         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1345         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1346
1347         if (!ops)
1348                 return 0;
1349
1350         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1351         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1352                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1353                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1354         } else {
1355                 ret = ops->iova_to_phys(ops, iova);
1356         }
1357
1358         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1359
1360         return ret;
1361 }
1362
1363 static bool arm_smmu_capable(enum iommu_cap cap)
1364 {
1365         switch (cap) {
1366         case IOMMU_CAP_CACHE_COHERENCY:
1367                 /*
1368                  * Return true here as the SMMU can always send out coherent
1369                  * requests.
1370                  */
1371                 return true;
1372         case IOMMU_CAP_INTR_REMAP:
1373                 return true; /* MSIs are just memory writes */
1374         case IOMMU_CAP_NOEXEC:
1375                 return true;
1376         default:
1377                 return false;
1378         }
1379 }
1380
1381 static int arm_smmu_match_node(struct device *dev, void *data)
1382 {
1383         return dev->of_node == data;
1384 }
1385
1386 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1387 {
1388         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1389                                                 np, arm_smmu_match_node);
1390         put_device(dev);
1391         return dev ? dev_get_drvdata(dev) : NULL;
1392 }
1393
1394 static int arm_smmu_add_device(struct device *dev)
1395 {
1396         struct arm_smmu_device *smmu;
1397         struct arm_smmu_master_cfg *cfg;
1398         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1399         int i, ret;
1400
1401         if (using_legacy_binding) {
1402                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1403                 fwspec = dev->iommu_fwspec;
1404                 if (ret)
1405                         goto out_free;
1406         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1407                 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1408         } else {
1409                 return -ENODEV;
1410         }
1411
1412         ret = -EINVAL;
1413         for (i = 0; i < fwspec->num_ids; i++) {
1414                 u16 sid = fwspec->ids[i];
1415                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1416
1417                 if (sid & ~smmu->streamid_mask) {
1418                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1419                                 sid, smmu->streamid_mask);
1420                         goto out_free;
1421                 }
1422                 if (mask & ~smmu->smr_mask_mask) {
1423                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1424                                 sid, smmu->smr_mask_mask);
1425                         goto out_free;
1426                 }
1427         }
1428
1429         ret = -ENOMEM;
1430         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1431                       GFP_KERNEL);
1432         if (!cfg)
1433                 goto out_free;
1434
1435         cfg->smmu = smmu;
1436         fwspec->iommu_priv = cfg;
1437         while (i--)
1438                 cfg->smendx[i] = INVALID_SMENDX;
1439
1440         ret = arm_smmu_master_alloc_smes(dev);
1441         if (ret)
1442                 goto out_free;
1443
1444         return 0;
1445
1446 out_free:
1447         if (fwspec)
1448                 kfree(fwspec->iommu_priv);
1449         iommu_fwspec_free(dev);
1450         return ret;
1451 }
1452
1453 static void arm_smmu_remove_device(struct device *dev)
1454 {
1455         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1456
1457         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1458                 return;
1459
1460         arm_smmu_master_free_smes(fwspec);
1461         iommu_group_remove_device(dev);
1462         kfree(fwspec->iommu_priv);
1463         iommu_fwspec_free(dev);
1464 }
1465
1466 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1467 {
1468         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1469         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1470         struct iommu_group *group = NULL;
1471         int i, idx;
1472
1473         for_each_cfg_sme(fwspec, i, idx) {
1474                 if (group && smmu->s2crs[idx].group &&
1475                     group != smmu->s2crs[idx].group)
1476                         return ERR_PTR(-EINVAL);
1477
1478                 group = smmu->s2crs[idx].group;
1479         }
1480
1481         if (group)
1482                 return group;
1483
1484         if (dev_is_pci(dev))
1485                 group = pci_device_group(dev);
1486         else
1487                 group = generic_device_group(dev);
1488
1489         return group;
1490 }
1491
1492 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1493                                     enum iommu_attr attr, void *data)
1494 {
1495         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1496
1497         switch (attr) {
1498         case DOMAIN_ATTR_NESTING:
1499                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1500                 return 0;
1501         default:
1502                 return -ENODEV;
1503         }
1504 }
1505
1506 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1507                                     enum iommu_attr attr, void *data)
1508 {
1509         int ret = 0;
1510         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1511
1512         mutex_lock(&smmu_domain->init_mutex);
1513
1514         switch (attr) {
1515         case DOMAIN_ATTR_NESTING:
1516                 if (smmu_domain->smmu) {
1517                         ret = -EPERM;
1518                         goto out_unlock;
1519                 }
1520
1521                 if (*(int *)data)
1522                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1523                 else
1524                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1525
1526                 break;
1527         default:
1528                 ret = -ENODEV;
1529         }
1530
1531 out_unlock:
1532         mutex_unlock(&smmu_domain->init_mutex);
1533         return ret;
1534 }
1535
1536 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1537 {
1538         u32 fwid = 0;
1539
1540         if (args->args_count > 0)
1541                 fwid |= (u16)args->args[0];
1542
1543         if (args->args_count > 1)
1544                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1545
1546         return iommu_fwspec_add_ids(dev, &fwid, 1);
1547 }
1548
1549 static struct iommu_ops arm_smmu_ops = {
1550         .capable                = arm_smmu_capable,
1551         .domain_alloc           = arm_smmu_domain_alloc,
1552         .domain_free            = arm_smmu_domain_free,
1553         .attach_dev             = arm_smmu_attach_dev,
1554         .map                    = arm_smmu_map,
1555         .unmap                  = arm_smmu_unmap,
1556         .map_sg                 = default_iommu_map_sg,
1557         .iova_to_phys           = arm_smmu_iova_to_phys,
1558         .add_device             = arm_smmu_add_device,
1559         .remove_device          = arm_smmu_remove_device,
1560         .device_group           = arm_smmu_device_group,
1561         .domain_get_attr        = arm_smmu_domain_get_attr,
1562         .domain_set_attr        = arm_smmu_domain_set_attr,
1563         .of_xlate               = arm_smmu_of_xlate,
1564         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1565 };
1566
1567 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1568 {
1569         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1570         void __iomem *cb_base;
1571         int i;
1572         u32 reg, major;
1573
1574         /* clear global FSR */
1575         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1576         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1577
1578         /*
1579          * Reset stream mapping groups: Initial values mark all SMRn as
1580          * invalid and all S2CRn as bypass unless overridden.
1581          */
1582         for (i = 0; i < smmu->num_mapping_groups; ++i)
1583                 arm_smmu_write_sme(smmu, i);
1584
1585         /*
1586          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1587          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1588          * bit is only present in MMU-500r2 onwards.
1589          */
1590         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1591         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1592         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1593                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1594                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1595                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1596         }
1597
1598         /* Make sure all context banks are disabled and clear CB_FSR  */
1599         for (i = 0; i < smmu->num_context_banks; ++i) {
1600                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1601                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1602                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1603                 /*
1604                  * Disable MMU-500's not-particularly-beneficial next-page
1605                  * prefetcher for the sake of errata #841119 and #826419.
1606                  */
1607                 if (smmu->model == ARM_MMU500) {
1608                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1609                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1610                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1611                 }
1612         }
1613
1614         /* Invalidate the TLB, just in case */
1615         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1616         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1617
1618         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1619
1620         /* Enable fault reporting */
1621         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1622
1623         /* Disable TLB broadcasting. */
1624         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1625
1626         /* Enable client access, handling unmatched streams as appropriate */
1627         reg &= ~sCR0_CLIENTPD;
1628         if (disable_bypass)
1629                 reg |= sCR0_USFCFG;
1630         else
1631                 reg &= ~sCR0_USFCFG;
1632
1633         /* Disable forced broadcasting */
1634         reg &= ~sCR0_FB;
1635
1636         /* Don't upgrade barriers */
1637         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1638
1639         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1640                 reg |= sCR0_VMID16EN;
1641
1642         /* Push the button */
1643         __arm_smmu_tlb_sync(smmu);
1644         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1645 }
1646
1647 static int arm_smmu_id_size_to_bits(int size)
1648 {
1649         switch (size) {
1650         case 0:
1651                 return 32;
1652         case 1:
1653                 return 36;
1654         case 2:
1655                 return 40;
1656         case 3:
1657                 return 42;
1658         case 4:
1659                 return 44;
1660         case 5:
1661         default:
1662                 return 48;
1663         }
1664 }
1665
1666 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1667 {
1668         unsigned long size;
1669         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1670         u32 id;
1671         bool cttw_dt, cttw_reg;
1672         int i;
1673
1674         dev_notice(smmu->dev, "probing hardware configuration...\n");
1675         dev_notice(smmu->dev, "SMMUv%d with:\n",
1676                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1677
1678         /* ID0 */
1679         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1680
1681         /* Restrict available stages based on module parameter */
1682         if (force_stage == 1)
1683                 id &= ~(ID0_S2TS | ID0_NTS);
1684         else if (force_stage == 2)
1685                 id &= ~(ID0_S1TS | ID0_NTS);
1686
1687         if (id & ID0_S1TS) {
1688                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1689                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1690         }
1691
1692         if (id & ID0_S2TS) {
1693                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1694                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1695         }
1696
1697         if (id & ID0_NTS) {
1698                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1699                 dev_notice(smmu->dev, "\tnested translation\n");
1700         }
1701
1702         if (!(smmu->features &
1703                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1704                 dev_err(smmu->dev, "\tno translation support!\n");
1705                 return -ENODEV;
1706         }
1707
1708         if ((id & ID0_S1TS) &&
1709                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1710                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1711                 dev_notice(smmu->dev, "\taddress translation ops\n");
1712         }
1713
1714         /*
1715          * In order for DMA API calls to work properly, we must defer to what
1716          * the DT says about coherency, regardless of what the hardware claims.
1717          * Fortunately, this also opens up a workaround for systems where the
1718          * ID register value has ended up configured incorrectly.
1719          */
1720         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1721         cttw_reg = !!(id & ID0_CTTW);
1722         if (cttw_dt)
1723                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1724         if (cttw_dt || cttw_reg)
1725                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1726                            cttw_dt ? "" : "non-");
1727         if (cttw_dt != cttw_reg)
1728                 dev_notice(smmu->dev,
1729                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1730
1731         /* Max. number of entries we have for stream matching/indexing */
1732         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1733         smmu->streamid_mask = size - 1;
1734         if (id & ID0_SMS) {
1735                 u32 smr;
1736
1737                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1738                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1739                 if (size == 0) {
1740                         dev_err(smmu->dev,
1741                                 "stream-matching supported, but no SMRs present!\n");
1742                         return -ENODEV;
1743                 }
1744
1745                 /*
1746                  * SMR.ID bits may not be preserved if the corresponding MASK
1747                  * bits are set, so check each one separately. We can reject
1748                  * masters later if they try to claim IDs outside these masks.
1749                  */
1750                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1751                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1752                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1753                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1754
1755                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1756                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1757                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1758                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1759
1760                 /* Zero-initialised to mark as invalid */
1761                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1762                                           GFP_KERNEL);
1763                 if (!smmu->smrs)
1764                         return -ENOMEM;
1765
1766                 dev_notice(smmu->dev,
1767                            "\tstream matching with %lu register groups, mask 0x%x",
1768                            size, smmu->smr_mask_mask);
1769         }
1770         /* s2cr->type == 0 means translation, so initialise explicitly */
1771         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1772                                          GFP_KERNEL);
1773         if (!smmu->s2crs)
1774                 return -ENOMEM;
1775         for (i = 0; i < size; i++)
1776                 smmu->s2crs[i] = s2cr_init_val;
1777
1778         smmu->num_mapping_groups = size;
1779         mutex_init(&smmu->stream_map_mutex);
1780
1781         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1782                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1783                 if (!(id & ID0_PTFS_NO_AARCH32S))
1784                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1785         }
1786
1787         /* ID1 */
1788         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1789         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1790
1791         /* Check for size mismatch of SMMU address space from mapped region */
1792         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1793         size *= 2 << smmu->pgshift;
1794         if (smmu->size != size)
1795                 dev_warn(smmu->dev,
1796                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1797                         size, smmu->size);
1798
1799         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1800         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1801         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1802                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1803                 return -ENODEV;
1804         }
1805         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1806                    smmu->num_context_banks, smmu->num_s2_context_banks);
1807         /*
1808          * Cavium CN88xx erratum #27704.
1809          * Ensure ASID and VMID allocation is unique across all SMMUs in
1810          * the system.
1811          */
1812         if (smmu->model == CAVIUM_SMMUV2) {
1813                 smmu->cavium_id_base =
1814                         atomic_add_return(smmu->num_context_banks,
1815                                           &cavium_smmu_context_count);
1816                 smmu->cavium_id_base -= smmu->num_context_banks;
1817         }
1818
1819         /* ID2 */
1820         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1821         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1822         smmu->ipa_size = size;
1823
1824         /* The output mask is also applied for bypass */
1825         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1826         smmu->pa_size = size;
1827
1828         if (id & ID2_VMID16)
1829                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1830
1831         /*
1832          * What the page table walker can address actually depends on which
1833          * descriptor format is in use, but since a) we don't know that yet,
1834          * and b) it can vary per context bank, this will have to do...
1835          */
1836         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1837                 dev_warn(smmu->dev,
1838                          "failed to set DMA mask for table walker\n");
1839
1840         if (smmu->version < ARM_SMMU_V2) {
1841                 smmu->va_size = smmu->ipa_size;
1842                 if (smmu->version == ARM_SMMU_V1_64K)
1843                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1844         } else {
1845                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1846                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1847                 if (id & ID2_PTFS_4K)
1848                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1849                 if (id & ID2_PTFS_16K)
1850                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1851                 if (id & ID2_PTFS_64K)
1852                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1853         }
1854
1855         /* Now we've corralled the various formats, what'll it do? */
1856         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1857                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1858         if (smmu->features &
1859             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1860                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1861         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1862                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1863         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1864                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1865
1866         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1867                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1868         else
1869                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1870         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1871                    smmu->pgsize_bitmap);
1872
1873
1874         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1875                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1876                            smmu->va_size, smmu->ipa_size);
1877
1878         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1879                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1880                            smmu->ipa_size, smmu->pa_size);
1881
1882         return 0;
1883 }
1884
1885 struct arm_smmu_match_data {
1886         enum arm_smmu_arch_version version;
1887         enum arm_smmu_implementation model;
1888 };
1889
1890 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1891 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1892
1893 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1894 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1895 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1896 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1897 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1898 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1899
1900 static const struct of_device_id arm_smmu_of_match[] = {
1901         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1902         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1903         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1904         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1905         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1906         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1907         { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1908         { },
1909 };
1910 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1911
1912 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1913 {
1914         const struct arm_smmu_match_data *data;
1915         struct resource *res;
1916         struct arm_smmu_device *smmu;
1917         struct device *dev = &pdev->dev;
1918         int num_irqs, i, err;
1919         bool legacy_binding;
1920
1921         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1922         if (legacy_binding && !using_generic_binding) {
1923                 if (!using_legacy_binding)
1924                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1925                 using_legacy_binding = true;
1926         } else if (!legacy_binding && !using_legacy_binding) {
1927                 using_generic_binding = true;
1928         } else {
1929                 dev_err(dev, "not probing due to mismatched DT properties\n");
1930                 return -ENODEV;
1931         }
1932
1933         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1934         if (!smmu) {
1935                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1936                 return -ENOMEM;
1937         }
1938         smmu->dev = dev;
1939
1940         data = of_device_get_match_data(dev);
1941         smmu->version = data->version;
1942         smmu->model = data->model;
1943
1944         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1945         smmu->base = devm_ioremap_resource(dev, res);
1946         if (IS_ERR(smmu->base))
1947                 return PTR_ERR(smmu->base);
1948         smmu->size = resource_size(res);
1949
1950         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1951                                  &smmu->num_global_irqs)) {
1952                 dev_err(dev, "missing #global-interrupts property\n");
1953                 return -ENODEV;
1954         }
1955
1956         num_irqs = 0;
1957         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1958                 num_irqs++;
1959                 if (num_irqs > smmu->num_global_irqs)
1960                         smmu->num_context_irqs++;
1961         }
1962
1963         if (!smmu->num_context_irqs) {
1964                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1965                         num_irqs, smmu->num_global_irqs + 1);
1966                 return -ENODEV;
1967         }
1968
1969         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1970                                   GFP_KERNEL);
1971         if (!smmu->irqs) {
1972                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1973                 return -ENOMEM;
1974         }
1975
1976         for (i = 0; i < num_irqs; ++i) {
1977                 int irq = platform_get_irq(pdev, i);
1978
1979                 if (irq < 0) {
1980                         dev_err(dev, "failed to get irq index %d\n", i);
1981                         return -ENODEV;
1982                 }
1983                 smmu->irqs[i] = irq;
1984         }
1985
1986         err = arm_smmu_device_cfg_probe(smmu);
1987         if (err)
1988                 return err;
1989
1990         parse_driver_options(smmu);
1991
1992         if (smmu->version == ARM_SMMU_V2 &&
1993             smmu->num_context_banks != smmu->num_context_irqs) {
1994                 dev_err(dev,
1995                         "found only %d context interrupt(s) but %d required\n",
1996                         smmu->num_context_irqs, smmu->num_context_banks);
1997                 return -ENODEV;
1998         }
1999
2000         for (i = 0; i < smmu->num_global_irqs; ++i) {
2001                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
2002                                        arm_smmu_global_fault,
2003                                        IRQF_SHARED,
2004                                        "arm-smmu global fault",
2005                                        smmu);
2006                 if (err) {
2007                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2008                                 i, smmu->irqs[i]);
2009                         return err;
2010                 }
2011         }
2012
2013         of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
2014         platform_set_drvdata(pdev, smmu);
2015         arm_smmu_device_reset(smmu);
2016
2017         /* Oh, for a proper bus abstraction */
2018         if (!iommu_present(&platform_bus_type))
2019                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2020 #ifdef CONFIG_ARM_AMBA
2021         if (!iommu_present(&amba_bustype))
2022                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2023 #endif
2024 #ifdef CONFIG_PCI
2025         if (!iommu_present(&pci_bus_type)) {
2026                 pci_request_acs();
2027                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2028         }
2029 #endif
2030         return 0;
2031 }
2032
2033 static int arm_smmu_device_remove(struct platform_device *pdev)
2034 {
2035         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2036
2037         if (!smmu)
2038                 return -ENODEV;
2039
2040         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2041                 dev_err(&pdev->dev, "removing device with active domains!\n");
2042
2043         /* Turn the thing off */
2044         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2045         return 0;
2046 }
2047
2048 static struct platform_driver arm_smmu_driver = {
2049         .driver = {
2050                 .name           = "arm-smmu",
2051                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2052         },
2053         .probe  = arm_smmu_device_dt_probe,
2054         .remove = arm_smmu_device_remove,
2055 };
2056
2057 static int __init arm_smmu_init(void)
2058 {
2059         static bool registered;
2060         int ret = 0;
2061
2062         if (!registered) {
2063                 ret = platform_driver_register(&arm_smmu_driver);
2064                 registered = !ret;
2065         }
2066         return ret;
2067 }
2068
2069 static void __exit arm_smmu_exit(void)
2070 {
2071         return platform_driver_unregister(&arm_smmu_driver);
2072 }
2073
2074 subsys_initcall(arm_smmu_init);
2075 module_exit(arm_smmu_exit);
2076
2077 static int __init arm_smmu_of_init(struct device_node *np)
2078 {
2079         int ret = arm_smmu_init();
2080
2081         if (ret)
2082                 return ret;
2083
2084         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2085                 return -ENODEV;
2086
2087         return 0;
2088 }
2089 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2090 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2091 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2092 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2093 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2094 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2095
2096 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2097 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2098 MODULE_LICENSE("GPL v2");