drivers/iommu/arm-smmu.c

   1 /*
   2  * IOMMU API for ARM architected SMMU implementations.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  16  *
  17  * Copyright (C) 2013 ARM Limited
  18  *
  19  * Author: Will Deacon <will.deacon@arm.com>
  20  *
  21  * This driver currently supports:
  22  *      - SMMUv1 and v2 implementations
  23  *      - Stream-matching and stream-indexing
  24  *      - v7/v8 long-descriptor format
  25  *      - Non-secure access to the SMMU
  26  *      - Context fault reporting
  27  */
  28
  29 #define pr_fmt(fmt) "arm-smmu: " fmt
  30
  31 #include <linux/atomic.h>
  32 #include <linux/delay.h>
  33 #include <linux/dma-iommu.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/err.h>
  36 #include <linux/interrupt.h>
  37 #include <linux/io.h>
  38 #include <linux/io-64-nonatomic-hi-lo.h>
  39 #include <linux/iommu.h>
  40 #include <linux/iopoll.h>
  41 #include <linux/module.h>
  42 #include <linux/of.h>
  43 #include <linux/of_address.h>
  44 #include <linux/of_device.h>
  45 #include <linux/of_iommu.h>
  46 #include <linux/pci.h>
  47 #include <linux/platform_device.h>
  48 #include <linux/slab.h>
  49 #include <linux/spinlock.h>
  50
  51 #include <linux/amba/bus.h>
  52
  53 #include "io-pgtable.h"
  54
  55 /* Maximum number of context banks per SMMU */
  56 #define ARM_SMMU_MAX_CBS                128
  57
  58 /* SMMU global address space */
  59 #define ARM_SMMU_GR0(smmu)              ((smmu)->base)
  60 #define ARM_SMMU_GR1(smmu)              ((smmu)->base + (1 << (smmu)->pgshift))
  61
  62 /*
  63  * SMMU global address space with conditional offset to access secure
  64  * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
  65  * nsGFSYNR0: 0x450)
  66  */
  67 #define ARM_SMMU_GR0_NS(smmu)                                           \
  68         ((smmu)->base +                                                 \
  69                 ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)       \
  70                         ? 0x400 : 0))
  71
  72 /*
  73  * Some 64-bit registers only make sense to write atomically, but in such
  74  * cases all the data relevant to AArch32 formats lies within the lower word,
  75  * therefore this actually makes more sense than it might first appear.
  76  */
  77 #ifdef CONFIG_64BIT
  78 #define smmu_write_atomic_lq            writeq_relaxed
  79 #else
  80 #define smmu_write_atomic_lq            writel_relaxed
  81 #endif
  82
  83 /* Configuration registers */
  84 #define ARM_SMMU_GR0_sCR0               0x0
  85 #define sCR0_CLIENTPD                   (1 << 0)
  86 #define sCR0_GFRE                       (1 << 1)
  87 #define sCR0_GFIE                       (1 << 2)
  88 #define sCR0_GCFGFRE                    (1 << 4)
  89 #define sCR0_GCFGFIE                    (1 << 5)
  90 #define sCR0_USFCFG                     (1 << 10)
  91 #define sCR0_VMIDPNE                    (1 << 11)
  92 #define sCR0_PTM                        (1 << 12)
  93 #define sCR0_FB                         (1 << 13)
  94 #define sCR0_VMID16EN                   (1 << 31)
  95 #define sCR0_BSU_SHIFT                  14
  96 #define sCR0_BSU_MASK                   0x3
  97
  98 /* Auxiliary Configuration register */
  99 #define ARM_SMMU_GR0_sACR               0x10
 100
 101 /* Identification registers */
 102 #define ARM_SMMU_GR0_ID0                0x20
 103 #define ARM_SMMU_GR0_ID1                0x24
 104 #define ARM_SMMU_GR0_ID2                0x28
 105 #define ARM_SMMU_GR0_ID3                0x2c
 106 #define ARM_SMMU_GR0_ID4                0x30
 107 #define ARM_SMMU_GR0_ID5                0x34
 108 #define ARM_SMMU_GR0_ID6                0x38
 109 #define ARM_SMMU_GR0_ID7                0x3c
 110 #define ARM_SMMU_GR0_sGFSR              0x48
 111 #define ARM_SMMU_GR0_sGFSYNR0           0x50
 112 #define ARM_SMMU_GR0_sGFSYNR1           0x54
 113 #define ARM_SMMU_GR0_sGFSYNR2           0x58
 114
 115 #define ID0_S1TS                        (1 << 30)
 116 #define ID0_S2TS                        (1 << 29)
 117 #define ID0_NTS                         (1 << 28)
 118 #define ID0_SMS                         (1 << 27)
 119 #define ID0_ATOSNS                      (1 << 26)
 120 #define ID0_PTFS_NO_AARCH32             (1 << 25)
 121 #define ID0_PTFS_NO_AARCH32S            (1 << 24)
 122 #define ID0_CTTW                        (1 << 14)
 123 #define ID0_NUMIRPT_SHIFT               16
 124 #define ID0_NUMIRPT_MASK                0xff
 125 #define ID0_NUMSIDB_SHIFT               9
 126 #define ID0_NUMSIDB_MASK                0xf
 127 #define ID0_NUMSMRG_SHIFT               0
 128 #define ID0_NUMSMRG_MASK                0xff
 129
 130 #define ID1_PAGESIZE                    (1 << 31)
 131 #define ID1_NUMPAGENDXB_SHIFT           28
 132 #define ID1_NUMPAGENDXB_MASK            7
 133 #define ID1_NUMS2CB_SHIFT               16
 134 #define ID1_NUMS2CB_MASK                0xff
 135 #define ID1_NUMCB_SHIFT                 0
 136 #define ID1_NUMCB_MASK                  0xff
 137
 138 #define ID2_OAS_SHIFT                   4
 139 #define ID2_OAS_MASK                    0xf
 140 #define ID2_IAS_SHIFT                   0
 141 #define ID2_IAS_MASK                    0xf
 142 #define ID2_UBS_SHIFT                   8
 143 #define ID2_UBS_MASK                    0xf
 144 #define ID2_PTFS_4K                     (1 << 12)
 145 #define ID2_PTFS_16K                    (1 << 13)
 146 #define ID2_PTFS_64K                    (1 << 14)
 147 #define ID2_VMID16                      (1 << 15)
 148
 149 #define ID7_MAJOR_SHIFT                 4
 150 #define ID7_MAJOR_MASK                  0xf
 151
 152 /* Global TLB invalidation */
 153 #define ARM_SMMU_GR0_TLBIVMID           0x64
 154 #define ARM_SMMU_GR0_TLBIALLNSNH        0x68
 155 #define ARM_SMMU_GR0_TLBIALLH           0x6c
 156 #define ARM_SMMU_GR0_sTLBGSYNC          0x70
 157 #define ARM_SMMU_GR0_sTLBGSTATUS        0x74
 158 #define sTLBGSTATUS_GSACTIVE            (1 << 0)
 159 #define TLB_LOOP_TIMEOUT                1000000 /* 1s! */
 160
 161 /* Stream mapping registers */
 162 #define ARM_SMMU_GR0_SMR(n)             (0x800 + ((n) << 2))
 163 #define SMR_VALID                       (1 << 31)
 164 #define SMR_MASK_SHIFT                  16
 165 #define SMR_ID_SHIFT                    0
 166
 167 #define ARM_SMMU_GR0_S2CR(n)            (0xc00 + ((n) << 2))
 168 #define S2CR_CBNDX_SHIFT                0
 169 #define S2CR_CBNDX_MASK                 0xff
 170 #define S2CR_TYPE_SHIFT                 16
 171 #define S2CR_TYPE_MASK                  0x3
 172 enum arm_smmu_s2cr_type {
 173         S2CR_TYPE_TRANS,
 174         S2CR_TYPE_BYPASS,
 175         S2CR_TYPE_FAULT,
 176 };
 177
 178 #define S2CR_PRIVCFG_SHIFT              24
 179 #define S2CR_PRIVCFG_MASK               0x3
 180 enum arm_smmu_s2cr_privcfg {
 181         S2CR_PRIVCFG_DEFAULT,
 182         S2CR_PRIVCFG_DIPAN,
 183         S2CR_PRIVCFG_UNPRIV,
 184         S2CR_PRIVCFG_PRIV,
 185 };
 186
 187 /* Context bank attribute registers */
 188 #define ARM_SMMU_GR1_CBAR(n)            (0x0 + ((n) << 2))
 189 #define CBAR_VMID_SHIFT                 0
 190 #define CBAR_VMID_MASK                  0xff
 191 #define CBAR_S1_BPSHCFG_SHIFT           8
 192 #define CBAR_S1_BPSHCFG_MASK            3
 193 #define CBAR_S1_BPSHCFG_NSH             3
 194 #define CBAR_S1_MEMATTR_SHIFT           12
 195 #define CBAR_S1_MEMATTR_MASK            0xf
 196 #define CBAR_S1_MEMATTR_WB              0xf
 197 #define CBAR_TYPE_SHIFT                 16
 198 #define CBAR_TYPE_MASK                  0x3
 199 #define CBAR_TYPE_S2_TRANS              (0 << CBAR_TYPE_SHIFT)
 200 #define CBAR_TYPE_S1_TRANS_S2_BYPASS    (1 << CBAR_TYPE_SHIFT)
 201 #define CBAR_TYPE_S1_TRANS_S2_FAULT     (2 << CBAR_TYPE_SHIFT)
 202 #define CBAR_TYPE_S1_TRANS_S2_TRANS     (3 << CBAR_TYPE_SHIFT)
 203 #define CBAR_IRPTNDX_SHIFT              24
 204 #define CBAR_IRPTNDX_MASK               0xff
 205
 206 #define ARM_SMMU_GR1_CBA2R(n)           (0x800 + ((n) << 2))
 207 #define CBA2R_RW64_32BIT                (0 << 0)
 208 #define CBA2R_RW64_64BIT                (1 << 0)
 209 #define CBA2R_VMID_SHIFT                16
 210 #define CBA2R_VMID_MASK                 0xffff
 211
 212 /* Translation context bank */
 213 #define ARM_SMMU_CB_BASE(smmu)          ((smmu)->base + ((smmu)->size >> 1))
 214 #define ARM_SMMU_CB(smmu, n)            ((n) * (1 << (smmu)->pgshift))
 215
 216 #define ARM_SMMU_CB_SCTLR               0x0
 217 #define ARM_SMMU_CB_ACTLR               0x4
 218 #define ARM_SMMU_CB_RESUME              0x8
 219 #define ARM_SMMU_CB_TTBCR2              0x10
 220 #define ARM_SMMU_CB_TTBR0               0x20
 221 #define ARM_SMMU_CB_TTBR1               0x28
 222 #define ARM_SMMU_CB_TTBCR               0x30
 223 #define ARM_SMMU_CB_CONTEXTIDR          0x34
 224 #define ARM_SMMU_CB_S1_MAIR0            0x38
 225 #define ARM_SMMU_CB_S1_MAIR1            0x3c
 226 #define ARM_SMMU_CB_PAR                 0x50
 227 #define ARM_SMMU_CB_FSR                 0x58
 228 #define ARM_SMMU_CB_FAR                 0x60
 229 #define ARM_SMMU_CB_FSYNR0              0x68
 230 #define ARM_SMMU_CB_S1_TLBIVA           0x600
 231 #define ARM_SMMU_CB_S1_TLBIASID         0x610
 232 #define ARM_SMMU_CB_S1_TLBIVAL          0x620
 233 #define ARM_SMMU_CB_S2_TLBIIPAS2        0x630
 234 #define ARM_SMMU_CB_S2_TLBIIPAS2L       0x638
 235 #define ARM_SMMU_CB_ATS1PR              0x800
 236 #define ARM_SMMU_CB_ATSR                0x8f0
 237
 238 #define SCTLR_S1_ASIDPNE                (1 << 12)
 239 #define SCTLR_CFCFG                     (1 << 7)
 240 #define SCTLR_CFIE                      (1 << 6)
 241 #define SCTLR_CFRE                      (1 << 5)
 242 #define SCTLR_E                         (1 << 4)
 243 #define SCTLR_AFE                       (1 << 2)
 244 #define SCTLR_TRE                       (1 << 1)
 245 #define SCTLR_M                         (1 << 0)
 246
 247 #define ARM_MMU500_ACTLR_CPRE           (1 << 1)
 248
 249 #define ARM_MMU500_ACR_CACHE_LOCK       (1 << 26)
 250
 251 #define CB_PAR_F                        (1 << 0)
 252
 253 #define ATSR_ACTIVE                     (1 << 0)
 254
 255 #define RESUME_RETRY                    (0 << 0)
 256 #define RESUME_TERMINATE                (1 << 0)
 257
 258 #define TTBCR2_SEP_SHIFT                15
 259 #define TTBCR2_SEP_UPSTREAM             (0x7 << TTBCR2_SEP_SHIFT)
 260
 261 #define TTBRn_ASID_SHIFT                48
 262
 263 #define FSR_MULTI                       (1 << 31)
 264 #define FSR_SS                          (1 << 30)
 265 #define FSR_UUT                         (1 << 8)
 266 #define FSR_ASF                         (1 << 7)
 267 #define FSR_TLBLKF                      (1 << 6)
 268 #define FSR_TLBMCF                      (1 << 5)
 269 #define FSR_EF                          (1 << 4)
 270 #define FSR_PF                          (1 << 3)
 271 #define FSR_AFF                         (1 << 2)
 272 #define FSR_TF                          (1 << 1)
 273
 274 #define FSR_IGN                         (FSR_AFF | FSR_ASF | \
 275                                          FSR_TLBMCF | FSR_TLBLKF)
 276 #define FSR_FAULT                       (FSR_MULTI | FSR_SS | FSR_UUT | \
 277                                          FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
 278
 279 #define FSYNR0_WNR                      (1 << 4)
 280
 281 static int force_stage;
 282 module_param(force_stage, int, S_IRUGO);
 283 MODULE_PARM_DESC(force_stage,
 284         "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
 285 static bool disable_bypass;
 286 module_param(disable_bypass, bool, S_IRUGO);
 287 MODULE_PARM_DESC(disable_bypass,
 288         "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
 289
 290 enum arm_smmu_arch_version {
 291         ARM_SMMU_V1,
 292         ARM_SMMU_V1_64K,
 293         ARM_SMMU_V2,
 294 };
 295
 296 enum arm_smmu_implementation {
 297         GENERIC_SMMU,
 298         ARM_MMU500,
 299         CAVIUM_SMMUV2,
 300 };
 301
 302 struct arm_smmu_s2cr {
 303         struct iommu_group              *group;
 304         int                             count;
 305         enum arm_smmu_s2cr_type         type;
 306         enum arm_smmu_s2cr_privcfg      privcfg;
 307         u8                              cbndx;
 308 };
 309
 310 #define s2cr_init_val (struct arm_smmu_s2cr){                           \
 311         .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS,    \
 312 }
 313
 314 struct arm_smmu_smr {
 315         u16                             mask;
 316         u16                             id;
 317         bool                            valid;
 318 };
 319
 320 struct arm_smmu_master_cfg {
 321         struct arm_smmu_device          *smmu;
 322         s16                             smendx[];
 323 };
 324 #define INVALID_SMENDX                  -1
 325 #define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv)
 326 #define fwspec_smmu(fw)  (__fwspec_cfg(fw)->smmu)
 327 #define fwspec_smendx(fw, i) \
 328         (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i])
 329 #define for_each_cfg_sme(fw, i, idx) \
 330         for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i)
 331
 332 struct arm_smmu_device {
 333         struct device                   *dev;
 334
 335         void __iomem                    *base;
 336         unsigned long                   size;
 337         unsigned long                   pgshift;
 338
 339 #define ARM_SMMU_FEAT_COHERENT_WALK     (1 << 0)
 340 #define ARM_SMMU_FEAT_STREAM_MATCH      (1 << 1)
 341 #define ARM_SMMU_FEAT_TRANS_S1          (1 << 2)
 342 #define ARM_SMMU_FEAT_TRANS_S2          (1 << 3)
 343 #define ARM_SMMU_FEAT_TRANS_NESTED      (1 << 4)
 344 #define ARM_SMMU_FEAT_TRANS_OPS         (1 << 5)
 345 #define ARM_SMMU_FEAT_VMID16            (1 << 6)
 346 #define ARM_SMMU_FEAT_FMT_AARCH64_4K    (1 << 7)
 347 #define ARM_SMMU_FEAT_FMT_AARCH64_16K   (1 << 8)
 348 #define ARM_SMMU_FEAT_FMT_AARCH64_64K   (1 << 9)
 349 #define ARM_SMMU_FEAT_FMT_AARCH32_L     (1 << 10)
 350 #define ARM_SMMU_FEAT_FMT_AARCH32_S     (1 << 11)
 351         u32                             features;
 352
 353 #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
 354         u32                             options;
 355         enum arm_smmu_arch_version      version;
 356         enum arm_smmu_implementation    model;
 357
 358         u32                             num_context_banks;
 359         u32                             num_s2_context_banks;
 360         DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
 361         atomic_t                        irptndx;
 362
 363         u32                             num_mapping_groups;
 364         u16                             streamid_mask;
 365         u16                             smr_mask_mask;
 366         struct arm_smmu_smr             *smrs;
 367         struct arm_smmu_s2cr            *s2crs;
 368         struct mutex                    stream_map_mutex;
 369
 370         unsigned long                   va_size;
 371         unsigned long                   ipa_size;
 372         unsigned long                   pa_size;
 373         unsigned long                   pgsize_bitmap;
 374
 375         u32                             num_global_irqs;
 376         u32                             num_context_irqs;
 377         unsigned int                    *irqs;
 378
 379         u32                             cavium_id_base; /* Specific to Cavium */
 380 };
 381
 382 enum arm_smmu_context_fmt {
 383         ARM_SMMU_CTX_FMT_NONE,
 384         ARM_SMMU_CTX_FMT_AARCH64,
 385         ARM_SMMU_CTX_FMT_AARCH32_L,
 386         ARM_SMMU_CTX_FMT_AARCH32_S,
 387 };
 388
 389 struct arm_smmu_cfg {
 390         u8                              cbndx;
 391         u8                              irptndx;
 392         u32                             cbar;
 393         enum arm_smmu_context_fmt       fmt;
 394 };
 395 #define INVALID_IRPTNDX                 0xff
 396
 397 #define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx)
 398 #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1)
 399
 400 enum arm_smmu_domain_stage {
 401         ARM_SMMU_DOMAIN_S1 = 0,
 402         ARM_SMMU_DOMAIN_S2,
 403         ARM_SMMU_DOMAIN_NESTED,
 404 };
 405
 406 struct arm_smmu_domain {
 407         struct arm_smmu_device          *smmu;
 408         struct io_pgtable_ops           *pgtbl_ops;
 409         spinlock_t                      pgtbl_lock;
 410         struct arm_smmu_cfg             cfg;
 411         enum arm_smmu_domain_stage      stage;
 412         struct mutex                    init_mutex; /* Protects smmu pointer */
 413         struct iommu_domain             domain;
 414 };
 415
 416 struct arm_smmu_option_prop {
 417         u32 opt;
 418         const char *prop;
 419 };
 420
 421 static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0);
 422
 423 static bool using_legacy_binding, using_generic_binding;
 424
 425 static struct arm_smmu_option_prop arm_smmu_options[] = {
 426         { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
 427         { 0, NULL},
 428 };
 429
 430 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
 431 {
 432         return container_of(dom, struct arm_smmu_domain, domain);
 433 }
 434
 435 static void parse_driver_options(struct arm_smmu_device *smmu)
 436 {
 437         int i = 0;
 438
 439         do {
 440                 if (of_property_read_bool(smmu->dev->of_node,
 441                                                 arm_smmu_options[i].prop)) {
 442                         smmu->options |= arm_smmu_options[i].opt;
 443                         dev_notice(smmu->dev, "option %s\n",
 444                                 arm_smmu_options[i].prop);
 445                 }
 446         } while (arm_smmu_options[++i].opt);
 447 }
 448
 449 static struct device_node *dev_get_dev_node(struct device *dev)
 450 {
 451         if (dev_is_pci(dev)) {
 452                 struct pci_bus *bus = to_pci_dev(dev)->bus;
 453
 454                 while (!pci_is_root_bus(bus))
 455                         bus = bus->parent;
 456                 return of_node_get(bus->bridge->parent->of_node);
 457         }
 458
 459         return of_node_get(dev->of_node);
 460 }
 461
 462 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
 463 {
 464         *((__be32 *)data) = cpu_to_be32(alias);
 465         return 0; /* Continue walking */
 466 }
 467
 468 static int __find_legacy_master_phandle(struct device *dev, void *data)
 469 {
 470         struct of_phandle_iterator *it = *(void **)data;
 471         struct device_node *np = it->node;
 472         int err;
 473
 474         of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
 475                             "#stream-id-cells", 0)
 476                 if (it->node == np) {
 477                         *(void **)data = dev;
 478                         return 1;
 479                 }
 480         it->node = np;
 481         return err == -ENOENT ? 0 : err;
 482 }
 483
 484 static struct platform_driver arm_smmu_driver;
 485 static struct iommu_ops arm_smmu_ops;
 486
 487 static int arm_smmu_register_legacy_master(struct device *dev,
 488                                            struct arm_smmu_device **smmu)
 489 {
 490         struct device *smmu_dev;
 491         struct device_node *np;
 492         struct of_phandle_iterator it;
 493         void *data = &it;
 494         u32 *sids;
 495         __be32 pci_sid;
 496         int err;
 497
 498         np = dev_get_dev_node(dev);
 499         if (!np || !of_find_property(np, "#stream-id-cells", NULL)) {
 500                 of_node_put(np);
 501                 return -ENODEV;
 502         }
 503
 504         it.node = np;
 505         err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
 506                                      __find_legacy_master_phandle);
 507         smmu_dev = data;
 508         of_node_put(np);
 509         if (err == 0)
 510                 return -ENODEV;
 511         if (err < 0)
 512                 return err;
 513
 514         if (dev_is_pci(dev)) {
 515                 /* "mmu-masters" assumes Stream ID == Requester ID */
 516                 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
 517                                        &pci_sid);
 518                 it.cur = &pci_sid;
 519                 it.cur_count = 1;
 520         }
 521
 522         err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode,
 523                                 &arm_smmu_ops);
 524         if (err)
 525                 return err;
 526
 527         sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
 528         if (!sids)
 529                 return -ENOMEM;
 530
 531         *smmu = dev_get_drvdata(smmu_dev);
 532         of_phandle_iterator_args(&it, sids, it.cur_count);
 533         err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
 534         kfree(sids);
 535         return err;
 536 }
 537
 538 static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end)
 539 {
 540         int idx;
 541
 542         do {
 543                 idx = find_next_zero_bit(map, end, start);
 544                 if (idx == end)
 545                         return -ENOSPC;
 546         } while (test_and_set_bit(idx, map));
 547
 548         return idx;
 549 }
 550
 551 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
 552 {
 553         clear_bit(idx, map);
 554 }
 555
 556 /* Wait for any pending TLB invalidations to complete */
 557 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)
 558 {
 559         int count = 0;
 560         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
 561
 562         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC);
 563         while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS)
 564                & sTLBGSTATUS_GSACTIVE) {
 565                 cpu_relax();
 566                 if (++count == TLB_LOOP_TIMEOUT) {
 567                         dev_err_ratelimited(smmu->dev,
 568                         "TLB sync timed out -- SMMU may be deadlocked\n");
 569                         return;
 570                 }
 571                 udelay(1);
 572         }
 573 }
 574
 575 static void arm_smmu_tlb_sync(void *cookie)
 576 {
 577         struct arm_smmu_domain *smmu_domain = cookie;
 578         __arm_smmu_tlb_sync(smmu_domain->smmu);
 579 }
 580
 581 static void arm_smmu_tlb_inv_context(void *cookie)
 582 {
 583         struct arm_smmu_domain *smmu_domain = cookie;
 584         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 585         struct arm_smmu_device *smmu = smmu_domain->smmu;
 586         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 587         void __iomem *base;
 588
 589         if (stage1) {
 590                 base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 591                 writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg),
 592                                base + ARM_SMMU_CB_S1_TLBIASID);
 593         } else {
 594                 base = ARM_SMMU_GR0(smmu);
 595                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg),
 596                                base + ARM_SMMU_GR0_TLBIVMID);
 597         }
 598
 599         __arm_smmu_tlb_sync(smmu);
 600 }
 601
 602 static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size,
 603                                           size_t granule, bool leaf, void *cookie)
 604 {
 605         struct arm_smmu_domain *smmu_domain = cookie;
 606         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 607         struct arm_smmu_device *smmu = smmu_domain->smmu;
 608         bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 609         void __iomem *reg;
 610
 611         if (stage1) {
 612                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 613                 reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA;
 614
 615                 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
 616                         iova &= ~12UL;
 617                         iova |= ARM_SMMU_CB_ASID(smmu, cfg);
 618                         do {
 619                                 writel_relaxed(iova, reg);
 620                                 iova += granule;
 621                         } while (size -= granule);
 622                 } else {
 623                         iova >>= 12;
 624                         iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48;
 625                         do {
 626                                 writeq_relaxed(iova, reg);
 627                                 iova += granule >> 12;
 628                         } while (size -= granule);
 629                 }
 630         } else if (smmu->version == ARM_SMMU_V2) {
 631                 reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 632                 reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L :
 633                               ARM_SMMU_CB_S2_TLBIIPAS2;
 634                 iova >>= 12;
 635                 do {
 636                         smmu_write_atomic_lq(iova, reg);
 637                         iova += granule >> 12;
 638                 } while (size -= granule);
 639         } else {
 640                 reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;
 641                 writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg);
 642         }
 643 }
 644
 645 static struct iommu_gather_ops arm_smmu_gather_ops = {
 646         .tlb_flush_all  = arm_smmu_tlb_inv_context,
 647         .tlb_add_flush  = arm_smmu_tlb_inv_range_nosync,
 648         .tlb_sync       = arm_smmu_tlb_sync,
 649 };
 650
 651 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 652 {
 653         u32 fsr, fsynr;
 654         unsigned long iova;
 655         struct iommu_domain *domain = dev;
 656         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 657         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 658         struct arm_smmu_device *smmu = smmu_domain->smmu;
 659         void __iomem *cb_base;
 660
 661         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 662         fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR);
 663
 664         if (!(fsr & FSR_FAULT))
 665                 return IRQ_NONE;
 666
 667         fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0);
 668         iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR);
 669
 670         dev_err_ratelimited(smmu->dev,
 671         "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d\n",
 672                             fsr, iova, fsynr, cfg->cbndx);
 673
 674         writel(fsr, cb_base + ARM_SMMU_CB_FSR);
 675         return IRQ_HANDLED;
 676 }
 677
 678 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 679 {
 680         u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 681         struct arm_smmu_device *smmu = dev;
 682         void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 683
 684         gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
 685         gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 686         gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 687         gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 688
 689         if (!gfsr)
 690                 return IRQ_NONE;
 691
 692         dev_err_ratelimited(smmu->dev,
 693                 "Unexpected global fault, this could be serious\n");
 694         dev_err_ratelimited(smmu->dev,
 695                 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
 696                 gfsr, gfsynr0, gfsynr1, gfsynr2);
 697
 698         writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR);
 699         return IRQ_HANDLED;
 700 }
 701
 702 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
 703                                        struct io_pgtable_cfg *pgtbl_cfg)
 704 {
 705         u32 reg, reg2;
 706         u64 reg64;
 707         bool stage1;
 708         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 709         struct arm_smmu_device *smmu = smmu_domain->smmu;
 710         void __iomem *cb_base, *gr1_base;
 711
 712         gr1_base = ARM_SMMU_GR1(smmu);
 713         stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
 714         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 715
 716         if (smmu->version > ARM_SMMU_V1) {
 717                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
 718                         reg = CBA2R_RW64_64BIT;
 719                 else
 720                         reg = CBA2R_RW64_32BIT;
 721                 /* 16-bit VMIDs live in CBA2R */
 722                 if (smmu->features & ARM_SMMU_FEAT_VMID16)
 723                         reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT;
 724
 725                 writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx));
 726         }
 727
 728         /* CBAR */
 729         reg = cfg->cbar;
 730         if (smmu->version < ARM_SMMU_V2)
 731                 reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT;
 732
 733         /*
 734          * Use the weakest shareability/memory types, so they are
 735          * overridden by the ttbcr/pte.
 736          */
 737         if (stage1) {
 738                 reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) |
 739                         (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
 740         } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
 741                 /* 8-bit VMIDs live in CBAR */
 742                 reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT;
 743         }
 744         writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx));
 745
 746         /* TTBRs */
 747         if (stage1) {
 748                 u16 asid = ARM_SMMU_CB_ASID(smmu, cfg);
 749
 750                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 751                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0];
 752                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0);
 753                         reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1];
 754                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1);
 755                         writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR);
 756                 } else {
 757                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
 758                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 759                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 760                         reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1];
 761                         reg64 |= (u64)asid << TTBRn_ASID_SHIFT;
 762                         writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1);
 763                 }
 764         } else {
 765                 reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
 766                 writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0);
 767         }
 768
 769         /* TTBCR */
 770         if (stage1) {
 771                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 772                         reg = pgtbl_cfg->arm_v7s_cfg.tcr;
 773                         reg2 = 0;
 774                 } else {
 775                         reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
 776                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32;
 777                         reg2 |= TTBCR2_SEP_UPSTREAM;
 778                 }
 779                 if (smmu->version > ARM_SMMU_V1)
 780                         writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2);
 781         } else {
 782                 reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
 783         }
 784         writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR);
 785
 786         /* MAIRs (stage-1 only) */
 787         if (stage1) {
 788                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
 789                         reg = pgtbl_cfg->arm_v7s_cfg.prrr;
 790                         reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr;
 791                 } else {
 792                         reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
 793                         reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1];
 794                 }
 795                 writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 796                 writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1);
 797         }
 798
 799         /* SCTLR */
 800         reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M;
 801         if (stage1)
 802                 reg |= SCTLR_S1_ASIDPNE;
 803 #ifdef __BIG_ENDIAN
 804         reg |= SCTLR_E;
 805 #endif
 806         writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR);
 807 }
 808
 809 static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 810                                         struct arm_smmu_device *smmu)
 811 {
 812         int irq, start, ret = 0;
 813         unsigned long ias, oas;
 814         struct io_pgtable_ops *pgtbl_ops;
 815         struct io_pgtable_cfg pgtbl_cfg;
 816         enum io_pgtable_fmt fmt;
 817         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 818         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 819
 820         mutex_lock(&smmu_domain->init_mutex);
 821         if (smmu_domain->smmu)
 822                 goto out_unlock;
 823
 824         /*
 825          * Mapping the requested stage onto what we support is surprisingly
 826          * complicated, mainly because the spec allows S1+S2 SMMUs without
 827          * support for nested translation. That means we end up with the
 828          * following table:
 829          *
 830          * Requested        Supported        Actual
 831          *     S1               N              S1
 832          *     S1             S1+S2            S1
 833          *     S1               S2             S2
 834          *     S1               S1             S1
 835          *     N                N              N
 836          *     N              S1+S2            S2
 837          *     N                S2             S2
 838          *     N                S1             S1
 839          *
 840          * Note that you can't actually request stage-2 mappings.
 841          */
 842         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
 843                 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
 844         if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
 845                 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
 846
 847         /*
 848          * Choosing a suitable context format is even more fiddly. Until we
 849          * grow some way for the caller to express a preference, and/or move
 850          * the decision into the io-pgtable code where it arguably belongs,
 851          * just aim for the closest thing to the rest of the system, and hope
 852          * that the hardware isn't esoteric enough that we can't assume AArch64
 853          * support to be a superset of AArch32 support...
 854          */
 855         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
 856                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
 857         if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
 858             !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
 859             (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
 860             (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
 861                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
 862         if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
 863             (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
 864                                ARM_SMMU_FEAT_FMT_AARCH64_16K |
 865                                ARM_SMMU_FEAT_FMT_AARCH64_4K)))
 866                 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
 867
 868         if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
 869                 ret = -EINVAL;
 870                 goto out_unlock;
 871         }
 872
 873         switch (smmu_domain->stage) {
 874         case ARM_SMMU_DOMAIN_S1:
 875                 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
 876                 start = smmu->num_s2_context_banks;
 877                 ias = smmu->va_size;
 878                 oas = smmu->ipa_size;
 879                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 880                         fmt = ARM_64_LPAE_S1;
 881                 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
 882                         fmt = ARM_32_LPAE_S1;
 883                         ias = min(ias, 32UL);
 884                         oas = min(oas, 40UL);
 885                 } else {
 886                         fmt = ARM_V7S;
 887                         ias = min(ias, 32UL);
 888                         oas = min(oas, 32UL);
 889                 }
 890                 break;
 891         case ARM_SMMU_DOMAIN_NESTED:
 892                 /*
 893                  * We will likely want to change this if/when KVM gets
 894                  * involved.
 895                  */
 896         case ARM_SMMU_DOMAIN_S2:
 897                 cfg->cbar = CBAR_TYPE_S2_TRANS;
 898                 start = 0;
 899                 ias = smmu->ipa_size;
 900                 oas = smmu->pa_size;
 901                 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
 902                         fmt = ARM_64_LPAE_S2;
 903                 } else {
 904                         fmt = ARM_32_LPAE_S2;
 905                         ias = min(ias, 40UL);
 906                         oas = min(oas, 40UL);
 907                 }
 908                 break;
 909         default:
 910                 ret = -EINVAL;
 911                 goto out_unlock;
 912         }
 913
 914         ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,
 915                                       smmu->num_context_banks);
 916         if (ret < 0)
 917                 goto out_unlock;
 918
 919         cfg->cbndx = ret;
 920         if (smmu->version < ARM_SMMU_V2) {
 921                 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
 922                 cfg->irptndx %= smmu->num_context_irqs;
 923         } else {
 924                 cfg->irptndx = cfg->cbndx;
 925         }
 926
 927         pgtbl_cfg = (struct io_pgtable_cfg) {
 928                 .pgsize_bitmap  = smmu->pgsize_bitmap,
 929                 .ias            = ias,
 930                 .oas            = oas,
 931                 .tlb            = &arm_smmu_gather_ops,
 932                 .iommu_dev      = smmu->dev,
 933         };
 934
 935         smmu_domain->smmu = smmu;
 936         pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
 937         if (!pgtbl_ops) {
 938                 ret = -ENOMEM;
 939                 goto out_clear_smmu;
 940         }
 941
 942         /* Update the domain's page sizes to reflect the page table format */
 943         domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 944         domain->geometry.aperture_end = (1UL << ias) - 1;
 945         domain->geometry.force_aperture = true;
 946
 947         /* Initialise the context bank with our page table cfg */
 948         arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
 949
 950         /*
 951          * Request context fault interrupt. Do this last to avoid the
 952          * handler seeing a half-initialised domain state.
 953          */
 954         irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 955         ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault,
 956                                IRQF_SHARED, "arm-smmu-context-fault", domain);
 957         if (ret < 0) {
 958                 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
 959                         cfg->irptndx, irq);
 960                 cfg->irptndx = INVALID_IRPTNDX;
 961         }
 962
 963         mutex_unlock(&smmu_domain->init_mutex);
 964
 965         /* Publish page table ops for map/unmap */
 966         smmu_domain->pgtbl_ops = pgtbl_ops;
 967         return 0;
 968
 969 out_clear_smmu:
 970         smmu_domain->smmu = NULL;
 971 out_unlock:
 972         mutex_unlock(&smmu_domain->init_mutex);
 973         return ret;
 974 }
 975
 976 static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 977 {
 978         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
 979         struct arm_smmu_device *smmu = smmu_domain->smmu;
 980         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
 981         void __iomem *cb_base;
 982         int irq;
 983
 984         if (!smmu)
 985                 return;
 986
 987         /*
 988          * Disable the context bank and free the page tables before freeing
 989          * it.
 990          */
 991         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
 992         writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
 993
 994         if (cfg->irptndx != INVALID_IRPTNDX) {
 995                 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
 996                 devm_free_irq(smmu->dev, irq, domain);
 997         }
 998
 999         free_io_pgtable_ops(smmu_domain->pgtbl_ops);
1000         __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
1001 }
1002
1003 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
1004 {
1005         struct arm_smmu_domain *smmu_domain;
1006
1007         if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
1008                 return NULL;
1009         /*
1010          * Allocate the domain and initialise some of its data structures.
1011          * We can't really do anything meaningful until we've added a
1012          * master.
1013          */
1014         smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
1015         if (!smmu_domain)
1016                 return NULL;
1017
1018         if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding ||
1019             iommu_get_dma_cookie(&smmu_domain->domain))) {
1020                 kfree(smmu_domain);
1021                 return NULL;
1022         }
1023
1024         mutex_init(&smmu_domain->init_mutex);
1025         spin_lock_init(&smmu_domain->pgtbl_lock);
1026
1027         return &smmu_domain->domain;
1028 }
1029
1030 static void arm_smmu_domain_free(struct iommu_domain *domain)
1031 {
1032         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1033
1034         /*
1035          * Free the domain resources. We assume that all devices have
1036          * already been detached.
1037          */
1038         iommu_put_dma_cookie(domain);
1039         arm_smmu_destroy_domain_context(domain);
1040         kfree(smmu_domain);
1041 }
1042
1043 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
1044 {
1045         struct arm_smmu_smr *smr = smmu->smrs + idx;
1046         u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT;
1047
1048         if (smr->valid)
1049                 reg |= SMR_VALID;
1050         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx));
1051 }
1052
1053 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
1054 {
1055         struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
1056         u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT |
1057                   (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT |
1058                   (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT;
1059
1060         writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx));
1061 }
1062
1063 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
1064 {
1065         arm_smmu_write_s2cr(smmu, idx);
1066         if (smmu->smrs)
1067                 arm_smmu_write_smr(smmu, idx);
1068 }
1069
1070 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1071 {
1072         struct arm_smmu_smr *smrs = smmu->smrs;
1073         int i, free_idx = -ENOSPC;
1074
1075         /* Stream indexing is blissfully easy */
1076         if (!smrs)
1077                 return id;
1078
1079         /* Validating SMRs is... less so */
1080         for (i = 0; i < smmu->num_mapping_groups; ++i) {
1081                 if (!smrs[i].valid) {
1082                         /*
1083                          * Note the first free entry we come across, which
1084                          * we'll claim in the end if nothing else matches.
1085                          */
1086                         if (free_idx < 0)
1087                                 free_idx = i;
1088                         continue;
1089                 }
1090                 /*
1091                  * If the new entry is _entirely_ matched by an existing entry,
1092                  * then reuse that, with the guarantee that there also cannot
1093                  * be any subsequent conflicting entries. In normal use we'd
1094                  * expect simply identical entries for this case, but there's
1095                  * no harm in accommodating the generalisation.
1096                  */
1097                 if ((mask & smrs[i].mask) == mask &&
1098                     !((id ^ smrs[i].id) & ~smrs[i].mask))
1099                         return i;
1100                 /*
1101                  * If the new entry has any other overlap with an existing one,
1102                  * though, then there always exists at least one stream ID
1103                  * which would cause a conflict, and we can't allow that risk.
1104                  */
1105                 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1106                         return -EINVAL;
1107         }
1108
1109         return free_idx;
1110 }
1111
1112 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1113 {
1114         if (--smmu->s2crs[idx].count)
1115                 return false;
1116
1117         smmu->s2crs[idx] = s2cr_init_val;
1118         if (smmu->smrs)
1119                 smmu->smrs[idx].valid = false;
1120
1121         return true;
1122 }
1123
1124 static int arm_smmu_master_alloc_smes(struct device *dev)
1125 {
1126         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1127         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1128         struct arm_smmu_device *smmu = cfg->smmu;
1129         struct arm_smmu_smr *smrs = smmu->smrs;
1130         struct iommu_group *group;
1131         int i, idx, ret;
1132
1133         mutex_lock(&smmu->stream_map_mutex);
1134         /* Figure out a viable stream map entry allocation */
1135         for_each_cfg_sme(fwspec, i, idx) {
1136                 u16 sid = fwspec->ids[i];
1137                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1138
1139                 if (idx != INVALID_SMENDX) {
1140                         ret = -EEXIST;
1141                         goto out_err;
1142                 }
1143
1144                 ret = arm_smmu_find_sme(smmu, sid, mask);
1145                 if (ret < 0)
1146                         goto out_err;
1147
1148                 idx = ret;
1149                 if (smrs && smmu->s2crs[idx].count == 0) {
1150                         smrs[idx].id = sid;
1151                         smrs[idx].mask = mask;
1152                         smrs[idx].valid = true;
1153                 }
1154                 smmu->s2crs[idx].count++;
1155                 cfg->smendx[i] = (s16)idx;
1156         }
1157
1158         group = iommu_group_get_for_dev(dev);
1159         if (!group)
1160                 group = ERR_PTR(-ENOMEM);
1161         if (IS_ERR(group)) {
1162                 ret = PTR_ERR(group);
1163                 goto out_err;
1164         }
1165         iommu_group_put(group);
1166
1167         /* It worked! Now, poke the actual hardware */
1168         for_each_cfg_sme(fwspec, i, idx) {
1169                 arm_smmu_write_sme(smmu, idx);
1170                 smmu->s2crs[idx].group = group;
1171         }
1172
1173         mutex_unlock(&smmu->stream_map_mutex);
1174         return 0;
1175
1176 out_err:
1177         while (i--) {
1178                 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1179                 cfg->smendx[i] = INVALID_SMENDX;
1180         }
1181         mutex_unlock(&smmu->stream_map_mutex);
1182         return ret;
1183 }
1184
1185 static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec)
1186 {
1187         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1188         struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv;
1189         int i, idx;
1190
1191         mutex_lock(&smmu->stream_map_mutex);
1192         for_each_cfg_sme(fwspec, i, idx) {
1193                 if (arm_smmu_free_sme(smmu, idx))
1194                         arm_smmu_write_sme(smmu, idx);
1195                 cfg->smendx[i] = INVALID_SMENDX;
1196         }
1197         mutex_unlock(&smmu->stream_map_mutex);
1198 }
1199
1200 static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain,
1201                                       struct iommu_fwspec *fwspec)
1202 {
1203         struct arm_smmu_device *smmu = smmu_domain->smmu;
1204         struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1205         enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS;
1206         u8 cbndx = smmu_domain->cfg.cbndx;
1207         int i, idx;
1208
1209         for_each_cfg_sme(fwspec, i, idx) {
1210                 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1211                         continue;
1212
1213                 s2cr[idx].type = type;
1214                 s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV;
1215                 s2cr[idx].cbndx = cbndx;
1216                 arm_smmu_write_s2cr(smmu, idx);
1217         }
1218         return 0;
1219 }
1220
1221 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1222 {
1223         int ret;
1224         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1225         struct arm_smmu_device *smmu;
1226         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1227
1228         if (!fwspec || fwspec->ops != &arm_smmu_ops) {
1229                 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");
1230                 return -ENXIO;
1231         }
1232
1233         /*
1234          * FIXME: The arch/arm DMA API code tries to attach devices to its own
1235          * domains between of_xlate() and add_device() - we have no way to cope
1236          * with that, so until ARM gets converted to rely on groups and default
1237          * domains, just say no (but more politely than by dereferencing NULL).
1238          * This should be at least a WARN_ON once that's sorted.
1239          */
1240         if (!fwspec->iommu_priv)
1241                 return -ENODEV;
1242
1243         smmu = fwspec_smmu(fwspec);
1244         /* Ensure that the domain is finalised */
1245         ret = arm_smmu_init_domain_context(domain, smmu);
1246         if (ret < 0)
1247                 return ret;
1248
1249         /*
1250          * Sanity check the domain. We don't support domains across
1251          * different SMMUs.
1252          */
1253         if (smmu_domain->smmu != smmu) {
1254                 dev_err(dev,
1255                         "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",
1256                         dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));
1257                 return -EINVAL;
1258         }
1259
1260         /* Looks ok, so add the device to the domain */
1261         return arm_smmu_domain_add_master(smmu_domain, fwspec);
1262 }
1263
1264 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
1265                         phys_addr_t paddr, size_t size, int prot)
1266 {
1267         int ret;
1268         unsigned long flags;
1269         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1270         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1271
1272         if (!ops)
1273                 return -ENODEV;
1274
1275         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1276         ret = ops->map(ops, iova, paddr, size, prot);
1277         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1278         return ret;
1279 }
1280
1281 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
1282                              size_t size)
1283 {
1284         size_t ret;
1285         unsigned long flags;
1286         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1287         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1288
1289         if (!ops)
1290                 return 0;
1291
1292         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1293         ret = ops->unmap(ops, iova, size);
1294         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1295         return ret;
1296 }
1297
1298 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1299                                               dma_addr_t iova)
1300 {
1301         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1302         struct arm_smmu_device *smmu = smmu_domain->smmu;
1303         struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1304         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1305         struct device *dev = smmu->dev;
1306         void __iomem *cb_base;
1307         u32 tmp;
1308         u64 phys;
1309         unsigned long va;
1310
1311         cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
1312
1313         /* ATS1 registers can only be written atomically */
1314         va = iova & ~0xfffUL;
1315         if (smmu->version == ARM_SMMU_V2)
1316                 smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR);
1317         else /* Register is only 32-bit in v1 */
1318                 writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR);
1319
1320         if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
1321                                       !(tmp & ATSR_ACTIVE), 5, 50)) {
1322                 dev_err(dev,
1323                         "iova to phys timed out on %pad. Falling back to software table walk.\n",
1324                         &iova);
1325                 return ops->iova_to_phys(ops, iova);
1326         }
1327
1328         phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
1329         if (phys & CB_PAR_F) {
1330                 dev_err(dev, "translation fault!\n");
1331                 dev_err(dev, "PAR = 0x%llx\n", phys);
1332                 return 0;
1333         }
1334
1335         return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1336 }
1337
1338 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1339                                         dma_addr_t iova)
1340 {
1341         phys_addr_t ret;
1342         unsigned long flags;
1343         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1344         struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1345
1346         if (!ops)
1347                 return 0;
1348
1349         spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
1350         if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1351                         smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1352                 ret = arm_smmu_iova_to_phys_hard(domain, iova);
1353         } else {
1354                 ret = ops->iova_to_phys(ops, iova);
1355         }
1356
1357         spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
1358
1359         return ret;
1360 }
1361
1362 static bool arm_smmu_capable(enum iommu_cap cap)
1363 {
1364         switch (cap) {
1365         case IOMMU_CAP_CACHE_COHERENCY:
1366                 /*
1367                  * Return true here as the SMMU can always send out coherent
1368                  * requests.
1369                  */
1370                 return true;
1371         case IOMMU_CAP_INTR_REMAP:
1372                 return true; /* MSIs are just memory writes */
1373         case IOMMU_CAP_NOEXEC:
1374                 return true;
1375         default:
1376                 return false;
1377         }
1378 }
1379
1380 static int arm_smmu_match_node(struct device *dev, void *data)
1381 {
1382         return dev->of_node == data;
1383 }
1384
1385 static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np)
1386 {
1387         struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL,
1388                                                 np, arm_smmu_match_node);
1389         put_device(dev);
1390         return dev ? dev_get_drvdata(dev) : NULL;
1391 }
1392
1393 static int arm_smmu_add_device(struct device *dev)
1394 {
1395         struct arm_smmu_device *smmu;
1396         struct arm_smmu_master_cfg *cfg;
1397         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1398         int i, ret;
1399
1400         if (using_legacy_binding) {
1401                 ret = arm_smmu_register_legacy_master(dev, &smmu);
1402                 fwspec = dev->iommu_fwspec;
1403                 if (ret)
1404                         goto out_free;
1405         } else if (fwspec && fwspec->ops == &arm_smmu_ops) {
1406                 smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode));
1407         } else {
1408                 return -ENODEV;
1409         }
1410
1411         ret = -EINVAL;
1412         for (i = 0; i < fwspec->num_ids; i++) {
1413                 u16 sid = fwspec->ids[i];
1414                 u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT;
1415
1416                 if (sid & ~smmu->streamid_mask) {
1417                         dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1418                                 sid, smmu->streamid_mask);
1419                         goto out_free;
1420                 }
1421                 if (mask & ~smmu->smr_mask_mask) {
1422                         dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1423                                 sid, smmu->smr_mask_mask);
1424                         goto out_free;
1425                 }
1426         }
1427
1428         ret = -ENOMEM;
1429         cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1430                       GFP_KERNEL);
1431         if (!cfg)
1432                 goto out_free;
1433
1434         cfg->smmu = smmu;
1435         fwspec->iommu_priv = cfg;
1436         while (i--)
1437                 cfg->smendx[i] = INVALID_SMENDX;
1438
1439         ret = arm_smmu_master_alloc_smes(dev);
1440         if (ret)
1441                 goto out_free;
1442
1443         return 0;
1444
1445 out_free:
1446         if (fwspec)
1447                 kfree(fwspec->iommu_priv);
1448         iommu_fwspec_free(dev);
1449         return ret;
1450 }
1451
1452 static void arm_smmu_remove_device(struct device *dev)
1453 {
1454         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1455
1456         if (!fwspec || fwspec->ops != &arm_smmu_ops)
1457                 return;
1458
1459         arm_smmu_master_free_smes(fwspec);
1460         iommu_group_remove_device(dev);
1461         kfree(fwspec->iommu_priv);
1462         iommu_fwspec_free(dev);
1463 }
1464
1465 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1466 {
1467         struct iommu_fwspec *fwspec = dev->iommu_fwspec;
1468         struct arm_smmu_device *smmu = fwspec_smmu(fwspec);
1469         struct iommu_group *group = NULL;
1470         int i, idx;
1471
1472         for_each_cfg_sme(fwspec, i, idx) {
1473                 if (group && smmu->s2crs[idx].group &&
1474                     group != smmu->s2crs[idx].group)
1475                         return ERR_PTR(-EINVAL);
1476
1477                 group = smmu->s2crs[idx].group;
1478         }
1479
1480         if (group)
1481                 return group;
1482
1483         if (dev_is_pci(dev))
1484                 group = pci_device_group(dev);
1485         else
1486                 group = generic_device_group(dev);
1487
1488         return group;
1489 }
1490
1491 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
1492                                     enum iommu_attr attr, void *data)
1493 {
1494         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1495
1496         switch (attr) {
1497         case DOMAIN_ATTR_NESTING:
1498                 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
1499                 return 0;
1500         default:
1501                 return -ENODEV;
1502         }
1503 }
1504
1505 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
1506                                     enum iommu_attr attr, void *data)
1507 {
1508         int ret = 0;
1509         struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1510
1511         mutex_lock(&smmu_domain->init_mutex);
1512
1513         switch (attr) {
1514         case DOMAIN_ATTR_NESTING:
1515                 if (smmu_domain->smmu) {
1516                         ret = -EPERM;
1517                         goto out_unlock;
1518                 }
1519
1520                 if (*(int *)data)
1521                         smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
1522                 else
1523                         smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
1524
1525                 break;
1526         default:
1527                 ret = -ENODEV;
1528         }
1529
1530 out_unlock:
1531         mutex_unlock(&smmu_domain->init_mutex);
1532         return ret;
1533 }
1534
1535 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
1536 {
1537         u32 fwid = 0;
1538
1539         if (args->args_count > 0)
1540                 fwid |= (u16)args->args[0];
1541
1542         if (args->args_count > 1)
1543                 fwid |= (u16)args->args[1] << SMR_MASK_SHIFT;
1544
1545         return iommu_fwspec_add_ids(dev, &fwid, 1);
1546 }
1547
1548 static struct iommu_ops arm_smmu_ops = {
1549         .capable                = arm_smmu_capable,
1550         .domain_alloc           = arm_smmu_domain_alloc,
1551         .domain_free            = arm_smmu_domain_free,
1552         .attach_dev             = arm_smmu_attach_dev,
1553         .map                    = arm_smmu_map,
1554         .unmap                  = arm_smmu_unmap,
1555         .map_sg                 = default_iommu_map_sg,
1556         .iova_to_phys           = arm_smmu_iova_to_phys,
1557         .add_device             = arm_smmu_add_device,
1558         .remove_device          = arm_smmu_remove_device,
1559         .device_group           = arm_smmu_device_group,
1560         .domain_get_attr        = arm_smmu_domain_get_attr,
1561         .domain_set_attr        = arm_smmu_domain_set_attr,
1562         .of_xlate               = arm_smmu_of_xlate,
1563         .pgsize_bitmap          = -1UL, /* Restricted during device attach */
1564 };
1565
1566 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1567 {
1568         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1569         void __iomem *cb_base;
1570         int i;
1571         u32 reg, major;
1572
1573         /* clear global FSR */
1574         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1575         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
1576
1577         /*
1578          * Reset stream mapping groups: Initial values mark all SMRn as
1579          * invalid and all S2CRn as bypass unless overridden.
1580          */
1581         for (i = 0; i < smmu->num_mapping_groups; ++i)
1582                 arm_smmu_write_sme(smmu, i);
1583
1584         /*
1585          * Before clearing ARM_MMU500_ACTLR_CPRE, need to
1586          * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK
1587          * bit is only present in MMU-500r2 onwards.
1588          */
1589         reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7);
1590         major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK;
1591         if ((smmu->model == ARM_MMU500) && (major >= 2)) {
1592                 reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR);
1593                 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
1594                 writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR);
1595         }
1596
1597         /* Make sure all context banks are disabled and clear CB_FSR  */
1598         for (i = 0; i < smmu->num_context_banks; ++i) {
1599                 cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);
1600                 writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
1601                 writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR);
1602                 /*
1603                  * Disable MMU-500's not-particularly-beneficial next-page
1604                  * prefetcher for the sake of errata #841119 and #826419.
1605                  */
1606                 if (smmu->model == ARM_MMU500) {
1607                         reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR);
1608                         reg &= ~ARM_MMU500_ACTLR_CPRE;
1609                         writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR);
1610                 }
1611         }
1612
1613         /* Invalidate the TLB, just in case */
1614         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
1615         writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
1616
1617         reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1618
1619         /* Enable fault reporting */
1620         reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
1621
1622         /* Disable TLB broadcasting. */
1623         reg |= (sCR0_VMIDPNE | sCR0_PTM);
1624
1625         /* Enable client access, handling unmatched streams as appropriate */
1626         reg &= ~sCR0_CLIENTPD;
1627         if (disable_bypass)
1628                 reg |= sCR0_USFCFG;
1629         else
1630                 reg &= ~sCR0_USFCFG;
1631
1632         /* Disable forced broadcasting */
1633         reg &= ~sCR0_FB;
1634
1635         /* Don't upgrade barriers */
1636         reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT);
1637
1638         if (smmu->features & ARM_SMMU_FEAT_VMID16)
1639                 reg |= sCR0_VMID16EN;
1640
1641         /* Push the button */
1642         __arm_smmu_tlb_sync(smmu);
1643         writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
1644 }
1645
1646 static int arm_smmu_id_size_to_bits(int size)
1647 {
1648         switch (size) {
1649         case 0:
1650                 return 32;
1651         case 1:
1652                 return 36;
1653         case 2:
1654                 return 40;
1655         case 3:
1656                 return 42;
1657         case 4:
1658                 return 44;
1659         case 5:
1660         default:
1661                 return 48;
1662         }
1663 }
1664
1665 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1666 {
1667         unsigned long size;
1668         void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1669         u32 id;
1670         bool cttw_dt, cttw_reg;
1671         int i;
1672
1673         dev_notice(smmu->dev, "probing hardware configuration...\n");
1674         dev_notice(smmu->dev, "SMMUv%d with:\n",
1675                         smmu->version == ARM_SMMU_V2 ? 2 : 1);
1676
1677         /* ID0 */
1678         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0);
1679
1680         /* Restrict available stages based on module parameter */
1681         if (force_stage == 1)
1682                 id &= ~(ID0_S2TS | ID0_NTS);
1683         else if (force_stage == 2)
1684                 id &= ~(ID0_S1TS | ID0_NTS);
1685
1686         if (id & ID0_S1TS) {
1687                 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1688                 dev_notice(smmu->dev, "\tstage 1 translation\n");
1689         }
1690
1691         if (id & ID0_S2TS) {
1692                 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1693                 dev_notice(smmu->dev, "\tstage 2 translation\n");
1694         }
1695
1696         if (id & ID0_NTS) {
1697                 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1698                 dev_notice(smmu->dev, "\tnested translation\n");
1699         }
1700
1701         if (!(smmu->features &
1702                 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1703                 dev_err(smmu->dev, "\tno translation support!\n");
1704                 return -ENODEV;
1705         }
1706
1707         if ((id & ID0_S1TS) &&
1708                 ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) {
1709                 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1710                 dev_notice(smmu->dev, "\taddress translation ops\n");
1711         }
1712
1713         /*
1714          * In order for DMA API calls to work properly, we must defer to what
1715          * the DT says about coherency, regardless of what the hardware claims.
1716          * Fortunately, this also opens up a workaround for systems where the
1717          * ID register value has ended up configured incorrectly.
1718          */
1719         cttw_dt = of_dma_is_coherent(smmu->dev->of_node);
1720         cttw_reg = !!(id & ID0_CTTW);
1721         if (cttw_dt)
1722                 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
1723         if (cttw_dt || cttw_reg)
1724                 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1725                            cttw_dt ? "" : "non-");
1726         if (cttw_dt != cttw_reg)
1727                 dev_notice(smmu->dev,
1728                            "\t(IDR0.CTTW overridden by dma-coherent property)\n");
1729
1730         /* Max. number of entries we have for stream matching/indexing */
1731         size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK);
1732         smmu->streamid_mask = size - 1;
1733         if (id & ID0_SMS) {
1734                 u32 smr;
1735
1736                 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1737                 size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK;
1738                 if (size == 0) {
1739                         dev_err(smmu->dev,
1740                                 "stream-matching supported, but no SMRs present!\n");
1741                         return -ENODEV;
1742                 }
1743
1744                 /*
1745                  * SMR.ID bits may not be preserved if the corresponding MASK
1746                  * bits are set, so check each one separately. We can reject
1747                  * masters later if they try to claim IDs outside these masks.
1748                  */
1749                 smr = smmu->streamid_mask << SMR_ID_SHIFT;
1750                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1751                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1752                 smmu->streamid_mask = smr >> SMR_ID_SHIFT;
1753
1754                 smr = smmu->streamid_mask << SMR_MASK_SHIFT;
1755                 writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
1756                 smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
1757                 smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT;
1758
1759                 /* Zero-initialised to mark as invalid */
1760                 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1761                                           GFP_KERNEL);
1762                 if (!smmu->smrs)
1763                         return -ENOMEM;
1764
1765                 dev_notice(smmu->dev,
1766                            "\tstream matching with %lu register groups, mask 0x%x",
1767                            size, smmu->smr_mask_mask);
1768         }
1769         /* s2cr->type == 0 means translation, so initialise explicitly */
1770         smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1771                                          GFP_KERNEL);
1772         if (!smmu->s2crs)
1773                 return -ENOMEM;
1774         for (i = 0; i < size; i++)
1775                 smmu->s2crs[i] = s2cr_init_val;
1776
1777         smmu->num_mapping_groups = size;
1778         mutex_init(&smmu->stream_map_mutex);
1779
1780         if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) {
1781                 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1782                 if (!(id & ID0_PTFS_NO_AARCH32S))
1783                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1784         }
1785
1786         /* ID1 */
1787         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1);
1788         smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;
1789
1790         /* Check for size mismatch of SMMU address space from mapped region */
1791         size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1);
1792         size *= 2 << smmu->pgshift;
1793         if (smmu->size != size)
1794                 dev_warn(smmu->dev,
1795                         "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n",
1796                         size, smmu->size);
1797
1798         smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;
1799         smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;
1800         if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1801                 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1802                 return -ENODEV;
1803         }
1804         dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1805                    smmu->num_context_banks, smmu->num_s2_context_banks);
1806         /*
1807          * Cavium CN88xx erratum #27704.
1808          * Ensure ASID and VMID allocation is unique across all SMMUs in
1809          * the system.
1810          */
1811         if (smmu->model == CAVIUM_SMMUV2) {
1812                 smmu->cavium_id_base =
1813                         atomic_add_return(smmu->num_context_banks,
1814                                           &cavium_smmu_context_count);
1815                 smmu->cavium_id_base -= smmu->num_context_banks;
1816         }
1817
1818         /* ID2 */
1819         id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2);
1820         size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK);
1821         smmu->ipa_size = size;
1822
1823         /* The output mask is also applied for bypass */
1824         size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK);
1825         smmu->pa_size = size;
1826
1827         if (id & ID2_VMID16)
1828                 smmu->features |= ARM_SMMU_FEAT_VMID16;
1829
1830         /*
1831          * What the page table walker can address actually depends on which
1832          * descriptor format is in use, but since a) we don't know that yet,
1833          * and b) it can vary per context bank, this will have to do...
1834          */
1835         if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1836                 dev_warn(smmu->dev,
1837                          "failed to set DMA mask for table walker\n");
1838
1839         if (smmu->version < ARM_SMMU_V2) {
1840                 smmu->va_size = smmu->ipa_size;
1841                 if (smmu->version == ARM_SMMU_V1_64K)
1842                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1843         } else {
1844                 size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK;
1845                 smmu->va_size = arm_smmu_id_size_to_bits(size);
1846                 if (id & ID2_PTFS_4K)
1847                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1848                 if (id & ID2_PTFS_16K)
1849                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1850                 if (id & ID2_PTFS_64K)
1851                         smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1852         }
1853
1854         /* Now we've corralled the various formats, what'll it do? */
1855         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1856                 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1857         if (smmu->features &
1858             (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1859                 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1860         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1861                 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1862         if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1863                 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1864
1865         if (arm_smmu_ops.pgsize_bitmap == -1UL)
1866                 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1867         else
1868                 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1869         dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1870                    smmu->pgsize_bitmap);
1871
1872
1873         if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1874                 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1875                            smmu->va_size, smmu->ipa_size);
1876
1877         if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1878                 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1879                            smmu->ipa_size, smmu->pa_size);
1880
1881         return 0;
1882 }
1883
1884 struct arm_smmu_match_data {
1885         enum arm_smmu_arch_version version;
1886         enum arm_smmu_implementation model;
1887 };
1888
1889 #define ARM_SMMU_MATCH_DATA(name, ver, imp)     \
1890 static struct arm_smmu_match_data name = { .version = ver, .model = imp }
1891
1892 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1893 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1894 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1895 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1896 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1897
1898 static const struct of_device_id arm_smmu_of_match[] = {
1899         { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1900         { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1901         { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1902         { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1903         { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1904         { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1905         { },
1906 };
1907 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1908
1909 static int arm_smmu_device_dt_probe(struct platform_device *pdev)
1910 {
1911         const struct arm_smmu_match_data *data;
1912         struct resource *res;
1913         struct arm_smmu_device *smmu;
1914         struct device *dev = &pdev->dev;
1915         int num_irqs, i, err;
1916         bool legacy_binding;
1917
1918         legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
1919         if (legacy_binding && !using_generic_binding) {
1920                 if (!using_legacy_binding)
1921                         pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable\n");
1922                 using_legacy_binding = true;
1923         } else if (!legacy_binding && !using_legacy_binding) {
1924                 using_generic_binding = true;
1925         } else {
1926                 dev_err(dev, "not probing due to mismatched DT properties\n");
1927                 return -ENODEV;
1928         }
1929
1930         smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
1931         if (!smmu) {
1932                 dev_err(dev, "failed to allocate arm_smmu_device\n");
1933                 return -ENOMEM;
1934         }
1935         smmu->dev = dev;
1936
1937         data = of_device_get_match_data(dev);
1938         smmu->version = data->version;
1939         smmu->model = data->model;
1940
1941         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1942         smmu->base = devm_ioremap_resource(dev, res);
1943         if (IS_ERR(smmu->base))
1944                 return PTR_ERR(smmu->base);
1945         smmu->size = resource_size(res);
1946
1947         if (of_property_read_u32(dev->of_node, "#global-interrupts",
1948                                  &smmu->num_global_irqs)) {
1949                 dev_err(dev, "missing #global-interrupts property\n");
1950                 return -ENODEV;
1951         }
1952
1953         num_irqs = 0;
1954         while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
1955                 num_irqs++;
1956                 if (num_irqs > smmu->num_global_irqs)
1957                         smmu->num_context_irqs++;
1958         }
1959
1960         if (!smmu->num_context_irqs) {
1961                 dev_err(dev, "found %d interrupts but expected at least %d\n",
1962                         num_irqs, smmu->num_global_irqs + 1);
1963                 return -ENODEV;
1964         }
1965
1966         smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,
1967                                   GFP_KERNEL);
1968         if (!smmu->irqs) {
1969                 dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
1970                 return -ENOMEM;
1971         }
1972
1973         for (i = 0; i < num_irqs; ++i) {
1974                 int irq = platform_get_irq(pdev, i);
1975
1976                 if (irq < 0) {
1977                         dev_err(dev, "failed to get irq index %d\n", i);
1978                         return -ENODEV;
1979                 }
1980                 smmu->irqs[i] = irq;
1981         }
1982
1983         err = arm_smmu_device_cfg_probe(smmu);
1984         if (err)
1985                 return err;
1986
1987         parse_driver_options(smmu);
1988
1989         if (smmu->version == ARM_SMMU_V2 &&
1990             smmu->num_context_banks != smmu->num_context_irqs) {
1991                 dev_err(dev,
1992                         "found only %d context interrupt(s) but %d required\n",
1993                         smmu->num_context_irqs, smmu->num_context_banks);
1994                 return -ENODEV;
1995         }
1996
1997         for (i = 0; i < smmu->num_global_irqs; ++i) {
1998                 err = devm_request_irq(smmu->dev, smmu->irqs[i],
1999                                        arm_smmu_global_fault,
2000                                        IRQF_SHARED,
2001                                        "arm-smmu global fault",
2002                                        smmu);
2003                 if (err) {
2004                         dev_err(dev, "failed to request global IRQ %d (%u)\n",
2005                                 i, smmu->irqs[i]);
2006                         return err;
2007                 }
2008         }
2009
2010         of_iommu_set_ops(dev->of_node, &arm_smmu_ops);
2011         platform_set_drvdata(pdev, smmu);
2012         arm_smmu_device_reset(smmu);
2013
2014         /* Oh, for a proper bus abstraction */
2015         if (!iommu_present(&platform_bus_type))
2016                 bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
2017 #ifdef CONFIG_ARM_AMBA
2018         if (!iommu_present(&amba_bustype))
2019                 bus_set_iommu(&amba_bustype, &arm_smmu_ops);
2020 #endif
2021 #ifdef CONFIG_PCI
2022         if (!iommu_present(&pci_bus_type)) {
2023                 pci_request_acs();
2024                 bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
2025         }
2026 #endif
2027         return 0;
2028 }
2029
2030 static int arm_smmu_device_remove(struct platform_device *pdev)
2031 {
2032         struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2033
2034         if (!smmu)
2035                 return -ENODEV;
2036
2037         if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2038                 dev_err(&pdev->dev, "removing device with active domains!\n");
2039
2040         /* Turn the thing off */
2041         writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
2042         return 0;
2043 }
2044
2045 static struct platform_driver arm_smmu_driver = {
2046         .driver = {
2047                 .name           = "arm-smmu",
2048                 .of_match_table = of_match_ptr(arm_smmu_of_match),
2049         },
2050         .probe  = arm_smmu_device_dt_probe,
2051         .remove = arm_smmu_device_remove,
2052 };
2053
2054 static int __init arm_smmu_init(void)
2055 {
2056         static bool registered;
2057         int ret = 0;
2058
2059         if (!registered) {
2060                 ret = platform_driver_register(&arm_smmu_driver);
2061                 registered = !ret;
2062         }
2063         return ret;
2064 }
2065
2066 static void __exit arm_smmu_exit(void)
2067 {
2068         return platform_driver_unregister(&arm_smmu_driver);
2069 }
2070
2071 subsys_initcall(arm_smmu_init);
2072 module_exit(arm_smmu_exit);
2073
2074 static int __init arm_smmu_of_init(struct device_node *np)
2075 {
2076         int ret = arm_smmu_init();
2077
2078         if (ret)
2079                 return ret;
2080
2081         if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root))
2082                 return -ENODEV;
2083
2084         return 0;
2085 }
2086 IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init);
2087 IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init);
2088 IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init);
2089 IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init);
2090 IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init);
2091 IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init);
2092
2093 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2094 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
2095 MODULE_LICENSE("GPL v2");