drivers/irqchip/irq-gic-v3-its.c

   1 /*
   2  * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/bitmap.h>
  19 #include <linux/cpu.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/log2.h>
  23 #include <linux/mm.h>
  24 #include <linux/msi.h>
  25 #include <linux/of.h>
  26 #include <linux/of_address.h>
  27 #include <linux/of_irq.h>
  28 #include <linux/of_pci.h>
  29 #include <linux/of_platform.h>
  30 #include <linux/percpu.h>
  31 #include <linux/slab.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35
  36 #include <asm/cacheflush.h>
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45
  46 #define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING     (1 << 0)
  47
  48 /*
  49  * Collection structure - just an ID, and a redistributor address to
  50  * ping. We use one per CPU as a bag of interrupts assigned to this
  51  * CPU.
  52  */
  53 struct its_collection {
  54         u64                     target_address;
  55         u16                     col_id;
  56 };
  57
  58 /*
  59  * The ITS structure - contains most of the infrastructure, with the
  60  * top-level MSI domain, the command queue, the collections, and the
  61  * list of devices writing to it.
  62  */
  63 struct its_node {
  64         raw_spinlock_t          lock;
  65         struct list_head        entry;
  66         void __iomem            *base;
  67         unsigned long           phys_base;
  68         struct its_cmd_block    *cmd_base;
  69         struct its_cmd_block    *cmd_write;
  70         struct {
  71                 void            *base;
  72                 u32             order;
  73         } tables[GITS_BASER_NR_REGS];
  74         struct its_collection   *collections;
  75         struct list_head        its_device_list;
  76         u64                     flags;
  77         u32                     ite_size;
  78         int                     numa_node;
  79 };
  80
  81 #define ITS_ITT_ALIGN           SZ_256
  82
  83 /* Convert page order to size in bytes */
  84 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
  85
  86 struct event_lpi_map {
  87         unsigned long           *lpi_map;
  88         u16                     *col_map;
  89         irq_hw_number_t         lpi_base;
  90         int                     nr_lpis;
  91 };
  92
  93 /*
  94  * The ITS view of a device - belongs to an ITS, a collection, owns an
  95  * interrupt translation table, and a list of interrupts.
  96  */
  97 struct its_device {
  98         struct list_head        entry;
  99         struct its_node         *its;
 100         struct event_lpi_map    event_map;
 101         void                    *itt;
 102         u32                     nr_ites;
 103         u32                     device_id;
 104 };
 105
 106 static LIST_HEAD(its_nodes);
 107 static DEFINE_SPINLOCK(its_lock);
 108 static struct device_node *gic_root_node;
 109 static struct rdists *gic_rdists;
 110
 111 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 112 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 113
 114 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 115                                                u32 event)
 116 {
 117         struct its_node *its = its_dev->its;
 118
 119         return its->collections + its_dev->event_map.col_map[event];
 120 }
 121
 122 /*
 123  * ITS command descriptors - parameters to be encoded in a command
 124  * block.
 125  */
 126 struct its_cmd_desc {
 127         union {
 128                 struct {
 129                         struct its_device *dev;
 130                         u32 event_id;
 131                 } its_inv_cmd;
 132
 133                 struct {
 134                         struct its_device *dev;
 135                         u32 event_id;
 136                 } its_int_cmd;
 137
 138                 struct {
 139                         struct its_device *dev;
 140                         int valid;
 141                 } its_mapd_cmd;
 142
 143                 struct {
 144                         struct its_collection *col;
 145                         int valid;
 146                 } its_mapc_cmd;
 147
 148                 struct {
 149                         struct its_device *dev;
 150                         u32 phys_id;
 151                         u32 event_id;
 152                 } its_mapvi_cmd;
 153
 154                 struct {
 155                         struct its_device *dev;
 156                         struct its_collection *col;
 157                         u32 event_id;
 158                 } its_movi_cmd;
 159
 160                 struct {
 161                         struct its_device *dev;
 162                         u32 event_id;
 163                 } its_discard_cmd;
 164
 165                 struct {
 166                         struct its_collection *col;
 167                 } its_invall_cmd;
 168         };
 169 };
 170
 171 /*
 172  * The ITS command block, which is what the ITS actually parses.
 173  */
 174 struct its_cmd_block {
 175         u64     raw_cmd[4];
 176 };
 177
 178 #define ITS_CMD_QUEUE_SZ                SZ_64K
 179 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 180
 181 typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *,
 182                                                     struct its_cmd_desc *);
 183
 184 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 185 {
 186         cmd->raw_cmd[0] &= ~0xffUL;
 187         cmd->raw_cmd[0] |= cmd_nr;
 188 }
 189
 190 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 191 {
 192         cmd->raw_cmd[0] &= BIT_ULL(32) - 1;
 193         cmd->raw_cmd[0] |= ((u64)devid) << 32;
 194 }
 195
 196 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 197 {
 198         cmd->raw_cmd[1] &= ~0xffffffffUL;
 199         cmd->raw_cmd[1] |= id;
 200 }
 201
 202 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 203 {
 204         cmd->raw_cmd[1] &= 0xffffffffUL;
 205         cmd->raw_cmd[1] |= ((u64)phys_id) << 32;
 206 }
 207
 208 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 209 {
 210         cmd->raw_cmd[1] &= ~0x1fUL;
 211         cmd->raw_cmd[1] |= size & 0x1f;
 212 }
 213
 214 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 215 {
 216         cmd->raw_cmd[2] &= ~0xffffffffffffUL;
 217         cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL;
 218 }
 219
 220 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 221 {
 222         cmd->raw_cmd[2] &= ~(1UL << 63);
 223         cmd->raw_cmd[2] |= ((u64)!!valid) << 63;
 224 }
 225
 226 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 227 {
 228         cmd->raw_cmd[2] &= ~(0xffffffffUL << 16);
 229         cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16));
 230 }
 231
 232 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 233 {
 234         cmd->raw_cmd[2] &= ~0xffffUL;
 235         cmd->raw_cmd[2] |= col;
 236 }
 237
 238 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 239 {
 240         /* Let's fixup BE commands */
 241         cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
 242         cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
 243         cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
 244         cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
 245 }
 246
 247 static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd,
 248                                                  struct its_cmd_desc *desc)
 249 {
 250         unsigned long itt_addr;
 251         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 252
 253         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 254         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 255
 256         its_encode_cmd(cmd, GITS_CMD_MAPD);
 257         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 258         its_encode_size(cmd, size - 1);
 259         its_encode_itt(cmd, itt_addr);
 260         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 261
 262         its_fixup_cmd(cmd);
 263
 264         return NULL;
 265 }
 266
 267 static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd,
 268                                                  struct its_cmd_desc *desc)
 269 {
 270         its_encode_cmd(cmd, GITS_CMD_MAPC);
 271         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 272         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 273         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 274
 275         its_fixup_cmd(cmd);
 276
 277         return desc->its_mapc_cmd.col;
 278 }
 279
 280 static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd,
 281                                                   struct its_cmd_desc *desc)
 282 {
 283         struct its_collection *col;
 284
 285         col = dev_event_to_col(desc->its_mapvi_cmd.dev,
 286                                desc->its_mapvi_cmd.event_id);
 287
 288         its_encode_cmd(cmd, GITS_CMD_MAPVI);
 289         its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id);
 290         its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id);
 291         its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id);
 292         its_encode_collection(cmd, col->col_id);
 293
 294         its_fixup_cmd(cmd);
 295
 296         return col;
 297 }
 298
 299 static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd,
 300                                                  struct its_cmd_desc *desc)
 301 {
 302         struct its_collection *col;
 303
 304         col = dev_event_to_col(desc->its_movi_cmd.dev,
 305                                desc->its_movi_cmd.event_id);
 306
 307         its_encode_cmd(cmd, GITS_CMD_MOVI);
 308         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 309         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 310         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 311
 312         its_fixup_cmd(cmd);
 313
 314         return col;
 315 }
 316
 317 static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd,
 318                                                     struct its_cmd_desc *desc)
 319 {
 320         struct its_collection *col;
 321
 322         col = dev_event_to_col(desc->its_discard_cmd.dev,
 323                                desc->its_discard_cmd.event_id);
 324
 325         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 326         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 327         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 328
 329         its_fixup_cmd(cmd);
 330
 331         return col;
 332 }
 333
 334 static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd,
 335                                                 struct its_cmd_desc *desc)
 336 {
 337         struct its_collection *col;
 338
 339         col = dev_event_to_col(desc->its_inv_cmd.dev,
 340                                desc->its_inv_cmd.event_id);
 341
 342         its_encode_cmd(cmd, GITS_CMD_INV);
 343         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 344         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 345
 346         its_fixup_cmd(cmd);
 347
 348         return col;
 349 }
 350
 351 static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd,
 352                                                    struct its_cmd_desc *desc)
 353 {
 354         its_encode_cmd(cmd, GITS_CMD_INVALL);
 355         its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 356
 357         its_fixup_cmd(cmd);
 358
 359         return NULL;
 360 }
 361
 362 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 363                                  struct its_cmd_block *ptr)
 364 {
 365         return (ptr - its->cmd_base) * sizeof(*ptr);
 366 }
 367
 368 static int its_queue_full(struct its_node *its)
 369 {
 370         int widx;
 371         int ridx;
 372
 373         widx = its->cmd_write - its->cmd_base;
 374         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
 375
 376         /* This is incredibly unlikely to happen, unless the ITS locks up. */
 377         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
 378                 return 1;
 379
 380         return 0;
 381 }
 382
 383 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
 384 {
 385         struct its_cmd_block *cmd;
 386         u32 count = 1000000;    /* 1s! */
 387
 388         while (its_queue_full(its)) {
 389                 count--;
 390                 if (!count) {
 391                         pr_err_ratelimited("ITS queue not draining\n");
 392                         return NULL;
 393                 }
 394                 cpu_relax();
 395                 udelay(1);
 396         }
 397
 398         cmd = its->cmd_write++;
 399
 400         /* Handle queue wrapping */
 401         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
 402                 its->cmd_write = its->cmd_base;
 403
 404         return cmd;
 405 }
 406
 407 static struct its_cmd_block *its_post_commands(struct its_node *its)
 408 {
 409         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
 410
 411         writel_relaxed(wr, its->base + GITS_CWRITER);
 412
 413         return its->cmd_write;
 414 }
 415
 416 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
 417 {
 418         /*
 419          * Make sure the commands written to memory are observable by
 420          * the ITS.
 421          */
 422         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
 423                 __flush_dcache_area(cmd, sizeof(*cmd));
 424         else
 425                 dsb(ishst);
 426 }
 427
 428 static void its_wait_for_range_completion(struct its_node *its,
 429                                           struct its_cmd_block *from,
 430                                           struct its_cmd_block *to)
 431 {
 432         u64 rd_idx, from_idx, to_idx;
 433         u32 count = 1000000;    /* 1s! */
 434
 435         from_idx = its_cmd_ptr_to_offset(its, from);
 436         to_idx = its_cmd_ptr_to_offset(its, to);
 437
 438         while (1) {
 439                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
 440                 if (rd_idx >= to_idx || rd_idx < from_idx)
 441                         break;
 442
 443                 count--;
 444                 if (!count) {
 445                         pr_err_ratelimited("ITS queue timeout\n");
 446                         return;
 447                 }
 448                 cpu_relax();
 449                 udelay(1);
 450         }
 451 }
 452
 453 static void its_send_single_command(struct its_node *its,
 454                                     its_cmd_builder_t builder,
 455                                     struct its_cmd_desc *desc)
 456 {
 457         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;
 458         struct its_collection *sync_col;
 459         unsigned long flags;
 460
 461         raw_spin_lock_irqsave(&its->lock, flags);
 462
 463         cmd = its_allocate_entry(its);
 464         if (!cmd) {             /* We're soooooo screewed... */
 465                 pr_err_ratelimited("ITS can't allocate, dropping command\n");
 466                 raw_spin_unlock_irqrestore(&its->lock, flags);
 467                 return;
 468         }
 469         sync_col = builder(cmd, desc);
 470         its_flush_cmd(its, cmd);
 471
 472         if (sync_col) {
 473                 sync_cmd = its_allocate_entry(its);
 474                 if (!sync_cmd) {
 475                         pr_err_ratelimited("ITS can't SYNC, skipping\n");
 476                         goto post;
 477                 }
 478                 its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
 479                 its_encode_target(sync_cmd, sync_col->target_address);
 480                 its_fixup_cmd(sync_cmd);
 481                 its_flush_cmd(its, sync_cmd);
 482         }
 483
 484 post:
 485         next_cmd = its_post_commands(its);
 486         raw_spin_unlock_irqrestore(&its->lock, flags);
 487
 488         its_wait_for_range_completion(its, cmd, next_cmd);
 489 }
 490
 491 static void its_send_inv(struct its_device *dev, u32 event_id)
 492 {
 493         struct its_cmd_desc desc;
 494
 495         desc.its_inv_cmd.dev = dev;
 496         desc.its_inv_cmd.event_id = event_id;
 497
 498         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
 499 }
 500
 501 static void its_send_mapd(struct its_device *dev, int valid)
 502 {
 503         struct its_cmd_desc desc;
 504
 505         desc.its_mapd_cmd.dev = dev;
 506         desc.its_mapd_cmd.valid = !!valid;
 507
 508         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
 509 }
 510
 511 static void its_send_mapc(struct its_node *its, struct its_collection *col,
 512                           int valid)
 513 {
 514         struct its_cmd_desc desc;
 515
 516         desc.its_mapc_cmd.col = col;
 517         desc.its_mapc_cmd.valid = !!valid;
 518
 519         its_send_single_command(its, its_build_mapc_cmd, &desc);
 520 }
 521
 522 static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id)
 523 {
 524         struct its_cmd_desc desc;
 525
 526         desc.its_mapvi_cmd.dev = dev;
 527         desc.its_mapvi_cmd.phys_id = irq_id;
 528         desc.its_mapvi_cmd.event_id = id;
 529
 530         its_send_single_command(dev->its, its_build_mapvi_cmd, &desc);
 531 }
 532
 533 static void its_send_movi(struct its_device *dev,
 534                           struct its_collection *col, u32 id)
 535 {
 536         struct its_cmd_desc desc;
 537
 538         desc.its_movi_cmd.dev = dev;
 539         desc.its_movi_cmd.col = col;
 540         desc.its_movi_cmd.event_id = id;
 541
 542         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
 543 }
 544
 545 static void its_send_discard(struct its_device *dev, u32 id)
 546 {
 547         struct its_cmd_desc desc;
 548
 549         desc.its_discard_cmd.dev = dev;
 550         desc.its_discard_cmd.event_id = id;
 551
 552         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
 553 }
 554
 555 static void its_send_invall(struct its_node *its, struct its_collection *col)
 556 {
 557         struct its_cmd_desc desc;
 558
 559         desc.its_invall_cmd.col = col;
 560
 561         its_send_single_command(its, its_build_invall_cmd, &desc);
 562 }
 563
 564 /*
 565  * irqchip functions - assumes MSI, mostly.
 566  */
 567
 568 static inline u32 its_get_event_id(struct irq_data *d)
 569 {
 570         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 571         return d->hwirq - its_dev->event_map.lpi_base;
 572 }
 573
 574 static void lpi_set_config(struct irq_data *d, bool enable)
 575 {
 576         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 577         irq_hw_number_t hwirq = d->hwirq;
 578         u32 id = its_get_event_id(d);
 579         u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192;
 580
 581         if (enable)
 582                 *cfg |= LPI_PROP_ENABLED;
 583         else
 584                 *cfg &= ~LPI_PROP_ENABLED;
 585
 586         /*
 587          * Make the above write visible to the redistributors.
 588          * And yes, we're flushing exactly: One. Single. Byte.
 589          * Humpf...
 590          */
 591         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
 592                 __flush_dcache_area(cfg, sizeof(*cfg));
 593         else
 594                 dsb(ishst);
 595         its_send_inv(its_dev, id);
 596 }
 597
 598 static void its_mask_irq(struct irq_data *d)
 599 {
 600         lpi_set_config(d, false);
 601 }
 602
 603 static void its_unmask_irq(struct irq_data *d)
 604 {
 605         lpi_set_config(d, true);
 606 }
 607
 608 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 609                             bool force)
 610 {
 611         unsigned int cpu;
 612         const struct cpumask *cpu_mask = cpu_online_mask;
 613         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 614         struct its_collection *target_col;
 615         u32 id = its_get_event_id(d);
 616
 617        /* lpi cannot be routed to a redistributor that is on a foreign node */
 618         if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
 619                 if (its_dev->its->numa_node >= 0) {
 620                         cpu_mask = cpumask_of_node(its_dev->its->numa_node);
 621                         if (!cpumask_intersects(mask_val, cpu_mask))
 622                                 return -EINVAL;
 623                 }
 624         }
 625
 626         cpu = cpumask_any_and(mask_val, cpu_mask);
 627
 628         if (cpu >= nr_cpu_ids)
 629                 return -EINVAL;
 630
 631         target_col = &its_dev->its->collections[cpu];
 632         its_send_movi(its_dev, target_col, id);
 633         its_dev->event_map.col_map[id] = cpu;
 634
 635         return IRQ_SET_MASK_OK_DONE;
 636 }
 637
 638 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
 639 {
 640         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 641         struct its_node *its;
 642         u64 addr;
 643
 644         its = its_dev->its;
 645         addr = its->phys_base + GITS_TRANSLATER;
 646
 647         msg->address_lo         = addr & ((1UL << 32) - 1);
 648         msg->address_hi         = addr >> 32;
 649         msg->data               = its_get_event_id(d);
 650 }
 651
 652 static struct irq_chip its_irq_chip = {
 653         .name                   = "ITS",
 654         .irq_mask               = its_mask_irq,
 655         .irq_unmask             = its_unmask_irq,
 656         .irq_eoi                = irq_chip_eoi_parent,
 657         .irq_set_affinity       = its_set_affinity,
 658         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
 659 };
 660
 661 /*
 662  * How we allocate LPIs:
 663  *
 664  * The GIC has id_bits bits for interrupt identifiers. From there, we
 665  * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as
 666  * we allocate LPIs by chunks of 32, we can shift the whole thing by 5
 667  * bits to the right.
 668  *
 669  * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations.
 670  */
 671 #define IRQS_PER_CHUNK_SHIFT    5
 672 #define IRQS_PER_CHUNK          (1UL << IRQS_PER_CHUNK_SHIFT)
 673
 674 static unsigned long *lpi_bitmap;
 675 static u32 lpi_chunks;
 676 static DEFINE_SPINLOCK(lpi_lock);
 677
 678 static int its_lpi_to_chunk(int lpi)
 679 {
 680         return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT;
 681 }
 682
 683 static int its_chunk_to_lpi(int chunk)
 684 {
 685         return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192;
 686 }
 687
 688 static int its_lpi_init(u32 id_bits)
 689 {
 690         lpi_chunks = its_lpi_to_chunk(1UL << id_bits);
 691
 692         lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long),
 693                              GFP_KERNEL);
 694         if (!lpi_bitmap) {
 695                 lpi_chunks = 0;
 696                 return -ENOMEM;
 697         }
 698
 699         pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks);
 700         return 0;
 701 }
 702
 703 static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids)
 704 {
 705         unsigned long *bitmap = NULL;
 706         int chunk_id;
 707         int nr_chunks;
 708         int i;
 709
 710         nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK);
 711
 712         spin_lock(&lpi_lock);
 713
 714         do {
 715                 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks,
 716                                                       0, nr_chunks, 0);
 717                 if (chunk_id < lpi_chunks)
 718                         break;
 719
 720                 nr_chunks--;
 721         } while (nr_chunks > 0);
 722
 723         if (!nr_chunks)
 724                 goto out;
 725
 726         bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long),
 727                          GFP_ATOMIC);
 728         if (!bitmap)
 729                 goto out;
 730
 731         for (i = 0; i < nr_chunks; i++)
 732                 set_bit(chunk_id + i, lpi_bitmap);
 733
 734         *base = its_chunk_to_lpi(chunk_id);
 735         *nr_ids = nr_chunks * IRQS_PER_CHUNK;
 736
 737 out:
 738         spin_unlock(&lpi_lock);
 739
 740         if (!bitmap)
 741                 *base = *nr_ids = 0;
 742
 743         return bitmap;
 744 }
 745
 746 static void its_lpi_free(struct event_lpi_map *map)
 747 {
 748         int base = map->lpi_base;
 749         int nr_ids = map->nr_lpis;
 750         int lpi;
 751
 752         spin_lock(&lpi_lock);
 753
 754         for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) {
 755                 int chunk = its_lpi_to_chunk(lpi);
 756                 BUG_ON(chunk > lpi_chunks);
 757                 if (test_bit(chunk, lpi_bitmap)) {
 758                         clear_bit(chunk, lpi_bitmap);
 759                 } else {
 760                         pr_err("Bad LPI chunk %d\n", chunk);
 761                 }
 762         }
 763
 764         spin_unlock(&lpi_lock);
 765
 766         kfree(map->lpi_map);
 767         kfree(map->col_map);
 768 }
 769
 770 /*
 771  * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to
 772  * deal with (one configuration byte per interrupt). PENDBASE has to
 773  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
 774  */
 775 #define LPI_PROPBASE_SZ         SZ_64K
 776 #define LPI_PENDBASE_SZ         (LPI_PROPBASE_SZ / 8 + SZ_1K)
 777
 778 /*
 779  * This is how many bits of ID we need, including the useless ones.
 780  */
 781 #define LPI_NRBITS              ilog2(LPI_PROPBASE_SZ + SZ_8K)
 782
 783 #define LPI_PROP_DEFAULT_PRIO   0xa0
 784
 785 static int __init its_alloc_lpi_tables(void)
 786 {
 787         phys_addr_t paddr;
 788
 789         gic_rdists->prop_page = alloc_pages(GFP_NOWAIT,
 790                                            get_order(LPI_PROPBASE_SZ));
 791         if (!gic_rdists->prop_page) {
 792                 pr_err("Failed to allocate PROPBASE\n");
 793                 return -ENOMEM;
 794         }
 795
 796         paddr = page_to_phys(gic_rdists->prop_page);
 797         pr_info("GIC: using LPI property table @%pa\n", &paddr);
 798
 799         /* Priority 0xa0, Group-1, disabled */
 800         memset(page_address(gic_rdists->prop_page),
 801                LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
 802                LPI_PROPBASE_SZ);
 803
 804         /* Make sure the GIC will observe the written configuration */
 805         __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ);
 806
 807         return 0;
 808 }
 809
 810 static const char *its_base_type_string[] = {
 811         [GITS_BASER_TYPE_DEVICE]        = "Devices",
 812         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
 813         [GITS_BASER_TYPE_CPU]           = "Physical CPUs",
 814         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
 815         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
 816         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
 817         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
 818 };
 819
 820 static void its_free_tables(struct its_node *its)
 821 {
 822         int i;
 823
 824         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
 825                 if (its->tables[i].base) {
 826                         free_pages((unsigned long)its->tables[i].base,
 827                                    its->tables[i].order);
 828                         its->tables[i].base = NULL;
 829                 }
 830         }
 831 }
 832
 833 static int its_alloc_tables(const char *node_name, struct its_node *its)
 834 {
 835         int err;
 836         int i;
 837         int psz = SZ_64K;
 838         u64 shr = GITS_BASER_InnerShareable;
 839         u64 cache;
 840         u64 typer;
 841         u32 ids;
 842
 843         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) {
 844                 /*
 845                  * erratum 22375: only alloc 8MB table size
 846                  * erratum 24313: ignore memory access type
 847                  */
 848                 cache   = 0;
 849                 ids     = 0x14;                 /* 20 bits, 8MB */
 850         } else {
 851                 cache   = GITS_BASER_WaWb;
 852                 typer   = readq_relaxed(its->base + GITS_TYPER);
 853                 ids     = GITS_TYPER_DEVBITS(typer);
 854         }
 855
 856         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
 857                 u64 val = readq_relaxed(its->base + GITS_BASER + i * 8);
 858                 u64 type = GITS_BASER_TYPE(val);
 859                 u64 entry_size = GITS_BASER_ENTRY_SIZE(val);
 860                 int order = get_order(psz);
 861                 int alloc_pages;
 862                 u64 tmp;
 863                 void *base;
 864
 865                 if (type == GITS_BASER_TYPE_NONE)
 866                         continue;
 867
 868                 /*
 869                  * Allocate as many entries as required to fit the
 870                  * range of device IDs that the ITS can grok... The ID
 871                  * space being incredibly sparse, this results in a
 872                  * massive waste of memory.
 873                  *
 874                  * For other tables, only allocate a single page.
 875                  */
 876                 if (type == GITS_BASER_TYPE_DEVICE) {
 877                         /*
 878                          * 'order' was initialized earlier to the default page
 879                          * granule of the the ITS.  We can't have an allocation
 880                          * smaller than that.  If the requested allocation
 881                          * is smaller, round up to the default page granule.
 882                          */
 883                         order = max(get_order((1UL << ids) * entry_size),
 884                                     order);
 885                         if (order >= MAX_ORDER) {
 886                                 order = MAX_ORDER - 1;
 887                                 pr_warn("%s: Device Table too large, reduce its page order to %u\n",
 888                                         node_name, order);
 889                         }
 890                 }
 891
 892 retry_alloc_baser:
 893                 alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
 894                 if (alloc_pages > GITS_BASER_PAGES_MAX) {
 895                         alloc_pages = GITS_BASER_PAGES_MAX;
 896                         order = get_order(GITS_BASER_PAGES_MAX * psz);
 897                         pr_warn("%s: Device Table too large, reduce its page order to %u (%u pages)\n",
 898                                 node_name, order, alloc_pages);
 899                 }
 900
 901                 base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
 902                 if (!base) {
 903                         err = -ENOMEM;
 904                         goto out_free;
 905                 }
 906
 907                 its->tables[i].base = base;
 908                 its->tables[i].order = order;
 909
 910 retry_baser:
 911                 val = (virt_to_phys(base)                                |
 912                        (type << GITS_BASER_TYPE_SHIFT)                   |
 913                        ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
 914                        cache                                             |
 915                        shr                                               |
 916                        GITS_BASER_VALID);
 917
 918                 switch (psz) {
 919                 case SZ_4K:
 920                         val |= GITS_BASER_PAGE_SIZE_4K;
 921                         break;
 922                 case SZ_16K:
 923                         val |= GITS_BASER_PAGE_SIZE_16K;
 924                         break;
 925                 case SZ_64K:
 926                         val |= GITS_BASER_PAGE_SIZE_64K;
 927                         break;
 928                 }
 929
 930                 val |= alloc_pages - 1;
 931
 932                 writeq_relaxed(val, its->base + GITS_BASER + i * 8);
 933                 tmp = readq_relaxed(its->base + GITS_BASER + i * 8);
 934
 935                 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
 936                         /*
 937                          * Shareability didn't stick. Just use
 938                          * whatever the read reported, which is likely
 939                          * to be the only thing this redistributor
 940                          * supports. If that's zero, make it
 941                          * non-cacheable as well.
 942                          */
 943                         shr = tmp & GITS_BASER_SHAREABILITY_MASK;
 944                         if (!shr) {
 945                                 cache = GITS_BASER_nC;
 946                                 __flush_dcache_area(base, PAGE_ORDER_TO_SIZE(order));
 947                         }
 948                         goto retry_baser;
 949                 }
 950
 951                 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
 952                         /*
 953                          * Page size didn't stick. Let's try a smaller
 954                          * size and retry. If we reach 4K, then
 955                          * something is horribly wrong...
 956                          */
 957                         free_pages((unsigned long)base, order);
 958                         its->tables[i].base = NULL;
 959
 960                         switch (psz) {
 961                         case SZ_16K:
 962                                 psz = SZ_4K;
 963                                 goto retry_alloc_baser;
 964                         case SZ_64K:
 965                                 psz = SZ_16K;
 966                                 goto retry_alloc_baser;
 967                         }
 968                 }
 969
 970                 if (val != tmp) {
 971                         pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n",
 972                                node_name, i,
 973                                (unsigned long) val, (unsigned long) tmp);
 974                         err = -ENXIO;
 975                         goto out_free;
 976                 }
 977
 978                 pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n",
 979                         (int)(PAGE_ORDER_TO_SIZE(order) / entry_size),
 980                         its_base_type_string[type],
 981                         (unsigned long)virt_to_phys(base),
 982                         psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
 983         }
 984
 985         return 0;
 986
 987 out_free:
 988         its_free_tables(its);
 989
 990         return err;
 991 }
 992
 993 static int its_alloc_collections(struct its_node *its)
 994 {
 995         its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections),
 996                                    GFP_KERNEL);
 997         if (!its->collections)
 998                 return -ENOMEM;
 999
1000         return 0;
1001 }
1002
1003 static void its_cpu_init_lpis(void)
1004 {
1005         void __iomem *rbase = gic_data_rdist_rd_base();
1006         struct page *pend_page;
1007         u64 val, tmp;
1008
1009         /* If we didn't allocate the pending table yet, do it now */
1010         pend_page = gic_data_rdist()->pend_page;
1011         if (!pend_page) {
1012                 phys_addr_t paddr;
1013                 /*
1014                  * The pending pages have to be at least 64kB aligned,
1015                  * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1016                  */
1017                 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO,
1018                                         get_order(max(LPI_PENDBASE_SZ, SZ_64K)));
1019                 if (!pend_page) {
1020                         pr_err("Failed to allocate PENDBASE for CPU%d\n",
1021                                smp_processor_id());
1022                         return;
1023                 }
1024
1025                 /* Make sure the GIC will observe the zero-ed page */
1026                 __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ);
1027
1028                 paddr = page_to_phys(pend_page);
1029                 pr_info("CPU%d: using LPI pending table @%pa\n",
1030                         smp_processor_id(), &paddr);
1031                 gic_data_rdist()->pend_page = pend_page;
1032         }
1033
1034         /* Disable LPIs */
1035         val = readl_relaxed(rbase + GICR_CTLR);
1036         val &= ~GICR_CTLR_ENABLE_LPIS;
1037         writel_relaxed(val, rbase + GICR_CTLR);
1038
1039         /*
1040          * Make sure any change to the table is observable by the GIC.
1041          */
1042         dsb(sy);
1043
1044         /* set PROPBASE */
1045         val = (page_to_phys(gic_rdists->prop_page) |
1046                GICR_PROPBASER_InnerShareable |
1047                GICR_PROPBASER_WaWb |
1048                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
1049
1050         writeq_relaxed(val, rbase + GICR_PROPBASER);
1051         tmp = readq_relaxed(rbase + GICR_PROPBASER);
1052
1053         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
1054                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
1055                         /*
1056                          * The HW reports non-shareable, we must
1057                          * remove the cacheability attributes as
1058                          * well.
1059                          */
1060                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
1061                                  GICR_PROPBASER_CACHEABILITY_MASK);
1062                         val |= GICR_PROPBASER_nC;
1063                         writeq_relaxed(val, rbase + GICR_PROPBASER);
1064                 }
1065                 pr_info_once("GIC: using cache flushing for LPI property table\n");
1066                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
1067         }
1068
1069         /* set PENDBASE */
1070         val = (page_to_phys(pend_page) |
1071                GICR_PENDBASER_InnerShareable |
1072                GICR_PENDBASER_WaWb);
1073
1074         writeq_relaxed(val, rbase + GICR_PENDBASER);
1075         tmp = readq_relaxed(rbase + GICR_PENDBASER);
1076
1077         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
1078                 /*
1079                  * The HW reports non-shareable, we must remove the
1080                  * cacheability attributes as well.
1081                  */
1082                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
1083                          GICR_PENDBASER_CACHEABILITY_MASK);
1084                 val |= GICR_PENDBASER_nC;
1085                 writeq_relaxed(val, rbase + GICR_PENDBASER);
1086         }
1087
1088         /* Enable LPIs */
1089         val = readl_relaxed(rbase + GICR_CTLR);
1090         val |= GICR_CTLR_ENABLE_LPIS;
1091         writel_relaxed(val, rbase + GICR_CTLR);
1092
1093         /* Make sure the GIC has seen the above */
1094         dsb(sy);
1095 }
1096
1097 static void its_cpu_init_collection(void)
1098 {
1099         struct its_node *its;
1100         int cpu;
1101
1102         spin_lock(&its_lock);
1103         cpu = smp_processor_id();
1104
1105         list_for_each_entry(its, &its_nodes, entry) {
1106                 u64 target;
1107
1108                 /* avoid cross node collections and its mapping */
1109                 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
1110                         struct device_node *cpu_node;
1111
1112                         cpu_node = of_get_cpu_node(cpu, NULL);
1113                         if (its->numa_node != NUMA_NO_NODE &&
1114                                 its->numa_node != of_node_to_nid(cpu_node))
1115                                 continue;
1116                 }
1117
1118                 /*
1119                  * We now have to bind each collection to its target
1120                  * redistributor.
1121                  */
1122                 if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
1123                         /*
1124                          * This ITS wants the physical address of the
1125                          * redistributor.
1126                          */
1127                         target = gic_data_rdist()->phys_base;
1128                 } else {
1129                         /*
1130                          * This ITS wants a linear CPU number.
1131                          */
1132                         target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER);
1133                         target = GICR_TYPER_CPU_NUMBER(target) << 16;
1134                 }
1135
1136                 /* Perform collection mapping */
1137                 its->collections[cpu].target_address = target;
1138                 its->collections[cpu].col_id = cpu;
1139
1140                 its_send_mapc(its, &its->collections[cpu], 1);
1141                 its_send_invall(its, &its->collections[cpu]);
1142         }
1143
1144         spin_unlock(&its_lock);
1145 }
1146
1147 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
1148 {
1149         struct its_device *its_dev = NULL, *tmp;
1150         unsigned long flags;
1151
1152         raw_spin_lock_irqsave(&its->lock, flags);
1153
1154         list_for_each_entry(tmp, &its->its_device_list, entry) {
1155                 if (tmp->device_id == dev_id) {
1156                         its_dev = tmp;
1157                         break;
1158                 }
1159         }
1160
1161         raw_spin_unlock_irqrestore(&its->lock, flags);
1162
1163         return its_dev;
1164 }
1165
1166 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
1167                                             int nvecs)
1168 {
1169         struct its_device *dev;
1170         unsigned long *lpi_map;
1171         unsigned long flags;
1172         u16 *col_map = NULL;
1173         void *itt;
1174         int lpi_base;
1175         int nr_lpis;
1176         int nr_ites;
1177         int sz;
1178
1179         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1180         /*
1181          * We allocate at least one chunk worth of LPIs bet device,
1182          * and thus that many ITEs. The device may require less though.
1183          */
1184         nr_ites = max(IRQS_PER_CHUNK, roundup_pow_of_two(nvecs));
1185         sz = nr_ites * its->ite_size;
1186         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
1187         itt = kzalloc(sz, GFP_KERNEL);
1188         lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis);
1189         if (lpi_map)
1190                 col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL);
1191
1192         if (!dev || !itt || !lpi_map || !col_map) {
1193                 kfree(dev);
1194                 kfree(itt);
1195                 kfree(lpi_map);
1196                 kfree(col_map);
1197                 return NULL;
1198         }
1199
1200         __flush_dcache_area(itt, sz);
1201
1202         dev->its = its;
1203         dev->itt = itt;
1204         dev->nr_ites = nr_ites;
1205         dev->event_map.lpi_map = lpi_map;
1206         dev->event_map.col_map = col_map;
1207         dev->event_map.lpi_base = lpi_base;
1208         dev->event_map.nr_lpis = nr_lpis;
1209         dev->device_id = dev_id;
1210         INIT_LIST_HEAD(&dev->entry);
1211
1212         raw_spin_lock_irqsave(&its->lock, flags);
1213         list_add(&dev->entry, &its->its_device_list);
1214         raw_spin_unlock_irqrestore(&its->lock, flags);
1215
1216         /* Map device to its ITT */
1217         its_send_mapd(dev, 1);
1218
1219         return dev;
1220 }
1221
1222 static void its_free_device(struct its_device *its_dev)
1223 {
1224         unsigned long flags;
1225
1226         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
1227         list_del(&its_dev->entry);
1228         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
1229         kfree(its_dev->itt);
1230         kfree(its_dev);
1231 }
1232
1233 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
1234 {
1235         int idx;
1236
1237         idx = bitmap_find_free_region(dev->event_map.lpi_map,
1238                                       dev->event_map.nr_lpis,
1239                                       get_count_order(nvecs));
1240         if (idx < 0)
1241                 return -ENOSPC;
1242
1243         *hwirq = dev->event_map.lpi_base + idx;
1244         set_bit(idx, dev->event_map.lpi_map);
1245
1246         return 0;
1247 }
1248
1249 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
1250                            int nvec, msi_alloc_info_t *info)
1251 {
1252         struct its_node *its;
1253         struct its_device *its_dev;
1254         struct msi_domain_info *msi_info;
1255         u32 dev_id;
1256
1257         /*
1258          * We ignore "dev" entierely, and rely on the dev_id that has
1259          * been passed via the scratchpad. This limits this domain's
1260          * usefulness to upper layers that definitely know that they
1261          * are built on top of the ITS.
1262          */
1263         dev_id = info->scratchpad[0].ul;
1264
1265         msi_info = msi_get_domain_info(domain);
1266         its = msi_info->data;
1267
1268         its_dev = its_find_device(its, dev_id);
1269         if (its_dev) {
1270                 /*
1271                  * We already have seen this ID, probably through
1272                  * another alias (PCI bridge of some sort). No need to
1273                  * create the device.
1274                  */
1275                 pr_debug("Reusing ITT for devID %x\n", dev_id);
1276                 goto out;
1277         }
1278
1279         its_dev = its_create_device(its, dev_id, nvec);
1280         if (!its_dev)
1281                 return -ENOMEM;
1282
1283         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
1284 out:
1285         info->scratchpad[0].ptr = its_dev;
1286         return 0;
1287 }
1288
1289 static struct msi_domain_ops its_msi_domain_ops = {
1290         .msi_prepare    = its_msi_prepare,
1291 };
1292
1293 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
1294                                     unsigned int virq,
1295                                     irq_hw_number_t hwirq)
1296 {
1297         struct irq_fwspec fwspec;
1298
1299         if (irq_domain_get_of_node(domain->parent)) {
1300                 fwspec.fwnode = domain->parent->fwnode;
1301                 fwspec.param_count = 3;
1302                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
1303                 fwspec.param[1] = hwirq;
1304                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
1305         } else {
1306                 return -EINVAL;
1307         }
1308
1309         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
1310 }
1311
1312 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
1313                                 unsigned int nr_irqs, void *args)
1314 {
1315         msi_alloc_info_t *info = args;
1316         struct its_device *its_dev = info->scratchpad[0].ptr;
1317         irq_hw_number_t hwirq;
1318         int err;
1319         int i;
1320
1321         err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
1322         if (err)
1323                 return err;
1324
1325         for (i = 0; i < nr_irqs; i++) {
1326                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
1327                 if (err)
1328                         return err;
1329
1330                 irq_domain_set_hwirq_and_chip(domain, virq + i,
1331                                               hwirq + i, &its_irq_chip, its_dev);
1332                 pr_debug("ID:%d pID:%d vID:%d\n",
1333                          (int)(hwirq + i - its_dev->event_map.lpi_base),
1334                          (int)(hwirq + i), virq + i);
1335         }
1336
1337         return 0;
1338 }
1339
1340 static void its_irq_domain_activate(struct irq_domain *domain,
1341                                     struct irq_data *d)
1342 {
1343         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1344         u32 event = its_get_event_id(d);
1345         const struct cpumask *cpu_mask = cpu_online_mask;
1346
1347         /* get the cpu_mask of local node */
1348         if (its_dev->its->numa_node >= 0)
1349                 cpu_mask = cpumask_of_node(its_dev->its->numa_node);
1350
1351         /* Bind the LPI to the first possible CPU */
1352         its_dev->event_map.col_map[event] = cpumask_first(cpu_mask);
1353
1354         /* Map the GIC IRQ and event to the device */
1355         its_send_mapvi(its_dev, d->hwirq, event);
1356 }
1357
1358 static void its_irq_domain_deactivate(struct irq_domain *domain,
1359                                       struct irq_data *d)
1360 {
1361         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1362         u32 event = its_get_event_id(d);
1363
1364         /* Stop the delivery of interrupts */
1365         its_send_discard(its_dev, event);
1366 }
1367
1368 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
1369                                 unsigned int nr_irqs)
1370 {
1371         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1372         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1373         int i;
1374
1375         bitmap_release_region(its_dev->event_map.lpi_map,
1376                               its_get_event_id(irq_domain_get_irq_data(domain, virq)),
1377                               get_count_order(nr_irqs));
1378
1379         for (i = 0; i < nr_irqs; i++) {
1380                 struct irq_data *data = irq_domain_get_irq_data(domain,
1381                                                                 virq + i);
1382                 /* Nuke the entry in the domain */
1383                 irq_domain_reset_irq_data(data);
1384         }
1385
1386         /* If all interrupts have been freed, start mopping the floor */
1387         if (bitmap_empty(its_dev->event_map.lpi_map,
1388                          its_dev->event_map.nr_lpis)) {
1389                 its_lpi_free(&its_dev->event_map);
1390
1391                 /* Unmap device/itt */
1392                 its_send_mapd(its_dev, 0);
1393                 its_free_device(its_dev);
1394         }
1395
1396         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
1397 }
1398
1399 static const struct irq_domain_ops its_domain_ops = {
1400         .alloc                  = its_irq_domain_alloc,
1401         .free                   = its_irq_domain_free,
1402         .activate               = its_irq_domain_activate,
1403         .deactivate             = its_irq_domain_deactivate,
1404 };
1405
1406 static int its_force_quiescent(void __iomem *base)
1407 {
1408         u32 count = 1000000;    /* 1s */
1409         u32 val;
1410
1411         val = readl_relaxed(base + GITS_CTLR);
1412         if (val & GITS_CTLR_QUIESCENT)
1413                 return 0;
1414
1415         /* Disable the generation of all interrupts to this ITS */
1416         val &= ~GITS_CTLR_ENABLE;
1417         writel_relaxed(val, base + GITS_CTLR);
1418
1419         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
1420         while (1) {
1421                 val = readl_relaxed(base + GITS_CTLR);
1422                 if (val & GITS_CTLR_QUIESCENT)
1423                         return 0;
1424
1425                 count--;
1426                 if (!count)
1427                         return -EBUSY;
1428
1429                 cpu_relax();
1430                 udelay(1);
1431         }
1432 }
1433
1434 static void __maybe_unused its_enable_quirk_cavium_22375(void *data)
1435 {
1436         struct its_node *its = data;
1437
1438         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
1439 }
1440
1441 static void __maybe_unused its_enable_quirk_cavium_23144(void *data)
1442 {
1443         struct its_node *its = data;
1444
1445         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
1446 }
1447
1448 static const struct gic_quirk its_quirks[] = {
1449 #ifdef CONFIG_CAVIUM_ERRATUM_22375
1450         {
1451                 .desc   = "ITS: Cavium errata 22375, 24313",
1452                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
1453                 .mask   = 0xffff0fff,
1454                 .init   = its_enable_quirk_cavium_22375,
1455         },
1456 #endif
1457 #ifdef CONFIG_CAVIUM_ERRATUM_23144
1458         {
1459                 .desc   = "ITS: Cavium erratum 23144",
1460                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
1461                 .mask   = 0xffff0fff,
1462                 .init   = its_enable_quirk_cavium_23144,
1463         },
1464 #endif
1465         {
1466         }
1467 };
1468
1469 static void its_enable_quirks(struct its_node *its)
1470 {
1471         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
1472
1473         gic_enable_quirks(iidr, its_quirks, its);
1474 }
1475
1476 static int its_probe(struct device_node *node, struct irq_domain *parent)
1477 {
1478         struct resource res;
1479         struct its_node *its;
1480         void __iomem *its_base;
1481         struct irq_domain *inner_domain;
1482         u32 val;
1483         u64 baser, tmp;
1484         int err;
1485
1486         err = of_address_to_resource(node, 0, &res);
1487         if (err) {
1488                 pr_warn("%s: no regs?\n", node->full_name);
1489                 return -ENXIO;
1490         }
1491
1492         its_base = ioremap(res.start, resource_size(&res));
1493         if (!its_base) {
1494                 pr_warn("%s: unable to map registers\n", node->full_name);
1495                 return -ENOMEM;
1496         }
1497
1498         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
1499         if (val != 0x30 && val != 0x40) {
1500                 pr_warn("%s: no ITS detected, giving up\n", node->full_name);
1501                 err = -ENODEV;
1502                 goto out_unmap;
1503         }
1504
1505         err = its_force_quiescent(its_base);
1506         if (err) {
1507                 pr_warn("%s: failed to quiesce, giving up\n",
1508                         node->full_name);
1509                 goto out_unmap;
1510         }
1511
1512         pr_info("ITS: %s\n", node->full_name);
1513
1514         its = kzalloc(sizeof(*its), GFP_KERNEL);
1515         if (!its) {
1516                 err = -ENOMEM;
1517                 goto out_unmap;
1518         }
1519
1520         raw_spin_lock_init(&its->lock);
1521         INIT_LIST_HEAD(&its->entry);
1522         INIT_LIST_HEAD(&its->its_device_list);
1523         its->base = its_base;
1524         its->phys_base = res.start;
1525         its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;
1526         its->numa_node = of_node_to_nid(node);
1527
1528         its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);
1529         if (!its->cmd_base) {
1530                 err = -ENOMEM;
1531                 goto out_free_its;
1532         }
1533         its->cmd_write = its->cmd_base;
1534
1535         its_enable_quirks(its);
1536
1537         err = its_alloc_tables(node->full_name, its);
1538         if (err)
1539                 goto out_free_cmd;
1540
1541         err = its_alloc_collections(its);
1542         if (err)
1543                 goto out_free_tables;
1544
1545         baser = (virt_to_phys(its->cmd_base)    |
1546                  GITS_CBASER_WaWb               |
1547                  GITS_CBASER_InnerShareable     |
1548                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
1549                  GITS_CBASER_VALID);
1550
1551         writeq_relaxed(baser, its->base + GITS_CBASER);
1552         tmp = readq_relaxed(its->base + GITS_CBASER);
1553
1554         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
1555                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
1556                         /*
1557                          * The HW reports non-shareable, we must
1558                          * remove the cacheability attributes as
1559                          * well.
1560                          */
1561                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
1562                                    GITS_CBASER_CACHEABILITY_MASK);
1563                         baser |= GITS_CBASER_nC;
1564                         writeq_relaxed(baser, its->base + GITS_CBASER);
1565                 }
1566                 pr_info("ITS: using cache flushing for cmd queue\n");
1567                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
1568         }
1569
1570         writeq_relaxed(0, its->base + GITS_CWRITER);
1571         writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);
1572
1573         if (of_property_read_bool(node, "msi-controller")) {
1574                 struct msi_domain_info *info;
1575
1576                 info = kzalloc(sizeof(*info), GFP_KERNEL);
1577                 if (!info) {
1578                         err = -ENOMEM;
1579                         goto out_free_tables;
1580                 }
1581
1582                 inner_domain = irq_domain_add_tree(node, &its_domain_ops, its);
1583                 if (!inner_domain) {
1584                         err = -ENOMEM;
1585                         kfree(info);
1586                         goto out_free_tables;
1587                 }
1588
1589                 inner_domain->parent = parent;
1590                 inner_domain->bus_token = DOMAIN_BUS_NEXUS;
1591                 info->ops = &its_msi_domain_ops;
1592                 info->data = its;
1593                 inner_domain->host_data = info;
1594         }
1595
1596         spin_lock(&its_lock);
1597         list_add(&its->entry, &its_nodes);
1598         spin_unlock(&its_lock);
1599
1600         return 0;
1601
1602 out_free_tables:
1603         its_free_tables(its);
1604 out_free_cmd:
1605         kfree(its->cmd_base);
1606 out_free_its:
1607         kfree(its);
1608 out_unmap:
1609         iounmap(its_base);
1610         pr_err("ITS: failed probing %s (%d)\n", node->full_name, err);
1611         return err;
1612 }
1613
1614 static bool gic_rdists_supports_plpis(void)
1615 {
1616         return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
1617 }
1618
1619 int its_cpu_init(void)
1620 {
1621         if (!list_empty(&its_nodes)) {
1622                 if (!gic_rdists_supports_plpis()) {
1623                         pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
1624                         return -ENXIO;
1625                 }
1626                 its_cpu_init_lpis();
1627                 its_cpu_init_collection();
1628         }
1629
1630         return 0;
1631 }
1632
1633 static struct of_device_id its_device_id[] = {
1634         {       .compatible     = "arm,gic-v3-its",     },
1635         {},
1636 };
1637
1638 int its_init(struct device_node *node, struct rdists *rdists,
1639              struct irq_domain *parent_domain)
1640 {
1641         struct device_node *np;
1642
1643         for (np = of_find_matching_node(node, its_device_id); np;
1644              np = of_find_matching_node(np, its_device_id)) {
1645                 its_probe(np, parent_domain);
1646         }
1647
1648         if (list_empty(&its_nodes)) {
1649                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
1650                 return -ENXIO;
1651         }
1652
1653         gic_rdists = rdists;
1654         gic_root_node = node;
1655
1656         its_alloc_lpi_tables();
1657         its_lpi_init(rdists->id_bits);
1658
1659         return 0;
1660 }