virt/kvm/arm/vgic/vgic-its.c

   1 /*
   2  * GICv3 ITS emulation
   3  *
   4  * Copyright (C) 2015,2016 ARM Ltd.
   5  * Author: Andre Przywara <andre.przywara@arm.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <linux/cpu.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/list.h>
  25 #include <linux/uaccess.h>
  26 #include <linux/list_sort.h>
  27
  28 #include <linux/irqchip/arm-gic-v3.h>
  29
  30 #include <asm/kvm_emulate.h>
  31 #include <asm/kvm_arm.h>
  32 #include <asm/kvm_mmu.h>
  33
  34 #include "vgic.h"
  35 #include "vgic-mmio.h"
  36
  37 static int vgic_its_save_tables_v0(struct vgic_its *its);
  38 static int vgic_its_restore_tables_v0(struct vgic_its *its);
  39 static int vgic_its_commit_v0(struct vgic_its *its);
  40 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
  41                              struct kvm_vcpu *filter_vcpu);
  42
  43 /*
  44  * Creates a new (reference to a) struct vgic_irq for a given LPI.
  45  * If this LPI is already mapped on another ITS, we increase its refcount
  46  * and return a pointer to the existing structure.
  47  * If this is a "new" LPI, we allocate and initialize a new struct vgic_irq.
  48  * This function returns a pointer to the _unlocked_ structure.
  49  */
  50 static struct vgic_irq *vgic_add_lpi(struct kvm *kvm, u32 intid,
  51                                      struct kvm_vcpu *vcpu)
  52 {
  53         struct vgic_dist *dist = &kvm->arch.vgic;
  54         struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intid), *oldirq;
  55         int ret;
  56
  57         /* In this case there is no put, since we keep the reference. */
  58         if (irq)
  59                 return irq;
  60
  61         irq = kzalloc(sizeof(struct vgic_irq), GFP_KERNEL);
  62         if (!irq)
  63                 return ERR_PTR(-ENOMEM);
  64
  65         INIT_LIST_HEAD(&irq->lpi_list);
  66         INIT_LIST_HEAD(&irq->ap_list);
  67         spin_lock_init(&irq->irq_lock);
  68
  69         irq->config = VGIC_CONFIG_EDGE;
  70         kref_init(&irq->refcount);
  71         irq->intid = intid;
  72         irq->target_vcpu = vcpu;
  73
  74         spin_lock(&dist->lpi_list_lock);
  75
  76         /*
  77          * There could be a race with another vgic_add_lpi(), so we need to
  78          * check that we don't add a second list entry with the same LPI.
  79          */
  80         list_for_each_entry(oldirq, &dist->lpi_list_head, lpi_list) {
  81                 if (oldirq->intid != intid)
  82                         continue;
  83
  84                 /* Someone was faster with adding this LPI, lets use that. */
  85                 kfree(irq);
  86                 irq = oldirq;
  87
  88                 /*
  89                  * This increases the refcount, the caller is expected to
  90                  * call vgic_put_irq() on the returned pointer once it's
  91                  * finished with the IRQ.
  92                  */
  93                 vgic_get_irq_kref(irq);
  94
  95                 goto out_unlock;
  96         }
  97
  98         list_add_tail(&irq->lpi_list, &dist->lpi_list_head);
  99         dist->lpi_list_count++;
 100
 101 out_unlock:
 102         spin_unlock(&dist->lpi_list_lock);
 103
 104         /*
 105          * We "cache" the configuration table entries in our struct vgic_irq's.
 106          * However we only have those structs for mapped IRQs, so we read in
 107          * the respective config data from memory here upon mapping the LPI.
 108          */
 109         ret = update_lpi_config(kvm, irq, NULL);
 110         if (ret)
 111                 return ERR_PTR(ret);
 112
 113         ret = vgic_v3_lpi_sync_pending_status(kvm, irq);
 114         if (ret)
 115                 return ERR_PTR(ret);
 116
 117         return irq;
 118 }
 119
 120 struct its_device {
 121         struct list_head dev_list;
 122
 123         /* the head for the list of ITTEs */
 124         struct list_head itt_head;
 125         u32 num_eventid_bits;
 126         gpa_t itt_addr;
 127         u32 device_id;
 128 };
 129
 130 #define COLLECTION_NOT_MAPPED ((u32)~0)
 131
 132 struct its_collection {
 133         struct list_head coll_list;
 134
 135         u32 collection_id;
 136         u32 target_addr;
 137 };
 138
 139 #define its_is_collection_mapped(coll) ((coll) && \
 140                                 ((coll)->target_addr != COLLECTION_NOT_MAPPED))
 141
 142 struct its_ite {
 143         struct list_head ite_list;
 144
 145         struct vgic_irq *irq;
 146         struct its_collection *collection;
 147         u32 event_id;
 148 };
 149
 150 /**
 151  * struct vgic_its_abi - ITS abi ops and settings
 152  * @cte_esz: collection table entry size
 153  * @dte_esz: device table entry size
 154  * @ite_esz: interrupt translation table entry size
 155  * @save tables: save the ITS tables into guest RAM
 156  * @restore_tables: restore the ITS internal structs from tables
 157  *  stored in guest RAM
 158  * @commit: initialize the registers which expose the ABI settings,
 159  *  especially the entry sizes
 160  */
 161 struct vgic_its_abi {
 162         int cte_esz;
 163         int dte_esz;
 164         int ite_esz;
 165         int (*save_tables)(struct vgic_its *its);
 166         int (*restore_tables)(struct vgic_its *its);
 167         int (*commit)(struct vgic_its *its);
 168 };
 169
 170 static const struct vgic_its_abi its_table_abi_versions[] = {
 171         [0] = {.cte_esz = 8, .dte_esz = 8, .ite_esz = 8,
 172          .save_tables = vgic_its_save_tables_v0,
 173          .restore_tables = vgic_its_restore_tables_v0,
 174          .commit = vgic_its_commit_v0,
 175         },
 176 };
 177
 178 #define NR_ITS_ABIS     ARRAY_SIZE(its_table_abi_versions)
 179
 180 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)
 181 {
 182         return &its_table_abi_versions[its->abi_rev];
 183 }
 184
 185 int vgic_its_set_abi(struct vgic_its *its, int rev)
 186 {
 187         const struct vgic_its_abi *abi;
 188
 189         its->abi_rev = rev;
 190         abi = vgic_its_get_abi(its);
 191         return abi->commit(its);
 192 }
 193
 194 /*
 195  * Find and returns a device in the device table for an ITS.
 196  * Must be called with the its_lock mutex held.
 197  */
 198 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)
 199 {
 200         struct its_device *device;
 201
 202         list_for_each_entry(device, &its->device_list, dev_list)
 203                 if (device_id == device->device_id)
 204                         return device;
 205
 206         return NULL;
 207 }
 208
 209 /*
 210  * Find and returns an interrupt translation table entry (ITTE) for a given
 211  * Device ID/Event ID pair on an ITS.
 212  * Must be called with the its_lock mutex held.
 213  */
 214 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,
 215                                   u32 event_id)
 216 {
 217         struct its_device *device;
 218         struct its_ite *ite;
 219
 220         device = find_its_device(its, device_id);
 221         if (device == NULL)
 222                 return NULL;
 223
 224         list_for_each_entry(ite, &device->itt_head, ite_list)
 225                 if (ite->event_id == event_id)
 226                         return ite;
 227
 228         return NULL;
 229 }
 230
 231 /* To be used as an iterator this macro misses the enclosing parentheses */
 232 #define for_each_lpi_its(dev, ite, its) \
 233         list_for_each_entry(dev, &(its)->device_list, dev_list) \
 234                 list_for_each_entry(ite, &(dev)->itt_head, ite_list)
 235
 236 /*
 237  * We only implement 48 bits of PA at the moment, although the ITS
 238  * supports more. Let's be restrictive here.
 239  */
 240 #define BASER_ADDRESS(x)        ((x) & GENMASK_ULL(47, 16))
 241 #define CBASER_ADDRESS(x)       ((x) & GENMASK_ULL(47, 12))
 242
 243 #define GIC_LPI_OFFSET 8192
 244
 245 #define VITS_TYPER_IDBITS 16
 246 #define VITS_TYPER_DEVBITS 16
 247 #define VITS_DTE_MAX_DEVID_OFFSET       (BIT(14) - 1)
 248 #define VITS_ITE_MAX_EVENTID_OFFSET     (BIT(16) - 1)
 249
 250 /*
 251  * Finds and returns a collection in the ITS collection table.
 252  * Must be called with the its_lock mutex held.
 253  */
 254 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)
 255 {
 256         struct its_collection *collection;
 257
 258         list_for_each_entry(collection, &its->collection_list, coll_list) {
 259                 if (coll_id == collection->collection_id)
 260                         return collection;
 261         }
 262
 263         return NULL;
 264 }
 265
 266 #define LPI_PROP_ENABLE_BIT(p)  ((p) & LPI_PROP_ENABLED)
 267 #define LPI_PROP_PRIORITY(p)    ((p) & 0xfc)
 268
 269 /*
 270  * Reads the configuration data for a given LPI from guest memory and
 271  * updates the fields in struct vgic_irq.
 272  * If filter_vcpu is not NULL, applies only if the IRQ is targeting this
 273  * VCPU. Unconditionally applies if filter_vcpu is NULL.
 274  */
 275 static int update_lpi_config(struct kvm *kvm, struct vgic_irq *irq,
 276                              struct kvm_vcpu *filter_vcpu)
 277 {
 278         u64 propbase = GICR_PROPBASER_ADDRESS(kvm->arch.vgic.propbaser);
 279         u8 prop;
 280         int ret;
 281
 282         ret = kvm_read_guest_lock(kvm, propbase + irq->intid - GIC_LPI_OFFSET,
 283                                   &prop, 1);
 284
 285         if (ret)
 286                 return ret;
 287
 288         spin_lock(&irq->irq_lock);
 289
 290         if (!filter_vcpu || filter_vcpu == irq->target_vcpu) {
 291                 irq->priority = LPI_PROP_PRIORITY(prop);
 292                 irq->enabled = LPI_PROP_ENABLE_BIT(prop);
 293
 294                 vgic_queue_irq_unlock(kvm, irq);
 295         } else {
 296                 spin_unlock(&irq->irq_lock);
 297         }
 298
 299         return 0;
 300 }
 301
 302 /*
 303  * Create a snapshot of the current LPIs targeting @vcpu, so that we can
 304  * enumerate those LPIs without holding any lock.
 305  * Returns their number and puts the kmalloc'ed array into intid_ptr.
 306  */
 307 static int vgic_copy_lpi_list(struct kvm_vcpu *vcpu, u32 **intid_ptr)
 308 {
 309         struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
 310         struct vgic_irq *irq;
 311         u32 *intids;
 312         int irq_count, i = 0;
 313
 314         /*
 315          * There is an obvious race between allocating the array and LPIs
 316          * being mapped/unmapped. If we ended up here as a result of a
 317          * command, we're safe (locks are held, preventing another
 318          * command). If coming from another path (such as enabling LPIs),
 319          * we must be careful not to overrun the array.
 320          */
 321         irq_count = READ_ONCE(dist->lpi_list_count);
 322         intids = kmalloc_array(irq_count, sizeof(intids[0]), GFP_KERNEL);
 323         if (!intids)
 324                 return -ENOMEM;
 325
 326         spin_lock(&dist->lpi_list_lock);
 327         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
 328                 if (i == irq_count)
 329                         break;
 330                 /* We don't need to "get" the IRQ, as we hold the list lock. */
 331                 if (irq->target_vcpu != vcpu)
 332                         continue;
 333                 intids[i++] = irq->intid;
 334         }
 335         spin_unlock(&dist->lpi_list_lock);
 336
 337         *intid_ptr = intids;
 338         return i;
 339 }
 340
 341 /*
 342  * Promotes the ITS view of affinity of an ITTE (which redistributor this LPI
 343  * is targeting) to the VGIC's view, which deals with target VCPUs.
 344  * Needs to be called whenever either the collection for a LPIs has
 345  * changed or the collection itself got retargeted.
 346  */
 347 static void update_affinity_ite(struct kvm *kvm, struct its_ite *ite)
 348 {
 349         struct kvm_vcpu *vcpu;
 350
 351         if (!its_is_collection_mapped(ite->collection))
 352                 return;
 353
 354         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 355
 356         spin_lock(&ite->irq->irq_lock);
 357         ite->irq->target_vcpu = vcpu;
 358         spin_unlock(&ite->irq->irq_lock);
 359 }
 360
 361 /*
 362  * Updates the target VCPU for every LPI targeting this collection.
 363  * Must be called with the its_lock mutex held.
 364  */
 365 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,
 366                                        struct its_collection *coll)
 367 {
 368         struct its_device *device;
 369         struct its_ite *ite;
 370
 371         for_each_lpi_its(device, ite, its) {
 372                 if (!ite->collection || coll != ite->collection)
 373                         continue;
 374
 375                 update_affinity_ite(kvm, ite);
 376         }
 377 }
 378
 379 static u32 max_lpis_propbaser(u64 propbaser)
 380 {
 381         int nr_idbits = (propbaser & 0x1f) + 1;
 382
 383         return 1U << min(nr_idbits, INTERRUPT_ID_BITS_ITS);
 384 }
 385
 386 /*
 387  * Sync the pending table pending bit of LPIs targeting @vcpu
 388  * with our own data structures. This relies on the LPI being
 389  * mapped before.
 390  */
 391 static int its_sync_lpi_pending_table(struct kvm_vcpu *vcpu)
 392 {
 393         gpa_t pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser);
 394         struct vgic_irq *irq;
 395         int last_byte_offset = -1;
 396         int ret = 0;
 397         u32 *intids;
 398         int nr_irqs, i;
 399         u8 pendmask;
 400
 401         nr_irqs = vgic_copy_lpi_list(vcpu, &intids);
 402         if (nr_irqs < 0)
 403                 return nr_irqs;
 404
 405         for (i = 0; i < nr_irqs; i++) {
 406                 int byte_offset, bit_nr;
 407
 408                 byte_offset = intids[i] / BITS_PER_BYTE;
 409                 bit_nr = intids[i] % BITS_PER_BYTE;
 410
 411                 /*
 412                  * For contiguously allocated LPIs chances are we just read
 413                  * this very same byte in the last iteration. Reuse that.
 414                  */
 415                 if (byte_offset != last_byte_offset) {
 416                         ret = kvm_read_guest_lock(vcpu->kvm,
 417                                                   pendbase + byte_offset,
 418                                                   &pendmask, 1);
 419                         if (ret) {
 420                                 kfree(intids);
 421                                 return ret;
 422                         }
 423                         last_byte_offset = byte_offset;
 424                 }
 425
 426                 irq = vgic_get_irq(vcpu->kvm, NULL, intids[i]);
 427                 spin_lock(&irq->irq_lock);
 428                 irq->pending_latch = pendmask & (1U << bit_nr);
 429                 vgic_queue_irq_unlock(vcpu->kvm, irq);
 430                 vgic_put_irq(vcpu->kvm, irq);
 431         }
 432
 433         kfree(intids);
 434
 435         return ret;
 436 }
 437
 438 static unsigned long vgic_mmio_read_its_typer(struct kvm *kvm,
 439                                               struct vgic_its *its,
 440                                               gpa_t addr, unsigned int len)
 441 {
 442         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
 443         u64 reg = GITS_TYPER_PLPIS;
 444
 445         /*
 446          * We use linear CPU numbers for redistributor addressing,
 447          * so GITS_TYPER.PTA is 0.
 448          * Also we force all PROPBASER registers to be the same, so
 449          * CommonLPIAff is 0 as well.
 450          * To avoid memory waste in the guest, we keep the number of IDBits and
 451          * DevBits low - as least for the time being.
 452          */
 453         reg |= GIC_ENCODE_SZ(VITS_TYPER_DEVBITS, 5) << GITS_TYPER_DEVBITS_SHIFT;
 454         reg |= GIC_ENCODE_SZ(VITS_TYPER_IDBITS, 5) << GITS_TYPER_IDBITS_SHIFT;
 455         reg |= GIC_ENCODE_SZ(abi->ite_esz, 4) << GITS_TYPER_ITT_ENTRY_SIZE_SHIFT;
 456
 457         return extract_bytes(reg, addr & 7, len);
 458 }
 459
 460 static unsigned long vgic_mmio_read_its_iidr(struct kvm *kvm,
 461                                              struct vgic_its *its,
 462                                              gpa_t addr, unsigned int len)
 463 {
 464         u32 val;
 465
 466         val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;
 467         val |= (PRODUCT_ID_KVM << GITS_IIDR_PRODUCTID_SHIFT) | IMPLEMENTER_ARM;
 468         return val;
 469 }
 470
 471 static int vgic_mmio_uaccess_write_its_iidr(struct kvm *kvm,
 472                                             struct vgic_its *its,
 473                                             gpa_t addr, unsigned int len,
 474                                             unsigned long val)
 475 {
 476         u32 rev = GITS_IIDR_REV(val);
 477
 478         if (rev >= NR_ITS_ABIS)
 479                 return -EINVAL;
 480         return vgic_its_set_abi(its, rev);
 481 }
 482
 483 static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,
 484                                                struct vgic_its *its,
 485                                                gpa_t addr, unsigned int len)
 486 {
 487         switch (addr & 0xffff) {
 488         case GITS_PIDR0:
 489                 return 0x92;    /* part number, bits[7:0] */
 490         case GITS_PIDR1:
 491                 return 0xb4;    /* part number, bits[11:8] */
 492         case GITS_PIDR2:
 493                 return GIC_PIDR2_ARCH_GICv3 | 0x0b;
 494         case GITS_PIDR4:
 495                 return 0x40;    /* This is a 64K software visible page */
 496         /* The following are the ID registers for (any) GIC. */
 497         case GITS_CIDR0:
 498                 return 0x0d;
 499         case GITS_CIDR1:
 500                 return 0xf0;
 501         case GITS_CIDR2:
 502                 return 0x05;
 503         case GITS_CIDR3:
 504                 return 0xb1;
 505         }
 506
 507         return 0;
 508 }
 509
 510 /*
 511  * Find the target VCPU and the LPI number for a given devid/eventid pair
 512  * and make this IRQ pending, possibly injecting it.
 513  * Must be called with the its_lock mutex held.
 514  * Returns 0 on success, a positive error value for any ITS mapping
 515  * related errors and negative error values for generic errors.
 516  */
 517 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,
 518                                 u32 devid, u32 eventid)
 519 {
 520         struct kvm_vcpu *vcpu;
 521         struct its_ite *ite;
 522
 523         if (!its->enabled)
 524                 return -EBUSY;
 525
 526         ite = find_ite(its, devid, eventid);
 527         if (!ite || !its_is_collection_mapped(ite->collection))
 528                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 529
 530         vcpu = kvm_get_vcpu(kvm, ite->collection->target_addr);
 531         if (!vcpu)
 532                 return E_ITS_INT_UNMAPPED_INTERRUPT;
 533
 534         if (!vcpu->arch.vgic_cpu.lpis_enabled)
 535                 return -EBUSY;
 536
 537         spin_lock(&ite->irq->irq_lock);
 538         ite->irq->pending_latch = true;
 539         vgic_queue_irq_unlock(kvm, ite->irq);
 540
 541         return 0;
 542 }
 543
 544 static struct vgic_io_device *vgic_get_its_iodev(struct kvm_io_device *dev)
 545 {
 546         struct vgic_io_device *iodev;
 547
 548         if (dev->ops != &kvm_io_gic_ops)
 549                 return NULL;
 550
 551         iodev = container_of(dev, struct vgic_io_device, dev);
 552
 553         if (iodev->iodev_type != IODEV_ITS)
 554                 return NULL;
 555
 556         return iodev;
 557 }
 558
 559 /*
 560  * Queries the KVM IO bus framework to get the ITS pointer from the given
 561  * doorbell address.
 562  * We then call vgic_its_trigger_msi() with the decoded data.
 563  * According to the KVM_SIGNAL_MSI API description returns 1 on success.
 564  */
 565 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)
 566 {
 567         u64 address;
 568         struct kvm_io_device *kvm_io_dev;
 569         struct vgic_io_device *iodev;
 570         int ret;
 571
 572         if (!vgic_has_its(kvm))
 573                 return -ENODEV;
 574
 575         if (!(msi->flags & KVM_MSI_VALID_DEVID))
 576                 return -EINVAL;
 577
 578         address = (u64)msi->address_hi << 32 | msi->address_lo;
 579
 580         kvm_io_dev = kvm_io_bus_get_dev(kvm, KVM_MMIO_BUS, address);
 581         if (!kvm_io_dev)
 582                 return -EINVAL;
 583
 584         iodev = vgic_get_its_iodev(kvm_io_dev);
 585         if (!iodev)
 586                 return -EINVAL;
 587
 588         mutex_lock(&iodev->its->its_lock);
 589         ret = vgic_its_trigger_msi(kvm, iodev->its, msi->devid, msi->data);
 590         mutex_unlock(&iodev->its->its_lock);
 591
 592         if (ret < 0)
 593                 return ret;
 594
 595         /*
 596          * KVM_SIGNAL_MSI demands a return value > 0 for success and 0
 597          * if the guest has blocked the MSI. So we map any LPI mapping
 598          * related error to that.
 599          */
 600         if (ret)
 601                 return 0;
 602         else
 603                 return 1;
 604 }
 605
 606 /* Requires the its_lock to be held. */
 607 static void its_free_ite(struct kvm *kvm, struct its_ite *ite)
 608 {
 609         list_del(&ite->ite_list);
 610
 611         /* This put matches the get in vgic_add_lpi. */
 612         if (ite->irq)
 613                 vgic_put_irq(kvm, ite->irq);
 614
 615         kfree(ite);
 616 }
 617
 618 static u64 its_cmd_mask_field(u64 *its_cmd, int word, int shift, int size)
 619 {
 620         return (le64_to_cpu(its_cmd[word]) >> shift) & (BIT_ULL(size) - 1);
 621 }
 622
 623 #define its_cmd_get_command(cmd)        its_cmd_mask_field(cmd, 0,  0,  8)
 624 #define its_cmd_get_deviceid(cmd)       its_cmd_mask_field(cmd, 0, 32, 32)
 625 #define its_cmd_get_size(cmd)           (its_cmd_mask_field(cmd, 1,  0,  5) + 1)
 626 #define its_cmd_get_id(cmd)             its_cmd_mask_field(cmd, 1,  0, 32)
 627 #define its_cmd_get_physical_id(cmd)    its_cmd_mask_field(cmd, 1, 32, 32)
 628 #define its_cmd_get_collection(cmd)     its_cmd_mask_field(cmd, 2,  0, 16)
 629 #define its_cmd_get_ittaddr(cmd)        (its_cmd_mask_field(cmd, 2,  8, 44) << 8)
 630 #define its_cmd_get_target_addr(cmd)    its_cmd_mask_field(cmd, 2, 16, 32)
 631 #define its_cmd_get_validbit(cmd)       its_cmd_mask_field(cmd, 2, 63,  1)
 632
 633 /*
 634  * The DISCARD command frees an Interrupt Translation Table Entry (ITTE).
 635  * Must be called with the its_lock mutex held.
 636  */
 637 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,
 638                                        u64 *its_cmd)
 639 {
 640         u32 device_id = its_cmd_get_deviceid(its_cmd);
 641         u32 event_id = its_cmd_get_id(its_cmd);
 642         struct its_ite *ite;
 643
 644
 645         ite = find_ite(its, device_id, event_id);
 646         if (ite && ite->collection) {
 647                 /*
 648                  * Though the spec talks about removing the pending state, we
 649                  * don't bother here since we clear the ITTE anyway and the
 650                  * pending state is a property of the ITTE struct.
 651                  */
 652                 its_free_ite(kvm, ite);
 653                 return 0;
 654         }
 655
 656         return E_ITS_DISCARD_UNMAPPED_INTERRUPT;
 657 }
 658
 659 /*
 660  * The MOVI command moves an ITTE to a different collection.
 661  * Must be called with the its_lock mutex held.
 662  */
 663 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,
 664                                     u64 *its_cmd)
 665 {
 666         u32 device_id = its_cmd_get_deviceid(its_cmd);
 667         u32 event_id = its_cmd_get_id(its_cmd);
 668         u32 coll_id = its_cmd_get_collection(its_cmd);
 669         struct kvm_vcpu *vcpu;
 670         struct its_ite *ite;
 671         struct its_collection *collection;
 672
 673         ite = find_ite(its, device_id, event_id);
 674         if (!ite)
 675                 return E_ITS_MOVI_UNMAPPED_INTERRUPT;
 676
 677         if (!its_is_collection_mapped(ite->collection))
 678                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 679
 680         collection = find_collection(its, coll_id);
 681         if (!its_is_collection_mapped(collection))
 682                 return E_ITS_MOVI_UNMAPPED_COLLECTION;
 683
 684         ite->collection = collection;
 685         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 686
 687         spin_lock(&ite->irq->irq_lock);
 688         ite->irq->target_vcpu = vcpu;
 689         spin_unlock(&ite->irq->irq_lock);
 690
 691         return 0;
 692 }
 693
 694 /*
 695  * Check whether an ID can be stored into the corresponding guest table.
 696  * For a direct table this is pretty easy, but gets a bit nasty for
 697  * indirect tables. We check whether the resulting guest physical address
 698  * is actually valid (covered by a memslot and guest accessible).
 699  * For this we have to read the respective first level entry.
 700  */
 701 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,
 702                               gpa_t *eaddr)
 703 {
 704         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
 705         u64 indirect_ptr, type = GITS_BASER_TYPE(baser);
 706         int esz = GITS_BASER_ENTRY_SIZE(baser);
 707         int index, idx;
 708         gfn_t gfn;
 709         bool ret;
 710
 711         switch (type) {
 712         case GITS_BASER_TYPE_DEVICE:
 713                 if (id >= BIT_ULL(VITS_TYPER_DEVBITS))
 714                         return false;
 715                 break;
 716         case GITS_BASER_TYPE_COLLECTION:
 717                 /* as GITS_TYPER.CIL == 0, ITS supports 16-bit collection ID */
 718                 if (id >= BIT_ULL(16))
 719                         return false;
 720                 break;
 721         default:
 722                 return false;
 723         }
 724
 725         if (!(baser & GITS_BASER_INDIRECT)) {
 726                 phys_addr_t addr;
 727
 728                 if (id >= (l1_tbl_size / esz))
 729                         return false;
 730
 731                 addr = BASER_ADDRESS(baser) + id * esz;
 732                 gfn = addr >> PAGE_SHIFT;
 733
 734                 if (eaddr)
 735                         *eaddr = addr;
 736
 737                 goto out;
 738         }
 739
 740         /* calculate and check the index into the 1st level */
 741         index = id / (SZ_64K / esz);
 742         if (index >= (l1_tbl_size / sizeof(u64)))
 743                 return false;
 744
 745         /* Each 1st level entry is represented by a 64-bit value. */
 746         if (kvm_read_guest_lock(its->dev->kvm,
 747                            BASER_ADDRESS(baser) + index * sizeof(indirect_ptr),
 748                            &indirect_ptr, sizeof(indirect_ptr)))
 749                 return false;
 750
 751         indirect_ptr = le64_to_cpu(indirect_ptr);
 752
 753         /* check the valid bit of the first level entry */
 754         if (!(indirect_ptr & BIT_ULL(63)))
 755                 return false;
 756
 757         /*
 758          * Mask the guest physical address and calculate the frame number.
 759          * Any address beyond our supported 48 bits of PA will be caught
 760          * by the actual check in the final step.
 761          */
 762         indirect_ptr &= GENMASK_ULL(51, 16);
 763
 764         /* Find the address of the actual entry */
 765         index = id % (SZ_64K / esz);
 766         indirect_ptr += index * esz;
 767         gfn = indirect_ptr >> PAGE_SHIFT;
 768
 769         if (eaddr)
 770                 *eaddr = indirect_ptr;
 771
 772 out:
 773         idx = srcu_read_lock(&its->dev->kvm->srcu);
 774         ret = kvm_is_visible_gfn(its->dev->kvm, gfn);
 775         srcu_read_unlock(&its->dev->kvm->srcu, idx);
 776         return ret;
 777 }
 778
 779 static int vgic_its_alloc_collection(struct vgic_its *its,
 780                                      struct its_collection **colp,
 781                                      u32 coll_id)
 782 {
 783         struct its_collection *collection;
 784
 785         if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))
 786                 return E_ITS_MAPC_COLLECTION_OOR;
 787
 788         collection = kzalloc(sizeof(*collection), GFP_KERNEL);
 789         if (!collection)
 790                 return -ENOMEM;
 791
 792         collection->collection_id = coll_id;
 793         collection->target_addr = COLLECTION_NOT_MAPPED;
 794
 795         list_add_tail(&collection->coll_list, &its->collection_list);
 796         *colp = collection;
 797
 798         return 0;
 799 }
 800
 801 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)
 802 {
 803         struct its_collection *collection;
 804         struct its_device *device;
 805         struct its_ite *ite;
 806
 807         /*
 808          * Clearing the mapping for that collection ID removes the
 809          * entry from the list. If there wasn't any before, we can
 810          * go home early.
 811          */
 812         collection = find_collection(its, coll_id);
 813         if (!collection)
 814                 return;
 815
 816         for_each_lpi_its(device, ite, its)
 817                 if (ite->collection &&
 818                     ite->collection->collection_id == coll_id)
 819                         ite->collection = NULL;
 820
 821         list_del(&collection->coll_list);
 822         kfree(collection);
 823 }
 824
 825 /* Must be called with its_lock mutex held */
 826 static struct its_ite *vgic_its_alloc_ite(struct its_device *device,
 827                                           struct its_collection *collection,
 828                                           u32 event_id)
 829 {
 830         struct its_ite *ite;
 831
 832         ite = kzalloc(sizeof(*ite), GFP_KERNEL);
 833         if (!ite)
 834                 return ERR_PTR(-ENOMEM);
 835
 836         ite->event_id   = event_id;
 837         ite->collection = collection;
 838
 839         list_add_tail(&ite->ite_list, &device->itt_head);
 840         return ite;
 841 }
 842
 843 /*
 844  * The MAPTI and MAPI commands map LPIs to ITTEs.
 845  * Must be called with its_lock mutex held.
 846  */
 847 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,
 848                                     u64 *its_cmd)
 849 {
 850         u32 device_id = its_cmd_get_deviceid(its_cmd);
 851         u32 event_id = its_cmd_get_id(its_cmd);
 852         u32 coll_id = its_cmd_get_collection(its_cmd);
 853         struct its_ite *ite;
 854         struct kvm_vcpu *vcpu = NULL;
 855         struct its_device *device;
 856         struct its_collection *collection, *new_coll = NULL;
 857         struct vgic_irq *irq;
 858         int lpi_nr;
 859
 860         device = find_its_device(its, device_id);
 861         if (!device)
 862                 return E_ITS_MAPTI_UNMAPPED_DEVICE;
 863
 864         if (event_id >= BIT_ULL(device->num_eventid_bits))
 865                 return E_ITS_MAPTI_ID_OOR;
 866
 867         if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI)
 868                 lpi_nr = its_cmd_get_physical_id(its_cmd);
 869         else
 870                 lpi_nr = event_id;
 871         if (lpi_nr < GIC_LPI_OFFSET ||
 872             lpi_nr >= max_lpis_propbaser(kvm->arch.vgic.propbaser))
 873                 return E_ITS_MAPTI_PHYSICALID_OOR;
 874
 875         /* If there is an existing mapping, behavior is UNPREDICTABLE. */
 876         if (find_ite(its, device_id, event_id))
 877                 return 0;
 878
 879         collection = find_collection(its, coll_id);
 880         if (!collection) {
 881                 int ret = vgic_its_alloc_collection(its, &collection, coll_id);
 882                 if (ret)
 883                         return ret;
 884                 new_coll = collection;
 885         }
 886
 887         ite = vgic_its_alloc_ite(device, collection, event_id);
 888         if (IS_ERR(ite)) {
 889                 if (new_coll)
 890                         vgic_its_free_collection(its, coll_id);
 891                 return PTR_ERR(ite);
 892         }
 893
 894         if (its_is_collection_mapped(collection))
 895                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
 896
 897         irq = vgic_add_lpi(kvm, lpi_nr, vcpu);
 898         if (IS_ERR(irq)) {
 899                 if (new_coll)
 900                         vgic_its_free_collection(its, coll_id);
 901                 its_free_ite(kvm, ite);
 902                 return PTR_ERR(irq);
 903         }
 904         ite->irq = irq;
 905
 906         return 0;
 907 }
 908
 909 /* Requires the its_lock to be held. */
 910 static void vgic_its_unmap_device(struct kvm *kvm, struct its_device *device)
 911 {
 912         struct its_ite *ite, *temp;
 913
 914         /*
 915          * The spec says that unmapping a device with still valid
 916          * ITTEs associated is UNPREDICTABLE. We remove all ITTEs,
 917          * since we cannot leave the memory unreferenced.
 918          */
 919         list_for_each_entry_safe(ite, temp, &device->itt_head, ite_list)
 920                 its_free_ite(kvm, ite);
 921
 922         list_del(&device->dev_list);
 923         kfree(device);
 924 }
 925
 926 /* Must be called with its_lock mutex held */
 927 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,
 928                                                 u32 device_id, gpa_t itt_addr,
 929                                                 u8 num_eventid_bits)
 930 {
 931         struct its_device *device;
 932
 933         device = kzalloc(sizeof(*device), GFP_KERNEL);
 934         if (!device)
 935                 return ERR_PTR(-ENOMEM);
 936
 937         device->device_id = device_id;
 938         device->itt_addr = itt_addr;
 939         device->num_eventid_bits = num_eventid_bits;
 940         INIT_LIST_HEAD(&device->itt_head);
 941
 942         list_add_tail(&device->dev_list, &its->device_list);
 943         return device;
 944 }
 945
 946 /*
 947  * MAPD maps or unmaps a device ID to Interrupt Translation Tables (ITTs).
 948  * Must be called with the its_lock mutex held.
 949  */
 950 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,
 951                                     u64 *its_cmd)
 952 {
 953         u32 device_id = its_cmd_get_deviceid(its_cmd);
 954         bool valid = its_cmd_get_validbit(its_cmd);
 955         u8 num_eventid_bits = its_cmd_get_size(its_cmd);
 956         gpa_t itt_addr = its_cmd_get_ittaddr(its_cmd);
 957         struct its_device *device;
 958
 959         if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))
 960                 return E_ITS_MAPD_DEVICE_OOR;
 961
 962         if (valid && num_eventid_bits > VITS_TYPER_IDBITS)
 963                 return E_ITS_MAPD_ITTSIZE_OOR;
 964
 965         device = find_its_device(its, device_id);
 966
 967         /*
 968          * The spec says that calling MAPD on an already mapped device
 969          * invalidates all cached data for this device. We implement this
 970          * by removing the mapping and re-establishing it.
 971          */
 972         if (device)
 973                 vgic_its_unmap_device(kvm, device);
 974
 975         /*
 976          * The spec does not say whether unmapping a not-mapped device
 977          * is an error, so we are done in any case.
 978          */
 979         if (!valid)
 980                 return 0;
 981
 982         device = vgic_its_alloc_device(its, device_id, itt_addr,
 983                                        num_eventid_bits);
 984         if (IS_ERR(device))
 985                 return PTR_ERR(device);
 986
 987         return 0;
 988 }
 989
 990 /*
 991  * The MAPC command maps collection IDs to redistributors.
 992  * Must be called with the its_lock mutex held.
 993  */
 994 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,
 995                                     u64 *its_cmd)
 996 {
 997         u16 coll_id;
 998         u32 target_addr;
 999         struct its_collection *collection;
1000         bool valid;
1001
1002         valid = its_cmd_get_validbit(its_cmd);
1003         coll_id = its_cmd_get_collection(its_cmd);
1004         target_addr = its_cmd_get_target_addr(its_cmd);
1005
1006         if (target_addr >= atomic_read(&kvm->online_vcpus))
1007                 return E_ITS_MAPC_PROCNUM_OOR;
1008
1009         if (!valid) {
1010                 vgic_its_free_collection(its, coll_id);
1011         } else {
1012                 collection = find_collection(its, coll_id);
1013
1014                 if (!collection) {
1015                         int ret;
1016
1017                         ret = vgic_its_alloc_collection(its, &collection,
1018                                                         coll_id);
1019                         if (ret)
1020                                 return ret;
1021                         collection->target_addr = target_addr;
1022                 } else {
1023                         collection->target_addr = target_addr;
1024                         update_affinity_collection(kvm, its, collection);
1025                 }
1026         }
1027
1028         return 0;
1029 }
1030
1031 /*
1032  * The CLEAR command removes the pending state for a particular LPI.
1033  * Must be called with the its_lock mutex held.
1034  */
1035 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,
1036                                      u64 *its_cmd)
1037 {
1038         u32 device_id = its_cmd_get_deviceid(its_cmd);
1039         u32 event_id = its_cmd_get_id(its_cmd);
1040         struct its_ite *ite;
1041
1042
1043         ite = find_ite(its, device_id, event_id);
1044         if (!ite)
1045                 return E_ITS_CLEAR_UNMAPPED_INTERRUPT;
1046
1047         ite->irq->pending_latch = false;
1048
1049         return 0;
1050 }
1051
1052 /*
1053  * The INV command syncs the configuration bits from the memory table.
1054  * Must be called with the its_lock mutex held.
1055  */
1056 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,
1057                                    u64 *its_cmd)
1058 {
1059         u32 device_id = its_cmd_get_deviceid(its_cmd);
1060         u32 event_id = its_cmd_get_id(its_cmd);
1061         struct its_ite *ite;
1062
1063
1064         ite = find_ite(its, device_id, event_id);
1065         if (!ite)
1066                 return E_ITS_INV_UNMAPPED_INTERRUPT;
1067
1068         return update_lpi_config(kvm, ite->irq, NULL);
1069 }
1070
1071 /*
1072  * The INVALL command requests flushing of all IRQ data in this collection.
1073  * Find the VCPU mapped to that collection, then iterate over the VM's list
1074  * of mapped LPIs and update the configuration for each IRQ which targets
1075  * the specified vcpu. The configuration will be read from the in-memory
1076  * configuration table.
1077  * Must be called with the its_lock mutex held.
1078  */
1079 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,
1080                                       u64 *its_cmd)
1081 {
1082         u32 coll_id = its_cmd_get_collection(its_cmd);
1083         struct its_collection *collection;
1084         struct kvm_vcpu *vcpu;
1085         struct vgic_irq *irq;
1086         u32 *intids;
1087         int irq_count, i;
1088
1089         collection = find_collection(its, coll_id);
1090         if (!its_is_collection_mapped(collection))
1091                 return E_ITS_INVALL_UNMAPPED_COLLECTION;
1092
1093         vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1094
1095         irq_count = vgic_copy_lpi_list(vcpu, &intids);
1096         if (irq_count < 0)
1097                 return irq_count;
1098
1099         for (i = 0; i < irq_count; i++) {
1100                 irq = vgic_get_irq(kvm, NULL, intids[i]);
1101                 if (!irq)
1102                         continue;
1103                 update_lpi_config(kvm, irq, vcpu);
1104                 vgic_put_irq(kvm, irq);
1105         }
1106
1107         kfree(intids);
1108
1109         return 0;
1110 }
1111
1112 /*
1113  * The MOVALL command moves the pending state of all IRQs targeting one
1114  * redistributor to another. We don't hold the pending state in the VCPUs,
1115  * but in the IRQs instead, so there is really not much to do for us here.
1116  * However the spec says that no IRQ must target the old redistributor
1117  * afterwards, so we make sure that no LPI is using the associated target_vcpu.
1118  * This command affects all LPIs in the system that target that redistributor.
1119  */
1120 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,
1121                                       u64 *its_cmd)
1122 {
1123         struct vgic_dist *dist = &kvm->arch.vgic;
1124         u32 target1_addr = its_cmd_get_target_addr(its_cmd);
1125         u32 target2_addr = its_cmd_mask_field(its_cmd, 3, 16, 32);
1126         struct kvm_vcpu *vcpu1, *vcpu2;
1127         struct vgic_irq *irq;
1128
1129         if (target1_addr >= atomic_read(&kvm->online_vcpus) ||
1130             target2_addr >= atomic_read(&kvm->online_vcpus))
1131                 return E_ITS_MOVALL_PROCNUM_OOR;
1132
1133         if (target1_addr == target2_addr)
1134                 return 0;
1135
1136         vcpu1 = kvm_get_vcpu(kvm, target1_addr);
1137         vcpu2 = kvm_get_vcpu(kvm, target2_addr);
1138
1139         spin_lock(&dist->lpi_list_lock);
1140
1141         list_for_each_entry(irq, &dist->lpi_list_head, lpi_list) {
1142                 spin_lock(&irq->irq_lock);
1143
1144                 if (irq->target_vcpu == vcpu1)
1145                         irq->target_vcpu = vcpu2;
1146
1147                 spin_unlock(&irq->irq_lock);
1148         }
1149
1150         spin_unlock(&dist->lpi_list_lock);
1151
1152         return 0;
1153 }
1154
1155 /*
1156  * The INT command injects the LPI associated with that DevID/EvID pair.
1157  * Must be called with the its_lock mutex held.
1158  */
1159 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,
1160                                    u64 *its_cmd)
1161 {
1162         u32 msi_data = its_cmd_get_id(its_cmd);
1163         u64 msi_devid = its_cmd_get_deviceid(its_cmd);
1164
1165         return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);
1166 }
1167
1168 /*
1169  * This function is called with the its_cmd lock held, but the ITS data
1170  * structure lock dropped.
1171  */
1172 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,
1173                                    u64 *its_cmd)
1174 {
1175         int ret = -ENODEV;
1176
1177         mutex_lock(&its->its_lock);
1178         switch (its_cmd_get_command(its_cmd)) {
1179         case GITS_CMD_MAPD:
1180                 ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);
1181                 break;
1182         case GITS_CMD_MAPC:
1183                 ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);
1184                 break;
1185         case GITS_CMD_MAPI:
1186                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1187                 break;
1188         case GITS_CMD_MAPTI:
1189                 ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);
1190                 break;
1191         case GITS_CMD_MOVI:
1192                 ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);
1193                 break;
1194         case GITS_CMD_DISCARD:
1195                 ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);
1196                 break;
1197         case GITS_CMD_CLEAR:
1198                 ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);
1199                 break;
1200         case GITS_CMD_MOVALL:
1201                 ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);
1202                 break;
1203         case GITS_CMD_INT:
1204                 ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);
1205                 break;
1206         case GITS_CMD_INV:
1207                 ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);
1208                 break;
1209         case GITS_CMD_INVALL:
1210                 ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);
1211                 break;
1212         case GITS_CMD_SYNC:
1213                 /* we ignore this command: we are in sync all of the time */
1214                 ret = 0;
1215                 break;
1216         }
1217         mutex_unlock(&its->its_lock);
1218
1219         return ret;
1220 }
1221
1222 static u64 vgic_sanitise_its_baser(u64 reg)
1223 {
1224         reg = vgic_sanitise_field(reg, GITS_BASER_SHAREABILITY_MASK,
1225                                   GITS_BASER_SHAREABILITY_SHIFT,
1226                                   vgic_sanitise_shareability);
1227         reg = vgic_sanitise_field(reg, GITS_BASER_INNER_CACHEABILITY_MASK,
1228                                   GITS_BASER_INNER_CACHEABILITY_SHIFT,
1229                                   vgic_sanitise_inner_cacheability);
1230         reg = vgic_sanitise_field(reg, GITS_BASER_OUTER_CACHEABILITY_MASK,
1231                                   GITS_BASER_OUTER_CACHEABILITY_SHIFT,
1232                                   vgic_sanitise_outer_cacheability);
1233
1234         /* Bits 15:12 contain bits 51:48 of the PA, which we don't support. */
1235         reg &= ~GENMASK_ULL(15, 12);
1236
1237         /* We support only one (ITS) page size: 64K */
1238         reg = (reg & ~GITS_BASER_PAGE_SIZE_MASK) | GITS_BASER_PAGE_SIZE_64K;
1239
1240         return reg;
1241 }
1242
1243 static u64 vgic_sanitise_its_cbaser(u64 reg)
1244 {
1245         reg = vgic_sanitise_field(reg, GITS_CBASER_SHAREABILITY_MASK,
1246                                   GITS_CBASER_SHAREABILITY_SHIFT,
1247                                   vgic_sanitise_shareability);
1248         reg = vgic_sanitise_field(reg, GITS_CBASER_INNER_CACHEABILITY_MASK,
1249                                   GITS_CBASER_INNER_CACHEABILITY_SHIFT,
1250                                   vgic_sanitise_inner_cacheability);
1251         reg = vgic_sanitise_field(reg, GITS_CBASER_OUTER_CACHEABILITY_MASK,
1252                                   GITS_CBASER_OUTER_CACHEABILITY_SHIFT,
1253                                   vgic_sanitise_outer_cacheability);
1254
1255         /*
1256          * Sanitise the physical address to be 64k aligned.
1257          * Also limit the physical addresses to 48 bits.
1258          */
1259         reg &= ~(GENMASK_ULL(51, 48) | GENMASK_ULL(15, 12));
1260
1261         return reg;
1262 }
1263
1264 static unsigned long vgic_mmio_read_its_cbaser(struct kvm *kvm,
1265                                                struct vgic_its *its,
1266                                                gpa_t addr, unsigned int len)
1267 {
1268         return extract_bytes(its->cbaser, addr & 7, len);
1269 }
1270
1271 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,
1272                                        gpa_t addr, unsigned int len,
1273                                        unsigned long val)
1274 {
1275         /* When GITS_CTLR.Enable is 1, this register is RO. */
1276         if (its->enabled)
1277                 return;
1278
1279         mutex_lock(&its->cmd_lock);
1280         its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);
1281         its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);
1282         its->creadr = 0;
1283         /*
1284          * CWRITER is architecturally UNKNOWN on reset, but we need to reset
1285          * it to CREADR to make sure we start with an empty command buffer.
1286          */
1287         its->cwriter = its->creadr;
1288         mutex_unlock(&its->cmd_lock);
1289 }
1290
1291 #define ITS_CMD_BUFFER_SIZE(baser)      ((((baser) & 0xff) + 1) << 12)
1292 #define ITS_CMD_SIZE                    32
1293 #define ITS_CMD_OFFSET(reg)             ((reg) & GENMASK(19, 5))
1294
1295 /* Must be called with the cmd_lock held. */
1296 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)
1297 {
1298         gpa_t cbaser;
1299         u64 cmd_buf[4];
1300
1301         /* Commands are only processed when the ITS is enabled. */
1302         if (!its->enabled)
1303                 return;
1304
1305         cbaser = CBASER_ADDRESS(its->cbaser);
1306
1307         while (its->cwriter != its->creadr) {
1308                 int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,
1309                                               cmd_buf, ITS_CMD_SIZE);
1310                 /*
1311                  * If kvm_read_guest() fails, this could be due to the guest
1312                  * programming a bogus value in CBASER or something else going
1313                  * wrong from which we cannot easily recover.
1314                  * According to section 6.3.2 in the GICv3 spec we can just
1315                  * ignore that command then.
1316                  */
1317                 if (!ret)
1318                         vgic_its_handle_command(kvm, its, cmd_buf);
1319
1320                 its->creadr += ITS_CMD_SIZE;
1321                 if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))
1322                         its->creadr = 0;
1323         }
1324 }
1325
1326 /*
1327  * By writing to CWRITER the guest announces new commands to be processed.
1328  * To avoid any races in the first place, we take the its_cmd lock, which
1329  * protects our ring buffer variables, so that there is only one user
1330  * per ITS handling commands at a given time.
1331  */
1332 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,
1333                                         gpa_t addr, unsigned int len,
1334                                         unsigned long val)
1335 {
1336         u64 reg;
1337
1338         if (!its)
1339                 return;
1340
1341         mutex_lock(&its->cmd_lock);
1342
1343         reg = update_64bit_reg(its->cwriter, addr & 7, len, val);
1344         reg = ITS_CMD_OFFSET(reg);
1345         if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1346                 mutex_unlock(&its->cmd_lock);
1347                 return;
1348         }
1349         its->cwriter = reg;
1350
1351         vgic_its_process_commands(kvm, its);
1352
1353         mutex_unlock(&its->cmd_lock);
1354 }
1355
1356 static unsigned long vgic_mmio_read_its_cwriter(struct kvm *kvm,
1357                                                 struct vgic_its *its,
1358                                                 gpa_t addr, unsigned int len)
1359 {
1360         return extract_bytes(its->cwriter, addr & 0x7, len);
1361 }
1362
1363 static unsigned long vgic_mmio_read_its_creadr(struct kvm *kvm,
1364                                                struct vgic_its *its,
1365                                                gpa_t addr, unsigned int len)
1366 {
1367         return extract_bytes(its->creadr, addr & 0x7, len);
1368 }
1369
1370 static int vgic_mmio_uaccess_write_its_creadr(struct kvm *kvm,
1371                                               struct vgic_its *its,
1372                                               gpa_t addr, unsigned int len,
1373                                               unsigned long val)
1374 {
1375         u32 cmd_offset;
1376         int ret = 0;
1377
1378         mutex_lock(&its->cmd_lock);
1379
1380         if (its->enabled) {
1381                 ret = -EBUSY;
1382                 goto out;
1383         }
1384
1385         cmd_offset = ITS_CMD_OFFSET(val);
1386         if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {
1387                 ret = -EINVAL;
1388                 goto out;
1389         }
1390
1391         its->creadr = cmd_offset;
1392 out:
1393         mutex_unlock(&its->cmd_lock);
1394         return ret;
1395 }
1396
1397 #define BASER_INDEX(addr) (((addr) / sizeof(u64)) & 0x7)
1398 static unsigned long vgic_mmio_read_its_baser(struct kvm *kvm,
1399                                               struct vgic_its *its,
1400                                               gpa_t addr, unsigned int len)
1401 {
1402         u64 reg;
1403
1404         switch (BASER_INDEX(addr)) {
1405         case 0:
1406                 reg = its->baser_device_table;
1407                 break;
1408         case 1:
1409                 reg = its->baser_coll_table;
1410                 break;
1411         default:
1412                 reg = 0;
1413                 break;
1414         }
1415
1416         return extract_bytes(reg, addr & 7, len);
1417 }
1418
1419 #define GITS_BASER_RO_MASK      (GENMASK_ULL(52, 48) | GENMASK_ULL(58, 56))
1420 static void vgic_mmio_write_its_baser(struct kvm *kvm,
1421                                       struct vgic_its *its,
1422                                       gpa_t addr, unsigned int len,
1423                                       unsigned long val)
1424 {
1425         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1426         u64 entry_size, device_type;
1427         u64 reg, *regptr, clearbits = 0;
1428
1429         /* When GITS_CTLR.Enable is 1, we ignore write accesses. */
1430         if (its->enabled)
1431                 return;
1432
1433         switch (BASER_INDEX(addr)) {
1434         case 0:
1435                 regptr = &its->baser_device_table;
1436                 entry_size = abi->dte_esz;
1437                 device_type = GITS_BASER_TYPE_DEVICE;
1438                 break;
1439         case 1:
1440                 regptr = &its->baser_coll_table;
1441                 entry_size = abi->cte_esz;
1442                 device_type = GITS_BASER_TYPE_COLLECTION;
1443                 clearbits = GITS_BASER_INDIRECT;
1444                 break;
1445         default:
1446                 return;
1447         }
1448
1449         reg = update_64bit_reg(*regptr, addr & 7, len, val);
1450         reg &= ~GITS_BASER_RO_MASK;
1451         reg &= ~clearbits;
1452
1453         reg |= (entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT;
1454         reg |= device_type << GITS_BASER_TYPE_SHIFT;
1455         reg = vgic_sanitise_its_baser(reg);
1456
1457         *regptr = reg;
1458 }
1459
1460 static unsigned long vgic_mmio_read_its_ctlr(struct kvm *vcpu,
1461                                              struct vgic_its *its,
1462                                              gpa_t addr, unsigned int len)
1463 {
1464         u32 reg = 0;
1465
1466         mutex_lock(&its->cmd_lock);
1467         if (its->creadr == its->cwriter)
1468                 reg |= GITS_CTLR_QUIESCENT;
1469         if (its->enabled)
1470                 reg |= GITS_CTLR_ENABLE;
1471         mutex_unlock(&its->cmd_lock);
1472
1473         return reg;
1474 }
1475
1476 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,
1477                                      gpa_t addr, unsigned int len,
1478                                      unsigned long val)
1479 {
1480         mutex_lock(&its->cmd_lock);
1481
1482         /*
1483          * It is UNPREDICTABLE to enable the ITS if any of the CBASER or
1484          * device/collection BASER are invalid
1485          */
1486         if (!its->enabled && (val & GITS_CTLR_ENABLE) &&
1487                 (!(its->baser_device_table & GITS_BASER_VALID) ||
1488                  !(its->baser_coll_table & GITS_BASER_VALID) ||
1489                  !(its->cbaser & GITS_CBASER_VALID)))
1490                 goto out;
1491
1492         its->enabled = !!(val & GITS_CTLR_ENABLE);
1493
1494         /*
1495          * Try to process any pending commands. This function bails out early
1496          * if the ITS is disabled or no commands have been queued.
1497          */
1498         vgic_its_process_commands(kvm, its);
1499
1500 out:
1501         mutex_unlock(&its->cmd_lock);
1502 }
1503
1504 #define REGISTER_ITS_DESC(off, rd, wr, length, acc)             \
1505 {                                                               \
1506         .reg_offset = off,                                      \
1507         .len = length,                                          \
1508         .access_flags = acc,                                    \
1509         .its_read = rd,                                         \
1510         .its_write = wr,                                        \
1511 }
1512
1513 #define REGISTER_ITS_DESC_UACCESS(off, rd, wr, uwr, length, acc)\
1514 {                                                               \
1515         .reg_offset = off,                                      \
1516         .len = length,                                          \
1517         .access_flags = acc,                                    \
1518         .its_read = rd,                                         \
1519         .its_write = wr,                                        \
1520         .uaccess_its_write = uwr,                               \
1521 }
1522
1523 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,
1524                               gpa_t addr, unsigned int len, unsigned long val)
1525 {
1526         /* Ignore */
1527 }
1528
1529 static struct vgic_register_region its_registers[] = {
1530         REGISTER_ITS_DESC(GITS_CTLR,
1531                 vgic_mmio_read_its_ctlr, vgic_mmio_write_its_ctlr, 4,
1532                 VGIC_ACCESS_32bit),
1533         REGISTER_ITS_DESC_UACCESS(GITS_IIDR,
1534                 vgic_mmio_read_its_iidr, its_mmio_write_wi,
1535                 vgic_mmio_uaccess_write_its_iidr, 4,
1536                 VGIC_ACCESS_32bit),
1537         REGISTER_ITS_DESC(GITS_TYPER,
1538                 vgic_mmio_read_its_typer, its_mmio_write_wi, 8,
1539                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1540         REGISTER_ITS_DESC(GITS_CBASER,
1541                 vgic_mmio_read_its_cbaser, vgic_mmio_write_its_cbaser, 8,
1542                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1543         REGISTER_ITS_DESC(GITS_CWRITER,
1544                 vgic_mmio_read_its_cwriter, vgic_mmio_write_its_cwriter, 8,
1545                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1546         REGISTER_ITS_DESC_UACCESS(GITS_CREADR,
1547                 vgic_mmio_read_its_creadr, its_mmio_write_wi,
1548                 vgic_mmio_uaccess_write_its_creadr, 8,
1549                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1550         REGISTER_ITS_DESC(GITS_BASER,
1551                 vgic_mmio_read_its_baser, vgic_mmio_write_its_baser, 0x40,
1552                 VGIC_ACCESS_64bit | VGIC_ACCESS_32bit),
1553         REGISTER_ITS_DESC(GITS_IDREGS_BASE,
1554                 vgic_mmio_read_its_idregs, its_mmio_write_wi, 0x30,
1555                 VGIC_ACCESS_32bit),
1556 };
1557
1558 /* This is called on setting the LPI enable bit in the redistributor. */
1559 void vgic_enable_lpis(struct kvm_vcpu *vcpu)
1560 {
1561         if (!(vcpu->arch.vgic_cpu.pendbaser & GICR_PENDBASER_PTZ))
1562                 its_sync_lpi_pending_table(vcpu);
1563 }
1564
1565 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,
1566                                    u64 addr)
1567 {
1568         struct vgic_io_device *iodev = &its->iodev;
1569         int ret;
1570
1571         mutex_lock(&kvm->slots_lock);
1572         if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1573                 ret = -EBUSY;
1574                 goto out;
1575         }
1576
1577         its->vgic_its_base = addr;
1578         iodev->regions = its_registers;
1579         iodev->nr_regions = ARRAY_SIZE(its_registers);
1580         kvm_iodevice_init(&iodev->dev, &kvm_io_gic_ops);
1581
1582         iodev->base_addr = its->vgic_its_base;
1583         iodev->iodev_type = IODEV_ITS;
1584         iodev->its = its;
1585         ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, iodev->base_addr,
1586                                       KVM_VGIC_V3_ITS_SIZE, &iodev->dev);
1587 out:
1588         mutex_unlock(&kvm->slots_lock);
1589
1590         return ret;
1591 }
1592
1593 #define INITIAL_BASER_VALUE                                               \
1594         (GIC_BASER_CACHEABILITY(GITS_BASER, INNER, RaWb)                | \
1595          GIC_BASER_CACHEABILITY(GITS_BASER, OUTER, SameAsInner)         | \
1596          GIC_BASER_SHAREABILITY(GITS_BASER, InnerShareable)             | \
1597          GITS_BASER_PAGE_SIZE_64K)
1598
1599 #define INITIAL_PROPBASER_VALUE                                           \
1600         (GIC_BASER_CACHEABILITY(GICR_PROPBASER, INNER, RaWb)            | \
1601          GIC_BASER_CACHEABILITY(GICR_PROPBASER, OUTER, SameAsInner)     | \
1602          GIC_BASER_SHAREABILITY(GICR_PROPBASER, InnerShareable))
1603
1604 static int vgic_its_create(struct kvm_device *dev, u32 type)
1605 {
1606         struct vgic_its *its;
1607
1608         if (type != KVM_DEV_TYPE_ARM_VGIC_ITS)
1609                 return -ENODEV;
1610
1611         its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL);
1612         if (!its)
1613                 return -ENOMEM;
1614
1615         mutex_init(&its->its_lock);
1616         mutex_init(&its->cmd_lock);
1617
1618         its->vgic_its_base = VGIC_ADDR_UNDEF;
1619
1620         INIT_LIST_HEAD(&its->device_list);
1621         INIT_LIST_HEAD(&its->collection_list);
1622
1623         dev->kvm->arch.vgic.msis_require_devid = true;
1624         dev->kvm->arch.vgic.has_its = true;
1625         its->enabled = false;
1626         its->dev = dev;
1627
1628         its->baser_device_table = INITIAL_BASER_VALUE                   |
1629                 ((u64)GITS_BASER_TYPE_DEVICE << GITS_BASER_TYPE_SHIFT);
1630         its->baser_coll_table = INITIAL_BASER_VALUE |
1631                 ((u64)GITS_BASER_TYPE_COLLECTION << GITS_BASER_TYPE_SHIFT);
1632         dev->kvm->arch.vgic.propbaser = INITIAL_PROPBASER_VALUE;
1633
1634         dev->private = its;
1635
1636         return vgic_its_set_abi(its, NR_ITS_ABIS - 1);
1637 }
1638
1639 static void vgic_its_free_device(struct kvm *kvm, struct its_device *dev)
1640 {
1641         struct its_ite *ite, *tmp;
1642
1643         list_for_each_entry_safe(ite, tmp, &dev->itt_head, ite_list)
1644                 its_free_ite(kvm, ite);
1645         list_del(&dev->dev_list);
1646         kfree(dev);
1647 }
1648
1649 static void vgic_its_destroy(struct kvm_device *kvm_dev)
1650 {
1651         struct kvm *kvm = kvm_dev->kvm;
1652         struct vgic_its *its = kvm_dev->private;
1653         struct list_head *cur, *temp;
1654
1655         /*
1656          * We may end up here without the lists ever having been initialized.
1657          * Check this and bail out early to avoid dereferencing a NULL pointer.
1658          */
1659         if (!its->device_list.next)
1660                 return;
1661
1662         mutex_lock(&its->its_lock);
1663         list_for_each_safe(cur, temp, &its->device_list) {
1664                 struct its_device *dev;
1665
1666                 dev = list_entry(cur, struct its_device, dev_list);
1667                 vgic_its_free_device(kvm, dev);
1668         }
1669
1670         list_for_each_safe(cur, temp, &its->collection_list) {
1671                 struct its_collection *coll;
1672
1673                 coll = list_entry(cur, struct its_collection, coll_list);
1674                 list_del(cur);
1675                 kfree(coll);
1676         }
1677         mutex_unlock(&its->its_lock);
1678
1679         kfree(its);
1680         kfree(kvm_dev);/* alloc by kvm_ioctl_create_device, free by .destroy */
1681 }
1682
1683 int vgic_its_has_attr_regs(struct kvm_device *dev,
1684                            struct kvm_device_attr *attr)
1685 {
1686         const struct vgic_register_region *region;
1687         gpa_t offset = attr->attr;
1688         int align;
1689
1690         align = (offset < GITS_TYPER) || (offset >= GITS_PIDR4) ? 0x3 : 0x7;
1691
1692         if (offset & align)
1693                 return -EINVAL;
1694
1695         region = vgic_find_mmio_region(its_registers,
1696                                        ARRAY_SIZE(its_registers),
1697                                        offset);
1698         if (!region)
1699                 return -ENXIO;
1700
1701         return 0;
1702 }
1703
1704 int vgic_its_attr_regs_access(struct kvm_device *dev,
1705                               struct kvm_device_attr *attr,
1706                               u64 *reg, bool is_write)
1707 {
1708         const struct vgic_register_region *region;
1709         struct vgic_its *its;
1710         gpa_t addr, offset;
1711         unsigned int len;
1712         int align, ret = 0;
1713
1714         its = dev->private;
1715         offset = attr->attr;
1716
1717         /*
1718          * Although the spec supports upper/lower 32-bit accesses to
1719          * 64-bit ITS registers, the userspace ABI requires 64-bit
1720          * accesses to all 64-bit wide registers. We therefore only
1721          * support 32-bit accesses to GITS_CTLR, GITS_IIDR and GITS ID
1722          * registers
1723          */
1724         if ((offset < GITS_TYPER) || (offset >= GITS_PIDR4))
1725                 align = 0x3;
1726         else
1727                 align = 0x7;
1728
1729         if (offset & align)
1730                 return -EINVAL;
1731
1732         mutex_lock(&dev->kvm->lock);
1733
1734         if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {
1735                 ret = -ENXIO;
1736                 goto out;
1737         }
1738
1739         region = vgic_find_mmio_region(its_registers,
1740                                        ARRAY_SIZE(its_registers),
1741                                        offset);
1742         if (!region) {
1743                 ret = -ENXIO;
1744                 goto out;
1745         }
1746
1747         if (!lock_all_vcpus(dev->kvm)) {
1748                 ret = -EBUSY;
1749                 goto out;
1750         }
1751
1752         addr = its->vgic_its_base + offset;
1753
1754         len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4;
1755
1756         if (is_write) {
1757                 if (region->uaccess_its_write)
1758                         ret = region->uaccess_its_write(dev->kvm, its, addr,
1759                                                         len, *reg);
1760                 else
1761                         region->its_write(dev->kvm, its, addr, len, *reg);
1762         } else {
1763                 *reg = region->its_read(dev->kvm, its, addr, len);
1764         }
1765         unlock_all_vcpus(dev->kvm);
1766 out:
1767         mutex_unlock(&dev->kvm->lock);
1768         return ret;
1769 }
1770
1771 static u32 compute_next_devid_offset(struct list_head *h,
1772                                      struct its_device *dev)
1773 {
1774         struct its_device *next;
1775         u32 next_offset;
1776
1777         if (list_is_last(&dev->dev_list, h))
1778                 return 0;
1779         next = list_next_entry(dev, dev_list);
1780         next_offset = next->device_id - dev->device_id;
1781
1782         return min_t(u32, next_offset, VITS_DTE_MAX_DEVID_OFFSET);
1783 }
1784
1785 static u32 compute_next_eventid_offset(struct list_head *h, struct its_ite *ite)
1786 {
1787         struct its_ite *next;
1788         u32 next_offset;
1789
1790         if (list_is_last(&ite->ite_list, h))
1791                 return 0;
1792         next = list_next_entry(ite, ite_list);
1793         next_offset = next->event_id - ite->event_id;
1794
1795         return min_t(u32, next_offset, VITS_ITE_MAX_EVENTID_OFFSET);
1796 }
1797
1798 /**
1799  * entry_fn_t - Callback called on a table entry restore path
1800  * @its: its handle
1801  * @id: id of the entry
1802  * @entry: pointer to the entry
1803  * @opaque: pointer to an opaque data
1804  *
1805  * Return: < 0 on error, 0 if last element was identified, id offset to next
1806  * element otherwise
1807  */
1808 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
1809                           void *opaque);
1810
1811 /**
1812  * scan_its_table - Scan a contiguous table in guest RAM and applies a function
1813  * to each entry
1814  *
1815  * @its: its handle
1816  * @base: base gpa of the table
1817  * @size: size of the table in bytes
1818  * @esz: entry size in bytes
1819  * @start_id: the ID of the first entry in the table
1820  * (non zero for 2d level tables)
1821  * @fn: function to apply on each entry
1822  *
1823  * Return: < 0 on error, 0 if last element was identified, 1 otherwise
1824  * (the last element may not be found on second level tables)
1825  */
1826 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, int esz,
1827                           int start_id, entry_fn_t fn, void *opaque)
1828 {
1829         struct kvm *kvm = its->dev->kvm;
1830         unsigned long len = size;
1831         int id = start_id;
1832         gpa_t gpa = base;
1833         char entry[esz];
1834         int ret;
1835
1836         memset(entry, 0, esz);
1837
1838         while (len > 0) {
1839                 int next_offset;
1840                 size_t byte_offset;
1841
1842                 ret = kvm_read_guest_lock(kvm, gpa, entry, esz);
1843                 if (ret)
1844                         return ret;
1845
1846                 next_offset = fn(its, id, entry, opaque);
1847                 if (next_offset <= 0)
1848                         return next_offset;
1849
1850                 byte_offset = next_offset * esz;
1851                 id += next_offset;
1852                 gpa += byte_offset;
1853                 len -= byte_offset;
1854         }
1855         return 1;
1856 }
1857
1858 /**
1859  * vgic_its_save_ite - Save an interrupt translation entry at @gpa
1860  */
1861 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
1862                               struct its_ite *ite, gpa_t gpa, int ite_esz)
1863 {
1864         struct kvm *kvm = its->dev->kvm;
1865         u32 next_offset;
1866         u64 val;
1867
1868         next_offset = compute_next_eventid_offset(&dev->itt_head, ite);
1869         val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) |
1870                ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) |
1871                 ite->collection->collection_id;
1872         val = cpu_to_le64(val);
1873         return kvm_write_guest(kvm, gpa, &val, ite_esz);
1874 }
1875
1876 /**
1877  * vgic_its_restore_ite - restore an interrupt translation entry
1878  * @event_id: id used for indexing
1879  * @ptr: pointer to the ITE entry
1880  * @opaque: pointer to the its_device
1881  */
1882 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,
1883                                 void *ptr, void *opaque)
1884 {
1885         struct its_device *dev = (struct its_device *)opaque;
1886         struct its_collection *collection;
1887         struct kvm *kvm = its->dev->kvm;
1888         struct kvm_vcpu *vcpu = NULL;
1889         u64 val;
1890         u64 *p = (u64 *)ptr;
1891         struct vgic_irq *irq;
1892         u32 coll_id, lpi_id;
1893         struct its_ite *ite;
1894         u32 offset;
1895
1896         val = *p;
1897
1898         val = le64_to_cpu(val);
1899
1900         coll_id = val & KVM_ITS_ITE_ICID_MASK;
1901         lpi_id = (val & KVM_ITS_ITE_PINTID_MASK) >> KVM_ITS_ITE_PINTID_SHIFT;
1902
1903         if (!lpi_id)
1904                 return 1; /* invalid entry, no choice but to scan next entry */
1905
1906         if (lpi_id < VGIC_MIN_LPI)
1907                 return -EINVAL;
1908
1909         offset = val >> KVM_ITS_ITE_NEXT_SHIFT;
1910         if (event_id + offset >= BIT_ULL(dev->num_eventid_bits))
1911                 return -EINVAL;
1912
1913         collection = find_collection(its, coll_id);
1914         if (!collection)
1915                 return -EINVAL;
1916
1917         ite = vgic_its_alloc_ite(dev, collection, event_id);
1918         if (IS_ERR(ite))
1919                 return PTR_ERR(ite);
1920
1921         if (its_is_collection_mapped(collection))
1922                 vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1923
1924         irq = vgic_add_lpi(kvm, lpi_id, vcpu);
1925         if (IS_ERR(irq))
1926                 return PTR_ERR(irq);
1927         ite->irq = irq;
1928
1929         return offset;
1930 }
1931
1932 static int vgic_its_ite_cmp(void *priv, struct list_head *a,
1933                             struct list_head *b)
1934 {
1935         struct its_ite *itea = container_of(a, struct its_ite, ite_list);
1936         struct its_ite *iteb = container_of(b, struct its_ite, ite_list);
1937
1938         if (itea->event_id < iteb->event_id)
1939                 return -1;
1940         else
1941                 return 1;
1942 }
1943
1944 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)
1945 {
1946         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1947         gpa_t base = device->itt_addr;
1948         struct its_ite *ite;
1949         int ret;
1950         int ite_esz = abi->ite_esz;
1951
1952         list_sort(NULL, &device->itt_head, vgic_its_ite_cmp);
1953
1954         list_for_each_entry(ite, &device->itt_head, ite_list) {
1955                 gpa_t gpa = base + ite->event_id * ite_esz;
1956
1957                 ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);
1958                 if (ret)
1959                         return ret;
1960         }
1961         return 0;
1962 }
1963
1964 /**
1965  * vgic_its_restore_itt - restore the ITT of a device
1966  *
1967  * @its: its handle
1968  * @dev: device handle
1969  *
1970  * Return 0 on success, < 0 on error
1971  */
1972 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)
1973 {
1974         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
1975         gpa_t base = dev->itt_addr;
1976         int ret;
1977         int ite_esz = abi->ite_esz;
1978         size_t max_size = BIT_ULL(dev->num_eventid_bits) * ite_esz;
1979
1980         ret = scan_its_table(its, base, max_size, ite_esz, 0,
1981                              vgic_its_restore_ite, dev);
1982
1983         /* scan_its_table returns +1 if all ITEs are invalid */
1984         if (ret > 0)
1985                 ret = 0;
1986
1987         return ret;
1988 }
1989
1990 /**
1991  * vgic_its_save_dte - Save a device table entry at a given GPA
1992  *
1993  * @its: ITS handle
1994  * @dev: ITS device
1995  * @ptr: GPA
1996  */
1997 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,
1998                              gpa_t ptr, int dte_esz)
1999 {
2000         struct kvm *kvm = its->dev->kvm;
2001         u64 val, itt_addr_field;
2002         u32 next_offset;
2003
2004         itt_addr_field = dev->itt_addr >> 8;
2005         next_offset = compute_next_devid_offset(&its->device_list, dev);
2006         val = (1ULL << KVM_ITS_DTE_VALID_SHIFT |
2007                ((u64)next_offset << KVM_ITS_DTE_NEXT_SHIFT) |
2008                (itt_addr_field << KVM_ITS_DTE_ITTADDR_SHIFT) |
2009                 (dev->num_eventid_bits - 1));
2010         val = cpu_to_le64(val);
2011         return kvm_write_guest(kvm, ptr, &val, dte_esz);
2012 }
2013
2014 /**
2015  * vgic_its_restore_dte - restore a device table entry
2016  *
2017  * @its: its handle
2018  * @id: device id the DTE corresponds to
2019  * @ptr: kernel VA where the 8 byte DTE is located
2020  * @opaque: unused
2021  *
2022  * Return: < 0 on error, 0 if the dte is the last one, id offset to the
2023  * next dte otherwise
2024  */
2025 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
2026                                 void *ptr, void *opaque)
2027 {
2028         struct its_device *dev;
2029         gpa_t itt_addr;
2030         u8 num_eventid_bits;
2031         u64 entry = *(u64 *)ptr;
2032         bool valid;
2033         u32 offset;
2034         int ret;
2035
2036         entry = le64_to_cpu(entry);
2037
2038         valid = entry >> KVM_ITS_DTE_VALID_SHIFT;
2039         num_eventid_bits = (entry & KVM_ITS_DTE_SIZE_MASK) + 1;
2040         itt_addr = ((entry & KVM_ITS_DTE_ITTADDR_MASK)
2041                         >> KVM_ITS_DTE_ITTADDR_SHIFT) << 8;
2042
2043         if (!valid)
2044                 return 1;
2045
2046         /* dte entry is valid */
2047         offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
2048
2049         dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);
2050         if (IS_ERR(dev))
2051                 return PTR_ERR(dev);
2052
2053         ret = vgic_its_restore_itt(its, dev);
2054         if (ret) {
2055                 vgic_its_free_device(its->dev->kvm, dev);
2056                 return ret;
2057         }
2058
2059         return offset;
2060 }
2061
2062 static int vgic_its_device_cmp(void *priv, struct list_head *a,
2063                                struct list_head *b)
2064 {
2065         struct its_device *deva = container_of(a, struct its_device, dev_list);
2066         struct its_device *devb = container_of(b, struct its_device, dev_list);
2067
2068         if (deva->device_id < devb->device_id)
2069                 return -1;
2070         else
2071                 return 1;
2072 }
2073
2074 /**
2075  * vgic_its_save_device_tables - Save the device table and all ITT
2076  * into guest RAM
2077  *
2078  * L1/L2 handling is hidden by vgic_its_check_id() helper which directly
2079  * returns the GPA of the device entry
2080  */
2081 static int vgic_its_save_device_tables(struct vgic_its *its)
2082 {
2083         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2084         u64 baser = its->baser_device_table;
2085         struct its_device *dev;
2086         int dte_esz = abi->dte_esz;
2087
2088         if (!(baser & GITS_BASER_VALID))
2089                 return 0;
2090
2091         list_sort(NULL, &its->device_list, vgic_its_device_cmp);
2092
2093         list_for_each_entry(dev, &its->device_list, dev_list) {
2094                 int ret;
2095                 gpa_t eaddr;
2096
2097                 if (!vgic_its_check_id(its, baser,
2098                                        dev->device_id, &eaddr))
2099                         return -EINVAL;
2100
2101                 ret = vgic_its_save_itt(its, dev);
2102                 if (ret)
2103                         return ret;
2104
2105                 ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);
2106                 if (ret)
2107                         return ret;
2108         }
2109         return 0;
2110 }
2111
2112 /**
2113  * handle_l1_dte - callback used for L1 device table entries (2 stage case)
2114  *
2115  * @its: its handle
2116  * @id: index of the entry in the L1 table
2117  * @addr: kernel VA
2118  * @opaque: unused
2119  *
2120  * L1 table entries are scanned by steps of 1 entry
2121  * Return < 0 if error, 0 if last dte was found when scanning the L2
2122  * table, +1 otherwise (meaning next L1 entry must be scanned)
2123  */
2124 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,
2125                          void *opaque)
2126 {
2127         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2128         int l2_start_id = id * (SZ_64K / abi->dte_esz);
2129         u64 entry = *(u64 *)addr;
2130         int dte_esz = abi->dte_esz;
2131         gpa_t gpa;
2132         int ret;
2133
2134         entry = le64_to_cpu(entry);
2135
2136         if (!(entry & KVM_ITS_L1E_VALID_MASK))
2137                 return 1;
2138
2139         gpa = entry & KVM_ITS_L1E_ADDR_MASK;
2140
2141         ret = scan_its_table(its, gpa, SZ_64K, dte_esz,
2142                              l2_start_id, vgic_its_restore_dte, NULL);
2143
2144         return ret;
2145 }
2146
2147 /**
2148  * vgic_its_restore_device_tables - Restore the device table and all ITT
2149  * from guest RAM to internal data structs
2150  */
2151 static int vgic_its_restore_device_tables(struct vgic_its *its)
2152 {
2153         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2154         u64 baser = its->baser_device_table;
2155         int l1_esz, ret;
2156         int l1_tbl_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2157         gpa_t l1_gpa;
2158
2159         if (!(baser & GITS_BASER_VALID))
2160                 return 0;
2161
2162         l1_gpa = BASER_ADDRESS(baser);
2163
2164         if (baser & GITS_BASER_INDIRECT) {
2165                 l1_esz = GITS_LVL1_ENTRY_SIZE;
2166                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2167                                      handle_l1_dte, NULL);
2168         } else {
2169                 l1_esz = abi->dte_esz;
2170                 ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,
2171                                      vgic_its_restore_dte, NULL);
2172         }
2173
2174         /* scan_its_table returns +1 if all entries are invalid */
2175         if (ret > 0)
2176                 ret = 0;
2177
2178         return ret;
2179 }
2180
2181 static int vgic_its_save_cte(struct vgic_its *its,
2182                              struct its_collection *collection,
2183                              gpa_t gpa, int esz)
2184 {
2185         u64 val;
2186
2187         val = (1ULL << KVM_ITS_CTE_VALID_SHIFT |
2188                ((u64)collection->target_addr << KVM_ITS_CTE_RDBASE_SHIFT) |
2189                collection->collection_id);
2190         val = cpu_to_le64(val);
2191         return kvm_write_guest(its->dev->kvm, gpa, &val, esz);
2192 }
2193
2194 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)
2195 {
2196         struct its_collection *collection;
2197         struct kvm *kvm = its->dev->kvm;
2198         u32 target_addr, coll_id;
2199         u64 val;
2200         int ret;
2201
2202         BUG_ON(esz > sizeof(val));
2203         ret = kvm_read_guest_lock(kvm, gpa, &val, esz);
2204         if (ret)
2205                 return ret;
2206         val = le64_to_cpu(val);
2207         if (!(val & KVM_ITS_CTE_VALID_MASK))
2208                 return 0;
2209
2210         target_addr = (u32)(val >> KVM_ITS_CTE_RDBASE_SHIFT);
2211         coll_id = val & KVM_ITS_CTE_ICID_MASK;
2212
2213         if (target_addr != COLLECTION_NOT_MAPPED &&
2214             target_addr >= atomic_read(&kvm->online_vcpus))
2215                 return -EINVAL;
2216
2217         collection = find_collection(its, coll_id);
2218         if (collection)
2219                 return -EEXIST;
2220         ret = vgic_its_alloc_collection(its, &collection, coll_id);
2221         if (ret)
2222                 return ret;
2223         collection->target_addr = target_addr;
2224         return 1;
2225 }
2226
2227 /**
2228  * vgic_its_save_collection_table - Save the collection table into
2229  * guest RAM
2230  */
2231 static int vgic_its_save_collection_table(struct vgic_its *its)
2232 {
2233         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2234         u64 baser = its->baser_coll_table;
2235         gpa_t gpa = BASER_ADDRESS(baser);
2236         struct its_collection *collection;
2237         u64 val;
2238         size_t max_size, filled = 0;
2239         int ret, cte_esz = abi->cte_esz;
2240
2241         if (!(baser & GITS_BASER_VALID))
2242                 return 0;
2243
2244         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2245
2246         list_for_each_entry(collection, &its->collection_list, coll_list) {
2247                 ret = vgic_its_save_cte(its, collection, gpa, cte_esz);
2248                 if (ret)
2249                         return ret;
2250                 gpa += cte_esz;
2251                 filled += cte_esz;
2252         }
2253
2254         if (filled == max_size)
2255                 return 0;
2256
2257         /*
2258          * table is not fully filled, add a last dummy element
2259          * with valid bit unset
2260          */
2261         val = 0;
2262         BUG_ON(cte_esz > sizeof(val));
2263         ret = kvm_write_guest(its->dev->kvm, gpa, &val, cte_esz);
2264         return ret;
2265 }
2266
2267 /**
2268  * vgic_its_restore_collection_table - reads the collection table
2269  * in guest memory and restores the ITS internal state. Requires the
2270  * BASER registers to be restored before.
2271  */
2272 static int vgic_its_restore_collection_table(struct vgic_its *its)
2273 {
2274         const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2275         u64 baser = its->baser_coll_table;
2276         int cte_esz = abi->cte_esz;
2277         size_t max_size, read = 0;
2278         gpa_t gpa;
2279         int ret;
2280
2281         if (!(baser & GITS_BASER_VALID))
2282                 return 0;
2283
2284         gpa = BASER_ADDRESS(baser);
2285
2286         max_size = GITS_BASER_NR_PAGES(baser) * SZ_64K;
2287
2288         while (read < max_size) {
2289                 ret = vgic_its_restore_cte(its, gpa, cte_esz);
2290                 if (ret <= 0)
2291                         break;
2292                 gpa += cte_esz;
2293                 read += cte_esz;
2294         }
2295
2296         if (ret > 0)
2297                 return 0;
2298
2299         return ret;
2300 }
2301
2302 /**
2303  * vgic_its_save_tables_v0 - Save the ITS tables into guest ARM
2304  * according to v0 ABI
2305  */
2306 static int vgic_its_save_tables_v0(struct vgic_its *its)
2307 {
2308         struct kvm *kvm = its->dev->kvm;
2309         int ret;
2310
2311         mutex_lock(&kvm->lock);
2312         mutex_lock(&its->its_lock);
2313
2314         if (!lock_all_vcpus(kvm)) {
2315                 mutex_unlock(&its->its_lock);
2316                 mutex_unlock(&kvm->lock);
2317                 return -EBUSY;
2318         }
2319
2320         ret = vgic_its_save_device_tables(its);
2321         if (ret)
2322                 goto out;
2323
2324         ret = vgic_its_save_collection_table(its);
2325
2326 out:
2327         unlock_all_vcpus(kvm);
2328         mutex_unlock(&its->its_lock);
2329         mutex_unlock(&kvm->lock);
2330         return ret;
2331 }
2332
2333 /**
2334  * vgic_its_restore_tables_v0 - Restore the ITS tables from guest RAM
2335  * to internal data structs according to V0 ABI
2336  *
2337  */
2338 static int vgic_its_restore_tables_v0(struct vgic_its *its)
2339 {
2340         struct kvm *kvm = its->dev->kvm;
2341         int ret;
2342
2343         mutex_lock(&kvm->lock);
2344         mutex_lock(&its->its_lock);
2345
2346         if (!lock_all_vcpus(kvm)) {
2347                 mutex_unlock(&its->its_lock);
2348                 mutex_unlock(&kvm->lock);
2349                 return -EBUSY;
2350         }
2351
2352         ret = vgic_its_restore_collection_table(its);
2353         if (ret)
2354                 goto out;
2355
2356         ret = vgic_its_restore_device_tables(its);
2357 out:
2358         unlock_all_vcpus(kvm);
2359         mutex_unlock(&its->its_lock);
2360         mutex_unlock(&kvm->lock);
2361
2362         return ret;
2363 }
2364
2365 static int vgic_its_commit_v0(struct vgic_its *its)
2366 {
2367         const struct vgic_its_abi *abi;
2368
2369         abi = vgic_its_get_abi(its);
2370         its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2371         its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;
2372
2373         its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)
2374                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2375
2376         its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)
2377                                         << GITS_BASER_ENTRY_SIZE_SHIFT);
2378         return 0;
2379 }
2380
2381 static int vgic_its_has_attr(struct kvm_device *dev,
2382                              struct kvm_device_attr *attr)
2383 {
2384         switch (attr->group) {
2385         case KVM_DEV_ARM_VGIC_GRP_ADDR:
2386                 switch (attr->attr) {
2387                 case KVM_VGIC_ITS_ADDR_TYPE:
2388                         return 0;
2389                 }
2390                 break;
2391         case KVM_DEV_ARM_VGIC_GRP_CTRL:
2392                 switch (attr->attr) {
2393                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2394                         return 0;
2395                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2396                         return 0;
2397                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2398                         return 0;
2399                 }
2400                 break;
2401         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS:
2402                 return vgic_its_has_attr_regs(dev, attr);
2403         }
2404         return -ENXIO;
2405 }
2406
2407 static int vgic_its_set_attr(struct kvm_device *dev,
2408                              struct kvm_device_attr *attr)
2409 {
2410         struct vgic_its *its = dev->private;
2411         int ret;
2412
2413         switch (attr->group) {
2414         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2415                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2416                 unsigned long type = (unsigned long)attr->attr;
2417                 u64 addr;
2418
2419                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2420                         return -ENODEV;
2421
2422                 if (copy_from_user(&addr, uaddr, sizeof(addr)))
2423                         return -EFAULT;
2424
2425                 ret = vgic_check_ioaddr(dev->kvm, &its->vgic_its_base,
2426                                         addr, SZ_64K);
2427                 if (ret)
2428                         return ret;
2429
2430                 return vgic_register_its_iodev(dev->kvm, its, addr);
2431         }
2432         case KVM_DEV_ARM_VGIC_GRP_CTRL: {
2433                 const struct vgic_its_abi *abi = vgic_its_get_abi(its);
2434
2435                 switch (attr->attr) {
2436                 case KVM_DEV_ARM_VGIC_CTRL_INIT:
2437                         /* Nothing to do */
2438                         return 0;
2439                 case KVM_DEV_ARM_ITS_SAVE_TABLES:
2440                         return abi->save_tables(its);
2441                 case KVM_DEV_ARM_ITS_RESTORE_TABLES:
2442                         return abi->restore_tables(its);
2443                 }
2444         }
2445         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2446                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2447                 u64 reg;
2448
2449                 if (get_user(reg, uaddr))
2450                         return -EFAULT;
2451
2452                 return vgic_its_attr_regs_access(dev, attr, &reg, true);
2453         }
2454         }
2455         return -ENXIO;
2456 }
2457
2458 static int vgic_its_get_attr(struct kvm_device *dev,
2459                              struct kvm_device_attr *attr)
2460 {
2461         switch (attr->group) {
2462         case KVM_DEV_ARM_VGIC_GRP_ADDR: {
2463                 struct vgic_its *its = dev->private;
2464                 u64 addr = its->vgic_its_base;
2465                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2466                 unsigned long type = (unsigned long)attr->attr;
2467
2468                 if (type != KVM_VGIC_ITS_ADDR_TYPE)
2469                         return -ENODEV;
2470
2471                 if (copy_to_user(uaddr, &addr, sizeof(addr)))
2472                         return -EFAULT;
2473                 break;
2474         }
2475         case KVM_DEV_ARM_VGIC_GRP_ITS_REGS: {
2476                 u64 __user *uaddr = (u64 __user *)(long)attr->addr;
2477                 u64 reg;
2478                 int ret;
2479
2480                 ret = vgic_its_attr_regs_access(dev, attr, &reg, false);
2481                 if (ret)
2482                         return ret;
2483                 return put_user(reg, uaddr);
2484         }
2485         default:
2486                 return -ENXIO;
2487         }
2488
2489         return 0;
2490 }
2491
2492 static struct kvm_device_ops kvm_arm_vgic_its_ops = {
2493         .name = "kvm-arm-vgic-its",
2494         .create = vgic_its_create,
2495         .destroy = vgic_its_destroy,
2496         .set_attr = vgic_its_set_attr,
2497         .get_attr = vgic_its_get_attr,
2498         .has_attr = vgic_its_has_attr,
2499 };
2500
2501 int kvm_vgic_register_its_device(void)
2502 {
2503         return kvm_register_device_ops(&kvm_arm_vgic_its_ops,
2504                                        KVM_DEV_TYPE_ARM_VGIC_ITS);
2505 }