arch/powerpc/kvm/book3s_64_mmu_hv.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright 2010 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  16  */
  17
  18 #include <linux/types.h>
  19 #include <linux/string.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/highmem.h>
  23 #include <linux/gfp.h>
  24 #include <linux/slab.h>
  25 #include <linux/hugetlb.h>
  26 #include <linux/vmalloc.h>
  27 #include <linux/srcu.h>
  28 #include <linux/anon_inodes.h>
  29 #include <linux/file.h>
  30 #include <linux/debugfs.h>
  31
  32 #include <asm/kvm_ppc.h>
  33 #include <asm/kvm_book3s.h>
  34 #include <asm/book3s/64/mmu-hash.h>
  35 #include <asm/hvcall.h>
  36 #include <asm/synch.h>
  37 #include <asm/ppc-opcode.h>
  38 #include <asm/cputable.h>
  39 #include <asm/pte-walk.h>
  40
  41 #include "trace_hv.h"
  42
  43 //#define DEBUG_RESIZE_HPT      1
  44
  45 #ifdef DEBUG_RESIZE_HPT
  46 #define resize_hpt_debug(resize, ...)                           \
  47         do {                                                    \
  48                 printk(KERN_DEBUG "RESIZE HPT %p: ", resize);   \
  49                 printk(__VA_ARGS__);                            \
  50         } while (0)
  51 #else
  52 #define resize_hpt_debug(resize, ...)                           \
  53         do { } while (0)
  54 #endif
  55
  56 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
  57                                 long pte_index, unsigned long pteh,
  58                                 unsigned long ptel, unsigned long *pte_idx_ret);
  59
  60 struct kvm_resize_hpt {
  61         /* These fields read-only after init */
  62         struct kvm *kvm;
  63         struct work_struct work;
  64         u32 order;
  65
  66         /* These fields protected by kvm->lock */
  67
  68         /* Possible values and their usage:
  69          *  <0     an error occurred during allocation,
  70          *  -EBUSY allocation is in the progress,
  71          *  0      allocation made successfuly.
  72          */
  73         int error;
  74
  75         /* Private to the work thread, until error != -EBUSY,
  76          * then protected by kvm->lock.
  77          */
  78         struct kvm_hpt_info hpt;
  79 };
  80
  81 int kvmppc_allocate_hpt(struct kvm_hpt_info *info, u32 order)
  82 {
  83         unsigned long hpt = 0;
  84         int cma = 0;
  85         struct page *page = NULL;
  86         struct revmap_entry *rev;
  87         unsigned long npte;
  88
  89         if ((order < PPC_MIN_HPT_ORDER) || (order > PPC_MAX_HPT_ORDER))
  90                 return -EINVAL;
  91
  92         page = kvm_alloc_hpt_cma(1ul << (order - PAGE_SHIFT));
  93         if (page) {
  94                 hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page));
  95                 memset((void *)hpt, 0, (1ul << order));
  96                 cma = 1;
  97         }
  98
  99         if (!hpt)
 100                 hpt = __get_free_pages(GFP_KERNEL|__GFP_ZERO|__GFP_RETRY_MAYFAIL
 101                                        |__GFP_NOWARN, order - PAGE_SHIFT);
 102
 103         if (!hpt)
 104                 return -ENOMEM;
 105
 106         /* HPTEs are 2**4 bytes long */
 107         npte = 1ul << (order - 4);
 108
 109         /* Allocate reverse map array */
 110         rev = vmalloc(array_size(npte, sizeof(struct revmap_entry)));
 111         if (!rev) {
 112                 if (cma)
 113                         kvm_free_hpt_cma(page, 1 << (order - PAGE_SHIFT));
 114                 else
 115                         free_pages(hpt, order - PAGE_SHIFT);
 116                 return -ENOMEM;
 117         }
 118
 119         info->order = order;
 120         info->virt = hpt;
 121         info->cma = cma;
 122         info->rev = rev;
 123
 124         return 0;
 125 }
 126
 127 void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info)
 128 {
 129         atomic64_set(&kvm->arch.mmio_update, 0);
 130         kvm->arch.hpt = *info;
 131         kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18);
 132
 133         pr_debug("KVM guest htab at %lx (order %ld), LPID %x\n",
 134                  info->virt, (long)info->order, kvm->arch.lpid);
 135 }
 136
 137 long kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)
 138 {
 139         long err = -EBUSY;
 140         struct kvm_hpt_info info;
 141
 142         mutex_lock(&kvm->lock);
 143         if (kvm->arch.mmu_ready) {
 144                 kvm->arch.mmu_ready = 0;
 145                 /* order mmu_ready vs. vcpus_running */
 146                 smp_mb();
 147                 if (atomic_read(&kvm->arch.vcpus_running)) {
 148                         kvm->arch.mmu_ready = 1;
 149                         goto out;
 150                 }
 151         }
 152         if (kvm_is_radix(kvm)) {
 153                 err = kvmppc_switch_mmu_to_hpt(kvm);
 154                 if (err)
 155                         goto out;
 156         }
 157
 158         if (kvm->arch.hpt.order == order) {
 159                 /* We already have a suitable HPT */
 160
 161                 /* Set the entire HPT to 0, i.e. invalid HPTEs */
 162                 memset((void *)kvm->arch.hpt.virt, 0, 1ul << order);
 163                 /*
 164                  * Reset all the reverse-mapping chains for all memslots
 165                  */
 166                 kvmppc_rmap_reset(kvm);
 167                 err = 0;
 168                 goto out;
 169         }
 170
 171         if (kvm->arch.hpt.virt) {
 172                 kvmppc_free_hpt(&kvm->arch.hpt);
 173                 kvmppc_rmap_reset(kvm);
 174         }
 175
 176         err = kvmppc_allocate_hpt(&info, order);
 177         if (err < 0)
 178                 goto out;
 179         kvmppc_set_hpt(kvm, &info);
 180
 181 out:
 182         if (err == 0)
 183                 /* Ensure that each vcpu will flush its TLB on next entry. */
 184                 cpumask_setall(&kvm->arch.need_tlb_flush);
 185
 186         mutex_unlock(&kvm->lock);
 187         return err;
 188 }
 189
 190 void kvmppc_free_hpt(struct kvm_hpt_info *info)
 191 {
 192         vfree(info->rev);
 193         info->rev = NULL;
 194         if (info->cma)
 195                 kvm_free_hpt_cma(virt_to_page(info->virt),
 196                                  1 << (info->order - PAGE_SHIFT));
 197         else if (info->virt)
 198                 free_pages(info->virt, info->order - PAGE_SHIFT);
 199         info->virt = 0;
 200         info->order = 0;
 201 }
 202
 203 /* Bits in first HPTE dword for pagesize 4k, 64k or 16M */
 204 static inline unsigned long hpte0_pgsize_encoding(unsigned long pgsize)
 205 {
 206         return (pgsize > 0x1000) ? HPTE_V_LARGE : 0;
 207 }
 208
 209 /* Bits in second HPTE dword for pagesize 4k, 64k or 16M */
 210 static inline unsigned long hpte1_pgsize_encoding(unsigned long pgsize)
 211 {
 212         return (pgsize == 0x10000) ? 0x1000 : 0;
 213 }
 214
 215 void kvmppc_map_vrma(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
 216                      unsigned long porder)
 217 {
 218         unsigned long i;
 219         unsigned long npages;
 220         unsigned long hp_v, hp_r;
 221         unsigned long addr, hash;
 222         unsigned long psize;
 223         unsigned long hp0, hp1;
 224         unsigned long idx_ret;
 225         long ret;
 226         struct kvm *kvm = vcpu->kvm;
 227
 228         psize = 1ul << porder;
 229         npages = memslot->npages >> (porder - PAGE_SHIFT);
 230
 231         /* VRMA can't be > 1TB */
 232         if (npages > 1ul << (40 - porder))
 233                 npages = 1ul << (40 - porder);
 234         /* Can't use more than 1 HPTE per HPTEG */
 235         if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1)
 236                 npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1;
 237
 238         hp0 = HPTE_V_1TB_SEG | (VRMA_VSID << (40 - 16)) |
 239                 HPTE_V_BOLTED | hpte0_pgsize_encoding(psize);
 240         hp1 = hpte1_pgsize_encoding(psize) |
 241                 HPTE_R_R | HPTE_R_C | HPTE_R_M | PP_RWXX;
 242
 243         for (i = 0; i < npages; ++i) {
 244                 addr = i << porder;
 245                 /* can't use hpt_hash since va > 64 bits */
 246                 hash = (i ^ (VRMA_VSID ^ (VRMA_VSID << 25)))
 247                         & kvmppc_hpt_mask(&kvm->arch.hpt);
 248                 /*
 249                  * We assume that the hash table is empty and no
 250                  * vcpus are using it at this stage.  Since we create
 251                  * at most one HPTE per HPTEG, we just assume entry 7
 252                  * is available and use it.
 253                  */
 254                 hash = (hash << 3) + 7;
 255                 hp_v = hp0 | ((addr >> 16) & ~0x7fUL);
 256                 hp_r = hp1 | addr;
 257                 ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,
 258                                                  &idx_ret);
 259                 if (ret != H_SUCCESS) {
 260                         pr_err("KVM: map_vrma at %lx failed, ret=%ld\n",
 261                                addr, ret);
 262                         break;
 263                 }
 264         }
 265 }
 266
 267 int kvmppc_mmu_hv_init(void)
 268 {
 269         unsigned long host_lpid, rsvd_lpid;
 270
 271         if (!cpu_has_feature(CPU_FTR_HVMODE))
 272                 return -EINVAL;
 273
 274         if (!mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE))
 275                 return -EINVAL;
 276
 277         /* POWER7 has 10-bit LPIDs (12-bit in POWER8) */
 278         host_lpid = mfspr(SPRN_LPID);
 279         rsvd_lpid = LPID_RSVD;
 280
 281         kvmppc_init_lpid(rsvd_lpid + 1);
 282
 283         kvmppc_claim_lpid(host_lpid);
 284         /* rsvd_lpid is reserved for use in partition switching */
 285         kvmppc_claim_lpid(rsvd_lpid);
 286
 287         return 0;
 288 }
 289
 290 static void kvmppc_mmu_book3s_64_hv_reset_msr(struct kvm_vcpu *vcpu)
 291 {
 292         unsigned long msr = vcpu->arch.intr_msr;
 293
 294         /* If transactional, change to suspend mode on IRQ delivery */
 295         if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
 296                 msr |= MSR_TS_S;
 297         else
 298                 msr |= vcpu->arch.shregs.msr & MSR_TS_MASK;
 299         kvmppc_set_msr(vcpu, msr);
 300 }
 301
 302 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
 303                                 long pte_index, unsigned long pteh,
 304                                 unsigned long ptel, unsigned long *pte_idx_ret)
 305 {
 306         long ret;
 307
 308         /* Protect linux PTE lookup from page table destruction */
 309         rcu_read_lock_sched();  /* this disables preemption too */
 310         ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,
 311                                 current->mm->pgd, false, pte_idx_ret);
 312         rcu_read_unlock_sched();
 313         if (ret == H_TOO_HARD) {
 314                 /* this can't happen */
 315                 pr_err("KVM: Oops, kvmppc_h_enter returned too hard!\n");
 316                 ret = H_RESOURCE;       /* or something */
 317         }
 318         return ret;
 319
 320 }
 321
 322 static struct kvmppc_slb *kvmppc_mmu_book3s_hv_find_slbe(struct kvm_vcpu *vcpu,
 323                                                          gva_t eaddr)
 324 {
 325         u64 mask;
 326         int i;
 327
 328         for (i = 0; i < vcpu->arch.slb_nr; i++) {
 329                 if (!(vcpu->arch.slb[i].orige & SLB_ESID_V))
 330                         continue;
 331
 332                 if (vcpu->arch.slb[i].origv & SLB_VSID_B_1T)
 333                         mask = ESID_MASK_1T;
 334                 else
 335                         mask = ESID_MASK;
 336
 337                 if (((vcpu->arch.slb[i].orige ^ eaddr) & mask) == 0)
 338                         return &vcpu->arch.slb[i];
 339         }
 340         return NULL;
 341 }
 342
 343 static unsigned long kvmppc_mmu_get_real_addr(unsigned long v, unsigned long r,
 344                         unsigned long ea)
 345 {
 346         unsigned long ra_mask;
 347
 348         ra_mask = kvmppc_actual_pgsz(v, r) - 1;
 349         return (r & HPTE_R_RPN & ~ra_mask) | (ea & ra_mask);
 350 }
 351
 352 static int kvmppc_mmu_book3s_64_hv_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
 353                         struct kvmppc_pte *gpte, bool data, bool iswrite)
 354 {
 355         struct kvm *kvm = vcpu->kvm;
 356         struct kvmppc_slb *slbe;
 357         unsigned long slb_v;
 358         unsigned long pp, key;
 359         unsigned long v, orig_v, gr;
 360         __be64 *hptep;
 361         long int index;
 362         int virtmode = vcpu->arch.shregs.msr & (data ? MSR_DR : MSR_IR);
 363
 364         if (kvm_is_radix(vcpu->kvm))
 365                 return kvmppc_mmu_radix_xlate(vcpu, eaddr, gpte, data, iswrite);
 366
 367         /* Get SLB entry */
 368         if (virtmode) {
 369                 slbe = kvmppc_mmu_book3s_hv_find_slbe(vcpu, eaddr);
 370                 if (!slbe)
 371                         return -EINVAL;
 372                 slb_v = slbe->origv;
 373         } else {
 374                 /* real mode access */
 375                 slb_v = vcpu->kvm->arch.vrma_slb_v;
 376         }
 377
 378         preempt_disable();
 379         /* Find the HPTE in the hash table */
 380         index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,
 381                                          HPTE_V_VALID | HPTE_V_ABSENT);
 382         if (index < 0) {
 383                 preempt_enable();
 384                 return -ENOENT;
 385         }
 386         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 387         v = orig_v = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 388         if (cpu_has_feature(CPU_FTR_ARCH_300))
 389                 v = hpte_new_to_old_v(v, be64_to_cpu(hptep[1]));
 390         gr = kvm->arch.hpt.rev[index].guest_rpte;
 391
 392         unlock_hpte(hptep, orig_v);
 393         preempt_enable();
 394
 395         gpte->eaddr = eaddr;
 396         gpte->vpage = ((v & HPTE_V_AVPN) << 4) | ((eaddr >> 12) & 0xfff);
 397
 398         /* Get PP bits and key for permission check */
 399         pp = gr & (HPTE_R_PP0 | HPTE_R_PP);
 400         key = (vcpu->arch.shregs.msr & MSR_PR) ? SLB_VSID_KP : SLB_VSID_KS;
 401         key &= slb_v;
 402
 403         /* Calculate permissions */
 404         gpte->may_read = hpte_read_permission(pp, key);
 405         gpte->may_write = hpte_write_permission(pp, key);
 406         gpte->may_execute = gpte->may_read && !(gr & (HPTE_R_N | HPTE_R_G));
 407
 408         /* Storage key permission check for POWER7 */
 409         if (data && virtmode) {
 410                 int amrfield = hpte_get_skey_perm(gr, vcpu->arch.amr);
 411                 if (amrfield & 1)
 412                         gpte->may_read = 0;
 413                 if (amrfield & 2)
 414                         gpte->may_write = 0;
 415         }
 416
 417         /* Get the guest physical address */
 418         gpte->raddr = kvmppc_mmu_get_real_addr(v, gr, eaddr);
 419         return 0;
 420 }
 421
 422 /*
 423  * Quick test for whether an instruction is a load or a store.
 424  * If the instruction is a load or a store, then this will indicate
 425  * which it is, at least on server processors.  (Embedded processors
 426  * have some external PID instructions that don't follow the rule
 427  * embodied here.)  If the instruction isn't a load or store, then
 428  * this doesn't return anything useful.
 429  */
 430 static int instruction_is_store(unsigned int instr)
 431 {
 432         unsigned int mask;
 433
 434         mask = 0x10000000;
 435         if ((instr & 0xfc000000) == 0x7c000000)
 436                 mask = 0x100;           /* major opcode 31 */
 437         return (instr & mask) != 0;
 438 }
 439
 440 int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu,
 441                            unsigned long gpa, gva_t ea, int is_store)
 442 {
 443         u32 last_inst;
 444
 445         /*
 446          * If we fail, we just return to the guest and try executing it again.
 447          */
 448         if (kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst) !=
 449                 EMULATE_DONE)
 450                 return RESUME_GUEST;
 451
 452         /*
 453          * WARNING: We do not know for sure whether the instruction we just
 454          * read from memory is the same that caused the fault in the first
 455          * place.  If the instruction we read is neither an load or a store,
 456          * then it can't access memory, so we don't need to worry about
 457          * enforcing access permissions.  So, assuming it is a load or
 458          * store, we just check that its direction (load or store) is
 459          * consistent with the original fault, since that's what we
 460          * checked the access permissions against.  If there is a mismatch
 461          * we just return and retry the instruction.
 462          */
 463
 464         if (instruction_is_store(last_inst) != !!is_store)
 465                 return RESUME_GUEST;
 466
 467         /*
 468          * Emulated accesses are emulated by looking at the hash for
 469          * translation once, then performing the access later. The
 470          * translation could be invalidated in the meantime in which
 471          * point performing the subsequent memory access on the old
 472          * physical address could possibly be a security hole for the
 473          * guest (but not the host).
 474          *
 475          * This is less of an issue for MMIO stores since they aren't
 476          * globally visible. It could be an issue for MMIO loads to
 477          * a certain extent but we'll ignore it for now.
 478          */
 479
 480         vcpu->arch.paddr_accessed = gpa;
 481         vcpu->arch.vaddr_accessed = ea;
 482         return kvmppc_emulate_mmio(run, vcpu);
 483 }
 484
 485 int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
 486                                 unsigned long ea, unsigned long dsisr)
 487 {
 488         struct kvm *kvm = vcpu->kvm;
 489         unsigned long hpte[3], r;
 490         unsigned long hnow_v, hnow_r;
 491         __be64 *hptep;
 492         unsigned long mmu_seq, psize, pte_size;
 493         unsigned long gpa_base, gfn_base;
 494         unsigned long gpa, gfn, hva, pfn;
 495         struct kvm_memory_slot *memslot;
 496         unsigned long *rmap;
 497         struct revmap_entry *rev;
 498         struct page *page, *pages[1];
 499         long index, ret, npages;
 500         bool is_ci;
 501         unsigned int writing, write_ok;
 502         struct vm_area_struct *vma;
 503         unsigned long rcbits;
 504         long mmio_update;
 505
 506         if (kvm_is_radix(kvm))
 507                 return kvmppc_book3s_radix_page_fault(run, vcpu, ea, dsisr);
 508
 509         /*
 510          * Real-mode code has already searched the HPT and found the
 511          * entry we're interested in.  Lock the entry and check that
 512          * it hasn't changed.  If it has, just return and re-execute the
 513          * instruction.
 514          */
 515         if (ea != vcpu->arch.pgfault_addr)
 516                 return RESUME_GUEST;
 517
 518         if (vcpu->arch.pgfault_cache) {
 519                 mmio_update = atomic64_read(&kvm->arch.mmio_update);
 520                 if (mmio_update == vcpu->arch.pgfault_cache->mmio_update) {
 521                         r = vcpu->arch.pgfault_cache->rpte;
 522                         psize = kvmppc_actual_pgsz(vcpu->arch.pgfault_hpte[0],
 523                                                    r);
 524                         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 525                         gfn_base = gpa_base >> PAGE_SHIFT;
 526                         gpa = gpa_base | (ea & (psize - 1));
 527                         return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 528                                                 dsisr & DSISR_ISSTORE);
 529                 }
 530         }
 531         index = vcpu->arch.pgfault_index;
 532         hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));
 533         rev = &kvm->arch.hpt.rev[index];
 534         preempt_disable();
 535         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 536                 cpu_relax();
 537         hpte[0] = be64_to_cpu(hptep[0]) & ~HPTE_V_HVLOCK;
 538         hpte[1] = be64_to_cpu(hptep[1]);
 539         hpte[2] = r = rev->guest_rpte;
 540         unlock_hpte(hptep, hpte[0]);
 541         preempt_enable();
 542
 543         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 544                 hpte[0] = hpte_new_to_old_v(hpte[0], hpte[1]);
 545                 hpte[1] = hpte_new_to_old_r(hpte[1]);
 546         }
 547         if (hpte[0] != vcpu->arch.pgfault_hpte[0] ||
 548             hpte[1] != vcpu->arch.pgfault_hpte[1])
 549                 return RESUME_GUEST;
 550
 551         /* Translate the logical address and get the page */
 552         psize = kvmppc_actual_pgsz(hpte[0], r);
 553         gpa_base = r & HPTE_R_RPN & ~(psize - 1);
 554         gfn_base = gpa_base >> PAGE_SHIFT;
 555         gpa = gpa_base | (ea & (psize - 1));
 556         gfn = gpa >> PAGE_SHIFT;
 557         memslot = gfn_to_memslot(kvm, gfn);
 558
 559         trace_kvm_page_fault_enter(vcpu, hpte, memslot, ea, dsisr);
 560
 561         /* No memslot means it's an emulated MMIO region */
 562         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
 563                 return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea,
 564                                               dsisr & DSISR_ISSTORE);
 565
 566         /*
 567          * This should never happen, because of the slot_is_aligned()
 568          * check in kvmppc_do_h_enter().
 569          */
 570         if (gfn_base < memslot->base_gfn)
 571                 return -EFAULT;
 572
 573         /* used to check for invalidations in progress */
 574         mmu_seq = kvm->mmu_notifier_seq;
 575         smp_rmb();
 576
 577         ret = -EFAULT;
 578         is_ci = false;
 579         pfn = 0;
 580         page = NULL;
 581         pte_size = PAGE_SIZE;
 582         writing = (dsisr & DSISR_ISSTORE) != 0;
 583         /* If writing != 0, then the HPTE must allow writing, if we get here */
 584         write_ok = writing;
 585         hva = gfn_to_hva_memslot(memslot, gfn);
 586         npages = get_user_pages_fast(hva, 1, writing, pages);
 587         if (npages < 1) {
 588                 /* Check if it's an I/O mapping */
 589                 down_read(&current->mm->mmap_sem);
 590                 vma = find_vma(current->mm, hva);
 591                 if (vma && vma->vm_start <= hva && hva + psize <= vma->vm_end &&
 592                     (vma->vm_flags & VM_PFNMAP)) {
 593                         pfn = vma->vm_pgoff +
 594                                 ((hva - vma->vm_start) >> PAGE_SHIFT);
 595                         pte_size = psize;
 596                         is_ci = pte_ci(__pte((pgprot_val(vma->vm_page_prot))));
 597                         write_ok = vma->vm_flags & VM_WRITE;
 598                 }
 599                 up_read(&current->mm->mmap_sem);
 600                 if (!pfn)
 601                         goto out_put;
 602         } else {
 603                 page = pages[0];
 604                 pfn = page_to_pfn(page);
 605                 if (PageHuge(page)) {
 606                         page = compound_head(page);
 607                         pte_size <<= compound_order(page);
 608                 }
 609                 /* if the guest wants write access, see if that is OK */
 610                 if (!writing && hpte_is_writable(r)) {
 611                         pte_t *ptep, pte;
 612                         unsigned long flags;
 613                         /*
 614                          * We need to protect against page table destruction
 615                          * hugepage split and collapse.
 616                          */
 617                         local_irq_save(flags);
 618                         ptep = find_current_mm_pte(current->mm->pgd,
 619                                                    hva, NULL, NULL);
 620                         if (ptep) {
 621                                 pte = kvmppc_read_update_linux_pte(ptep, 1);
 622                                 if (__pte_write(pte))
 623                                         write_ok = 1;
 624                         }
 625                         local_irq_restore(flags);
 626                 }
 627         }
 628
 629         if (psize > pte_size)
 630                 goto out_put;
 631
 632         /* Check WIMG vs. the actual page we're accessing */
 633         if (!hpte_cache_flags_ok(r, is_ci)) {
 634                 if (is_ci)
 635                         goto out_put;
 636                 /*
 637                  * Allow guest to map emulated device memory as
 638                  * uncacheable, but actually make it cacheable.
 639                  */
 640                 r = (r & ~(HPTE_R_W|HPTE_R_I|HPTE_R_G)) | HPTE_R_M;
 641         }
 642
 643         /*
 644          * Set the HPTE to point to pfn.
 645          * Since the pfn is at PAGE_SIZE granularity, make sure we
 646          * don't mask out lower-order bits if psize < PAGE_SIZE.
 647          */
 648         if (psize < PAGE_SIZE)
 649                 psize = PAGE_SIZE;
 650         r = (r & HPTE_R_KEY_HI) | (r & ~(HPTE_R_PP0 - psize)) |
 651                                         ((pfn << PAGE_SHIFT) & ~(psize - 1));
 652         if (hpte_is_writable(r) && !write_ok)
 653                 r = hpte_make_readonly(r);
 654         ret = RESUME_GUEST;
 655         preempt_disable();
 656         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
 657                 cpu_relax();
 658         hnow_v = be64_to_cpu(hptep[0]);
 659         hnow_r = be64_to_cpu(hptep[1]);
 660         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 661                 hnow_v = hpte_new_to_old_v(hnow_v, hnow_r);
 662                 hnow_r = hpte_new_to_old_r(hnow_r);
 663         }
 664
 665         /*
 666          * If the HPT is being resized, don't update the HPTE,
 667          * instead let the guest retry after the resize operation is complete.
 668          * The synchronization for mmu_ready test vs. set is provided
 669          * by the HPTE lock.
 670          */
 671         if (!kvm->arch.mmu_ready)
 672                 goto out_unlock;
 673
 674         if ((hnow_v & ~HPTE_V_HVLOCK) != hpte[0] || hnow_r != hpte[1] ||
 675             rev->guest_rpte != hpte[2])
 676                 /* HPTE has been changed under us; let the guest retry */
 677                 goto out_unlock;
 678         hpte[0] = (hpte[0] & ~HPTE_V_ABSENT) | HPTE_V_VALID;
 679
 680         /* Always put the HPTE in the rmap chain for the page base address */
 681         rmap = &memslot->arch.rmap[gfn_base - memslot->base_gfn];
 682         lock_rmap(rmap);
 683
 684         /* Check if we might have been invalidated; let the guest retry if so */
 685         ret = RESUME_GUEST;
 686         if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) {
 687                 unlock_rmap(rmap);
 688                 goto out_unlock;
 689         }
 690
 691         /* Only set R/C in real HPTE if set in both *rmap and guest_rpte */
 692         rcbits = *rmap >> KVMPPC_RMAP_RC_SHIFT;
 693         r &= rcbits | ~(HPTE_R_R | HPTE_R_C);
 694
 695         if (be64_to_cpu(hptep[0]) & HPTE_V_VALID) {
 696                 /* HPTE was previously valid, so we need to invalidate it */
 697                 unlock_rmap(rmap);
 698                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 699                 kvmppc_invalidate_hpte(kvm, hptep, index);
 700                 /* don't lose previous R and C bits */
 701                 r |= be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 702         } else {
 703                 kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);
 704         }
 705
 706         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 707                 r = hpte_old_to_new_r(hpte[0], r);
 708                 hpte[0] = hpte_old_to_new_v(hpte[0]);
 709         }
 710         hptep[1] = cpu_to_be64(r);
 711         eieio();
 712         __unlock_hpte(hptep, hpte[0]);
 713         asm volatile("ptesync" : : : "memory");
 714         preempt_enable();
 715         if (page && hpte_is_writable(r))
 716                 SetPageDirty(page);
 717
 718  out_put:
 719         trace_kvm_page_fault_exit(vcpu, hpte, ret);
 720
 721         if (page) {
 722                 /*
 723                  * We drop pages[0] here, not page because page might
 724                  * have been set to the head page of a compound, but
 725                  * we have to drop the reference on the correct tail
 726                  * page to match the get inside gup()
 727                  */
 728                 put_page(pages[0]);
 729         }
 730         return ret;
 731
 732  out_unlock:
 733         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 734         preempt_enable();
 735         goto out_put;
 736 }
 737
 738 void kvmppc_rmap_reset(struct kvm *kvm)
 739 {
 740         struct kvm_memslots *slots;
 741         struct kvm_memory_slot *memslot;
 742         int srcu_idx;
 743
 744         srcu_idx = srcu_read_lock(&kvm->srcu);
 745         slots = kvm_memslots(kvm);
 746         kvm_for_each_memslot(memslot, slots) {
 747                 /* Mutual exclusion with kvm_unmap_hva_range etc. */
 748                 spin_lock(&kvm->mmu_lock);
 749                 /*
 750                  * This assumes it is acceptable to lose reference and
 751                  * change bits across a reset.
 752                  */
 753                 memset(memslot->arch.rmap, 0,
 754                        memslot->npages * sizeof(*memslot->arch.rmap));
 755                 spin_unlock(&kvm->mmu_lock);
 756         }
 757         srcu_read_unlock(&kvm->srcu, srcu_idx);
 758 }
 759
 760 typedef int (*hva_handler_fn)(struct kvm *kvm, struct kvm_memory_slot *memslot,
 761                               unsigned long gfn);
 762
 763 static int kvm_handle_hva_range(struct kvm *kvm,
 764                                 unsigned long start,
 765                                 unsigned long end,
 766                                 hva_handler_fn handler)
 767 {
 768         int ret;
 769         int retval = 0;
 770         struct kvm_memslots *slots;
 771         struct kvm_memory_slot *memslot;
 772
 773         slots = kvm_memslots(kvm);
 774         kvm_for_each_memslot(memslot, slots) {
 775                 unsigned long hva_start, hva_end;
 776                 gfn_t gfn, gfn_end;
 777
 778                 hva_start = max(start, memslot->userspace_addr);
 779                 hva_end = min(end, memslot->userspace_addr +
 780                                         (memslot->npages << PAGE_SHIFT));
 781                 if (hva_start >= hva_end)
 782                         continue;
 783                 /*
 784                  * {gfn(page) | page intersects with [hva_start, hva_end)} =
 785                  * {gfn, gfn+1, ..., gfn_end-1}.
 786                  */
 787                 gfn = hva_to_gfn_memslot(hva_start, memslot);
 788                 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
 789
 790                 for (; gfn < gfn_end; ++gfn) {
 791                         ret = handler(kvm, memslot, gfn);
 792                         retval |= ret;
 793                 }
 794         }
 795
 796         return retval;
 797 }
 798
 799 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 800                           hva_handler_fn handler)
 801 {
 802         return kvm_handle_hva_range(kvm, hva, hva + 1, handler);
 803 }
 804
 805 /* Must be called with both HPTE and rmap locked */
 806 static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i,
 807                               struct kvm_memory_slot *memslot,
 808                               unsigned long *rmapp, unsigned long gfn)
 809 {
 810         __be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 811         struct revmap_entry *rev = kvm->arch.hpt.rev;
 812         unsigned long j, h;
 813         unsigned long ptel, psize, rcbits;
 814
 815         j = rev[i].forw;
 816         if (j == i) {
 817                 /* chain is now empty */
 818                 *rmapp &= ~(KVMPPC_RMAP_PRESENT | KVMPPC_RMAP_INDEX);
 819         } else {
 820                 /* remove i from chain */
 821                 h = rev[i].back;
 822                 rev[h].forw = j;
 823                 rev[j].back = h;
 824                 rev[i].forw = rev[i].back = i;
 825                 *rmapp = (*rmapp & ~KVMPPC_RMAP_INDEX) | j;
 826         }
 827
 828         /* Now check and modify the HPTE */
 829         ptel = rev[i].guest_rpte;
 830         psize = kvmppc_actual_pgsz(be64_to_cpu(hptep[0]), ptel);
 831         if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 832             hpte_rpn(ptel, psize) == gfn) {
 833                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
 834                 kvmppc_invalidate_hpte(kvm, hptep, i);
 835                 hptep[1] &= ~cpu_to_be64(HPTE_R_KEY_HI | HPTE_R_KEY_LO);
 836                 /* Harvest R and C */
 837                 rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C);
 838                 *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT;
 839                 if ((rcbits & HPTE_R_C) && memslot->dirty_bitmap)
 840                         kvmppc_update_dirty_map(memslot, gfn, psize);
 841                 if (rcbits & ~rev[i].guest_rpte) {
 842                         rev[i].guest_rpte = ptel | rcbits;
 843                         note_hpte_modification(kvm, &rev[i]);
 844                 }
 845         }
 846 }
 847
 848 static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 849                            unsigned long gfn)
 850 {
 851         unsigned long i;
 852         __be64 *hptep;
 853         unsigned long *rmapp;
 854
 855         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 856         for (;;) {
 857                 lock_rmap(rmapp);
 858                 if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 859                         unlock_rmap(rmapp);
 860                         break;
 861                 }
 862
 863                 /*
 864                  * To avoid an ABBA deadlock with the HPTE lock bit,
 865                  * we can't spin on the HPTE lock while holding the
 866                  * rmap chain lock.
 867                  */
 868                 i = *rmapp & KVMPPC_RMAP_INDEX;
 869                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 870                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 871                         /* unlock rmap before spinning on the HPTE lock */
 872                         unlock_rmap(rmapp);
 873                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 874                                 cpu_relax();
 875                         continue;
 876                 }
 877
 878                 kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);
 879                 unlock_rmap(rmapp);
 880                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 881         }
 882         return 0;
 883 }
 884
 885 int kvm_unmap_hva_range_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 886 {
 887         hva_handler_fn handler;
 888
 889         handler = kvm_is_radix(kvm) ? kvm_unmap_radix : kvm_unmap_rmapp;
 890         kvm_handle_hva_range(kvm, start, end, handler);
 891         return 0;
 892 }
 893
 894 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,
 895                                   struct kvm_memory_slot *memslot)
 896 {
 897         unsigned long gfn;
 898         unsigned long n;
 899         unsigned long *rmapp;
 900
 901         gfn = memslot->base_gfn;
 902         rmapp = memslot->arch.rmap;
 903         for (n = memslot->npages; n; --n, ++gfn) {
 904                 if (kvm_is_radix(kvm)) {
 905                         kvm_unmap_radix(kvm, memslot, gfn);
 906                         continue;
 907                 }
 908                 /*
 909                  * Testing the present bit without locking is OK because
 910                  * the memslot has been marked invalid already, and hence
 911                  * no new HPTEs referencing this page can be created,
 912                  * thus the present bit can't go from 0 to 1.
 913                  */
 914                 if (*rmapp & KVMPPC_RMAP_PRESENT)
 915                         kvm_unmap_rmapp(kvm, memslot, gfn);
 916                 ++rmapp;
 917         }
 918 }
 919
 920 static int kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 921                          unsigned long gfn)
 922 {
 923         struct revmap_entry *rev = kvm->arch.hpt.rev;
 924         unsigned long head, i, j;
 925         __be64 *hptep;
 926         int ret = 0;
 927         unsigned long *rmapp;
 928
 929         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 930  retry:
 931         lock_rmap(rmapp);
 932         if (*rmapp & KVMPPC_RMAP_REFERENCED) {
 933                 *rmapp &= ~KVMPPC_RMAP_REFERENCED;
 934                 ret = 1;
 935         }
 936         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
 937                 unlock_rmap(rmapp);
 938                 return ret;
 939         }
 940
 941         i = head = *rmapp & KVMPPC_RMAP_INDEX;
 942         do {
 943                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
 944                 j = rev[i].forw;
 945
 946                 /* If this HPTE isn't referenced, ignore it */
 947                 if (!(be64_to_cpu(hptep[1]) & HPTE_R_R))
 948                         continue;
 949
 950                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
 951                         /* unlock rmap before spinning on the HPTE lock */
 952                         unlock_rmap(rmapp);
 953                         while (be64_to_cpu(hptep[0]) & HPTE_V_HVLOCK)
 954                                 cpu_relax();
 955                         goto retry;
 956                 }
 957
 958                 /* Now check and modify the HPTE */
 959                 if ((be64_to_cpu(hptep[0]) & HPTE_V_VALID) &&
 960                     (be64_to_cpu(hptep[1]) & HPTE_R_R)) {
 961                         kvmppc_clear_ref_hpte(kvm, hptep, i);
 962                         if (!(rev[i].guest_rpte & HPTE_R_R)) {
 963                                 rev[i].guest_rpte |= HPTE_R_R;
 964                                 note_hpte_modification(kvm, &rev[i]);
 965                         }
 966                         ret = 1;
 967                 }
 968                 __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
 969         } while ((i = j) != head);
 970
 971         unlock_rmap(rmapp);
 972         return ret;
 973 }
 974
 975 int kvm_age_hva_hv(struct kvm *kvm, unsigned long start, unsigned long end)
 976 {
 977         hva_handler_fn handler;
 978
 979         handler = kvm_is_radix(kvm) ? kvm_age_radix : kvm_age_rmapp;
 980         return kvm_handle_hva_range(kvm, start, end, handler);
 981 }
 982
 983 static int kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,
 984                               unsigned long gfn)
 985 {
 986         struct revmap_entry *rev = kvm->arch.hpt.rev;
 987         unsigned long head, i, j;
 988         unsigned long *hp;
 989         int ret = 1;
 990         unsigned long *rmapp;
 991
 992         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
 993         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 994                 return 1;
 995
 996         lock_rmap(rmapp);
 997         if (*rmapp & KVMPPC_RMAP_REFERENCED)
 998                 goto out;
 999
1000         if (*rmapp & KVMPPC_RMAP_PRESENT) {
1001                 i = head = *rmapp & KVMPPC_RMAP_INDEX;
1002                 do {
1003                         hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4));
1004                         j = rev[i].forw;
1005                         if (be64_to_cpu(hp[1]) & HPTE_R_R)
1006                                 goto out;
1007                 } while ((i = j) != head);
1008         }
1009         ret = 0;
1010
1011  out:
1012         unlock_rmap(rmapp);
1013         return ret;
1014 }
1015
1016 int kvm_test_age_hva_hv(struct kvm *kvm, unsigned long hva)
1017 {
1018         hva_handler_fn handler;
1019
1020         handler = kvm_is_radix(kvm) ? kvm_test_age_radix : kvm_test_age_rmapp;
1021         return kvm_handle_hva(kvm, hva, handler);
1022 }
1023
1024 void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte)
1025 {
1026         hva_handler_fn handler;
1027
1028         handler = kvm_is_radix(kvm) ? kvm_unmap_radix : kvm_unmap_rmapp;
1029         kvm_handle_hva(kvm, hva, handler);
1030 }
1031
1032 static int vcpus_running(struct kvm *kvm)
1033 {
1034         return atomic_read(&kvm->arch.vcpus_running) != 0;
1035 }
1036
1037 /*
1038  * Returns the number of system pages that are dirty.
1039  * This can be more than 1 if we find a huge-page HPTE.
1040  */
1041 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)
1042 {
1043         struct revmap_entry *rev = kvm->arch.hpt.rev;
1044         unsigned long head, i, j;
1045         unsigned long n;
1046         unsigned long v, r;
1047         __be64 *hptep;
1048         int npages_dirty = 0;
1049
1050  retry:
1051         lock_rmap(rmapp);
1052         if (!(*rmapp & KVMPPC_RMAP_PRESENT)) {
1053                 unlock_rmap(rmapp);
1054                 return npages_dirty;
1055         }
1056
1057         i = head = *rmapp & KVMPPC_RMAP_INDEX;
1058         do {
1059                 unsigned long hptep1;
1060                 hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));
1061                 j = rev[i].forw;
1062
1063                 /*
1064                  * Checking the C (changed) bit here is racy since there
1065                  * is no guarantee about when the hardware writes it back.
1066                  * If the HPTE is not writable then it is stable since the
1067                  * page can't be written to, and we would have done a tlbie
1068                  * (which forces the hardware to complete any writeback)
1069                  * when making the HPTE read-only.
1070                  * If vcpus are running then this call is racy anyway
1071                  * since the page could get dirtied subsequently, so we
1072                  * expect there to be a further call which would pick up
1073                  * any delayed C bit writeback.
1074                  * Otherwise we need to do the tlbie even if C==0 in
1075                  * order to pick up any delayed writeback of C.
1076                  */
1077                 hptep1 = be64_to_cpu(hptep[1]);
1078                 if (!(hptep1 & HPTE_R_C) &&
1079                     (!hpte_is_writable(hptep1) || vcpus_running(kvm)))
1080                         continue;
1081
1082                 if (!try_lock_hpte(hptep, HPTE_V_HVLOCK)) {
1083                         /* unlock rmap before spinning on the HPTE lock */
1084                         unlock_rmap(rmapp);
1085                         while (hptep[0] & cpu_to_be64(HPTE_V_HVLOCK))
1086                                 cpu_relax();
1087                         goto retry;
1088                 }
1089
1090                 /* Now check and modify the HPTE */
1091                 if (!(hptep[0] & cpu_to_be64(HPTE_V_VALID))) {
1092                         __unlock_hpte(hptep, be64_to_cpu(hptep[0]));
1093                         continue;
1094                 }
1095
1096                 /* need to make it temporarily absent so C is stable */
1097                 hptep[0] |= cpu_to_be64(HPTE_V_ABSENT);
1098                 kvmppc_invalidate_hpte(kvm, hptep, i);
1099                 v = be64_to_cpu(hptep[0]);
1100                 r = be64_to_cpu(hptep[1]);
1101                 if (r & HPTE_R_C) {
1102                         hptep[1] = cpu_to_be64(r & ~HPTE_R_C);
1103                         if (!(rev[i].guest_rpte & HPTE_R_C)) {
1104                                 rev[i].guest_rpte |= HPTE_R_C;
1105                                 note_hpte_modification(kvm, &rev[i]);
1106                         }
1107                         n = kvmppc_actual_pgsz(v, r);
1108                         n = (n + PAGE_SIZE - 1) >> PAGE_SHIFT;
1109                         if (n > npages_dirty)
1110                                 npages_dirty = n;
1111                         eieio();
1112                 }
1113                 v &= ~HPTE_V_ABSENT;
1114                 v |= HPTE_V_VALID;
1115                 __unlock_hpte(hptep, v);
1116         } while ((i = j) != head);
1117
1118         unlock_rmap(rmapp);
1119         return npages_dirty;
1120 }
1121
1122 void kvmppc_harvest_vpa_dirty(struct kvmppc_vpa *vpa,
1123                               struct kvm_memory_slot *memslot,
1124                               unsigned long *map)
1125 {
1126         unsigned long gfn;
1127
1128         if (!vpa->dirty || !vpa->pinned_addr)
1129                 return;
1130         gfn = vpa->gpa >> PAGE_SHIFT;
1131         if (gfn < memslot->base_gfn ||
1132             gfn >= memslot->base_gfn + memslot->npages)
1133                 return;
1134
1135         vpa->dirty = false;
1136         if (map)
1137                 __set_bit_le(gfn - memslot->base_gfn, map);
1138 }
1139
1140 long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,
1141                         struct kvm_memory_slot *memslot, unsigned long *map)
1142 {
1143         unsigned long i;
1144         unsigned long *rmapp;
1145
1146         preempt_disable();
1147         rmapp = memslot->arch.rmap;
1148         for (i = 0; i < memslot->npages; ++i) {
1149                 int npages = kvm_test_clear_dirty_npages(kvm, rmapp);
1150                 /*
1151                  * Note that if npages > 0 then i must be a multiple of npages,
1152                  * since we always put huge-page HPTEs in the rmap chain
1153                  * corresponding to their page base address.
1154                  */
1155                 if (npages)
1156                         set_dirty_bits(map, i, npages);
1157                 ++rmapp;
1158         }
1159         preempt_enable();
1160         return 0;
1161 }
1162
1163 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,
1164                             unsigned long *nb_ret)
1165 {
1166         struct kvm_memory_slot *memslot;
1167         unsigned long gfn = gpa >> PAGE_SHIFT;
1168         struct page *page, *pages[1];
1169         int npages;
1170         unsigned long hva, offset;
1171         int srcu_idx;
1172
1173         srcu_idx = srcu_read_lock(&kvm->srcu);
1174         memslot = gfn_to_memslot(kvm, gfn);
1175         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID))
1176                 goto err;
1177         hva = gfn_to_hva_memslot(memslot, gfn);
1178         npages = get_user_pages_fast(hva, 1, 1, pages);
1179         if (npages < 1)
1180                 goto err;
1181         page = pages[0];
1182         srcu_read_unlock(&kvm->srcu, srcu_idx);
1183
1184         offset = gpa & (PAGE_SIZE - 1);
1185         if (nb_ret)
1186                 *nb_ret = PAGE_SIZE - offset;
1187         return page_address(page) + offset;
1188
1189  err:
1190         srcu_read_unlock(&kvm->srcu, srcu_idx);
1191         return NULL;
1192 }
1193
1194 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,
1195                              bool dirty)
1196 {
1197         struct page *page = virt_to_page(va);
1198         struct kvm_memory_slot *memslot;
1199         unsigned long gfn;
1200         int srcu_idx;
1201
1202         put_page(page);
1203
1204         if (!dirty)
1205                 return;
1206
1207         /* We need to mark this page dirty in the memslot dirty_bitmap, if any */
1208         gfn = gpa >> PAGE_SHIFT;
1209         srcu_idx = srcu_read_lock(&kvm->srcu);
1210         memslot = gfn_to_memslot(kvm, gfn);
1211         if (memslot && memslot->dirty_bitmap)
1212                 set_bit_le(gfn - memslot->base_gfn, memslot->dirty_bitmap);
1213         srcu_read_unlock(&kvm->srcu, srcu_idx);
1214 }
1215
1216 /*
1217  * HPT resizing
1218  */
1219 static int resize_hpt_allocate(struct kvm_resize_hpt *resize)
1220 {
1221         int rc;
1222
1223         rc = kvmppc_allocate_hpt(&resize->hpt, resize->order);
1224         if (rc < 0)
1225                 return rc;
1226
1227         resize_hpt_debug(resize, "resize_hpt_allocate(): HPT @ 0x%lx\n",
1228                          resize->hpt.virt);
1229
1230         return 0;
1231 }
1232
1233 static unsigned long resize_hpt_rehash_hpte(struct kvm_resize_hpt *resize,
1234                                             unsigned long idx)
1235 {
1236         struct kvm *kvm = resize->kvm;
1237         struct kvm_hpt_info *old = &kvm->arch.hpt;
1238         struct kvm_hpt_info *new = &resize->hpt;
1239         unsigned long old_hash_mask = (1ULL << (old->order - 7)) - 1;
1240         unsigned long new_hash_mask = (1ULL << (new->order - 7)) - 1;
1241         __be64 *hptep, *new_hptep;
1242         unsigned long vpte, rpte, guest_rpte;
1243         int ret;
1244         struct revmap_entry *rev;
1245         unsigned long apsize, avpn, pteg, hash;
1246         unsigned long new_idx, new_pteg, replace_vpte;
1247         int pshift;
1248
1249         hptep = (__be64 *)(old->virt + (idx << 4));
1250
1251         /* Guest is stopped, so new HPTEs can't be added or faulted
1252          * in, only unmapped or altered by host actions.  So, it's
1253          * safe to check this before we take the HPTE lock */
1254         vpte = be64_to_cpu(hptep[0]);
1255         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1256                 return 0; /* nothing to do */
1257
1258         while (!try_lock_hpte(hptep, HPTE_V_HVLOCK))
1259                 cpu_relax();
1260
1261         vpte = be64_to_cpu(hptep[0]);
1262
1263         ret = 0;
1264         if (!(vpte & HPTE_V_VALID) && !(vpte & HPTE_V_ABSENT))
1265                 /* Nothing to do */
1266                 goto out;
1267
1268         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1269                 rpte = be64_to_cpu(hptep[1]);
1270                 vpte = hpte_new_to_old_v(vpte, rpte);
1271         }
1272
1273         /* Unmap */
1274         rev = &old->rev[idx];
1275         guest_rpte = rev->guest_rpte;
1276
1277         ret = -EIO;
1278         apsize = kvmppc_actual_pgsz(vpte, guest_rpte);
1279         if (!apsize)
1280                 goto out;
1281
1282         if (vpte & HPTE_V_VALID) {
1283                 unsigned long gfn = hpte_rpn(guest_rpte, apsize);
1284                 int srcu_idx = srcu_read_lock(&kvm->srcu);
1285                 struct kvm_memory_slot *memslot =
1286                         __gfn_to_memslot(kvm_memslots(kvm), gfn);
1287
1288                 if (memslot) {
1289                         unsigned long *rmapp;
1290                         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1291
1292                         lock_rmap(rmapp);
1293                         kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn);
1294                         unlock_rmap(rmapp);
1295                 }
1296
1297                 srcu_read_unlock(&kvm->srcu, srcu_idx);
1298         }
1299
1300         /* Reload PTE after unmap */
1301         vpte = be64_to_cpu(hptep[0]);
1302         BUG_ON(vpte & HPTE_V_VALID);
1303         BUG_ON(!(vpte & HPTE_V_ABSENT));
1304
1305         ret = 0;
1306         if (!(vpte & HPTE_V_BOLTED))
1307                 goto out;
1308
1309         rpte = be64_to_cpu(hptep[1]);
1310
1311         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1312                 vpte = hpte_new_to_old_v(vpte, rpte);
1313                 rpte = hpte_new_to_old_r(rpte);
1314         }
1315
1316         pshift = kvmppc_hpte_base_page_shift(vpte, rpte);
1317         avpn = HPTE_V_AVPN_VAL(vpte) & ~(((1ul << pshift) - 1) >> 23);
1318         pteg = idx / HPTES_PER_GROUP;
1319         if (vpte & HPTE_V_SECONDARY)
1320                 pteg = ~pteg;
1321
1322         if (!(vpte & HPTE_V_1TB_SEG)) {
1323                 unsigned long offset, vsid;
1324
1325                 /* We only have 28 - 23 bits of offset in avpn */
1326                 offset = (avpn & 0x1f) << 23;
1327                 vsid = avpn >> 5;
1328                 /* We can find more bits from the pteg value */
1329                 if (pshift < 23)
1330                         offset |= ((vsid ^ pteg) & old_hash_mask) << pshift;
1331
1332                 hash = vsid ^ (offset >> pshift);
1333         } else {
1334                 unsigned long offset, vsid;
1335
1336                 /* We only have 40 - 23 bits of seg_off in avpn */
1337                 offset = (avpn & 0x1ffff) << 23;
1338                 vsid = avpn >> 17;
1339                 if (pshift < 23)
1340                         offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask) << pshift;
1341
1342                 hash = vsid ^ (vsid << 25) ^ (offset >> pshift);
1343         }
1344
1345         new_pteg = hash & new_hash_mask;
1346         if (vpte & HPTE_V_SECONDARY)
1347                 new_pteg = ~hash & new_hash_mask;
1348
1349         new_idx = new_pteg * HPTES_PER_GROUP + (idx % HPTES_PER_GROUP);
1350         new_hptep = (__be64 *)(new->virt + (new_idx << 4));
1351
1352         replace_vpte = be64_to_cpu(new_hptep[0]);
1353         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1354                 unsigned long replace_rpte = be64_to_cpu(new_hptep[1]);
1355                 replace_vpte = hpte_new_to_old_v(replace_vpte, replace_rpte);
1356         }
1357
1358         if (replace_vpte & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1359                 BUG_ON(new->order >= old->order);
1360
1361                 if (replace_vpte & HPTE_V_BOLTED) {
1362                         if (vpte & HPTE_V_BOLTED)
1363                                 /* Bolted collision, nothing we can do */
1364                                 ret = -ENOSPC;
1365                         /* Discard the new HPTE */
1366                         goto out;
1367                 }
1368
1369                 /* Discard the previous HPTE */
1370         }
1371
1372         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1373                 rpte = hpte_old_to_new_r(vpte, rpte);
1374                 vpte = hpte_old_to_new_v(vpte);
1375         }
1376
1377         new_hptep[1] = cpu_to_be64(rpte);
1378         new->rev[new_idx].guest_rpte = guest_rpte;
1379         /* No need for a barrier, since new HPT isn't active */
1380         new_hptep[0] = cpu_to_be64(vpte);
1381         unlock_hpte(new_hptep, vpte);
1382
1383 out:
1384         unlock_hpte(hptep, vpte);
1385         return ret;
1386 }
1387
1388 static int resize_hpt_rehash(struct kvm_resize_hpt *resize)
1389 {
1390         struct kvm *kvm = resize->kvm;
1391         unsigned  long i;
1392         int rc;
1393
1394         for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {
1395                 rc = resize_hpt_rehash_hpte(resize, i);
1396                 if (rc != 0)
1397                         return rc;
1398         }
1399
1400         return 0;
1401 }
1402
1403 static void resize_hpt_pivot(struct kvm_resize_hpt *resize)
1404 {
1405         struct kvm *kvm = resize->kvm;
1406         struct kvm_hpt_info hpt_tmp;
1407
1408         /* Exchange the pending tables in the resize structure with
1409          * the active tables */
1410
1411         resize_hpt_debug(resize, "resize_hpt_pivot()\n");
1412
1413         spin_lock(&kvm->mmu_lock);
1414         asm volatile("ptesync" : : : "memory");
1415
1416         hpt_tmp = kvm->arch.hpt;
1417         kvmppc_set_hpt(kvm, &resize->hpt);
1418         resize->hpt = hpt_tmp;
1419
1420         spin_unlock(&kvm->mmu_lock);
1421
1422         synchronize_srcu_expedited(&kvm->srcu);
1423
1424         if (cpu_has_feature(CPU_FTR_ARCH_300))
1425                 kvmppc_setup_partition_table(kvm);
1426
1427         resize_hpt_debug(resize, "resize_hpt_pivot() done\n");
1428 }
1429
1430 static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)
1431 {
1432         if (WARN_ON(!mutex_is_locked(&kvm->lock)))
1433                 return;
1434
1435         if (!resize)
1436                 return;
1437
1438         if (resize->error != -EBUSY) {
1439                 if (resize->hpt.virt)
1440                         kvmppc_free_hpt(&resize->hpt);
1441                 kfree(resize);
1442         }
1443
1444         if (kvm->arch.resize_hpt == resize)
1445                 kvm->arch.resize_hpt = NULL;
1446 }
1447
1448 static void resize_hpt_prepare_work(struct work_struct *work)
1449 {
1450         struct kvm_resize_hpt *resize = container_of(work,
1451                                                      struct kvm_resize_hpt,
1452                                                      work);
1453         struct kvm *kvm = resize->kvm;
1454         int err = 0;
1455
1456         if (WARN_ON(resize->error != -EBUSY))
1457                 return;
1458
1459         mutex_lock(&kvm->lock);
1460
1461         /* Request is still current? */
1462         if (kvm->arch.resize_hpt == resize) {
1463                 /* We may request large allocations here:
1464                  * do not sleep with kvm->lock held for a while.
1465                  */
1466                 mutex_unlock(&kvm->lock);
1467
1468                 resize_hpt_debug(resize, "resize_hpt_prepare_work(): order = %d\n",
1469                                  resize->order);
1470
1471                 err = resize_hpt_allocate(resize);
1472
1473                 /* We have strict assumption about -EBUSY
1474                  * when preparing for HPT resize.
1475                  */
1476                 if (WARN_ON(err == -EBUSY))
1477                         err = -EINPROGRESS;
1478
1479                 mutex_lock(&kvm->lock);
1480                 /* It is possible that kvm->arch.resize_hpt != resize
1481                  * after we grab kvm->lock again.
1482                  */
1483         }
1484
1485         resize->error = err;
1486
1487         if (kvm->arch.resize_hpt != resize)
1488                 resize_hpt_release(kvm, resize);
1489
1490         mutex_unlock(&kvm->lock);
1491 }
1492
1493 long kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,
1494                                      struct kvm_ppc_resize_hpt *rhpt)
1495 {
1496         unsigned long flags = rhpt->flags;
1497         unsigned long shift = rhpt->shift;
1498         struct kvm_resize_hpt *resize;
1499         int ret;
1500
1501         if (flags != 0 || kvm_is_radix(kvm))
1502                 return -EINVAL;
1503
1504         if (shift && ((shift < 18) || (shift > 46)))
1505                 return -EINVAL;
1506
1507         mutex_lock(&kvm->lock);
1508
1509         resize = kvm->arch.resize_hpt;
1510
1511         if (resize) {
1512                 if (resize->order == shift) {
1513                         /* Suitable resize in progress? */
1514                         ret = resize->error;
1515                         if (ret == -EBUSY)
1516                                 ret = 100; /* estimated time in ms */
1517                         else if (ret)
1518                                 resize_hpt_release(kvm, resize);
1519
1520                         goto out;
1521                 }
1522
1523                 /* not suitable, cancel it */
1524                 resize_hpt_release(kvm, resize);
1525         }
1526
1527         ret = 0;
1528         if (!shift)
1529                 goto out; /* nothing to do */
1530
1531         /* start new resize */
1532
1533         resize = kzalloc(sizeof(*resize), GFP_KERNEL);
1534         if (!resize) {
1535                 ret = -ENOMEM;
1536                 goto out;
1537         }
1538
1539         resize->error = -EBUSY;
1540         resize->order = shift;
1541         resize->kvm = kvm;
1542         INIT_WORK(&resize->work, resize_hpt_prepare_work);
1543         kvm->arch.resize_hpt = resize;
1544
1545         schedule_work(&resize->work);
1546
1547         ret = 100; /* estimated time in ms */
1548
1549 out:
1550         mutex_unlock(&kvm->lock);
1551         return ret;
1552 }
1553
1554 static void resize_hpt_boot_vcpu(void *opaque)
1555 {
1556         /* Nothing to do, just force a KVM exit */
1557 }
1558
1559 long kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,
1560                                     struct kvm_ppc_resize_hpt *rhpt)
1561 {
1562         unsigned long flags = rhpt->flags;
1563         unsigned long shift = rhpt->shift;
1564         struct kvm_resize_hpt *resize;
1565         long ret;
1566
1567         if (flags != 0 || kvm_is_radix(kvm))
1568                 return -EINVAL;
1569
1570         if (shift && ((shift < 18) || (shift > 46)))
1571                 return -EINVAL;
1572
1573         mutex_lock(&kvm->lock);
1574
1575         resize = kvm->arch.resize_hpt;
1576
1577         /* This shouldn't be possible */
1578         ret = -EIO;
1579         if (WARN_ON(!kvm->arch.mmu_ready))
1580                 goto out_no_hpt;
1581
1582         /* Stop VCPUs from running while we mess with the HPT */
1583         kvm->arch.mmu_ready = 0;
1584         smp_mb();
1585
1586         /* Boot all CPUs out of the guest so they re-read
1587          * mmu_ready */
1588         on_each_cpu(resize_hpt_boot_vcpu, NULL, 1);
1589
1590         ret = -ENXIO;
1591         if (!resize || (resize->order != shift))
1592                 goto out;
1593
1594         ret = resize->error;
1595         if (ret)
1596                 goto out;
1597
1598         ret = resize_hpt_rehash(resize);
1599         if (ret)
1600                 goto out;
1601
1602         resize_hpt_pivot(resize);
1603
1604 out:
1605         /* Let VCPUs run again */
1606         kvm->arch.mmu_ready = 1;
1607         smp_mb();
1608 out_no_hpt:
1609         resize_hpt_release(kvm, resize);
1610         mutex_unlock(&kvm->lock);
1611         return ret;
1612 }
1613
1614 /*
1615  * Functions for reading and writing the hash table via reads and
1616  * writes on a file descriptor.
1617  *
1618  * Reads return the guest view of the hash table, which has to be
1619  * pieced together from the real hash table and the guest_rpte
1620  * values in the revmap array.
1621  *
1622  * On writes, each HPTE written is considered in turn, and if it
1623  * is valid, it is written to the HPT as if an H_ENTER with the
1624  * exact flag set was done.  When the invalid count is non-zero
1625  * in the header written to the stream, the kernel will make
1626  * sure that that many HPTEs are invalid, and invalidate them
1627  * if not.
1628  */
1629
1630 struct kvm_htab_ctx {
1631         unsigned long   index;
1632         unsigned long   flags;
1633         struct kvm      *kvm;
1634         int             first_pass;
1635 };
1636
1637 #define HPTE_SIZE       (2 * sizeof(unsigned long))
1638
1639 /*
1640  * Returns 1 if this HPT entry has been modified or has pending
1641  * R/C bit changes.
1642  */
1643 static int hpte_dirty(struct revmap_entry *revp, __be64 *hptp)
1644 {
1645         unsigned long rcbits_unset;
1646
1647         if (revp->guest_rpte & HPTE_GR_MODIFIED)
1648                 return 1;
1649
1650         /* Also need to consider changes in reference and changed bits */
1651         rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1652         if ((be64_to_cpu(hptp[0]) & HPTE_V_VALID) &&
1653             (be64_to_cpu(hptp[1]) & rcbits_unset))
1654                 return 1;
1655
1656         return 0;
1657 }
1658
1659 static long record_hpte(unsigned long flags, __be64 *hptp,
1660                         unsigned long *hpte, struct revmap_entry *revp,
1661                         int want_valid, int first_pass)
1662 {
1663         unsigned long v, r, hr;
1664         unsigned long rcbits_unset;
1665         int ok = 1;
1666         int valid, dirty;
1667
1668         /* Unmodified entries are uninteresting except on the first pass */
1669         dirty = hpte_dirty(revp, hptp);
1670         if (!first_pass && !dirty)
1671                 return 0;
1672
1673         valid = 0;
1674         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)) {
1675                 valid = 1;
1676                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) &&
1677                     !(be64_to_cpu(hptp[0]) & HPTE_V_BOLTED))
1678                         valid = 0;
1679         }
1680         if (valid != want_valid)
1681                 return 0;
1682
1683         v = r = 0;
1684         if (valid || dirty) {
1685                 /* lock the HPTE so it's stable and read it */
1686                 preempt_disable();
1687                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
1688                         cpu_relax();
1689                 v = be64_to_cpu(hptp[0]);
1690                 hr = be64_to_cpu(hptp[1]);
1691                 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
1692                         v = hpte_new_to_old_v(v, hr);
1693                         hr = hpte_new_to_old_r(hr);
1694                 }
1695
1696                 /* re-evaluate valid and dirty from synchronized HPTE value */
1697                 valid = !!(v & HPTE_V_VALID);
1698                 dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED);
1699
1700                 /* Harvest R and C into guest view if necessary */
1701                 rcbits_unset = ~revp->guest_rpte & (HPTE_R_R | HPTE_R_C);
1702                 if (valid && (rcbits_unset & hr)) {
1703                         revp->guest_rpte |= (hr &
1704                                 (HPTE_R_R | HPTE_R_C)) | HPTE_GR_MODIFIED;
1705                         dirty = 1;
1706                 }
1707
1708                 if (v & HPTE_V_ABSENT) {
1709                         v &= ~HPTE_V_ABSENT;
1710                         v |= HPTE_V_VALID;
1711                         valid = 1;
1712                 }
1713                 if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED))
1714                         valid = 0;
1715
1716                 r = revp->guest_rpte;
1717                 /* only clear modified if this is the right sort of entry */
1718                 if (valid == want_valid && dirty) {
1719                         r &= ~HPTE_GR_MODIFIED;
1720                         revp->guest_rpte = r;
1721                 }
1722                 unlock_hpte(hptp, be64_to_cpu(hptp[0]));
1723                 preempt_enable();
1724                 if (!(valid == want_valid && (first_pass || dirty)))
1725                         ok = 0;
1726         }
1727         hpte[0] = cpu_to_be64(v);
1728         hpte[1] = cpu_to_be64(r);
1729         return ok;
1730 }
1731
1732 static ssize_t kvm_htab_read(struct file *file, char __user *buf,
1733                              size_t count, loff_t *ppos)
1734 {
1735         struct kvm_htab_ctx *ctx = file->private_data;
1736         struct kvm *kvm = ctx->kvm;
1737         struct kvm_get_htab_header hdr;
1738         __be64 *hptp;
1739         struct revmap_entry *revp;
1740         unsigned long i, nb, nw;
1741         unsigned long __user *lbuf;
1742         struct kvm_get_htab_header __user *hptr;
1743         unsigned long flags;
1744         int first_pass;
1745         unsigned long hpte[2];
1746
1747         if (!access_ok(VERIFY_WRITE, buf, count))
1748                 return -EFAULT;
1749         if (kvm_is_radix(kvm))
1750                 return 0;
1751
1752         first_pass = ctx->first_pass;
1753         flags = ctx->flags;
1754
1755         i = ctx->index;
1756         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1757         revp = kvm->arch.hpt.rev + i;
1758         lbuf = (unsigned long __user *)buf;
1759
1760         nb = 0;
1761         while (nb + sizeof(hdr) + HPTE_SIZE < count) {
1762                 /* Initialize header */
1763                 hptr = (struct kvm_get_htab_header __user *)buf;
1764                 hdr.n_valid = 0;
1765                 hdr.n_invalid = 0;
1766                 nw = nb;
1767                 nb += sizeof(hdr);
1768                 lbuf = (unsigned long __user *)(buf + sizeof(hdr));
1769
1770                 /* Skip uninteresting entries, i.e. clean on not-first pass */
1771                 if (!first_pass) {
1772                         while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1773                                !hpte_dirty(revp, hptp)) {
1774                                 ++i;
1775                                 hptp += 2;
1776                                 ++revp;
1777                         }
1778                 }
1779                 hdr.index = i;
1780
1781                 /* Grab a series of valid entries */
1782                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1783                        hdr.n_valid < 0xffff &&
1784                        nb + HPTE_SIZE < count &&
1785                        record_hpte(flags, hptp, hpte, revp, 1, first_pass)) {
1786                         /* valid entry, write it out */
1787                         ++hdr.n_valid;
1788                         if (__put_user(hpte[0], lbuf) ||
1789                             __put_user(hpte[1], lbuf + 1))
1790                                 return -EFAULT;
1791                         nb += HPTE_SIZE;
1792                         lbuf += 2;
1793                         ++i;
1794                         hptp += 2;
1795                         ++revp;
1796                 }
1797                 /* Now skip invalid entries while we can */
1798                 while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&
1799                        hdr.n_invalid < 0xffff &&
1800                        record_hpte(flags, hptp, hpte, revp, 0, first_pass)) {
1801                         /* found an invalid entry */
1802                         ++hdr.n_invalid;
1803                         ++i;
1804                         hptp += 2;
1805                         ++revp;
1806                 }
1807
1808                 if (hdr.n_valid || hdr.n_invalid) {
1809                         /* write back the header */
1810                         if (__copy_to_user(hptr, &hdr, sizeof(hdr)))
1811                                 return -EFAULT;
1812                         nw = nb;
1813                         buf = (char __user *)lbuf;
1814                 } else {
1815                         nb = nw;
1816                 }
1817
1818                 /* Check if we've wrapped around the hash table */
1819                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) {
1820                         i = 0;
1821                         ctx->first_pass = 0;
1822                         break;
1823                 }
1824         }
1825
1826         ctx->index = i;
1827
1828         return nb;
1829 }
1830
1831 static ssize_t kvm_htab_write(struct file *file, const char __user *buf,
1832                               size_t count, loff_t *ppos)
1833 {
1834         struct kvm_htab_ctx *ctx = file->private_data;
1835         struct kvm *kvm = ctx->kvm;
1836         struct kvm_get_htab_header hdr;
1837         unsigned long i, j;
1838         unsigned long v, r;
1839         unsigned long __user *lbuf;
1840         __be64 *hptp;
1841         unsigned long tmp[2];
1842         ssize_t nb;
1843         long int err, ret;
1844         int mmu_ready;
1845         int pshift;
1846
1847         if (!access_ok(VERIFY_READ, buf, count))
1848                 return -EFAULT;
1849         if (kvm_is_radix(kvm))
1850                 return -EINVAL;
1851
1852         /* lock out vcpus from running while we're doing this */
1853         mutex_lock(&kvm->lock);
1854         mmu_ready = kvm->arch.mmu_ready;
1855         if (mmu_ready) {
1856                 kvm->arch.mmu_ready = 0;        /* temporarily */
1857                 /* order mmu_ready vs. vcpus_running */
1858                 smp_mb();
1859                 if (atomic_read(&kvm->arch.vcpus_running)) {
1860                         kvm->arch.mmu_ready = 1;
1861                         mutex_unlock(&kvm->lock);
1862                         return -EBUSY;
1863                 }
1864         }
1865
1866         err = 0;
1867         for (nb = 0; nb + sizeof(hdr) <= count; ) {
1868                 err = -EFAULT;
1869                 if (__copy_from_user(&hdr, buf, sizeof(hdr)))
1870                         break;
1871
1872                 err = 0;
1873                 if (nb + hdr.n_valid * HPTE_SIZE > count)
1874                         break;
1875
1876                 nb += sizeof(hdr);
1877                 buf += sizeof(hdr);
1878
1879                 err = -EINVAL;
1880                 i = hdr.index;
1881                 if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) ||
1882                     i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt))
1883                         break;
1884
1885                 hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
1886                 lbuf = (unsigned long __user *)buf;
1887                 for (j = 0; j < hdr.n_valid; ++j) {
1888                         __be64 hpte_v;
1889                         __be64 hpte_r;
1890
1891                         err = -EFAULT;
1892                         if (__get_user(hpte_v, lbuf) ||
1893                             __get_user(hpte_r, lbuf + 1))
1894                                 goto out;
1895                         v = be64_to_cpu(hpte_v);
1896                         r = be64_to_cpu(hpte_r);
1897                         err = -EINVAL;
1898                         if (!(v & HPTE_V_VALID))
1899                                 goto out;
1900                         pshift = kvmppc_hpte_base_page_shift(v, r);
1901                         if (pshift <= 0)
1902                                 goto out;
1903                         lbuf += 2;
1904                         nb += HPTE_SIZE;
1905
1906                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1907                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1908                         err = -EIO;
1909                         ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,
1910                                                          tmp);
1911                         if (ret != H_SUCCESS) {
1912                                 pr_err("kvm_htab_write ret %ld i=%ld v=%lx "
1913                                        "r=%lx\n", ret, i, v, r);
1914                                 goto out;
1915                         }
1916                         if (!mmu_ready && is_vrma_hpte(v)) {
1917                                 unsigned long senc, lpcr;
1918
1919                                 senc = slb_pgsize_encoding(1ul << pshift);
1920                                 kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |
1921                                         (VRMA_VSID << SLB_VSID_SHIFT_1T);
1922                                 if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
1923                                         lpcr = senc << (LPCR_VRMASD_SH - 4);
1924                                         kvmppc_update_lpcr(kvm, lpcr,
1925                                                            LPCR_VRMASD);
1926                                 } else {
1927                                         kvmppc_setup_partition_table(kvm);
1928                                 }
1929                                 mmu_ready = 1;
1930                         }
1931                         ++i;
1932                         hptp += 2;
1933                 }
1934
1935                 for (j = 0; j < hdr.n_invalid; ++j) {
1936                         if (be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT))
1937                                 kvmppc_do_h_remove(kvm, 0, i, 0, tmp);
1938                         ++i;
1939                         hptp += 2;
1940                 }
1941                 err = 0;
1942         }
1943
1944  out:
1945         /* Order HPTE updates vs. mmu_ready */
1946         smp_wmb();
1947         kvm->arch.mmu_ready = mmu_ready;
1948         mutex_unlock(&kvm->lock);
1949
1950         if (err)
1951                 return err;
1952         return nb;
1953 }
1954
1955 static int kvm_htab_release(struct inode *inode, struct file *filp)
1956 {
1957         struct kvm_htab_ctx *ctx = filp->private_data;
1958
1959         filp->private_data = NULL;
1960         if (!(ctx->flags & KVM_GET_HTAB_WRITE))
1961                 atomic_dec(&ctx->kvm->arch.hpte_mod_interest);
1962         kvm_put_kvm(ctx->kvm);
1963         kfree(ctx);
1964         return 0;
1965 }
1966
1967 static const struct file_operations kvm_htab_fops = {
1968         .read           = kvm_htab_read,
1969         .write          = kvm_htab_write,
1970         .llseek         = default_llseek,
1971         .release        = kvm_htab_release,
1972 };
1973
1974 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)
1975 {
1976         int ret;
1977         struct kvm_htab_ctx *ctx;
1978         int rwflag;
1979
1980         /* reject flags we don't recognize */
1981         if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE))
1982                 return -EINVAL;
1983         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1984         if (!ctx)
1985                 return -ENOMEM;
1986         kvm_get_kvm(kvm);
1987         ctx->kvm = kvm;
1988         ctx->index = ghf->start_index;
1989         ctx->flags = ghf->flags;
1990         ctx->first_pass = 1;
1991
1992         rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY;
1993         ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag | O_CLOEXEC);
1994         if (ret < 0) {
1995                 kfree(ctx);
1996                 kvm_put_kvm(kvm);
1997                 return ret;
1998         }
1999
2000         if (rwflag == O_RDONLY) {
2001                 mutex_lock(&kvm->slots_lock);
2002                 atomic_inc(&kvm->arch.hpte_mod_interest);
2003                 /* make sure kvmppc_do_h_enter etc. see the increment */
2004                 synchronize_srcu_expedited(&kvm->srcu);
2005                 mutex_unlock(&kvm->slots_lock);
2006         }
2007
2008         return ret;
2009 }
2010
2011 struct debugfs_htab_state {
2012         struct kvm      *kvm;
2013         struct mutex    mutex;
2014         unsigned long   hpt_index;
2015         int             chars_left;
2016         int             buf_index;
2017         char            buf[64];
2018 };
2019
2020 static int debugfs_htab_open(struct inode *inode, struct file *file)
2021 {
2022         struct kvm *kvm = inode->i_private;
2023         struct debugfs_htab_state *p;
2024
2025         p = kzalloc(sizeof(*p), GFP_KERNEL);
2026         if (!p)
2027                 return -ENOMEM;
2028
2029         kvm_get_kvm(kvm);
2030         p->kvm = kvm;
2031         mutex_init(&p->mutex);
2032         file->private_data = p;
2033
2034         return nonseekable_open(inode, file);
2035 }
2036
2037 static int debugfs_htab_release(struct inode *inode, struct file *file)
2038 {
2039         struct debugfs_htab_state *p = file->private_data;
2040
2041         kvm_put_kvm(p->kvm);
2042         kfree(p);
2043         return 0;
2044 }
2045
2046 static ssize_t debugfs_htab_read(struct file *file, char __user *buf,
2047                                  size_t len, loff_t *ppos)
2048 {
2049         struct debugfs_htab_state *p = file->private_data;
2050         ssize_t ret, r;
2051         unsigned long i, n;
2052         unsigned long v, hr, gr;
2053         struct kvm *kvm;
2054         __be64 *hptp;
2055
2056         kvm = p->kvm;
2057         if (kvm_is_radix(kvm))
2058                 return 0;
2059
2060         ret = mutex_lock_interruptible(&p->mutex);
2061         if (ret)
2062                 return ret;
2063
2064         if (p->chars_left) {
2065                 n = p->chars_left;
2066                 if (n > len)
2067                         n = len;
2068                 r = copy_to_user(buf, p->buf + p->buf_index, n);
2069                 n -= r;
2070                 p->chars_left -= n;
2071                 p->buf_index += n;
2072                 buf += n;
2073                 len -= n;
2074                 ret = n;
2075                 if (r) {
2076                         if (!n)
2077                                 ret = -EFAULT;
2078                         goto out;
2079                 }
2080         }
2081
2082         i = p->hpt_index;
2083         hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));
2084         for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);
2085              ++i, hptp += 2) {
2086                 if (!(be64_to_cpu(hptp[0]) & (HPTE_V_VALID | HPTE_V_ABSENT)))
2087                         continue;
2088
2089                 /* lock the HPTE so it's stable and read it */
2090                 preempt_disable();
2091                 while (!try_lock_hpte(hptp, HPTE_V_HVLOCK))
2092                         cpu_relax();
2093                 v = be64_to_cpu(hptp[0]) & ~HPTE_V_HVLOCK;
2094                 hr = be64_to_cpu(hptp[1]);
2095                 gr = kvm->arch.hpt.rev[i].guest_rpte;
2096                 unlock_hpte(hptp, v);
2097                 preempt_enable();
2098
2099                 if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT)))
2100                         continue;
2101
2102                 n = scnprintf(p->buf, sizeof(p->buf),
2103                               "%6lx %.16lx %.16lx %.16lx\n",
2104                               i, v, hr, gr);
2105                 p->chars_left = n;
2106                 if (n > len)
2107                         n = len;
2108                 r = copy_to_user(buf, p->buf, n);
2109                 n -= r;
2110                 p->chars_left -= n;
2111                 p->buf_index = n;
2112                 buf += n;
2113                 len -= n;
2114                 ret += n;
2115                 if (r) {
2116                         if (!ret)
2117                                 ret = -EFAULT;
2118                         goto out;
2119                 }
2120         }
2121         p->hpt_index = i;
2122
2123  out:
2124         mutex_unlock(&p->mutex);
2125         return ret;
2126 }
2127
2128 static ssize_t debugfs_htab_write(struct file *file, const char __user *buf,
2129                            size_t len, loff_t *ppos)
2130 {
2131         return -EACCES;
2132 }
2133
2134 static const struct file_operations debugfs_htab_fops = {
2135         .owner   = THIS_MODULE,
2136         .open    = debugfs_htab_open,
2137         .release = debugfs_htab_release,
2138         .read    = debugfs_htab_read,
2139         .write   = debugfs_htab_write,
2140         .llseek  = generic_file_llseek,
2141 };
2142
2143 void kvmppc_mmu_debugfs_init(struct kvm *kvm)
2144 {
2145         kvm->arch.htab_dentry = debugfs_create_file("htab", 0400,
2146                                                     kvm->arch.debugfs_dir, kvm,
2147                                                     &debugfs_htab_fops);
2148 }
2149
2150 void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu)
2151 {
2152         struct kvmppc_mmu *mmu = &vcpu->arch.mmu;
2153
2154         vcpu->arch.slb_nr = 32;         /* POWER7/POWER8 */
2155
2156         mmu->xlate = kvmppc_mmu_book3s_64_hv_xlate;
2157         mmu->reset_msr = kvmppc_mmu_book3s_64_hv_reset_msr;
2158
2159         vcpu->arch.hflags |= BOOK3S_HFLAG_SLB;
2160 }