arch/powerpc/kvm/powerpc.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright IBM Corp. 2007
  16  *
  17  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  18  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
  19  */
  20
  21 #include <linux/errno.h>
  22 #include <linux/err.h>
  23 #include <linux/kvm_host.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/hrtimer.h>
  26 #include <linux/sched/signal.h>
  27 #include <linux/fs.h>
  28 #include <linux/slab.h>
  29 #include <linux/file.h>
  30 #include <linux/module.h>
  31 #include <linux/irqbypass.h>
  32 #include <linux/kvm_irqfd.h>
  33 #include <asm/cputable.h>
  34 #include <linux/uaccess.h>
  35 #include <asm/kvm_ppc.h>
  36 #include <asm/cputhreads.h>
  37 #include <asm/irqflags.h>
  38 #include <asm/iommu.h>
  39 #include <asm/switch_to.h>
  40 #include <asm/xive.h>
  41 #ifdef CONFIG_PPC_PSERIES
  42 #include <asm/hvcall.h>
  43 #include <asm/plpar_wrappers.h>
  44 #endif
  45
  46 #include "timing.h"
  47 #include "irq.h"
  48 #include "../mm/mmu_decl.h"
  49
  50 #define CREATE_TRACE_POINTS
  51 #include "trace.h"
  52
  53 struct kvmppc_ops *kvmppc_hv_ops;
  54 EXPORT_SYMBOL_GPL(kvmppc_hv_ops);
  55 struct kvmppc_ops *kvmppc_pr_ops;
  56 EXPORT_SYMBOL_GPL(kvmppc_pr_ops);
  57
  58
  59 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
  60 {
  61         return !!(v->arch.pending_exceptions) || kvm_request_pending(v);
  62 }
  63
  64 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
  65 {
  66         return kvm_arch_vcpu_runnable(vcpu);
  67 }
  68
  69 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
  70 {
  71         return false;
  72 }
  73
  74 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  75 {
  76         return 1;
  77 }
  78
  79 /*
  80  * Common checks before entering the guest world.  Call with interrupts
  81  * disabled.
  82  *
  83  * returns:
  84  *
  85  * == 1 if we're ready to go into guest state
  86  * <= 0 if we need to go back to the host with return value
  87  */
  88 int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
  89 {
  90         int r;
  91
  92         WARN_ON(irqs_disabled());
  93         hard_irq_disable();
  94
  95         while (true) {
  96                 if (need_resched()) {
  97                         local_irq_enable();
  98                         cond_resched();
  99                         hard_irq_disable();
 100                         continue;
 101                 }
 102
 103                 if (signal_pending(current)) {
 104                         kvmppc_account_exit(vcpu, SIGNAL_EXITS);
 105                         vcpu->run->exit_reason = KVM_EXIT_INTR;
 106                         r = -EINTR;
 107                         break;
 108                 }
 109
 110                 vcpu->mode = IN_GUEST_MODE;
 111
 112                 /*
 113                  * Reading vcpu->requests must happen after setting vcpu->mode,
 114                  * so we don't miss a request because the requester sees
 115                  * OUTSIDE_GUEST_MODE and assumes we'll be checking requests
 116                  * before next entering the guest (and thus doesn't IPI).
 117                  * This also orders the write to mode from any reads
 118                  * to the page tables done while the VCPU is running.
 119                  * Please see the comment in kvm_flush_remote_tlbs.
 120                  */
 121                 smp_mb();
 122
 123                 if (kvm_request_pending(vcpu)) {
 124                         /* Make sure we process requests preemptable */
 125                         local_irq_enable();
 126                         trace_kvm_check_requests(vcpu);
 127                         r = kvmppc_core_check_requests(vcpu);
 128                         hard_irq_disable();
 129                         if (r > 0)
 130                                 continue;
 131                         break;
 132                 }
 133
 134                 if (kvmppc_core_prepare_to_enter(vcpu)) {
 135                         /* interrupts got enabled in between, so we
 136                            are back at square 1 */
 137                         continue;
 138                 }
 139
 140                 guest_enter_irqoff();
 141                 return 1;
 142         }
 143
 144         /* return to host */
 145         local_irq_enable();
 146         return r;
 147 }
 148 EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter);
 149
 150 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
 151 static void kvmppc_swab_shared(struct kvm_vcpu *vcpu)
 152 {
 153         struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared;
 154         int i;
 155
 156         shared->sprg0 = swab64(shared->sprg0);
 157         shared->sprg1 = swab64(shared->sprg1);
 158         shared->sprg2 = swab64(shared->sprg2);
 159         shared->sprg3 = swab64(shared->sprg3);
 160         shared->srr0 = swab64(shared->srr0);
 161         shared->srr1 = swab64(shared->srr1);
 162         shared->dar = swab64(shared->dar);
 163         shared->msr = swab64(shared->msr);
 164         shared->dsisr = swab32(shared->dsisr);
 165         shared->int_pending = swab32(shared->int_pending);
 166         for (i = 0; i < ARRAY_SIZE(shared->sr); i++)
 167                 shared->sr[i] = swab32(shared->sr[i]);
 168 }
 169 #endif
 170
 171 int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
 172 {
 173         int nr = kvmppc_get_gpr(vcpu, 11);
 174         int r;
 175         unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3);
 176         unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4);
 177         unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5);
 178         unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6);
 179         unsigned long r2 = 0;
 180
 181         if (!(kvmppc_get_msr(vcpu) & MSR_SF)) {
 182                 /* 32 bit mode */
 183                 param1 &= 0xffffffff;
 184                 param2 &= 0xffffffff;
 185                 param3 &= 0xffffffff;
 186                 param4 &= 0xffffffff;
 187         }
 188
 189         switch (nr) {
 190         case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE):
 191         {
 192 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
 193                 /* Book3S can be little endian, find it out here */
 194                 int shared_big_endian = true;
 195                 if (vcpu->arch.intr_msr & MSR_LE)
 196                         shared_big_endian = false;
 197                 if (shared_big_endian != vcpu->arch.shared_big_endian)
 198                         kvmppc_swab_shared(vcpu);
 199                 vcpu->arch.shared_big_endian = shared_big_endian;
 200 #endif
 201
 202                 if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) {
 203                         /*
 204                          * Older versions of the Linux magic page code had
 205                          * a bug where they would map their trampoline code
 206                          * NX. If that's the case, remove !PR NX capability.
 207                          */
 208                         vcpu->arch.disable_kernel_nx = true;
 209                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
 210                 }
 211
 212                 vcpu->arch.magic_page_pa = param1 & ~0xfffULL;
 213                 vcpu->arch.magic_page_ea = param2 & ~0xfffULL;
 214
 215 #ifdef CONFIG_PPC_64K_PAGES
 216                 /*
 217                  * Make sure our 4k magic page is in the same window of a 64k
 218                  * page within the guest and within the host's page.
 219                  */
 220                 if ((vcpu->arch.magic_page_pa & 0xf000) !=
 221                     ((ulong)vcpu->arch.shared & 0xf000)) {
 222                         void *old_shared = vcpu->arch.shared;
 223                         ulong shared = (ulong)vcpu->arch.shared;
 224                         void *new_shared;
 225
 226                         shared &= PAGE_MASK;
 227                         shared |= vcpu->arch.magic_page_pa & 0xf000;
 228                         new_shared = (void*)shared;
 229                         memcpy(new_shared, old_shared, 0x1000);
 230                         vcpu->arch.shared = new_shared;
 231                 }
 232 #endif
 233
 234                 r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7;
 235
 236                 r = EV_SUCCESS;
 237                 break;
 238         }
 239         case KVM_HCALL_TOKEN(KVM_HC_FEATURES):
 240                 r = EV_SUCCESS;
 241 #if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2)
 242                 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE);
 243 #endif
 244
 245                 /* Second return value is in r4 */
 246                 break;
 247         case EV_HCALL_TOKEN(EV_IDLE):
 248                 r = EV_SUCCESS;
 249                 kvm_vcpu_block(vcpu);
 250                 kvm_clear_request(KVM_REQ_UNHALT, vcpu);
 251                 break;
 252         default:
 253                 r = EV_UNIMPLEMENTED;
 254                 break;
 255         }
 256
 257         kvmppc_set_gpr(vcpu, 4, r2);
 258
 259         return r;
 260 }
 261 EXPORT_SYMBOL_GPL(kvmppc_kvm_pv);
 262
 263 int kvmppc_sanity_check(struct kvm_vcpu *vcpu)
 264 {
 265         int r = false;
 266
 267         /* We have to know what CPU to virtualize */
 268         if (!vcpu->arch.pvr)
 269                 goto out;
 270
 271         /* PAPR only works with book3s_64 */
 272         if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled)
 273                 goto out;
 274
 275         /* HV KVM can only do PAPR mode for now */
 276         if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm))
 277                 goto out;
 278
 279 #ifdef CONFIG_KVM_BOOKE_HV
 280         if (!cpu_has_feature(CPU_FTR_EMB_HV))
 281                 goto out;
 282 #endif
 283
 284         r = true;
 285
 286 out:
 287         vcpu->arch.sane = r;
 288         return r ? 0 : -EINVAL;
 289 }
 290 EXPORT_SYMBOL_GPL(kvmppc_sanity_check);
 291
 292 int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu)
 293 {
 294         enum emulation_result er;
 295         int r;
 296
 297         er = kvmppc_emulate_loadstore(vcpu);
 298         switch (er) {
 299         case EMULATE_DONE:
 300                 /* Future optimization: only reload non-volatiles if they were
 301                  * actually modified. */
 302                 r = RESUME_GUEST_NV;
 303                 break;
 304         case EMULATE_AGAIN:
 305                 r = RESUME_GUEST;
 306                 break;
 307         case EMULATE_DO_MMIO:
 308                 run->exit_reason = KVM_EXIT_MMIO;
 309                 /* We must reload nonvolatiles because "update" load/store
 310                  * instructions modify register state. */
 311                 /* Future optimization: only reload non-volatiles if they were
 312                  * actually modified. */
 313                 r = RESUME_HOST_NV;
 314                 break;
 315         case EMULATE_FAIL:
 316         {
 317                 u32 last_inst;
 318
 319                 kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
 320                 /* XXX Deliver Program interrupt to guest. */
 321                 pr_emerg("%s: emulation failed (%08x)\n", __func__, last_inst);
 322                 r = RESUME_HOST;
 323                 break;
 324         }
 325         default:
 326                 WARN_ON(1);
 327                 r = RESUME_GUEST;
 328         }
 329
 330         return r;
 331 }
 332 EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio);
 333
 334 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 335               bool data)
 336 {
 337         ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
 338         struct kvmppc_pte pte;
 339         int r;
 340
 341         vcpu->stat.st++;
 342
 343         r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
 344                          XLATE_WRITE, &pte);
 345         if (r < 0)
 346                 return r;
 347
 348         *eaddr = pte.raddr;
 349
 350         if (!pte.may_write)
 351                 return -EPERM;
 352
 353         /* Magic page override */
 354         if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
 355             ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
 356             !(kvmppc_get_msr(vcpu) & MSR_PR)) {
 357                 void *magic = vcpu->arch.shared;
 358                 magic += pte.eaddr & 0xfff;
 359                 memcpy(magic, ptr, size);
 360                 return EMULATE_DONE;
 361         }
 362
 363         if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
 364                 return EMULATE_DO_MMIO;
 365
 366         return EMULATE_DONE;
 367 }
 368 EXPORT_SYMBOL_GPL(kvmppc_st);
 369
 370 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 371                       bool data)
 372 {
 373         ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
 374         struct kvmppc_pte pte;
 375         int rc;
 376
 377         vcpu->stat.ld++;
 378
 379         rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
 380                           XLATE_READ, &pte);
 381         if (rc)
 382                 return rc;
 383
 384         *eaddr = pte.raddr;
 385
 386         if (!pte.may_read)
 387                 return -EPERM;
 388
 389         if (!data && !pte.may_execute)
 390                 return -ENOEXEC;
 391
 392         /* Magic page override */
 393         if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
 394             ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
 395             !(kvmppc_get_msr(vcpu) & MSR_PR)) {
 396                 void *magic = vcpu->arch.shared;
 397                 magic += pte.eaddr & 0xfff;
 398                 memcpy(ptr, magic, size);
 399                 return EMULATE_DONE;
 400         }
 401
 402         if (kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size))
 403                 return EMULATE_DO_MMIO;
 404
 405         return EMULATE_DONE;
 406 }
 407 EXPORT_SYMBOL_GPL(kvmppc_ld);
 408
 409 int kvm_arch_hardware_enable(void)
 410 {
 411         return 0;
 412 }
 413
 414 int kvm_arch_hardware_setup(void)
 415 {
 416         return 0;
 417 }
 418
 419 void kvm_arch_check_processor_compat(void *rtn)
 420 {
 421         *(int *)rtn = kvmppc_core_check_processor_compat();
 422 }
 423
 424 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 425 {
 426         struct kvmppc_ops *kvm_ops = NULL;
 427         /*
 428          * if we have both HV and PR enabled, default is HV
 429          */
 430         if (type == 0) {
 431                 if (kvmppc_hv_ops)
 432                         kvm_ops = kvmppc_hv_ops;
 433                 else
 434                         kvm_ops = kvmppc_pr_ops;
 435                 if (!kvm_ops)
 436                         goto err_out;
 437         } else  if (type == KVM_VM_PPC_HV) {
 438                 if (!kvmppc_hv_ops)
 439                         goto err_out;
 440                 kvm_ops = kvmppc_hv_ops;
 441         } else if (type == KVM_VM_PPC_PR) {
 442                 if (!kvmppc_pr_ops)
 443                         goto err_out;
 444                 kvm_ops = kvmppc_pr_ops;
 445         } else
 446                 goto err_out;
 447
 448         if (kvm_ops->owner && !try_module_get(kvm_ops->owner))
 449                 return -ENOENT;
 450
 451         kvm->arch.kvm_ops = kvm_ops;
 452         return kvmppc_core_init_vm(kvm);
 453 err_out:
 454         return -EINVAL;
 455 }
 456
 457 bool kvm_arch_has_vcpu_debugfs(void)
 458 {
 459         return false;
 460 }
 461
 462 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
 463 {
 464         return 0;
 465 }
 466
 467 void kvm_arch_destroy_vm(struct kvm *kvm)
 468 {
 469         unsigned int i;
 470         struct kvm_vcpu *vcpu;
 471
 472 #ifdef CONFIG_KVM_XICS
 473         /*
 474          * We call kick_all_cpus_sync() to ensure that all
 475          * CPUs have executed any pending IPIs before we
 476          * continue and free VCPUs structures below.
 477          */
 478         if (is_kvmppc_hv_enabled(kvm))
 479                 kick_all_cpus_sync();
 480 #endif
 481
 482         kvm_for_each_vcpu(i, vcpu, kvm)
 483                 kvm_arch_vcpu_free(vcpu);
 484
 485         mutex_lock(&kvm->lock);
 486         for (i = 0; i < atomic_read(&kvm->online_vcpus); i++)
 487                 kvm->vcpus[i] = NULL;
 488
 489         atomic_set(&kvm->online_vcpus, 0);
 490
 491         kvmppc_core_destroy_vm(kvm);
 492
 493         mutex_unlock(&kvm->lock);
 494
 495         /* drop the module reference */
 496         module_put(kvm->arch.kvm_ops->owner);
 497 }
 498
 499 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 500 {
 501         int r;
 502         /* Assume we're using HV mode when the HV module is loaded */
 503         int hv_enabled = kvmppc_hv_ops ? 1 : 0;
 504
 505         if (kvm) {
 506                 /*
 507                  * Hooray - we know which VM type we're running on. Depend on
 508                  * that rather than the guess above.
 509                  */
 510                 hv_enabled = is_kvmppc_hv_enabled(kvm);
 511         }
 512
 513         switch (ext) {
 514 #ifdef CONFIG_BOOKE
 515         case KVM_CAP_PPC_BOOKE_SREGS:
 516         case KVM_CAP_PPC_BOOKE_WATCHDOG:
 517         case KVM_CAP_PPC_EPR:
 518 #else
 519         case KVM_CAP_PPC_SEGSTATE:
 520         case KVM_CAP_PPC_HIOR:
 521         case KVM_CAP_PPC_PAPR:
 522 #endif
 523         case KVM_CAP_PPC_UNSET_IRQ:
 524         case KVM_CAP_PPC_IRQ_LEVEL:
 525         case KVM_CAP_ENABLE_CAP:
 526         case KVM_CAP_ENABLE_CAP_VM:
 527         case KVM_CAP_ONE_REG:
 528         case KVM_CAP_IOEVENTFD:
 529         case KVM_CAP_DEVICE_CTRL:
 530         case KVM_CAP_IMMEDIATE_EXIT:
 531                 r = 1;
 532                 break;
 533         case KVM_CAP_PPC_PAIRED_SINGLES:
 534         case KVM_CAP_PPC_OSI:
 535         case KVM_CAP_PPC_GET_PVINFO:
 536 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
 537         case KVM_CAP_SW_TLB:
 538 #endif
 539                 /* We support this only for PR */
 540                 r = !hv_enabled;
 541                 break;
 542 #ifdef CONFIG_KVM_MPIC
 543         case KVM_CAP_IRQ_MPIC:
 544                 r = 1;
 545                 break;
 546 #endif
 547
 548 #ifdef CONFIG_PPC_BOOK3S_64
 549         case KVM_CAP_SPAPR_TCE:
 550         case KVM_CAP_SPAPR_TCE_64:
 551                 r = 1;
 552                 break;
 553         case KVM_CAP_SPAPR_TCE_VFIO:
 554                 r = !!cpu_has_feature(CPU_FTR_HVMODE);
 555                 break;
 556         case KVM_CAP_PPC_RTAS:
 557         case KVM_CAP_PPC_FIXUP_HCALL:
 558         case KVM_CAP_PPC_ENABLE_HCALL:
 559 #ifdef CONFIG_KVM_XICS
 560         case KVM_CAP_IRQ_XICS:
 561 #endif
 562         case KVM_CAP_PPC_GET_CPU_CHAR:
 563                 r = 1;
 564                 break;
 565
 566         case KVM_CAP_PPC_ALLOC_HTAB:
 567                 r = hv_enabled;
 568                 break;
 569 #endif /* CONFIG_PPC_BOOK3S_64 */
 570 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 571         case KVM_CAP_PPC_SMT:
 572                 r = 0;
 573                 if (kvm) {
 574                         if (kvm->arch.emul_smt_mode > 1)
 575                                 r = kvm->arch.emul_smt_mode;
 576                         else
 577                                 r = kvm->arch.smt_mode;
 578                 } else if (hv_enabled) {
 579                         if (cpu_has_feature(CPU_FTR_ARCH_300))
 580                                 r = 1;
 581                         else
 582                                 r = threads_per_subcore;
 583                 }
 584                 break;
 585         case KVM_CAP_PPC_SMT_POSSIBLE:
 586                 r = 1;
 587                 if (hv_enabled) {
 588                         if (!cpu_has_feature(CPU_FTR_ARCH_300))
 589                                 r = ((threads_per_subcore << 1) - 1);
 590                         else
 591                                 /* P9 can emulate dbells, so allow any mode */
 592                                 r = 8 | 4 | 2 | 1;
 593                 }
 594                 break;
 595         case KVM_CAP_PPC_RMA:
 596                 r = 0;
 597                 break;
 598         case KVM_CAP_PPC_HWRNG:
 599                 r = kvmppc_hwrng_present();
 600                 break;
 601         case KVM_CAP_PPC_MMU_RADIX:
 602                 r = !!(hv_enabled && radix_enabled());
 603                 break;
 604         case KVM_CAP_PPC_MMU_HASH_V3:
 605                 r = !!(hv_enabled && cpu_has_feature(CPU_FTR_ARCH_300));
 606                 break;
 607 #endif
 608         case KVM_CAP_SYNC_MMU:
 609 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 610                 r = hv_enabled;
 611 #elif defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 612                 r = 1;
 613 #else
 614                 r = 0;
 615 #endif
 616                 break;
 617 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 618         case KVM_CAP_PPC_HTAB_FD:
 619                 r = hv_enabled;
 620                 break;
 621 #endif
 622         case KVM_CAP_NR_VCPUS:
 623                 /*
 624                  * Recommending a number of CPUs is somewhat arbitrary; we
 625                  * return the number of present CPUs for -HV (since a host
 626                  * will have secondary threads "offline"), and for other KVM
 627                  * implementations just count online CPUs.
 628                  */
 629                 if (hv_enabled)
 630                         r = num_present_cpus();
 631                 else
 632                         r = num_online_cpus();
 633                 break;
 634         case KVM_CAP_NR_MEMSLOTS:
 635                 r = KVM_USER_MEM_SLOTS;
 636                 break;
 637         case KVM_CAP_MAX_VCPUS:
 638                 r = KVM_MAX_VCPUS;
 639                 break;
 640         case KVM_CAP_MAX_VCPU_ID:
 641                 r = KVM_MAX_VCPU_ID;
 642                 break;
 643 #ifdef CONFIG_PPC_BOOK3S_64
 644         case KVM_CAP_PPC_GET_SMMU_INFO:
 645                 r = 1;
 646                 break;
 647         case KVM_CAP_SPAPR_MULTITCE:
 648                 r = 1;
 649                 break;
 650         case KVM_CAP_SPAPR_RESIZE_HPT:
 651                 r = !!hv_enabled;
 652                 break;
 653 #endif
 654 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 655         case KVM_CAP_PPC_FWNMI:
 656                 r = hv_enabled;
 657                 break;
 658 #endif
 659 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 660         case KVM_CAP_PPC_HTM:
 661                 r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) ||
 662                      (hv_enabled && cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST));
 663                 break;
 664 #endif
 665         default:
 666                 r = 0;
 667                 break;
 668         }
 669         return r;
 670
 671 }
 672
 673 long kvm_arch_dev_ioctl(struct file *filp,
 674                         unsigned int ioctl, unsigned long arg)
 675 {
 676         return -EINVAL;
 677 }
 678
 679 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 680                            struct kvm_memory_slot *dont)
 681 {
 682         kvmppc_core_free_memslot(kvm, free, dont);
 683 }
 684
 685 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot,
 686                             unsigned long npages)
 687 {
 688         return kvmppc_core_create_memslot(kvm, slot, npages);
 689 }
 690
 691 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 692                                    struct kvm_memory_slot *memslot,
 693                                    const struct kvm_userspace_memory_region *mem,
 694                                    enum kvm_mr_change change)
 695 {
 696         return kvmppc_core_prepare_memory_region(kvm, memslot, mem);
 697 }
 698
 699 void kvm_arch_commit_memory_region(struct kvm *kvm,
 700                                    const struct kvm_userspace_memory_region *mem,
 701                                    const struct kvm_memory_slot *old,
 702                                    const struct kvm_memory_slot *new,
 703                                    enum kvm_mr_change change)
 704 {
 705         kvmppc_core_commit_memory_region(kvm, mem, old, new);
 706 }
 707
 708 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 709                                    struct kvm_memory_slot *slot)
 710 {
 711         kvmppc_core_flush_memslot(kvm, slot);
 712 }
 713
 714 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id)
 715 {
 716         struct kvm_vcpu *vcpu;
 717         vcpu = kvmppc_core_vcpu_create(kvm, id);
 718         if (!IS_ERR(vcpu)) {
 719                 vcpu->arch.wqp = &vcpu->wq;
 720                 kvmppc_create_vcpu_debugfs(vcpu, id);
 721         }
 722         return vcpu;
 723 }
 724
 725 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 726 {
 727 }
 728
 729 void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu)
 730 {
 731         /* Make sure we're not using the vcpu anymore */
 732         hrtimer_cancel(&vcpu->arch.dec_timer);
 733
 734         kvmppc_remove_vcpu_debugfs(vcpu);
 735
 736         switch (vcpu->arch.irq_type) {
 737         case KVMPPC_IRQ_MPIC:
 738                 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu);
 739                 break;
 740         case KVMPPC_IRQ_XICS:
 741                 if (xive_enabled())
 742                         kvmppc_xive_cleanup_vcpu(vcpu);
 743                 else
 744                         kvmppc_xics_free_icp(vcpu);
 745                 break;
 746         }
 747
 748         kvmppc_core_vcpu_free(vcpu);
 749 }
 750
 751 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 752 {
 753         kvm_arch_vcpu_free(vcpu);
 754 }
 755
 756 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 757 {
 758         return kvmppc_core_pending_dec(vcpu);
 759 }
 760
 761 static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer)
 762 {
 763         struct kvm_vcpu *vcpu;
 764
 765         vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer);
 766         kvmppc_decrementer_func(vcpu);
 767
 768         return HRTIMER_NORESTART;
 769 }
 770
 771 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
 772 {
 773         int ret;
 774
 775         hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 776         vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
 777         vcpu->arch.dec_expires = get_tb();
 778
 779 #ifdef CONFIG_KVM_EXIT_TIMING
 780         mutex_init(&vcpu->arch.exit_timing_lock);
 781 #endif
 782         ret = kvmppc_subarch_vcpu_init(vcpu);
 783         return ret;
 784 }
 785
 786 void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
 787 {
 788         kvmppc_mmu_destroy(vcpu);
 789         kvmppc_subarch_vcpu_uninit(vcpu);
 790 }
 791
 792 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 793 {
 794 #ifdef CONFIG_BOOKE
 795         /*
 796          * vrsave (formerly usprg0) isn't used by Linux, but may
 797          * be used by the guest.
 798          *
 799          * On non-booke this is associated with Altivec and
 800          * is handled by code in book3s.c.
 801          */
 802         mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
 803 #endif
 804         kvmppc_core_vcpu_load(vcpu, cpu);
 805 }
 806
 807 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 808 {
 809         kvmppc_core_vcpu_put(vcpu);
 810 #ifdef CONFIG_BOOKE
 811         vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
 812 #endif
 813 }
 814
 815 /*
 816  * irq_bypass_add_producer and irq_bypass_del_producer are only
 817  * useful if the architecture supports PCI passthrough.
 818  * irq_bypass_stop and irq_bypass_start are not needed and so
 819  * kvm_ops are not defined for them.
 820  */
 821 bool kvm_arch_has_irq_bypass(void)
 822 {
 823         return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) ||
 824                 (kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer));
 825 }
 826
 827 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
 828                                      struct irq_bypass_producer *prod)
 829 {
 830         struct kvm_kernel_irqfd *irqfd =
 831                 container_of(cons, struct kvm_kernel_irqfd, consumer);
 832         struct kvm *kvm = irqfd->kvm;
 833
 834         if (kvm->arch.kvm_ops->irq_bypass_add_producer)
 835                 return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod);
 836
 837         return 0;
 838 }
 839
 840 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
 841                                       struct irq_bypass_producer *prod)
 842 {
 843         struct kvm_kernel_irqfd *irqfd =
 844                 container_of(cons, struct kvm_kernel_irqfd, consumer);
 845         struct kvm *kvm = irqfd->kvm;
 846
 847         if (kvm->arch.kvm_ops->irq_bypass_del_producer)
 848                 kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod);
 849 }
 850
 851 #ifdef CONFIG_VSX
 852 static inline int kvmppc_get_vsr_dword_offset(int index)
 853 {
 854         int offset;
 855
 856         if ((index != 0) && (index != 1))
 857                 return -1;
 858
 859 #ifdef __BIG_ENDIAN
 860         offset =  index;
 861 #else
 862         offset = 1 - index;
 863 #endif
 864
 865         return offset;
 866 }
 867
 868 static inline int kvmppc_get_vsr_word_offset(int index)
 869 {
 870         int offset;
 871
 872         if ((index > 3) || (index < 0))
 873                 return -1;
 874
 875 #ifdef __BIG_ENDIAN
 876         offset = index;
 877 #else
 878         offset = 3 - index;
 879 #endif
 880         return offset;
 881 }
 882
 883 static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu,
 884         u64 gpr)
 885 {
 886         union kvmppc_one_reg val;
 887         int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
 888         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 889
 890         if (offset == -1)
 891                 return;
 892
 893         if (index >= 32) {
 894                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 895                 val.vsxval[offset] = gpr;
 896                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 897         } else {
 898                 VCPU_VSX_FPR(vcpu, index, offset) = gpr;
 899         }
 900 }
 901
 902 static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu,
 903         u64 gpr)
 904 {
 905         union kvmppc_one_reg val;
 906         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 907
 908         if (index >= 32) {
 909                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 910                 val.vsxval[0] = gpr;
 911                 val.vsxval[1] = gpr;
 912                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 913         } else {
 914                 VCPU_VSX_FPR(vcpu, index, 0) = gpr;
 915                 VCPU_VSX_FPR(vcpu, index, 1) = gpr;
 916         }
 917 }
 918
 919 static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu,
 920         u32 gpr)
 921 {
 922         union kvmppc_one_reg val;
 923         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 924
 925         if (index >= 32) {
 926                 val.vsx32val[0] = gpr;
 927                 val.vsx32val[1] = gpr;
 928                 val.vsx32val[2] = gpr;
 929                 val.vsx32val[3] = gpr;
 930                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 931         } else {
 932                 val.vsx32val[0] = gpr;
 933                 val.vsx32val[1] = gpr;
 934                 VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0];
 935                 VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0];
 936         }
 937 }
 938
 939 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
 940         u32 gpr32)
 941 {
 942         union kvmppc_one_reg val;
 943         int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
 944         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 945         int dword_offset, word_offset;
 946
 947         if (offset == -1)
 948                 return;
 949
 950         if (index >= 32) {
 951                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 952                 val.vsx32val[offset] = gpr32;
 953                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 954         } else {
 955                 dword_offset = offset / 2;
 956                 word_offset = offset % 2;
 957                 val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset);
 958                 val.vsx32val[word_offset] = gpr32;
 959                 VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0];
 960         }
 961 }
 962 #endif /* CONFIG_VSX */
 963
 964 #ifdef CONFIG_ALTIVEC
 965 static inline int kvmppc_get_vmx_offset_generic(struct kvm_vcpu *vcpu,
 966                 int index, int element_size)
 967 {
 968         int offset;
 969         int elts = sizeof(vector128)/element_size;
 970
 971         if ((index < 0) || (index >= elts))
 972                 return -1;
 973
 974         if (kvmppc_need_byteswap(vcpu))
 975                 offset = elts - index - 1;
 976         else
 977                 offset = index;
 978
 979         return offset;
 980 }
 981
 982 static inline int kvmppc_get_vmx_dword_offset(struct kvm_vcpu *vcpu,
 983                 int index)
 984 {
 985         return kvmppc_get_vmx_offset_generic(vcpu, index, 8);
 986 }
 987
 988 static inline int kvmppc_get_vmx_word_offset(struct kvm_vcpu *vcpu,
 989                 int index)
 990 {
 991         return kvmppc_get_vmx_offset_generic(vcpu, index, 4);
 992 }
 993
 994 static inline int kvmppc_get_vmx_hword_offset(struct kvm_vcpu *vcpu,
 995                 int index)
 996 {
 997         return kvmppc_get_vmx_offset_generic(vcpu, index, 2);
 998 }
 999
1000 static inline int kvmppc_get_vmx_byte_offset(struct kvm_vcpu *vcpu,
1001                 int index)
1002 {
1003         return kvmppc_get_vmx_offset_generic(vcpu, index, 1);
1004 }
1005
1006
1007 static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu,
1008         u64 gpr)
1009 {
1010         union kvmppc_one_reg val;
1011         int offset = kvmppc_get_vmx_dword_offset(vcpu,
1012                         vcpu->arch.mmio_vmx_offset);
1013         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1014
1015         if (offset == -1)
1016                 return;
1017
1018         val.vval = VCPU_VSX_VR(vcpu, index);
1019         val.vsxval[offset] = gpr;
1020         VCPU_VSX_VR(vcpu, index) = val.vval;
1021 }
1022
1023 static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu,
1024         u32 gpr32)
1025 {
1026         union kvmppc_one_reg val;
1027         int offset = kvmppc_get_vmx_word_offset(vcpu,
1028                         vcpu->arch.mmio_vmx_offset);
1029         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1030
1031         if (offset == -1)
1032                 return;
1033
1034         val.vval = VCPU_VSX_VR(vcpu, index);
1035         val.vsx32val[offset] = gpr32;
1036         VCPU_VSX_VR(vcpu, index) = val.vval;
1037 }
1038
1039 static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu,
1040         u16 gpr16)
1041 {
1042         union kvmppc_one_reg val;
1043         int offset = kvmppc_get_vmx_hword_offset(vcpu,
1044                         vcpu->arch.mmio_vmx_offset);
1045         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1046
1047         if (offset == -1)
1048                 return;
1049
1050         val.vval = VCPU_VSX_VR(vcpu, index);
1051         val.vsx16val[offset] = gpr16;
1052         VCPU_VSX_VR(vcpu, index) = val.vval;
1053 }
1054
1055 static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu,
1056         u8 gpr8)
1057 {
1058         union kvmppc_one_reg val;
1059         int offset = kvmppc_get_vmx_byte_offset(vcpu,
1060                         vcpu->arch.mmio_vmx_offset);
1061         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1062
1063         if (offset == -1)
1064                 return;
1065
1066         val.vval = VCPU_VSX_VR(vcpu, index);
1067         val.vsx8val[offset] = gpr8;
1068         VCPU_VSX_VR(vcpu, index) = val.vval;
1069 }
1070 #endif /* CONFIG_ALTIVEC */
1071
1072 #ifdef CONFIG_PPC_FPU
1073 static inline u64 sp_to_dp(u32 fprs)
1074 {
1075         u64 fprd;
1076
1077         preempt_disable();
1078         enable_kernel_fp();
1079         asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m" (fprd) : "m" (fprs)
1080              : "fr0");
1081         preempt_enable();
1082         return fprd;
1083 }
1084
1085 static inline u32 dp_to_sp(u64 fprd)
1086 {
1087         u32 fprs;
1088
1089         preempt_disable();
1090         enable_kernel_fp();
1091         asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m" (fprs) : "m" (fprd)
1092              : "fr0");
1093         preempt_enable();
1094         return fprs;
1095 }
1096
1097 #else
1098 #define sp_to_dp(x)     (x)
1099 #define dp_to_sp(x)     (x)
1100 #endif /* CONFIG_PPC_FPU */
1101
1102 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu,
1103                                       struct kvm_run *run)
1104 {
1105         u64 uninitialized_var(gpr);
1106
1107         if (run->mmio.len > sizeof(gpr)) {
1108                 printk(KERN_ERR "bad MMIO length: %d\n", run->mmio.len);
1109                 return;
1110         }
1111
1112         if (!vcpu->arch.mmio_host_swabbed) {
1113                 switch (run->mmio.len) {
1114                 case 8: gpr = *(u64 *)run->mmio.data; break;
1115                 case 4: gpr = *(u32 *)run->mmio.data; break;
1116                 case 2: gpr = *(u16 *)run->mmio.data; break;
1117                 case 1: gpr = *(u8 *)run->mmio.data; break;
1118                 }
1119         } else {
1120                 switch (run->mmio.len) {
1121                 case 8: gpr = swab64(*(u64 *)run->mmio.data); break;
1122                 case 4: gpr = swab32(*(u32 *)run->mmio.data); break;
1123                 case 2: gpr = swab16(*(u16 *)run->mmio.data); break;
1124                 case 1: gpr = *(u8 *)run->mmio.data; break;
1125                 }
1126         }
1127
1128         /* conversion between single and double precision */
1129         if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4))
1130                 gpr = sp_to_dp(gpr);
1131
1132         if (vcpu->arch.mmio_sign_extend) {
1133                 switch (run->mmio.len) {
1134 #ifdef CONFIG_PPC64
1135                 case 4:
1136                         gpr = (s64)(s32)gpr;
1137                         break;
1138 #endif
1139                 case 2:
1140                         gpr = (s64)(s16)gpr;
1141                         break;
1142                 case 1:
1143                         gpr = (s64)(s8)gpr;
1144                         break;
1145                 }
1146         }
1147
1148         switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) {
1149         case KVM_MMIO_REG_GPR:
1150                 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
1151                 break;
1152         case KVM_MMIO_REG_FPR:
1153                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1154                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP);
1155
1156                 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1157                 break;
1158 #ifdef CONFIG_PPC_BOOK3S
1159         case KVM_MMIO_REG_QPR:
1160                 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1161                 break;
1162         case KVM_MMIO_REG_FQPR:
1163                 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1164                 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1165                 break;
1166 #endif
1167 #ifdef CONFIG_VSX
1168         case KVM_MMIO_REG_VSX:
1169                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1170                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX);
1171
1172                 if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD)
1173                         kvmppc_set_vsr_dword(vcpu, gpr);
1174                 else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD)
1175                         kvmppc_set_vsr_word(vcpu, gpr);
1176                 else if (vcpu->arch.mmio_copy_type ==
1177                                 KVMPPC_VSX_COPY_DWORD_LOAD_DUMP)
1178                         kvmppc_set_vsr_dword_dump(vcpu, gpr);
1179                 else if (vcpu->arch.mmio_copy_type ==
1180                                 KVMPPC_VSX_COPY_WORD_LOAD_DUMP)
1181                         kvmppc_set_vsr_word_dump(vcpu, gpr);
1182                 break;
1183 #endif
1184 #ifdef CONFIG_ALTIVEC
1185         case KVM_MMIO_REG_VMX:
1186                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1187                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC);
1188
1189                 if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD)
1190                         kvmppc_set_vmx_dword(vcpu, gpr);
1191                 else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD)
1192                         kvmppc_set_vmx_word(vcpu, gpr);
1193                 else if (vcpu->arch.mmio_copy_type ==
1194                                 KVMPPC_VMX_COPY_HWORD)
1195                         kvmppc_set_vmx_hword(vcpu, gpr);
1196                 else if (vcpu->arch.mmio_copy_type ==
1197                                 KVMPPC_VMX_COPY_BYTE)
1198                         kvmppc_set_vmx_byte(vcpu, gpr);
1199                 break;
1200 #endif
1201         default:
1202                 BUG();
1203         }
1204 }
1205
1206 static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
1207                                 unsigned int rt, unsigned int bytes,
1208                                 int is_default_endian, int sign_extend)
1209 {
1210         int idx, ret;
1211         bool host_swabbed;
1212
1213         /* Pity C doesn't have a logical XOR operator */
1214         if (kvmppc_need_byteswap(vcpu)) {
1215                 host_swabbed = is_default_endian;
1216         } else {
1217                 host_swabbed = !is_default_endian;
1218         }
1219
1220         if (bytes > sizeof(run->mmio.data)) {
1221                 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
1222                        run->mmio.len);
1223         }
1224
1225         run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1226         run->mmio.len = bytes;
1227         run->mmio.is_write = 0;
1228
1229         vcpu->arch.io_gpr = rt;
1230         vcpu->arch.mmio_host_swabbed = host_swabbed;
1231         vcpu->mmio_needed = 1;
1232         vcpu->mmio_is_write = 0;
1233         vcpu->arch.mmio_sign_extend = sign_extend;
1234
1235         idx = srcu_read_lock(&vcpu->kvm->srcu);
1236
1237         ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1238                               bytes, &run->mmio.data);
1239
1240         srcu_read_unlock(&vcpu->kvm->srcu, idx);
1241
1242         if (!ret) {
1243                 kvmppc_complete_mmio_load(vcpu, run);
1244                 vcpu->mmio_needed = 0;
1245                 return EMULATE_DONE;
1246         }
1247
1248         return EMULATE_DO_MMIO;
1249 }
1250
1251 int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
1252                        unsigned int rt, unsigned int bytes,
1253                        int is_default_endian)
1254 {
1255         return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 0);
1256 }
1257 EXPORT_SYMBOL_GPL(kvmppc_handle_load);
1258
1259 /* Same as above, but sign extends */
1260 int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu,
1261                         unsigned int rt, unsigned int bytes,
1262                         int is_default_endian)
1263 {
1264         return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 1);
1265 }
1266
1267 #ifdef CONFIG_VSX
1268 int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
1269                         unsigned int rt, unsigned int bytes,
1270                         int is_default_endian, int mmio_sign_extend)
1271 {
1272         enum emulation_result emulated = EMULATE_DONE;
1273
1274         /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1275         if (vcpu->arch.mmio_vsx_copy_nums > 4)
1276                 return EMULATE_FAIL;
1277
1278         while (vcpu->arch.mmio_vsx_copy_nums) {
1279                 emulated = __kvmppc_handle_load(run, vcpu, rt, bytes,
1280                         is_default_endian, mmio_sign_extend);
1281
1282                 if (emulated != EMULATE_DONE)
1283                         break;
1284
1285                 vcpu->arch.paddr_accessed += run->mmio.len;
1286
1287                 vcpu->arch.mmio_vsx_copy_nums--;
1288                 vcpu->arch.mmio_vsx_offset++;
1289         }
1290         return emulated;
1291 }
1292 #endif /* CONFIG_VSX */
1293
1294 int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
1295                         u64 val, unsigned int bytes, int is_default_endian)
1296 {
1297         void *data = run->mmio.data;
1298         int idx, ret;
1299         bool host_swabbed;
1300
1301         /* Pity C doesn't have a logical XOR operator */
1302         if (kvmppc_need_byteswap(vcpu)) {
1303                 host_swabbed = is_default_endian;
1304         } else {
1305                 host_swabbed = !is_default_endian;
1306         }
1307
1308         if (bytes > sizeof(run->mmio.data)) {
1309                 printk(KERN_ERR "%s: bad MMIO length: %d\n", __func__,
1310                        run->mmio.len);
1311         }
1312
1313         run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1314         run->mmio.len = bytes;
1315         run->mmio.is_write = 1;
1316         vcpu->mmio_needed = 1;
1317         vcpu->mmio_is_write = 1;
1318
1319         if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4))
1320                 val = dp_to_sp(val);
1321
1322         /* Store the value at the lowest bytes in 'data'. */
1323         if (!host_swabbed) {
1324                 switch (bytes) {
1325                 case 8: *(u64 *)data = val; break;
1326                 case 4: *(u32 *)data = val; break;
1327                 case 2: *(u16 *)data = val; break;
1328                 case 1: *(u8  *)data = val; break;
1329                 }
1330         } else {
1331                 switch (bytes) {
1332                 case 8: *(u64 *)data = swab64(val); break;
1333                 case 4: *(u32 *)data = swab32(val); break;
1334                 case 2: *(u16 *)data = swab16(val); break;
1335                 case 1: *(u8  *)data = val; break;
1336                 }
1337         }
1338
1339         idx = srcu_read_lock(&vcpu->kvm->srcu);
1340
1341         ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1342                                bytes, &run->mmio.data);
1343
1344         srcu_read_unlock(&vcpu->kvm->srcu, idx);
1345
1346         if (!ret) {
1347                 vcpu->mmio_needed = 0;
1348                 return EMULATE_DONE;
1349         }
1350
1351         return EMULATE_DO_MMIO;
1352 }
1353 EXPORT_SYMBOL_GPL(kvmppc_handle_store);
1354
1355 #ifdef CONFIG_VSX
1356 static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
1357 {
1358         u32 dword_offset, word_offset;
1359         union kvmppc_one_reg reg;
1360         int vsx_offset = 0;
1361         int copy_type = vcpu->arch.mmio_copy_type;
1362         int result = 0;
1363
1364         switch (copy_type) {
1365         case KVMPPC_VSX_COPY_DWORD:
1366                 vsx_offset =
1367                         kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
1368
1369                 if (vsx_offset == -1) {
1370                         result = -1;
1371                         break;
1372                 }
1373
1374                 if (rs < 32) {
1375                         *val = VCPU_VSX_FPR(vcpu, rs, vsx_offset);
1376                 } else {
1377                         reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1378                         *val = reg.vsxval[vsx_offset];
1379                 }
1380                 break;
1381
1382         case KVMPPC_VSX_COPY_WORD:
1383                 vsx_offset =
1384                         kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
1385
1386                 if (vsx_offset == -1) {
1387                         result = -1;
1388                         break;
1389                 }
1390
1391                 if (rs < 32) {
1392                         dword_offset = vsx_offset / 2;
1393                         word_offset = vsx_offset % 2;
1394                         reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset);
1395                         *val = reg.vsx32val[word_offset];
1396                 } else {
1397                         reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1398                         *val = reg.vsx32val[vsx_offset];
1399                 }
1400                 break;
1401
1402         default:
1403                 result = -1;
1404                 break;
1405         }
1406
1407         return result;
1408 }
1409
1410 int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
1411                         int rs, unsigned int bytes, int is_default_endian)
1412 {
1413         u64 val;
1414         enum emulation_result emulated = EMULATE_DONE;
1415
1416         vcpu->arch.io_gpr = rs;
1417
1418         /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1419         if (vcpu->arch.mmio_vsx_copy_nums > 4)
1420                 return EMULATE_FAIL;
1421
1422         while (vcpu->arch.mmio_vsx_copy_nums) {
1423                 if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1)
1424                         return EMULATE_FAIL;
1425
1426                 emulated = kvmppc_handle_store(run, vcpu,
1427                          val, bytes, is_default_endian);
1428
1429                 if (emulated != EMULATE_DONE)
1430                         break;
1431
1432                 vcpu->arch.paddr_accessed += run->mmio.len;
1433
1434                 vcpu->arch.mmio_vsx_copy_nums--;
1435                 vcpu->arch.mmio_vsx_offset++;
1436         }
1437
1438         return emulated;
1439 }
1440
1441 static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu,
1442                         struct kvm_run *run)
1443 {
1444         enum emulation_result emulated = EMULATE_FAIL;
1445         int r;
1446
1447         vcpu->arch.paddr_accessed += run->mmio.len;
1448
1449         if (!vcpu->mmio_is_write) {
1450                 emulated = kvmppc_handle_vsx_load(run, vcpu, vcpu->arch.io_gpr,
1451                          run->mmio.len, 1, vcpu->arch.mmio_sign_extend);
1452         } else {
1453                 emulated = kvmppc_handle_vsx_store(run, vcpu,
1454                          vcpu->arch.io_gpr, run->mmio.len, 1);
1455         }
1456
1457         switch (emulated) {
1458         case EMULATE_DO_MMIO:
1459                 run->exit_reason = KVM_EXIT_MMIO;
1460                 r = RESUME_HOST;
1461                 break;
1462         case EMULATE_FAIL:
1463                 pr_info("KVM: MMIO emulation failed (VSX repeat)\n");
1464                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1465                 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1466                 r = RESUME_HOST;
1467                 break;
1468         default:
1469                 r = RESUME_GUEST;
1470                 break;
1471         }
1472         return r;
1473 }
1474 #endif /* CONFIG_VSX */
1475
1476 #ifdef CONFIG_ALTIVEC
1477 int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu,
1478                 unsigned int rt, unsigned int bytes, int is_default_endian)
1479 {
1480         enum emulation_result emulated = EMULATE_DONE;
1481
1482         if (vcpu->arch.mmio_vmx_copy_nums > 2)
1483                 return EMULATE_FAIL;
1484
1485         while (vcpu->arch.mmio_vmx_copy_nums) {
1486                 emulated = __kvmppc_handle_load(run, vcpu, rt, bytes,
1487                                 is_default_endian, 0);
1488
1489                 if (emulated != EMULATE_DONE)
1490                         break;
1491
1492                 vcpu->arch.paddr_accessed += run->mmio.len;
1493                 vcpu->arch.mmio_vmx_copy_nums--;
1494                 vcpu->arch.mmio_vmx_offset++;
1495         }
1496
1497         return emulated;
1498 }
1499
1500 static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val)
1501 {
1502         union kvmppc_one_reg reg;
1503         int vmx_offset = 0;
1504         int result = 0;
1505
1506         vmx_offset =
1507                 kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1508
1509         if (vmx_offset == -1)
1510                 return -1;
1511
1512         reg.vval = VCPU_VSX_VR(vcpu, index);
1513         *val = reg.vsxval[vmx_offset];
1514
1515         return result;
1516 }
1517
1518 static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val)
1519 {
1520         union kvmppc_one_reg reg;
1521         int vmx_offset = 0;
1522         int result = 0;
1523
1524         vmx_offset =
1525                 kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1526
1527         if (vmx_offset == -1)
1528                 return -1;
1529
1530         reg.vval = VCPU_VSX_VR(vcpu, index);
1531         *val = reg.vsx32val[vmx_offset];
1532
1533         return result;
1534 }
1535
1536 static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val)
1537 {
1538         union kvmppc_one_reg reg;
1539         int vmx_offset = 0;
1540         int result = 0;
1541
1542         vmx_offset =
1543                 kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1544
1545         if (vmx_offset == -1)
1546                 return -1;
1547
1548         reg.vval = VCPU_VSX_VR(vcpu, index);
1549         *val = reg.vsx16val[vmx_offset];
1550
1551         return result;
1552 }
1553
1554 static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val)
1555 {
1556         union kvmppc_one_reg reg;
1557         int vmx_offset = 0;
1558         int result = 0;
1559
1560         vmx_offset =
1561                 kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1562
1563         if (vmx_offset == -1)
1564                 return -1;
1565
1566         reg.vval = VCPU_VSX_VR(vcpu, index);
1567         *val = reg.vsx8val[vmx_offset];
1568
1569         return result;
1570 }
1571
1572 int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu,
1573                 unsigned int rs, unsigned int bytes, int is_default_endian)
1574 {
1575         u64 val = 0;
1576         unsigned int index = rs & KVM_MMIO_REG_MASK;
1577         enum emulation_result emulated = EMULATE_DONE;
1578
1579         if (vcpu->arch.mmio_vmx_copy_nums > 2)
1580                 return EMULATE_FAIL;
1581
1582         vcpu->arch.io_gpr = rs;
1583
1584         while (vcpu->arch.mmio_vmx_copy_nums) {
1585                 switch (vcpu->arch.mmio_copy_type) {
1586                 case KVMPPC_VMX_COPY_DWORD:
1587                         if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1)
1588                                 return EMULATE_FAIL;
1589
1590                         break;
1591                 case KVMPPC_VMX_COPY_WORD:
1592                         if (kvmppc_get_vmx_word(vcpu, index, &val) == -1)
1593                                 return EMULATE_FAIL;
1594                         break;
1595                 case KVMPPC_VMX_COPY_HWORD:
1596                         if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1)
1597                                 return EMULATE_FAIL;
1598                         break;
1599                 case KVMPPC_VMX_COPY_BYTE:
1600                         if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1)
1601                                 return EMULATE_FAIL;
1602                         break;
1603                 default:
1604                         return EMULATE_FAIL;
1605                 }
1606
1607                 emulated = kvmppc_handle_store(run, vcpu, val, bytes,
1608                                 is_default_endian);
1609                 if (emulated != EMULATE_DONE)
1610                         break;
1611
1612                 vcpu->arch.paddr_accessed += run->mmio.len;
1613                 vcpu->arch.mmio_vmx_copy_nums--;
1614                 vcpu->arch.mmio_vmx_offset++;
1615         }
1616
1617         return emulated;
1618 }
1619
1620 static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu,
1621                 struct kvm_run *run)
1622 {
1623         enum emulation_result emulated = EMULATE_FAIL;
1624         int r;
1625
1626         vcpu->arch.paddr_accessed += run->mmio.len;
1627
1628         if (!vcpu->mmio_is_write) {
1629                 emulated = kvmppc_handle_vmx_load(run, vcpu,
1630                                 vcpu->arch.io_gpr, run->mmio.len, 1);
1631         } else {
1632                 emulated = kvmppc_handle_vmx_store(run, vcpu,
1633                                 vcpu->arch.io_gpr, run->mmio.len, 1);
1634         }
1635
1636         switch (emulated) {
1637         case EMULATE_DO_MMIO:
1638                 run->exit_reason = KVM_EXIT_MMIO;
1639                 r = RESUME_HOST;
1640                 break;
1641         case EMULATE_FAIL:
1642                 pr_info("KVM: MMIO emulation failed (VMX repeat)\n");
1643                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1644                 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1645                 r = RESUME_HOST;
1646                 break;
1647         default:
1648                 r = RESUME_GUEST;
1649                 break;
1650         }
1651         return r;
1652 }
1653 #endif /* CONFIG_ALTIVEC */
1654
1655 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1656 {
1657         int r = 0;
1658         union kvmppc_one_reg val;
1659         int size;
1660
1661         size = one_reg_size(reg->id);
1662         if (size > sizeof(val))
1663                 return -EINVAL;
1664
1665         r = kvmppc_get_one_reg(vcpu, reg->id, &val);
1666         if (r == -EINVAL) {
1667                 r = 0;
1668                 switch (reg->id) {
1669 #ifdef CONFIG_ALTIVEC
1670                 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1671                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1672                                 r = -ENXIO;
1673                                 break;
1674                         }
1675                         val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0];
1676                         break;
1677                 case KVM_REG_PPC_VSCR:
1678                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1679                                 r = -ENXIO;
1680                                 break;
1681                         }
1682                         val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]);
1683                         break;
1684                 case KVM_REG_PPC_VRSAVE:
1685                         val = get_reg_val(reg->id, vcpu->arch.vrsave);
1686                         break;
1687 #endif /* CONFIG_ALTIVEC */
1688                 default:
1689                         r = -EINVAL;
1690                         break;
1691                 }
1692         }
1693
1694         if (r)
1695                 return r;
1696
1697         if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size))
1698                 r = -EFAULT;
1699
1700         return r;
1701 }
1702
1703 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1704 {
1705         int r;
1706         union kvmppc_one_reg val;
1707         int size;
1708
1709         size = one_reg_size(reg->id);
1710         if (size > sizeof(val))
1711                 return -EINVAL;
1712
1713         if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size))
1714                 return -EFAULT;
1715
1716         r = kvmppc_set_one_reg(vcpu, reg->id, &val);
1717         if (r == -EINVAL) {
1718                 r = 0;
1719                 switch (reg->id) {
1720 #ifdef CONFIG_ALTIVEC
1721                 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1722                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1723                                 r = -ENXIO;
1724                                 break;
1725                         }
1726                         vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval;
1727                         break;
1728                 case KVM_REG_PPC_VSCR:
1729                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1730                                 r = -ENXIO;
1731                                 break;
1732                         }
1733                         vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val);
1734                         break;
1735                 case KVM_REG_PPC_VRSAVE:
1736                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1737                                 r = -ENXIO;
1738                                 break;
1739                         }
1740                         vcpu->arch.vrsave = set_reg_val(reg->id, val);
1741                         break;
1742 #endif /* CONFIG_ALTIVEC */
1743                 default:
1744                         r = -EINVAL;
1745                         break;
1746                 }
1747         }
1748
1749         return r;
1750 }
1751
1752 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
1753 {
1754         int r;
1755
1756         vcpu_load(vcpu);
1757
1758         if (vcpu->mmio_needed) {
1759                 vcpu->mmio_needed = 0;
1760                 if (!vcpu->mmio_is_write)
1761                         kvmppc_complete_mmio_load(vcpu, run);
1762 #ifdef CONFIG_VSX
1763                 if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1764                         vcpu->arch.mmio_vsx_copy_nums--;
1765                         vcpu->arch.mmio_vsx_offset++;
1766                 }
1767
1768                 if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1769                         r = kvmppc_emulate_mmio_vsx_loadstore(vcpu, run);
1770                         if (r == RESUME_HOST) {
1771                                 vcpu->mmio_needed = 1;
1772                                 goto out;
1773                         }
1774                 }
1775 #endif
1776 #ifdef CONFIG_ALTIVEC
1777                 if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1778                         vcpu->arch.mmio_vmx_copy_nums--;
1779                         vcpu->arch.mmio_vmx_offset++;
1780                 }
1781
1782                 if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1783                         r = kvmppc_emulate_mmio_vmx_loadstore(vcpu, run);
1784                         if (r == RESUME_HOST) {
1785                                 vcpu->mmio_needed = 1;
1786                                 goto out;
1787                         }
1788                 }
1789 #endif
1790         } else if (vcpu->arch.osi_needed) {
1791                 u64 *gprs = run->osi.gprs;
1792                 int i;
1793
1794                 for (i = 0; i < 32; i++)
1795                         kvmppc_set_gpr(vcpu, i, gprs[i]);
1796                 vcpu->arch.osi_needed = 0;
1797         } else if (vcpu->arch.hcall_needed) {
1798                 int i;
1799
1800                 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret);
1801                 for (i = 0; i < 9; ++i)
1802                         kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]);
1803                 vcpu->arch.hcall_needed = 0;
1804 #ifdef CONFIG_BOOKE
1805         } else if (vcpu->arch.epr_needed) {
1806                 kvmppc_set_epr(vcpu, run->epr.epr);
1807                 vcpu->arch.epr_needed = 0;
1808 #endif
1809         }
1810
1811         kvm_sigset_activate(vcpu);
1812
1813         if (run->immediate_exit)
1814                 r = -EINTR;
1815         else
1816                 r = kvmppc_vcpu_run(run, vcpu);
1817
1818         kvm_sigset_deactivate(vcpu);
1819
1820 #ifdef CONFIG_ALTIVEC
1821 out:
1822 #endif
1823         vcpu_put(vcpu);
1824         return r;
1825 }
1826
1827 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1828 {
1829         if (irq->irq == KVM_INTERRUPT_UNSET) {
1830                 kvmppc_core_dequeue_external(vcpu);
1831                 return 0;
1832         }
1833
1834         kvmppc_core_queue_external(vcpu, irq);
1835
1836         kvm_vcpu_kick(vcpu);
1837
1838         return 0;
1839 }
1840
1841 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1842                                      struct kvm_enable_cap *cap)
1843 {
1844         int r;
1845
1846         if (cap->flags)
1847                 return -EINVAL;
1848
1849         switch (cap->cap) {
1850         case KVM_CAP_PPC_OSI:
1851                 r = 0;
1852                 vcpu->arch.osi_enabled = true;
1853                 break;
1854         case KVM_CAP_PPC_PAPR:
1855                 r = 0;
1856                 vcpu->arch.papr_enabled = true;
1857                 break;
1858         case KVM_CAP_PPC_EPR:
1859                 r = 0;
1860                 if (cap->args[0])
1861                         vcpu->arch.epr_flags |= KVMPPC_EPR_USER;
1862                 else
1863                         vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER;
1864                 break;
1865 #ifdef CONFIG_BOOKE
1866         case KVM_CAP_PPC_BOOKE_WATCHDOG:
1867                 r = 0;
1868                 vcpu->arch.watchdog_enabled = true;
1869                 break;
1870 #endif
1871 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
1872         case KVM_CAP_SW_TLB: {
1873                 struct kvm_config_tlb cfg;
1874                 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0];
1875
1876                 r = -EFAULT;
1877                 if (copy_from_user(&cfg, user_ptr, sizeof(cfg)))
1878                         break;
1879
1880                 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg);
1881                 break;
1882         }
1883 #endif
1884 #ifdef CONFIG_KVM_MPIC
1885         case KVM_CAP_IRQ_MPIC: {
1886                 struct fd f;
1887                 struct kvm_device *dev;
1888
1889                 r = -EBADF;
1890                 f = fdget(cap->args[0]);
1891                 if (!f.file)
1892                         break;
1893
1894                 r = -EPERM;
1895                 dev = kvm_device_from_filp(f.file);
1896                 if (dev)
1897                         r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]);
1898
1899                 fdput(f);
1900                 break;
1901         }
1902 #endif
1903 #ifdef CONFIG_KVM_XICS
1904         case KVM_CAP_IRQ_XICS: {
1905                 struct fd f;
1906                 struct kvm_device *dev;
1907
1908                 r = -EBADF;
1909                 f = fdget(cap->args[0]);
1910                 if (!f.file)
1911                         break;
1912
1913                 r = -EPERM;
1914                 dev = kvm_device_from_filp(f.file);
1915                 if (dev) {
1916                         if (xive_enabled())
1917                                 r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]);
1918                         else
1919                                 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]);
1920                 }
1921
1922                 fdput(f);
1923                 break;
1924         }
1925 #endif /* CONFIG_KVM_XICS */
1926 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1927         case KVM_CAP_PPC_FWNMI:
1928                 r = -EINVAL;
1929                 if (!is_kvmppc_hv_enabled(vcpu->kvm))
1930                         break;
1931                 r = 0;
1932                 vcpu->kvm->arch.fwnmi_enabled = true;
1933                 break;
1934 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
1935         default:
1936                 r = -EINVAL;
1937                 break;
1938         }
1939
1940         if (!r)
1941                 r = kvmppc_sanity_check(vcpu);
1942
1943         return r;
1944 }
1945
1946 bool kvm_arch_intc_initialized(struct kvm *kvm)
1947 {
1948 #ifdef CONFIG_KVM_MPIC
1949         if (kvm->arch.mpic)
1950                 return true;
1951 #endif
1952 #ifdef CONFIG_KVM_XICS
1953         if (kvm->arch.xics || kvm->arch.xive)
1954                 return true;
1955 #endif
1956         return false;
1957 }
1958
1959 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
1960                                     struct kvm_mp_state *mp_state)
1961 {
1962         return -EINVAL;
1963 }
1964
1965 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
1966                                     struct kvm_mp_state *mp_state)
1967 {
1968         return -EINVAL;
1969 }
1970
1971 long kvm_arch_vcpu_async_ioctl(struct file *filp,
1972                                unsigned int ioctl, unsigned long arg)
1973 {
1974         struct kvm_vcpu *vcpu = filp->private_data;
1975         void __user *argp = (void __user *)arg;
1976
1977         if (ioctl == KVM_INTERRUPT) {
1978                 struct kvm_interrupt irq;
1979                 if (copy_from_user(&irq, argp, sizeof(irq)))
1980                         return -EFAULT;
1981                 return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1982         }
1983         return -ENOIOCTLCMD;
1984 }
1985
1986 long kvm_arch_vcpu_ioctl(struct file *filp,
1987                          unsigned int ioctl, unsigned long arg)
1988 {
1989         struct kvm_vcpu *vcpu = filp->private_data;
1990         void __user *argp = (void __user *)arg;
1991         long r;
1992
1993         switch (ioctl) {
1994         case KVM_ENABLE_CAP:
1995         {
1996                 struct kvm_enable_cap cap;
1997                 r = -EFAULT;
1998                 if (copy_from_user(&cap, argp, sizeof(cap)))
1999                         goto out;
2000                 vcpu_load(vcpu);
2001                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2002                 vcpu_put(vcpu);
2003                 break;
2004         }
2005
2006         case KVM_SET_ONE_REG:
2007         case KVM_GET_ONE_REG:
2008         {
2009                 struct kvm_one_reg reg;
2010                 r = -EFAULT;
2011                 if (copy_from_user(&reg, argp, sizeof(reg)))
2012                         goto out;
2013                 if (ioctl == KVM_SET_ONE_REG)
2014                         r = kvm_vcpu_ioctl_set_one_reg(vcpu, &reg);
2015                 else
2016                         r = kvm_vcpu_ioctl_get_one_reg(vcpu, &reg);
2017                 break;
2018         }
2019
2020 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
2021         case KVM_DIRTY_TLB: {
2022                 struct kvm_dirty_tlb dirty;
2023                 r = -EFAULT;
2024                 if (copy_from_user(&dirty, argp, sizeof(dirty)))
2025                         goto out;
2026                 vcpu_load(vcpu);
2027                 r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty);
2028                 vcpu_put(vcpu);
2029                 break;
2030         }
2031 #endif
2032         default:
2033                 r = -EINVAL;
2034         }
2035
2036 out:
2037         return r;
2038 }
2039
2040 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2041 {
2042         return VM_FAULT_SIGBUS;
2043 }
2044
2045 static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo)
2046 {
2047         u32 inst_nop = 0x60000000;
2048 #ifdef CONFIG_KVM_BOOKE_HV
2049         u32 inst_sc1 = 0x44000022;
2050         pvinfo->hcall[0] = cpu_to_be32(inst_sc1);
2051         pvinfo->hcall[1] = cpu_to_be32(inst_nop);
2052         pvinfo->hcall[2] = cpu_to_be32(inst_nop);
2053         pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2054 #else
2055         u32 inst_lis = 0x3c000000;
2056         u32 inst_ori = 0x60000000;
2057         u32 inst_sc = 0x44000002;
2058         u32 inst_imm_mask = 0xffff;
2059
2060         /*
2061          * The hypercall to get into KVM from within guest context is as
2062          * follows:
2063          *
2064          *    lis r0, r0, KVM_SC_MAGIC_R0@h
2065          *    ori r0, KVM_SC_MAGIC_R0@l
2066          *    sc
2067          *    nop
2068          */
2069         pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask));
2070         pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask));
2071         pvinfo->hcall[2] = cpu_to_be32(inst_sc);
2072         pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2073 #endif
2074
2075         pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE;
2076
2077         return 0;
2078 }
2079
2080 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
2081                           bool line_status)
2082 {
2083         if (!irqchip_in_kernel(kvm))
2084                 return -ENXIO;
2085
2086         irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
2087                                         irq_event->irq, irq_event->level,
2088                                         line_status);
2089         return 0;
2090 }
2091
2092
2093 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
2094                                    struct kvm_enable_cap *cap)
2095 {
2096         int r;
2097
2098         if (cap->flags)
2099                 return -EINVAL;
2100
2101         switch (cap->cap) {
2102 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2103         case KVM_CAP_PPC_ENABLE_HCALL: {
2104                 unsigned long hcall = cap->args[0];
2105
2106                 r = -EINVAL;
2107                 if (hcall > MAX_HCALL_OPCODE || (hcall & 3) ||
2108                     cap->args[1] > 1)
2109                         break;
2110                 if (!kvmppc_book3s_hcall_implemented(kvm, hcall))
2111                         break;
2112                 if (cap->args[1])
2113                         set_bit(hcall / 4, kvm->arch.enabled_hcalls);
2114                 else
2115                         clear_bit(hcall / 4, kvm->arch.enabled_hcalls);
2116                 r = 0;
2117                 break;
2118         }
2119         case KVM_CAP_PPC_SMT: {
2120                 unsigned long mode = cap->args[0];
2121                 unsigned long flags = cap->args[1];
2122
2123                 r = -EINVAL;
2124                 if (kvm->arch.kvm_ops->set_smt_mode)
2125                         r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags);
2126                 break;
2127         }
2128 #endif
2129         default:
2130                 r = -EINVAL;
2131                 break;
2132         }
2133
2134         return r;
2135 }
2136
2137 #ifdef CONFIG_PPC_BOOK3S_64
2138 /*
2139  * These functions check whether the underlying hardware is safe
2140  * against attacks based on observing the effects of speculatively
2141  * executed instructions, and whether it supplies instructions for
2142  * use in workarounds.  The information comes from firmware, either
2143  * via the device tree on powernv platforms or from an hcall on
2144  * pseries platforms.
2145  */
2146 #ifdef CONFIG_PPC_PSERIES
2147 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2148 {
2149         struct h_cpu_char_result c;
2150         unsigned long rc;
2151
2152         if (!machine_is(pseries))
2153                 return -ENOTTY;
2154
2155         rc = plpar_get_cpu_characteristics(&c);
2156         if (rc == H_SUCCESS) {
2157                 cp->character = c.character;
2158                 cp->behaviour = c.behaviour;
2159                 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2160                         KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2161                         KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2162                         KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2163                         KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2164                         KVM_PPC_CPU_CHAR_BR_HINT_HONOURED |
2165                         KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF |
2166                         KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS;
2167                 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2168                         KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2169                         KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
2170         }
2171         return 0;
2172 }
2173 #else
2174 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2175 {
2176         return -ENOTTY;
2177 }
2178 #endif
2179
2180 static inline bool have_fw_feat(struct device_node *fw_features,
2181                                 const char *state, const char *name)
2182 {
2183         struct device_node *np;
2184         bool r = false;
2185
2186         np = of_get_child_by_name(fw_features, name);
2187         if (np) {
2188                 r = of_property_read_bool(np, state);
2189                 of_node_put(np);
2190         }
2191         return r;
2192 }
2193
2194 static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2195 {
2196         struct device_node *np, *fw_features;
2197         int r;
2198
2199         memset(cp, 0, sizeof(*cp));
2200         r = pseries_get_cpu_char(cp);
2201         if (r != -ENOTTY)
2202                 return r;
2203
2204         np = of_find_node_by_name(NULL, "ibm,opal");
2205         if (np) {
2206                 fw_features = of_get_child_by_name(np, "fw-features");
2207                 of_node_put(np);
2208                 if (!fw_features)
2209                         return 0;
2210                 if (have_fw_feat(fw_features, "enabled",
2211                                  "inst-spec-barrier-ori31,31,0"))
2212                         cp->character |= KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31;
2213                 if (have_fw_feat(fw_features, "enabled",
2214                                  "fw-bcctrl-serialized"))
2215                         cp->character |= KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED;
2216                 if (have_fw_feat(fw_features, "enabled",
2217                                  "inst-l1d-flush-ori30,30,0"))
2218                         cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30;
2219                 if (have_fw_feat(fw_features, "enabled",
2220                                  "inst-l1d-flush-trig2"))
2221                         cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2;
2222                 if (have_fw_feat(fw_features, "enabled",
2223                                  "fw-l1d-thread-split"))
2224                         cp->character |= KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV;
2225                 if (have_fw_feat(fw_features, "enabled",
2226                                  "fw-count-cache-disabled"))
2227                         cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS;
2228                 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2229                         KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2230                         KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2231                         KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2232                         KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2233                         KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS;
2234
2235                 if (have_fw_feat(fw_features, "enabled",
2236                                  "speculation-policy-favor-security"))
2237                         cp->behaviour |= KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY;
2238                 if (!have_fw_feat(fw_features, "disabled",
2239                                   "needs-l1d-flush-msr-pr-0-to-1"))
2240                         cp->behaviour |= KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR;
2241                 if (!have_fw_feat(fw_features, "disabled",
2242                                   "needs-spec-barrier-for-bound-checks"))
2243                         cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
2244                 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2245                         KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2246                         KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
2247
2248                 of_node_put(fw_features);
2249         }
2250
2251         return 0;
2252 }
2253 #endif
2254
2255 long kvm_arch_vm_ioctl(struct file *filp,
2256                        unsigned int ioctl, unsigned long arg)
2257 {
2258         struct kvm *kvm __maybe_unused = filp->private_data;
2259         void __user *argp = (void __user *)arg;
2260         long r;
2261
2262         switch (ioctl) {
2263         case KVM_PPC_GET_PVINFO: {
2264                 struct kvm_ppc_pvinfo pvinfo;
2265                 memset(&pvinfo, 0, sizeof(pvinfo));
2266                 r = kvm_vm_ioctl_get_pvinfo(&pvinfo);
2267                 if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) {
2268                         r = -EFAULT;
2269                         goto out;
2270                 }
2271
2272                 break;
2273         }
2274         case KVM_ENABLE_CAP:
2275         {
2276                 struct kvm_enable_cap cap;
2277                 r = -EFAULT;
2278                 if (copy_from_user(&cap, argp, sizeof(cap)))
2279                         goto out;
2280                 r = kvm_vm_ioctl_enable_cap(kvm, &cap);
2281                 break;
2282         }
2283 #ifdef CONFIG_SPAPR_TCE_IOMMU
2284         case KVM_CREATE_SPAPR_TCE_64: {
2285                 struct kvm_create_spapr_tce_64 create_tce_64;
2286
2287                 r = -EFAULT;
2288                 if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64)))
2289                         goto out;
2290                 if (create_tce_64.flags) {
2291                         r = -EINVAL;
2292                         goto out;
2293                 }
2294                 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2295                 goto out;
2296         }
2297         case KVM_CREATE_SPAPR_TCE: {
2298                 struct kvm_create_spapr_tce create_tce;
2299                 struct kvm_create_spapr_tce_64 create_tce_64;
2300
2301                 r = -EFAULT;
2302                 if (copy_from_user(&create_tce, argp, sizeof(create_tce)))
2303                         goto out;
2304
2305                 create_tce_64.liobn = create_tce.liobn;
2306                 create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K;
2307                 create_tce_64.offset = 0;
2308                 create_tce_64.size = create_tce.window_size >>
2309                                 IOMMU_PAGE_SHIFT_4K;
2310                 create_tce_64.flags = 0;
2311                 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2312                 goto out;
2313         }
2314 #endif
2315 #ifdef CONFIG_PPC_BOOK3S_64
2316         case KVM_PPC_GET_SMMU_INFO: {
2317                 struct kvm_ppc_smmu_info info;
2318                 struct kvm *kvm = filp->private_data;
2319
2320                 memset(&info, 0, sizeof(info));
2321                 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info);
2322                 if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2323                         r = -EFAULT;
2324                 break;
2325         }
2326         case KVM_PPC_RTAS_DEFINE_TOKEN: {
2327                 struct kvm *kvm = filp->private_data;
2328
2329                 r = kvm_vm_ioctl_rtas_define_token(kvm, argp);
2330                 break;
2331         }
2332         case KVM_PPC_CONFIGURE_V3_MMU: {
2333                 struct kvm *kvm = filp->private_data;
2334                 struct kvm_ppc_mmuv3_cfg cfg;
2335
2336                 r = -EINVAL;
2337                 if (!kvm->arch.kvm_ops->configure_mmu)
2338                         goto out;
2339                 r = -EFAULT;
2340                 if (copy_from_user(&cfg, argp, sizeof(cfg)))
2341                         goto out;
2342                 r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg);
2343                 break;
2344         }
2345         case KVM_PPC_GET_RMMU_INFO: {
2346                 struct kvm *kvm = filp->private_data;
2347                 struct kvm_ppc_rmmu_info info;
2348
2349                 r = -EINVAL;
2350                 if (!kvm->arch.kvm_ops->get_rmmu_info)
2351                         goto out;
2352                 r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info);
2353                 if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2354                         r = -EFAULT;
2355                 break;
2356         }
2357         case KVM_PPC_GET_CPU_CHAR: {
2358                 struct kvm_ppc_cpu_char cpuchar;
2359
2360                 r = kvmppc_get_cpu_char(&cpuchar);
2361                 if (r >= 0 && copy_to_user(argp, &cpuchar, sizeof(cpuchar)))
2362                         r = -EFAULT;
2363                 break;
2364         }
2365         default: {
2366                 struct kvm *kvm = filp->private_data;
2367                 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg);
2368         }
2369 #else /* CONFIG_PPC_BOOK3S_64 */
2370         default:
2371                 r = -ENOTTY;
2372 #endif
2373         }
2374 out:
2375         return r;
2376 }
2377
2378 static unsigned long lpid_inuse[BITS_TO_LONGS(KVMPPC_NR_LPIDS)];
2379 static unsigned long nr_lpids;
2380
2381 long kvmppc_alloc_lpid(void)
2382 {
2383         long lpid;
2384
2385         do {
2386                 lpid = find_first_zero_bit(lpid_inuse, KVMPPC_NR_LPIDS);
2387                 if (lpid >= nr_lpids) {
2388                         pr_err("%s: No LPIDs free\n", __func__);
2389                         return -ENOMEM;
2390                 }
2391         } while (test_and_set_bit(lpid, lpid_inuse));
2392
2393         return lpid;
2394 }
2395 EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid);
2396
2397 void kvmppc_claim_lpid(long lpid)
2398 {
2399         set_bit(lpid, lpid_inuse);
2400 }
2401 EXPORT_SYMBOL_GPL(kvmppc_claim_lpid);
2402
2403 void kvmppc_free_lpid(long lpid)
2404 {
2405         clear_bit(lpid, lpid_inuse);
2406 }
2407 EXPORT_SYMBOL_GPL(kvmppc_free_lpid);
2408
2409 void kvmppc_init_lpid(unsigned long nr_lpids_param)
2410 {
2411         nr_lpids = min_t(unsigned long, KVMPPC_NR_LPIDS, nr_lpids_param);
2412         memset(lpid_inuse, 0, sizeof(lpid_inuse));
2413 }
2414 EXPORT_SYMBOL_GPL(kvmppc_init_lpid);
2415
2416 int kvm_arch_init(void *opaque)
2417 {
2418         return 0;
2419 }
2420
2421 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr);