arch/x86/kvm/vmx.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "cpuid.h"
  22 #include "lapic.h"
  23
  24 #include <linux/kvm_host.h>
  25 #include <linux/module.h>
  26 #include <linux/kernel.h>
  27 #include <linux/mm.h>
  28 #include <linux/highmem.h>
  29 #include <linux/sched.h>
  30 #include <linux/sched/smt.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/mod_devicetable.h>
  33 #include <linux/trace_events.h>
  34 #include <linux/slab.h>
  35 #include <linux/tboot.h>
  36 #include <linux/hrtimer.h>
  37 #include <linux/frame.h>
  38 #include <linux/nospec.h>
  39 #include "kvm_cache_regs.h"
  40 #include "x86.h"
  41
  42 #include <asm/asm.h>
  43 #include <asm/cpu.h>
  44 #include <asm/io.h>
  45 #include <asm/desc.h>
  46 #include <asm/vmx.h>
  47 #include <asm/virtext.h>
  48 #include <asm/mce.h>
  49 #include <asm/fpu/internal.h>
  50 #include <asm/perf_event.h>
  51 #include <asm/debugreg.h>
  52 #include <asm/kexec.h>
  53 #include <asm/apic.h>
  54 #include <asm/irq_remapping.h>
  55 #include <asm/mmu_context.h>
  56 #include <asm/spec-ctrl.h>
  57 #include <asm/mshyperv.h>
  58
  59 #include "trace.h"
  60 #include "pmu.h"
  61 #include "vmx_evmcs.h"
  62
  63 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  64 #define __ex_clear(x, reg) \
  65         ____kvm_handle_fault_on_reboot(x, "xor " reg " , " reg)
  66
  67 MODULE_AUTHOR("Qumranet");
  68 MODULE_LICENSE("GPL");
  69
  70 static const struct x86_cpu_id vmx_cpu_id[] = {
  71         X86_FEATURE_MATCH(X86_FEATURE_VMX),
  72         {}
  73 };
  74 MODULE_DEVICE_TABLE(x86cpu, vmx_cpu_id);
  75
  76 static bool __read_mostly enable_vpid = 1;
  77 module_param_named(vpid, enable_vpid, bool, 0444);
  78
  79 static bool __read_mostly enable_vnmi = 1;
  80 module_param_named(vnmi, enable_vnmi, bool, S_IRUGO);
  81
  82 static bool __read_mostly flexpriority_enabled = 1;
  83 module_param_named(flexpriority, flexpriority_enabled, bool, S_IRUGO);
  84
  85 static bool __read_mostly enable_ept = 1;
  86 module_param_named(ept, enable_ept, bool, S_IRUGO);
  87
  88 static bool __read_mostly enable_unrestricted_guest = 1;
  89 module_param_named(unrestricted_guest,
  90                         enable_unrestricted_guest, bool, S_IRUGO);
  91
  92 static bool __read_mostly enable_ept_ad_bits = 1;
  93 module_param_named(eptad, enable_ept_ad_bits, bool, S_IRUGO);
  94
  95 static bool __read_mostly emulate_invalid_guest_state = true;
  96 module_param(emulate_invalid_guest_state, bool, S_IRUGO);
  97
  98 static bool __read_mostly fasteoi = 1;
  99 module_param(fasteoi, bool, S_IRUGO);
 100
 101 static bool __read_mostly enable_apicv = 1;
 102 module_param(enable_apicv, bool, S_IRUGO);
 103
 104 static bool __read_mostly enable_shadow_vmcs = 1;
 105 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
 106 /*
 107  * If nested=1, nested virtualization is supported, i.e., guests may use
 108  * VMX and be a hypervisor for its own guests. If nested=0, guests may not
 109  * use VMX instructions.
 110  */
 111 static bool __read_mostly nested = 0;
 112 module_param(nested, bool, S_IRUGO);
 113
 114 static u64 __read_mostly host_xss;
 115
 116 static bool __read_mostly enable_pml = 1;
 117 module_param_named(pml, enable_pml, bool, S_IRUGO);
 118
 119 #define MSR_TYPE_R      1
 120 #define MSR_TYPE_W      2
 121 #define MSR_TYPE_RW     3
 122
 123 #define MSR_BITMAP_MODE_X2APIC          1
 124 #define MSR_BITMAP_MODE_X2APIC_APICV    2
 125
 126 #define KVM_VMX_TSC_MULTIPLIER_MAX     0xffffffffffffffffULL
 127
 128 /* Guest_tsc -> host_tsc conversion requires 64-bit division.  */
 129 static int __read_mostly cpu_preemption_timer_multi;
 130 static bool __read_mostly enable_preemption_timer = 1;
 131 #ifdef CONFIG_X86_64
 132 module_param_named(preemption_timer, enable_preemption_timer, bool, S_IRUGO);
 133 #endif
 134
 135 #define KVM_GUEST_CR0_MASK (X86_CR0_NW | X86_CR0_CD)
 136 #define KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR0_NE
 137 #define KVM_VM_CR0_ALWAYS_ON                            \
 138         (KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST |      \
 139          X86_CR0_WP | X86_CR0_PG | X86_CR0_PE)
 140 #define KVM_CR4_GUEST_OWNED_BITS                                      \
 141         (X86_CR4_PVI | X86_CR4_DE | X86_CR4_PCE | X86_CR4_OSFXSR      \
 142          | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_TSD)
 143
 144 #define KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST X86_CR4_VMXE
 145 #define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE)
 146 #define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE)
 147
 148 #define RMODE_GUEST_OWNED_EFLAGS_BITS (~(X86_EFLAGS_IOPL | X86_EFLAGS_VM))
 149
 150 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
 151
 152 /*
 153  * Hyper-V requires all of these, so mark them as supported even though
 154  * they are just treated the same as all-context.
 155  */
 156 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
 157         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
 158         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
 159         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
 160         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
 161
 162 /*
 163  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 164  * ple_gap:    upper bound on the amount of time between two successive
 165  *             executions of PAUSE in a loop. Also indicate if ple enabled.
 166  *             According to test, this time is usually smaller than 128 cycles.
 167  * ple_window: upper bound on the amount of time a guest is allowed to execute
 168  *             in a PAUSE loop. Tests indicate that most spinlocks are held for
 169  *             less than 2^12 cycles
 170  * Time is measured based on a counter that runs at the same rate as the TSC,
 171  * refer SDM volume 3b section 21.6.13 & 22.1.3.
 172  */
 173 static unsigned int ple_gap = KVM_DEFAULT_PLE_GAP;
 174 module_param(ple_gap, uint, 0444);
 175
 176 static unsigned int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
 177 module_param(ple_window, uint, 0444);
 178
 179 /* Default doubles per-vcpu window every exit. */
 180 static unsigned int ple_window_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 181 module_param(ple_window_grow, uint, 0444);
 182
 183 /* Default resets per-vcpu window every exit to ple_window. */
 184 static unsigned int ple_window_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 185 module_param(ple_window_shrink, uint, 0444);
 186
 187 /* Default is to compute the maximum so we can never overflow. */
 188 static unsigned int ple_window_max        = KVM_VMX_DEFAULT_PLE_WINDOW_MAX;
 189 module_param(ple_window_max, uint, 0444);
 190
 191 extern const ulong vmx_return;
 192
 193 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_should_flush);
 194 static DEFINE_STATIC_KEY_FALSE(vmx_l1d_flush_cond);
 195 static DEFINE_MUTEX(vmx_l1d_flush_mutex);
 196
 197 /* Storage for pre module init parameter parsing */
 198 static enum vmx_l1d_flush_state __read_mostly vmentry_l1d_flush_param = VMENTER_L1D_FLUSH_AUTO;
 199
 200 static const struct {
 201         const char *option;
 202         bool for_parse;
 203 } vmentry_l1d_param[] = {
 204         [VMENTER_L1D_FLUSH_AUTO]         = {"auto", true},
 205         [VMENTER_L1D_FLUSH_NEVER]        = {"never", true},
 206         [VMENTER_L1D_FLUSH_COND]         = {"cond", true},
 207         [VMENTER_L1D_FLUSH_ALWAYS]       = {"always", true},
 208         [VMENTER_L1D_FLUSH_EPT_DISABLED] = {"EPT disabled", false},
 209         [VMENTER_L1D_FLUSH_NOT_REQUIRED] = {"not required", false},
 210 };
 211
 212 #define L1D_CACHE_ORDER 4
 213 static void *vmx_l1d_flush_pages;
 214
 215 /* Control for disabling CPU Fill buffer clear */
 216 static bool __read_mostly vmx_fb_clear_ctrl_available;
 217
 218 static int vmx_setup_l1d_flush(enum vmx_l1d_flush_state l1tf)
 219 {
 220         struct page *page;
 221         unsigned int i;
 222
 223         if (!enable_ept) {
 224                 l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_EPT_DISABLED;
 225                 return 0;
 226         }
 227
 228         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES)) {
 229                 u64 msr;
 230
 231                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
 232                 if (msr & ARCH_CAP_SKIP_VMENTRY_L1DFLUSH) {
 233                         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_NOT_REQUIRED;
 234                         return 0;
 235                 }
 236         }
 237
 238         /* If set to auto use the default l1tf mitigation method */
 239         if (l1tf == VMENTER_L1D_FLUSH_AUTO) {
 240                 switch (l1tf_mitigation) {
 241                 case L1TF_MITIGATION_OFF:
 242                         l1tf = VMENTER_L1D_FLUSH_NEVER;
 243                         break;
 244                 case L1TF_MITIGATION_FLUSH_NOWARN:
 245                 case L1TF_MITIGATION_FLUSH:
 246                 case L1TF_MITIGATION_FLUSH_NOSMT:
 247                         l1tf = VMENTER_L1D_FLUSH_COND;
 248                         break;
 249                 case L1TF_MITIGATION_FULL:
 250                 case L1TF_MITIGATION_FULL_FORCE:
 251                         l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 252                         break;
 253                 }
 254         } else if (l1tf_mitigation == L1TF_MITIGATION_FULL_FORCE) {
 255                 l1tf = VMENTER_L1D_FLUSH_ALWAYS;
 256         }
 257
 258         if (l1tf != VMENTER_L1D_FLUSH_NEVER && !vmx_l1d_flush_pages &&
 259             !boot_cpu_has(X86_FEATURE_FLUSH_L1D)) {
 260                 page = alloc_pages(GFP_KERNEL, L1D_CACHE_ORDER);
 261                 if (!page)
 262                         return -ENOMEM;
 263                 vmx_l1d_flush_pages = page_address(page);
 264
 265                 /*
 266                  * Initialize each page with a different pattern in
 267                  * order to protect against KSM in the nested
 268                  * virtualization case.
 269                  */
 270                 for (i = 0; i < 1u << L1D_CACHE_ORDER; ++i) {
 271                         memset(vmx_l1d_flush_pages + i * PAGE_SIZE, i + 1,
 272                                PAGE_SIZE);
 273                 }
 274         }
 275
 276         l1tf_vmx_mitigation = l1tf;
 277
 278         if (l1tf != VMENTER_L1D_FLUSH_NEVER)
 279                 static_branch_enable(&vmx_l1d_should_flush);
 280         else
 281                 static_branch_disable(&vmx_l1d_should_flush);
 282
 283         if (l1tf == VMENTER_L1D_FLUSH_COND)
 284                 static_branch_enable(&vmx_l1d_flush_cond);
 285         else
 286                 static_branch_disable(&vmx_l1d_flush_cond);
 287         return 0;
 288 }
 289
 290 static int vmentry_l1d_flush_parse(const char *s)
 291 {
 292         unsigned int i;
 293
 294         if (s) {
 295                 for (i = 0; i < ARRAY_SIZE(vmentry_l1d_param); i++) {
 296                         if (vmentry_l1d_param[i].for_parse &&
 297                             sysfs_streq(s, vmentry_l1d_param[i].option))
 298                                 return i;
 299                 }
 300         }
 301         return -EINVAL;
 302 }
 303
 304 static int vmentry_l1d_flush_set(const char *s, const struct kernel_param *kp)
 305 {
 306         int l1tf, ret;
 307
 308         l1tf = vmentry_l1d_flush_parse(s);
 309         if (l1tf < 0)
 310                 return l1tf;
 311
 312         if (!boot_cpu_has(X86_BUG_L1TF))
 313                 return 0;
 314
 315         /*
 316          * Has vmx_init() run already? If not then this is the pre init
 317          * parameter parsing. In that case just store the value and let
 318          * vmx_init() do the proper setup after enable_ept has been
 319          * established.
 320          */
 321         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO) {
 322                 vmentry_l1d_flush_param = l1tf;
 323                 return 0;
 324         }
 325
 326         mutex_lock(&vmx_l1d_flush_mutex);
 327         ret = vmx_setup_l1d_flush(l1tf);
 328         mutex_unlock(&vmx_l1d_flush_mutex);
 329         return ret;
 330 }
 331
 332 static int vmentry_l1d_flush_get(char *s, const struct kernel_param *kp)
 333 {
 334         if (WARN_ON_ONCE(l1tf_vmx_mitigation >= ARRAY_SIZE(vmentry_l1d_param)))
 335                 return sprintf(s, "???\n");
 336
 337         return sprintf(s, "%s\n", vmentry_l1d_param[l1tf_vmx_mitigation].option);
 338 }
 339
 340 static const struct kernel_param_ops vmentry_l1d_flush_ops = {
 341         .set = vmentry_l1d_flush_set,
 342         .get = vmentry_l1d_flush_get,
 343 };
 344 module_param_cb(vmentry_l1d_flush, &vmentry_l1d_flush_ops, NULL, 0644);
 345
 346 enum ept_pointers_status {
 347         EPT_POINTERS_CHECK = 0,
 348         EPT_POINTERS_MATCH = 1,
 349         EPT_POINTERS_MISMATCH = 2
 350 };
 351
 352 struct kvm_vmx {
 353         struct kvm kvm;
 354
 355         unsigned int tss_addr;
 356         bool ept_identity_pagetable_done;
 357         gpa_t ept_identity_map_addr;
 358
 359         enum ept_pointers_status ept_pointers_match;
 360         spinlock_t ept_pointer_lock;
 361 };
 362
 363 #define NR_AUTOLOAD_MSRS 8
 364
 365 struct vmcs_hdr {
 366         u32 revision_id:31;
 367         u32 shadow_vmcs:1;
 368 };
 369
 370 struct vmcs {
 371         struct vmcs_hdr hdr;
 372         u32 abort;
 373         char data[0];
 374 };
 375
 376 /*
 377  * vmcs_host_state tracks registers that are loaded from the VMCS on VMEXIT
 378  * and whose values change infrequently, but are not constant.  I.e. this is
 379  * used as a write-through cache of the corresponding VMCS fields.
 380  */
 381 struct vmcs_host_state {
 382         unsigned long cr3;      /* May not match real cr3 */
 383         unsigned long cr4;      /* May not match real cr4 */
 384         unsigned long gs_base;
 385         unsigned long fs_base;
 386
 387         u16           fs_sel, gs_sel, ldt_sel;
 388 #ifdef CONFIG_X86_64
 389         u16           ds_sel, es_sel;
 390 #endif
 391 };
 392
 393 /*
 394  * Track a VMCS that may be loaded on a certain CPU. If it is (cpu!=-1), also
 395  * remember whether it was VMLAUNCHed, and maintain a linked list of all VMCSs
 396  * loaded on this CPU (so we can clear them if the CPU goes down).
 397  */
 398 struct loaded_vmcs {
 399         struct vmcs *vmcs;
 400         struct vmcs *shadow_vmcs;
 401         int cpu;
 402         bool launched;
 403         bool nmi_known_unmasked;
 404         bool hv_timer_armed;
 405         /* Support for vnmi-less CPUs */
 406         int soft_vnmi_blocked;
 407         ktime_t entry_time;
 408         s64 vnmi_blocked_time;
 409         unsigned long *msr_bitmap;
 410         struct list_head loaded_vmcss_on_cpu_link;
 411         struct vmcs_host_state host_state;
 412 };
 413
 414 struct shared_msr_entry {
 415         unsigned index;
 416         u64 data;
 417         u64 mask;
 418 };
 419
 420 /*
 421  * struct vmcs12 describes the state that our guest hypervisor (L1) keeps for a
 422  * single nested guest (L2), hence the name vmcs12. Any VMX implementation has
 423  * a VMCS structure, and vmcs12 is our emulated VMX's VMCS. This structure is
 424  * stored in guest memory specified by VMPTRLD, but is opaque to the guest,
 425  * which must access it using VMREAD/VMWRITE/VMCLEAR instructions.
 426  * More than one of these structures may exist, if L1 runs multiple L2 guests.
 427  * nested_vmx_run() will use the data here to build the vmcs02: a VMCS for the
 428  * underlying hardware which will be used to run L2.
 429  * This structure is packed to ensure that its layout is identical across
 430  * machines (necessary for live migration).
 431  *
 432  * IMPORTANT: Changing the layout of existing fields in this structure
 433  * will break save/restore compatibility with older kvm releases. When
 434  * adding new fields, either use space in the reserved padding* arrays
 435  * or add the new fields to the end of the structure.
 436  */
 437 typedef u64 natural_width;
 438 struct __packed vmcs12 {
 439         /* According to the Intel spec, a VMCS region must start with the
 440          * following two fields. Then follow implementation-specific data.
 441          */
 442         struct vmcs_hdr hdr;
 443         u32 abort;
 444
 445         u32 launch_state; /* set to 0 by VMCLEAR, to 1 by VMLAUNCH */
 446         u32 padding[7]; /* room for future expansion */
 447
 448         u64 io_bitmap_a;
 449         u64 io_bitmap_b;
 450         u64 msr_bitmap;
 451         u64 vm_exit_msr_store_addr;
 452         u64 vm_exit_msr_load_addr;
 453         u64 vm_entry_msr_load_addr;
 454         u64 tsc_offset;
 455         u64 virtual_apic_page_addr;
 456         u64 apic_access_addr;
 457         u64 posted_intr_desc_addr;
 458         u64 ept_pointer;
 459         u64 eoi_exit_bitmap0;
 460         u64 eoi_exit_bitmap1;
 461         u64 eoi_exit_bitmap2;
 462         u64 eoi_exit_bitmap3;
 463         u64 xss_exit_bitmap;
 464         u64 guest_physical_address;
 465         u64 vmcs_link_pointer;
 466         u64 guest_ia32_debugctl;
 467         u64 guest_ia32_pat;
 468         u64 guest_ia32_efer;
 469         u64 guest_ia32_perf_global_ctrl;
 470         u64 guest_pdptr0;
 471         u64 guest_pdptr1;
 472         u64 guest_pdptr2;
 473         u64 guest_pdptr3;
 474         u64 guest_bndcfgs;
 475         u64 host_ia32_pat;
 476         u64 host_ia32_efer;
 477         u64 host_ia32_perf_global_ctrl;
 478         u64 vmread_bitmap;
 479         u64 vmwrite_bitmap;
 480         u64 vm_function_control;
 481         u64 eptp_list_address;
 482         u64 pml_address;
 483         u64 padding64[3]; /* room for future expansion */
 484         /*
 485          * To allow migration of L1 (complete with its L2 guests) between
 486          * machines of different natural widths (32 or 64 bit), we cannot have
 487          * unsigned long fields with no explict size. We use u64 (aliased
 488          * natural_width) instead. Luckily, x86 is little-endian.
 489          */
 490         natural_width cr0_guest_host_mask;
 491         natural_width cr4_guest_host_mask;
 492         natural_width cr0_read_shadow;
 493         natural_width cr4_read_shadow;
 494         natural_width cr3_target_value0;
 495         natural_width cr3_target_value1;
 496         natural_width cr3_target_value2;
 497         natural_width cr3_target_value3;
 498         natural_width exit_qualification;
 499         natural_width guest_linear_address;
 500         natural_width guest_cr0;
 501         natural_width guest_cr3;
 502         natural_width guest_cr4;
 503         natural_width guest_es_base;
 504         natural_width guest_cs_base;
 505         natural_width guest_ss_base;
 506         natural_width guest_ds_base;
 507         natural_width guest_fs_base;
 508         natural_width guest_gs_base;
 509         natural_width guest_ldtr_base;
 510         natural_width guest_tr_base;
 511         natural_width guest_gdtr_base;
 512         natural_width guest_idtr_base;
 513         natural_width guest_dr7;
 514         natural_width guest_rsp;
 515         natural_width guest_rip;
 516         natural_width guest_rflags;
 517         natural_width guest_pending_dbg_exceptions;
 518         natural_width guest_sysenter_esp;
 519         natural_width guest_sysenter_eip;
 520         natural_width host_cr0;
 521         natural_width host_cr3;
 522         natural_width host_cr4;
 523         natural_width host_fs_base;
 524         natural_width host_gs_base;
 525         natural_width host_tr_base;
 526         natural_width host_gdtr_base;
 527         natural_width host_idtr_base;
 528         natural_width host_ia32_sysenter_esp;
 529         natural_width host_ia32_sysenter_eip;
 530         natural_width host_rsp;
 531         natural_width host_rip;
 532         natural_width paddingl[8]; /* room for future expansion */
 533         u32 pin_based_vm_exec_control;
 534         u32 cpu_based_vm_exec_control;
 535         u32 exception_bitmap;
 536         u32 page_fault_error_code_mask;
 537         u32 page_fault_error_code_match;
 538         u32 cr3_target_count;
 539         u32 vm_exit_controls;
 540         u32 vm_exit_msr_store_count;
 541         u32 vm_exit_msr_load_count;
 542         u32 vm_entry_controls;
 543         u32 vm_entry_msr_load_count;
 544         u32 vm_entry_intr_info_field;
 545         u32 vm_entry_exception_error_code;
 546         u32 vm_entry_instruction_len;
 547         u32 tpr_threshold;
 548         u32 secondary_vm_exec_control;
 549         u32 vm_instruction_error;
 550         u32 vm_exit_reason;
 551         u32 vm_exit_intr_info;
 552         u32 vm_exit_intr_error_code;
 553         u32 idt_vectoring_info_field;
 554         u32 idt_vectoring_error_code;
 555         u32 vm_exit_instruction_len;
 556         u32 vmx_instruction_info;
 557         u32 guest_es_limit;
 558         u32 guest_cs_limit;
 559         u32 guest_ss_limit;
 560         u32 guest_ds_limit;
 561         u32 guest_fs_limit;
 562         u32 guest_gs_limit;
 563         u32 guest_ldtr_limit;
 564         u32 guest_tr_limit;
 565         u32 guest_gdtr_limit;
 566         u32 guest_idtr_limit;
 567         u32 guest_es_ar_bytes;
 568         u32 guest_cs_ar_bytes;
 569         u32 guest_ss_ar_bytes;
 570         u32 guest_ds_ar_bytes;
 571         u32 guest_fs_ar_bytes;
 572         u32 guest_gs_ar_bytes;
 573         u32 guest_ldtr_ar_bytes;
 574         u32 guest_tr_ar_bytes;
 575         u32 guest_interruptibility_info;
 576         u32 guest_activity_state;
 577         u32 guest_sysenter_cs;
 578         u32 host_ia32_sysenter_cs;
 579         u32 vmx_preemption_timer_value;
 580         u32 padding32[7]; /* room for future expansion */
 581         u16 virtual_processor_id;
 582         u16 posted_intr_nv;
 583         u16 guest_es_selector;
 584         u16 guest_cs_selector;
 585         u16 guest_ss_selector;
 586         u16 guest_ds_selector;
 587         u16 guest_fs_selector;
 588         u16 guest_gs_selector;
 589         u16 guest_ldtr_selector;
 590         u16 guest_tr_selector;
 591         u16 guest_intr_status;
 592         u16 host_es_selector;
 593         u16 host_cs_selector;
 594         u16 host_ss_selector;
 595         u16 host_ds_selector;
 596         u16 host_fs_selector;
 597         u16 host_gs_selector;
 598         u16 host_tr_selector;
 599         u16 guest_pml_index;
 600 };
 601
 602 /*
 603  * For save/restore compatibility, the vmcs12 field offsets must not change.
 604  */
 605 #define CHECK_OFFSET(field, loc)                                \
 606         BUILD_BUG_ON_MSG(offsetof(struct vmcs12, field) != (loc),       \
 607                 "Offset of " #field " in struct vmcs12 has changed.")
 608
 609 static inline void vmx_check_vmcs12_offsets(void) {
 610         CHECK_OFFSET(hdr, 0);
 611         CHECK_OFFSET(abort, 4);
 612         CHECK_OFFSET(launch_state, 8);
 613         CHECK_OFFSET(io_bitmap_a, 40);
 614         CHECK_OFFSET(io_bitmap_b, 48);
 615         CHECK_OFFSET(msr_bitmap, 56);
 616         CHECK_OFFSET(vm_exit_msr_store_addr, 64);
 617         CHECK_OFFSET(vm_exit_msr_load_addr, 72);
 618         CHECK_OFFSET(vm_entry_msr_load_addr, 80);
 619         CHECK_OFFSET(tsc_offset, 88);
 620         CHECK_OFFSET(virtual_apic_page_addr, 96);
 621         CHECK_OFFSET(apic_access_addr, 104);
 622         CHECK_OFFSET(posted_intr_desc_addr, 112);
 623         CHECK_OFFSET(ept_pointer, 120);
 624         CHECK_OFFSET(eoi_exit_bitmap0, 128);
 625         CHECK_OFFSET(eoi_exit_bitmap1, 136);
 626         CHECK_OFFSET(eoi_exit_bitmap2, 144);
 627         CHECK_OFFSET(eoi_exit_bitmap3, 152);
 628         CHECK_OFFSET(xss_exit_bitmap, 160);
 629         CHECK_OFFSET(guest_physical_address, 168);
 630         CHECK_OFFSET(vmcs_link_pointer, 176);
 631         CHECK_OFFSET(guest_ia32_debugctl, 184);
 632         CHECK_OFFSET(guest_ia32_pat, 192);
 633         CHECK_OFFSET(guest_ia32_efer, 200);
 634         CHECK_OFFSET(guest_ia32_perf_global_ctrl, 208);
 635         CHECK_OFFSET(guest_pdptr0, 216);
 636         CHECK_OFFSET(guest_pdptr1, 224);
 637         CHECK_OFFSET(guest_pdptr2, 232);
 638         CHECK_OFFSET(guest_pdptr3, 240);
 639         CHECK_OFFSET(guest_bndcfgs, 248);
 640         CHECK_OFFSET(host_ia32_pat, 256);
 641         CHECK_OFFSET(host_ia32_efer, 264);
 642         CHECK_OFFSET(host_ia32_perf_global_ctrl, 272);
 643         CHECK_OFFSET(vmread_bitmap, 280);
 644         CHECK_OFFSET(vmwrite_bitmap, 288);
 645         CHECK_OFFSET(vm_function_control, 296);
 646         CHECK_OFFSET(eptp_list_address, 304);
 647         CHECK_OFFSET(pml_address, 312);
 648         CHECK_OFFSET(cr0_guest_host_mask, 344);
 649         CHECK_OFFSET(cr4_guest_host_mask, 352);
 650         CHECK_OFFSET(cr0_read_shadow, 360);
 651         CHECK_OFFSET(cr4_read_shadow, 368);
 652         CHECK_OFFSET(cr3_target_value0, 376);
 653         CHECK_OFFSET(cr3_target_value1, 384);
 654         CHECK_OFFSET(cr3_target_value2, 392);
 655         CHECK_OFFSET(cr3_target_value3, 400);
 656         CHECK_OFFSET(exit_qualification, 408);
 657         CHECK_OFFSET(guest_linear_address, 416);
 658         CHECK_OFFSET(guest_cr0, 424);
 659         CHECK_OFFSET(guest_cr3, 432);
 660         CHECK_OFFSET(guest_cr4, 440);
 661         CHECK_OFFSET(guest_es_base, 448);
 662         CHECK_OFFSET(guest_cs_base, 456);
 663         CHECK_OFFSET(guest_ss_base, 464);
 664         CHECK_OFFSET(guest_ds_base, 472);
 665         CHECK_OFFSET(guest_fs_base, 480);
 666         CHECK_OFFSET(guest_gs_base, 488);
 667         CHECK_OFFSET(guest_ldtr_base, 496);
 668         CHECK_OFFSET(guest_tr_base, 504);
 669         CHECK_OFFSET(guest_gdtr_base, 512);
 670         CHECK_OFFSET(guest_idtr_base, 520);
 671         CHECK_OFFSET(guest_dr7, 528);
 672         CHECK_OFFSET(guest_rsp, 536);
 673         CHECK_OFFSET(guest_rip, 544);
 674         CHECK_OFFSET(guest_rflags, 552);
 675         CHECK_OFFSET(guest_pending_dbg_exceptions, 560);
 676         CHECK_OFFSET(guest_sysenter_esp, 568);
 677         CHECK_OFFSET(guest_sysenter_eip, 576);
 678         CHECK_OFFSET(host_cr0, 584);
 679         CHECK_OFFSET(host_cr3, 592);
 680         CHECK_OFFSET(host_cr4, 600);
 681         CHECK_OFFSET(host_fs_base, 608);
 682         CHECK_OFFSET(host_gs_base, 616);
 683         CHECK_OFFSET(host_tr_base, 624);
 684         CHECK_OFFSET(host_gdtr_base, 632);
 685         CHECK_OFFSET(host_idtr_base, 640);
 686         CHECK_OFFSET(host_ia32_sysenter_esp, 648);
 687         CHECK_OFFSET(host_ia32_sysenter_eip, 656);
 688         CHECK_OFFSET(host_rsp, 664);
 689         CHECK_OFFSET(host_rip, 672);
 690         CHECK_OFFSET(pin_based_vm_exec_control, 744);
 691         CHECK_OFFSET(cpu_based_vm_exec_control, 748);
 692         CHECK_OFFSET(exception_bitmap, 752);
 693         CHECK_OFFSET(page_fault_error_code_mask, 756);
 694         CHECK_OFFSET(page_fault_error_code_match, 760);
 695         CHECK_OFFSET(cr3_target_count, 764);
 696         CHECK_OFFSET(vm_exit_controls, 768);
 697         CHECK_OFFSET(vm_exit_msr_store_count, 772);
 698         CHECK_OFFSET(vm_exit_msr_load_count, 776);
 699         CHECK_OFFSET(vm_entry_controls, 780);
 700         CHECK_OFFSET(vm_entry_msr_load_count, 784);
 701         CHECK_OFFSET(vm_entry_intr_info_field, 788);
 702         CHECK_OFFSET(vm_entry_exception_error_code, 792);
 703         CHECK_OFFSET(vm_entry_instruction_len, 796);
 704         CHECK_OFFSET(tpr_threshold, 800);
 705         CHECK_OFFSET(secondary_vm_exec_control, 804);
 706         CHECK_OFFSET(vm_instruction_error, 808);
 707         CHECK_OFFSET(vm_exit_reason, 812);
 708         CHECK_OFFSET(vm_exit_intr_info, 816);
 709         CHECK_OFFSET(vm_exit_intr_error_code, 820);
 710         CHECK_OFFSET(idt_vectoring_info_field, 824);
 711         CHECK_OFFSET(idt_vectoring_error_code, 828);
 712         CHECK_OFFSET(vm_exit_instruction_len, 832);
 713         CHECK_OFFSET(vmx_instruction_info, 836);
 714         CHECK_OFFSET(guest_es_limit, 840);
 715         CHECK_OFFSET(guest_cs_limit, 844);
 716         CHECK_OFFSET(guest_ss_limit, 848);
 717         CHECK_OFFSET(guest_ds_limit, 852);
 718         CHECK_OFFSET(guest_fs_limit, 856);
 719         CHECK_OFFSET(guest_gs_limit, 860);
 720         CHECK_OFFSET(guest_ldtr_limit, 864);
 721         CHECK_OFFSET(guest_tr_limit, 868);
 722         CHECK_OFFSET(guest_gdtr_limit, 872);
 723         CHECK_OFFSET(guest_idtr_limit, 876);
 724         CHECK_OFFSET(guest_es_ar_bytes, 880);
 725         CHECK_OFFSET(guest_cs_ar_bytes, 884);
 726         CHECK_OFFSET(guest_ss_ar_bytes, 888);
 727         CHECK_OFFSET(guest_ds_ar_bytes, 892);
 728         CHECK_OFFSET(guest_fs_ar_bytes, 896);
 729         CHECK_OFFSET(guest_gs_ar_bytes, 900);
 730         CHECK_OFFSET(guest_ldtr_ar_bytes, 904);
 731         CHECK_OFFSET(guest_tr_ar_bytes, 908);
 732         CHECK_OFFSET(guest_interruptibility_info, 912);
 733         CHECK_OFFSET(guest_activity_state, 916);
 734         CHECK_OFFSET(guest_sysenter_cs, 920);
 735         CHECK_OFFSET(host_ia32_sysenter_cs, 924);
 736         CHECK_OFFSET(vmx_preemption_timer_value, 928);
 737         CHECK_OFFSET(virtual_processor_id, 960);
 738         CHECK_OFFSET(posted_intr_nv, 962);
 739         CHECK_OFFSET(guest_es_selector, 964);
 740         CHECK_OFFSET(guest_cs_selector, 966);
 741         CHECK_OFFSET(guest_ss_selector, 968);
 742         CHECK_OFFSET(guest_ds_selector, 970);
 743         CHECK_OFFSET(guest_fs_selector, 972);
 744         CHECK_OFFSET(guest_gs_selector, 974);
 745         CHECK_OFFSET(guest_ldtr_selector, 976);
 746         CHECK_OFFSET(guest_tr_selector, 978);
 747         CHECK_OFFSET(guest_intr_status, 980);
 748         CHECK_OFFSET(host_es_selector, 982);
 749         CHECK_OFFSET(host_cs_selector, 984);
 750         CHECK_OFFSET(host_ss_selector, 986);
 751         CHECK_OFFSET(host_ds_selector, 988);
 752         CHECK_OFFSET(host_fs_selector, 990);
 753         CHECK_OFFSET(host_gs_selector, 992);
 754         CHECK_OFFSET(host_tr_selector, 994);
 755         CHECK_OFFSET(guest_pml_index, 996);
 756 }
 757
 758 /*
 759  * VMCS12_REVISION is an arbitrary id that should be changed if the content or
 760  * layout of struct vmcs12 is changed. MSR_IA32_VMX_BASIC returns this id, and
 761  * VMPTRLD verifies that the VMCS region that L1 is loading contains this id.
 762  *
 763  * IMPORTANT: Changing this value will break save/restore compatibility with
 764  * older kvm releases.
 765  */
 766 #define VMCS12_REVISION 0x11e57ed0
 767
 768 /*
 769  * VMCS12_SIZE is the number of bytes L1 should allocate for the VMXON region
 770  * and any VMCS region. Although only sizeof(struct vmcs12) are used by the
 771  * current implementation, 4K are reserved to avoid future complications.
 772  */
 773 #define VMCS12_SIZE 0x1000
 774
 775 /*
 776  * VMCS12_MAX_FIELD_INDEX is the highest index value used in any
 777  * supported VMCS12 field encoding.
 778  */
 779 #define VMCS12_MAX_FIELD_INDEX 0x17
 780
 781 struct nested_vmx_msrs {
 782         /*
 783          * We only store the "true" versions of the VMX capability MSRs. We
 784          * generate the "non-true" versions by setting the must-be-1 bits
 785          * according to the SDM.
 786          */
 787         u32 procbased_ctls_low;
 788         u32 procbased_ctls_high;
 789         u32 secondary_ctls_low;
 790         u32 secondary_ctls_high;
 791         u32 pinbased_ctls_low;
 792         u32 pinbased_ctls_high;
 793         u32 exit_ctls_low;
 794         u32 exit_ctls_high;
 795         u32 entry_ctls_low;
 796         u32 entry_ctls_high;
 797         u32 misc_low;
 798         u32 misc_high;
 799         u32 ept_caps;
 800         u32 vpid_caps;
 801         u64 basic;
 802         u64 cr0_fixed0;
 803         u64 cr0_fixed1;
 804         u64 cr4_fixed0;
 805         u64 cr4_fixed1;
 806         u64 vmcs_enum;
 807         u64 vmfunc_controls;
 808 };
 809
 810 /*
 811  * The nested_vmx structure is part of vcpu_vmx, and holds information we need
 812  * for correct emulation of VMX (i.e., nested VMX) on this vcpu.
 813  */
 814 struct nested_vmx {
 815         /* Has the level1 guest done vmxon? */
 816         bool vmxon;
 817         gpa_t vmxon_ptr;
 818         bool pml_full;
 819
 820         /* The guest-physical address of the current VMCS L1 keeps for L2 */
 821         gpa_t current_vmptr;
 822         /*
 823          * Cache of the guest's VMCS, existing outside of guest memory.
 824          * Loaded from guest memory during VMPTRLD. Flushed to guest
 825          * memory during VMCLEAR and VMPTRLD.
 826          */
 827         struct vmcs12 *cached_vmcs12;
 828         /*
 829          * Cache of the guest's shadow VMCS, existing outside of guest
 830          * memory. Loaded from guest memory during VM entry. Flushed
 831          * to guest memory during VM exit.
 832          */
 833         struct vmcs12 *cached_shadow_vmcs12;
 834         /*
 835          * Indicates if the shadow vmcs must be updated with the
 836          * data hold by vmcs12
 837          */
 838         bool sync_shadow_vmcs;
 839         bool dirty_vmcs12;
 840
 841         bool change_vmcs01_virtual_apic_mode;
 842
 843         /* L2 must run next, and mustn't decide to exit to L1. */
 844         bool nested_run_pending;
 845
 846         struct loaded_vmcs vmcs02;
 847
 848         /*
 849          * Guest pages referred to in the vmcs02 with host-physical
 850          * pointers, so we must keep them pinned while L2 runs.
 851          */
 852         struct page *apic_access_page;
 853         struct page *virtual_apic_page;
 854         struct page *pi_desc_page;
 855         struct pi_desc *pi_desc;
 856         bool pi_pending;
 857         u16 posted_intr_nv;
 858
 859         struct hrtimer preemption_timer;
 860         bool preemption_timer_expired;
 861
 862         /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */
 863         u64 vmcs01_debugctl;
 864         u64 vmcs01_guest_bndcfgs;
 865
 866         u16 vpid02;
 867         u16 last_vpid;
 868
 869         struct nested_vmx_msrs msrs;
 870
 871         /* SMM related state */
 872         struct {
 873                 /* in VMX operation on SMM entry? */
 874                 bool vmxon;
 875                 /* in guest mode on SMM entry? */
 876                 bool guest_mode;
 877         } smm;
 878 };
 879
 880 #define POSTED_INTR_ON  0
 881 #define POSTED_INTR_SN  1
 882
 883 /* Posted-Interrupt Descriptor */
 884 struct pi_desc {
 885         u32 pir[8];     /* Posted interrupt requested */
 886         union {
 887                 struct {
 888                                 /* bit 256 - Outstanding Notification */
 889                         u16     on      : 1,
 890                                 /* bit 257 - Suppress Notification */
 891                                 sn      : 1,
 892                                 /* bit 271:258 - Reserved */
 893                                 rsvd_1  : 14;
 894                                 /* bit 279:272 - Notification Vector */
 895                         u8      nv;
 896                                 /* bit 287:280 - Reserved */
 897                         u8      rsvd_2;
 898                                 /* bit 319:288 - Notification Destination */
 899                         u32     ndst;
 900                 };
 901                 u64 control;
 902         };
 903         u32 rsvd[6];
 904 } __aligned(64);
 905
 906 static bool pi_test_and_set_on(struct pi_desc *pi_desc)
 907 {
 908         return test_and_set_bit(POSTED_INTR_ON,
 909                         (unsigned long *)&pi_desc->control);
 910 }
 911
 912 static bool pi_test_and_clear_on(struct pi_desc *pi_desc)
 913 {
 914         return test_and_clear_bit(POSTED_INTR_ON,
 915                         (unsigned long *)&pi_desc->control);
 916 }
 917
 918 static int pi_test_and_set_pir(int vector, struct pi_desc *pi_desc)
 919 {
 920         return test_and_set_bit(vector, (unsigned long *)pi_desc->pir);
 921 }
 922
 923 static inline void pi_clear_sn(struct pi_desc *pi_desc)
 924 {
 925         return clear_bit(POSTED_INTR_SN,
 926                         (unsigned long *)&pi_desc->control);
 927 }
 928
 929 static inline void pi_set_sn(struct pi_desc *pi_desc)
 930 {
 931         return set_bit(POSTED_INTR_SN,
 932                         (unsigned long *)&pi_desc->control);
 933 }
 934
 935 static inline void pi_clear_on(struct pi_desc *pi_desc)
 936 {
 937         clear_bit(POSTED_INTR_ON,
 938                   (unsigned long *)&pi_desc->control);
 939 }
 940
 941 static inline int pi_test_on(struct pi_desc *pi_desc)
 942 {
 943         return test_bit(POSTED_INTR_ON,
 944                         (unsigned long *)&pi_desc->control);
 945 }
 946
 947 static inline int pi_test_sn(struct pi_desc *pi_desc)
 948 {
 949         return test_bit(POSTED_INTR_SN,
 950                         (unsigned long *)&pi_desc->control);
 951 }
 952
 953 struct vmx_msrs {
 954         unsigned int            nr;
 955         struct vmx_msr_entry    val[NR_AUTOLOAD_MSRS];
 956 };
 957
 958 struct vcpu_vmx {
 959         struct kvm_vcpu       vcpu;
 960         unsigned long         host_rsp;
 961         u8                    fail;
 962         u8                    msr_bitmap_mode;
 963         u32                   exit_intr_info;
 964         u32                   idt_vectoring_info;
 965         ulong                 rflags;
 966         struct shared_msr_entry *guest_msrs;
 967         int                   nmsrs;
 968         int                   save_nmsrs;
 969         bool                  guest_msrs_dirty;
 970         unsigned long         host_idt_base;
 971 #ifdef CONFIG_X86_64
 972         u64                   msr_host_kernel_gs_base;
 973         u64                   msr_guest_kernel_gs_base;
 974 #endif
 975
 976         u64                   spec_ctrl;
 977
 978         u32 vm_entry_controls_shadow;
 979         u32 vm_exit_controls_shadow;
 980         u32 secondary_exec_control;
 981
 982         /*
 983          * loaded_vmcs points to the VMCS currently used in this vcpu. For a
 984          * non-nested (L1) guest, it always points to vmcs01. For a nested
 985          * guest (L2), it points to a different VMCS.  loaded_cpu_state points
 986          * to the VMCS whose state is loaded into the CPU registers that only
 987          * need to be switched when transitioning to/from the kernel; a NULL
 988          * value indicates that host state is loaded.
 989          */
 990         struct loaded_vmcs    vmcs01;
 991         struct loaded_vmcs   *loaded_vmcs;
 992         struct loaded_vmcs   *loaded_cpu_state;
 993         bool                  __launched; /* temporary, used in vmx_vcpu_run */
 994         struct msr_autoload {
 995                 struct vmx_msrs guest;
 996                 struct vmx_msrs host;
 997         } msr_autoload;
 998
 999         struct {
1000                 int vm86_active;
1001                 ulong save_rflags;
1002                 struct kvm_segment segs[8];
1003         } rmode;
1004         struct {
1005                 u32 bitmask; /* 4 bits per segment (1 bit per field) */
1006                 struct kvm_save_segment {
1007                         u16 selector;
1008                         unsigned long base;
1009                         u32 limit;
1010                         u32 ar;
1011                 } seg[8];
1012         } segment_cache;
1013         int vpid;
1014         bool emulation_required;
1015
1016         u32 exit_reason;
1017
1018         /* Posted interrupt descriptor */
1019         struct pi_desc pi_desc;
1020
1021         /* Support for a guest hypervisor (nested VMX) */
1022         struct nested_vmx nested;
1023
1024         /* Dynamic PLE window. */
1025         int ple_window;
1026         bool ple_window_dirty;
1027
1028         bool req_immediate_exit;
1029
1030         /* Support for PML */
1031 #define PML_ENTITY_NUM          512
1032         struct page *pml_pg;
1033
1034         /* apic deadline value in host tsc */
1035         u64 hv_deadline_tsc;
1036
1037         u64 current_tsc_ratio;
1038
1039         u32 host_pkru;
1040
1041         unsigned long host_debugctlmsr;
1042
1043         /*
1044          * Only bits masked by msr_ia32_feature_control_valid_bits can be set in
1045          * msr_ia32_feature_control. FEATURE_CONTROL_LOCKED is always included
1046          * in msr_ia32_feature_control_valid_bits.
1047          */
1048         u64 msr_ia32_feature_control;
1049         u64 msr_ia32_feature_control_valid_bits;
1050         u64 ept_pointer;
1051         u64 msr_ia32_mcu_opt_ctrl;
1052         bool disable_fb_clear;
1053 };
1054
1055 enum segment_cache_field {
1056         SEG_FIELD_SEL = 0,
1057         SEG_FIELD_BASE = 1,
1058         SEG_FIELD_LIMIT = 2,
1059         SEG_FIELD_AR = 3,
1060
1061         SEG_FIELD_NR = 4
1062 };
1063
1064 static inline struct kvm_vmx *to_kvm_vmx(struct kvm *kvm)
1065 {
1066         return container_of(kvm, struct kvm_vmx, kvm);
1067 }
1068
1069 static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
1070 {
1071         return container_of(vcpu, struct vcpu_vmx, vcpu);
1072 }
1073
1074 static struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu)
1075 {
1076         return &(to_vmx(vcpu)->pi_desc);
1077 }
1078
1079 #define ROL16(val, n) ((u16)(((u16)(val) << (n)) | ((u16)(val) >> (16 - (n)))))
1080 #define VMCS12_OFFSET(x) offsetof(struct vmcs12, x)
1081 #define FIELD(number, name)     [ROL16(number, 6)] = VMCS12_OFFSET(name)
1082 #define FIELD64(number, name)                                           \
1083         FIELD(number, name),                                            \
1084         [ROL16(number##_HIGH, 6)] = VMCS12_OFFSET(name) + sizeof(u32)
1085
1086
1087 static u16 shadow_read_only_fields[] = {
1088 #define SHADOW_FIELD_RO(x) x,
1089 #include "vmx_shadow_fields.h"
1090 };
1091 static int max_shadow_read_only_fields =
1092         ARRAY_SIZE(shadow_read_only_fields);
1093
1094 static u16 shadow_read_write_fields[] = {
1095 #define SHADOW_FIELD_RW(x) x,
1096 #include "vmx_shadow_fields.h"
1097 };
1098 static int max_shadow_read_write_fields =
1099         ARRAY_SIZE(shadow_read_write_fields);
1100
1101 static const unsigned short vmcs_field_to_offset_table[] = {
1102         FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
1103         FIELD(POSTED_INTR_NV, posted_intr_nv),
1104         FIELD(GUEST_ES_SELECTOR, guest_es_selector),
1105         FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
1106         FIELD(GUEST_SS_SELECTOR, guest_ss_selector),
1107         FIELD(GUEST_DS_SELECTOR, guest_ds_selector),
1108         FIELD(GUEST_FS_SELECTOR, guest_fs_selector),
1109         FIELD(GUEST_GS_SELECTOR, guest_gs_selector),
1110         FIELD(GUEST_LDTR_SELECTOR, guest_ldtr_selector),
1111         FIELD(GUEST_TR_SELECTOR, guest_tr_selector),
1112         FIELD(GUEST_INTR_STATUS, guest_intr_status),
1113         FIELD(GUEST_PML_INDEX, guest_pml_index),
1114         FIELD(HOST_ES_SELECTOR, host_es_selector),
1115         FIELD(HOST_CS_SELECTOR, host_cs_selector),
1116         FIELD(HOST_SS_SELECTOR, host_ss_selector),
1117         FIELD(HOST_DS_SELECTOR, host_ds_selector),
1118         FIELD(HOST_FS_SELECTOR, host_fs_selector),
1119         FIELD(HOST_GS_SELECTOR, host_gs_selector),
1120         FIELD(HOST_TR_SELECTOR, host_tr_selector),
1121         FIELD64(IO_BITMAP_A, io_bitmap_a),
1122         FIELD64(IO_BITMAP_B, io_bitmap_b),
1123         FIELD64(MSR_BITMAP, msr_bitmap),
1124         FIELD64(VM_EXIT_MSR_STORE_ADDR, vm_exit_msr_store_addr),
1125         FIELD64(VM_EXIT_MSR_LOAD_ADDR, vm_exit_msr_load_addr),
1126         FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr),
1127         FIELD64(PML_ADDRESS, pml_address),
1128         FIELD64(TSC_OFFSET, tsc_offset),
1129         FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr),
1130         FIELD64(APIC_ACCESS_ADDR, apic_access_addr),
1131         FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr),
1132         FIELD64(VM_FUNCTION_CONTROL, vm_function_control),
1133         FIELD64(EPT_POINTER, ept_pointer),
1134         FIELD64(EOI_EXIT_BITMAP0, eoi_exit_bitmap0),
1135         FIELD64(EOI_EXIT_BITMAP1, eoi_exit_bitmap1),
1136         FIELD64(EOI_EXIT_BITMAP2, eoi_exit_bitmap2),
1137         FIELD64(EOI_EXIT_BITMAP3, eoi_exit_bitmap3),
1138         FIELD64(EPTP_LIST_ADDRESS, eptp_list_address),
1139         FIELD64(VMREAD_BITMAP, vmread_bitmap),
1140         FIELD64(VMWRITE_BITMAP, vmwrite_bitmap),
1141         FIELD64(XSS_EXIT_BITMAP, xss_exit_bitmap),
1142         FIELD64(GUEST_PHYSICAL_ADDRESS, guest_physical_address),
1143         FIELD64(VMCS_LINK_POINTER, vmcs_link_pointer),
1144         FIELD64(GUEST_IA32_DEBUGCTL, guest_ia32_debugctl),
1145         FIELD64(GUEST_IA32_PAT, guest_ia32_pat),
1146         FIELD64(GUEST_IA32_EFER, guest_ia32_efer),
1147         FIELD64(GUEST_IA32_PERF_GLOBAL_CTRL, guest_ia32_perf_global_ctrl),
1148         FIELD64(GUEST_PDPTR0, guest_pdptr0),
1149         FIELD64(GUEST_PDPTR1, guest_pdptr1),
1150         FIELD64(GUEST_PDPTR2, guest_pdptr2),
1151         FIELD64(GUEST_PDPTR3, guest_pdptr3),
1152         FIELD64(GUEST_BNDCFGS, guest_bndcfgs),
1153         FIELD64(HOST_IA32_PAT, host_ia32_pat),
1154         FIELD64(HOST_IA32_EFER, host_ia32_efer),
1155         FIELD64(HOST_IA32_PERF_GLOBAL_CTRL, host_ia32_perf_global_ctrl),
1156         FIELD(PIN_BASED_VM_EXEC_CONTROL, pin_based_vm_exec_control),
1157         FIELD(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control),
1158         FIELD(EXCEPTION_BITMAP, exception_bitmap),
1159         FIELD(PAGE_FAULT_ERROR_CODE_MASK, page_fault_error_code_mask),
1160         FIELD(PAGE_FAULT_ERROR_CODE_MATCH, page_fault_error_code_match),
1161         FIELD(CR3_TARGET_COUNT, cr3_target_count),
1162         FIELD(VM_EXIT_CONTROLS, vm_exit_controls),
1163         FIELD(VM_EXIT_MSR_STORE_COUNT, vm_exit_msr_store_count),
1164         FIELD(VM_EXIT_MSR_LOAD_COUNT, vm_exit_msr_load_count),
1165         FIELD(VM_ENTRY_CONTROLS, vm_entry_controls),
1166         FIELD(VM_ENTRY_MSR_LOAD_COUNT, vm_entry_msr_load_count),
1167         FIELD(VM_ENTRY_INTR_INFO_FIELD, vm_entry_intr_info_field),
1168         FIELD(VM_ENTRY_EXCEPTION_ERROR_CODE, vm_entry_exception_error_code),
1169         FIELD(VM_ENTRY_INSTRUCTION_LEN, vm_entry_instruction_len),
1170         FIELD(TPR_THRESHOLD, tpr_threshold),
1171         FIELD(SECONDARY_VM_EXEC_CONTROL, secondary_vm_exec_control),
1172         FIELD(VM_INSTRUCTION_ERROR, vm_instruction_error),
1173         FIELD(VM_EXIT_REASON, vm_exit_reason),
1174         FIELD(VM_EXIT_INTR_INFO, vm_exit_intr_info),
1175         FIELD(VM_EXIT_INTR_ERROR_CODE, vm_exit_intr_error_code),
1176         FIELD(IDT_VECTORING_INFO_FIELD, idt_vectoring_info_field),
1177         FIELD(IDT_VECTORING_ERROR_CODE, idt_vectoring_error_code),
1178         FIELD(VM_EXIT_INSTRUCTION_LEN, vm_exit_instruction_len),
1179         FIELD(VMX_INSTRUCTION_INFO, vmx_instruction_info),
1180         FIELD(GUEST_ES_LIMIT, guest_es_limit),
1181         FIELD(GUEST_CS_LIMIT, guest_cs_limit),
1182         FIELD(GUEST_SS_LIMIT, guest_ss_limit),
1183         FIELD(GUEST_DS_LIMIT, guest_ds_limit),
1184         FIELD(GUEST_FS_LIMIT, guest_fs_limit),
1185         FIELD(GUEST_GS_LIMIT, guest_gs_limit),
1186         FIELD(GUEST_LDTR_LIMIT, guest_ldtr_limit),
1187         FIELD(GUEST_TR_LIMIT, guest_tr_limit),
1188         FIELD(GUEST_GDTR_LIMIT, guest_gdtr_limit),
1189         FIELD(GUEST_IDTR_LIMIT, guest_idtr_limit),
1190         FIELD(GUEST_ES_AR_BYTES, guest_es_ar_bytes),
1191         FIELD(GUEST_CS_AR_BYTES, guest_cs_ar_bytes),
1192         FIELD(GUEST_SS_AR_BYTES, guest_ss_ar_bytes),
1193         FIELD(GUEST_DS_AR_BYTES, guest_ds_ar_bytes),
1194         FIELD(GUEST_FS_AR_BYTES, guest_fs_ar_bytes),
1195         FIELD(GUEST_GS_AR_BYTES, guest_gs_ar_bytes),
1196         FIELD(GUEST_LDTR_AR_BYTES, guest_ldtr_ar_bytes),
1197         FIELD(GUEST_TR_AR_BYTES, guest_tr_ar_bytes),
1198         FIELD(GUEST_INTERRUPTIBILITY_INFO, guest_interruptibility_info),
1199         FIELD(GUEST_ACTIVITY_STATE, guest_activity_state),
1200         FIELD(GUEST_SYSENTER_CS, guest_sysenter_cs),
1201         FIELD(HOST_IA32_SYSENTER_CS, host_ia32_sysenter_cs),
1202         FIELD(VMX_PREEMPTION_TIMER_VALUE, vmx_preemption_timer_value),
1203         FIELD(CR0_GUEST_HOST_MASK, cr0_guest_host_mask),
1204         FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask),
1205         FIELD(CR0_READ_SHADOW, cr0_read_shadow),
1206         FIELD(CR4_READ_SHADOW, cr4_read_shadow),
1207         FIELD(CR3_TARGET_VALUE0, cr3_target_value0),
1208         FIELD(CR3_TARGET_VALUE1, cr3_target_value1),
1209         FIELD(CR3_TARGET_VALUE2, cr3_target_value2),
1210         FIELD(CR3_TARGET_VALUE3, cr3_target_value3),
1211         FIELD(EXIT_QUALIFICATION, exit_qualification),
1212         FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address),
1213         FIELD(GUEST_CR0, guest_cr0),
1214         FIELD(GUEST_CR3, guest_cr3),
1215         FIELD(GUEST_CR4, guest_cr4),
1216         FIELD(GUEST_ES_BASE, guest_es_base),
1217         FIELD(GUEST_CS_BASE, guest_cs_base),
1218         FIELD(GUEST_SS_BASE, guest_ss_base),
1219         FIELD(GUEST_DS_BASE, guest_ds_base),
1220         FIELD(GUEST_FS_BASE, guest_fs_base),
1221         FIELD(GUEST_GS_BASE, guest_gs_base),
1222         FIELD(GUEST_LDTR_BASE, guest_ldtr_base),
1223         FIELD(GUEST_TR_BASE, guest_tr_base),
1224         FIELD(GUEST_GDTR_BASE, guest_gdtr_base),
1225         FIELD(GUEST_IDTR_BASE, guest_idtr_base),
1226         FIELD(GUEST_DR7, guest_dr7),
1227         FIELD(GUEST_RSP, guest_rsp),
1228         FIELD(GUEST_RIP, guest_rip),
1229         FIELD(GUEST_RFLAGS, guest_rflags),
1230         FIELD(GUEST_PENDING_DBG_EXCEPTIONS, guest_pending_dbg_exceptions),
1231         FIELD(GUEST_SYSENTER_ESP, guest_sysenter_esp),
1232         FIELD(GUEST_SYSENTER_EIP, guest_sysenter_eip),
1233         FIELD(HOST_CR0, host_cr0),
1234         FIELD(HOST_CR3, host_cr3),
1235         FIELD(HOST_CR4, host_cr4),
1236         FIELD(HOST_FS_BASE, host_fs_base),
1237         FIELD(HOST_GS_BASE, host_gs_base),
1238         FIELD(HOST_TR_BASE, host_tr_base),
1239         FIELD(HOST_GDTR_BASE, host_gdtr_base),
1240         FIELD(HOST_IDTR_BASE, host_idtr_base),
1241         FIELD(HOST_IA32_SYSENTER_ESP, host_ia32_sysenter_esp),
1242         FIELD(HOST_IA32_SYSENTER_EIP, host_ia32_sysenter_eip),
1243         FIELD(HOST_RSP, host_rsp),
1244         FIELD(HOST_RIP, host_rip),
1245 };
1246
1247 static inline short vmcs_field_to_offset(unsigned long field)
1248 {
1249         const size_t size = ARRAY_SIZE(vmcs_field_to_offset_table);
1250         unsigned short offset;
1251         unsigned index;
1252
1253         if (field >> 15)
1254                 return -ENOENT;
1255
1256         index = ROL16(field, 6);
1257         if (index >= size)
1258                 return -ENOENT;
1259
1260         index = array_index_nospec(index, size);
1261         offset = vmcs_field_to_offset_table[index];
1262         if (offset == 0)
1263                 return -ENOENT;
1264         return offset;
1265 }
1266
1267 static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
1268 {
1269         return to_vmx(vcpu)->nested.cached_vmcs12;
1270 }
1271
1272 static inline struct vmcs12 *get_shadow_vmcs12(struct kvm_vcpu *vcpu)
1273 {
1274         return to_vmx(vcpu)->nested.cached_shadow_vmcs12;
1275 }
1276
1277 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu);
1278 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu);
1279 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa);
1280 static bool vmx_xsaves_supported(void);
1281 static void vmx_set_segment(struct kvm_vcpu *vcpu,
1282                             struct kvm_segment *var, int seg);
1283 static void vmx_get_segment(struct kvm_vcpu *vcpu,
1284                             struct kvm_segment *var, int seg);
1285 static bool guest_state_valid(struct kvm_vcpu *vcpu);
1286 static u32 vmx_segment_access_rights(struct kvm_segment *var);
1287 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx);
1288 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
1289 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
1290 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
1291                                             u16 error_code);
1292 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu);
1293 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
1294                                                           u32 msr, int type);
1295
1296 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
1297 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
1298 /*
1299  * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed
1300  * when a CPU is brought down, and we need to VMCLEAR all VMCSs loaded on it.
1301  */
1302 static DEFINE_PER_CPU(struct list_head, loaded_vmcss_on_cpu);
1303
1304 /*
1305  * We maintian a per-CPU linked-list of vCPU, so in wakeup_handler() we
1306  * can find which vCPU should be waken up.
1307  */
1308 static DEFINE_PER_CPU(struct list_head, blocked_vcpu_on_cpu);
1309 static DEFINE_PER_CPU(spinlock_t, blocked_vcpu_on_cpu_lock);
1310
1311 enum {
1312         VMX_VMREAD_BITMAP,
1313         VMX_VMWRITE_BITMAP,
1314         VMX_BITMAP_NR
1315 };
1316
1317 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
1318
1319 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
1320 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
1321
1322 static bool cpu_has_load_ia32_efer;
1323 static bool cpu_has_load_perf_global_ctrl;
1324
1325 static DECLARE_BITMAP(vmx_vpid_bitmap, VMX_NR_VPIDS);
1326 static DEFINE_SPINLOCK(vmx_vpid_lock);
1327
1328 static struct vmcs_config {
1329         int size;
1330         int order;
1331         u32 basic_cap;
1332         u32 revision_id;
1333         u32 pin_based_exec_ctrl;
1334         u32 cpu_based_exec_ctrl;
1335         u32 cpu_based_2nd_exec_ctrl;
1336         u32 vmexit_ctrl;
1337         u32 vmentry_ctrl;
1338         struct nested_vmx_msrs nested;
1339 } vmcs_config;
1340
1341 static struct vmx_capability {
1342         u32 ept;
1343         u32 vpid;
1344 } vmx_capability;
1345
1346 #define VMX_SEGMENT_FIELD(seg)                                  \
1347         [VCPU_SREG_##seg] = {                                   \
1348                 .selector = GUEST_##seg##_SELECTOR,             \
1349                 .base = GUEST_##seg##_BASE,                     \
1350                 .limit = GUEST_##seg##_LIMIT,                   \
1351                 .ar_bytes = GUEST_##seg##_AR_BYTES,             \
1352         }
1353
1354 static const struct kvm_vmx_segment_field {
1355         unsigned selector;
1356         unsigned base;
1357         unsigned limit;
1358         unsigned ar_bytes;
1359 } kvm_vmx_segment_fields[] = {
1360         VMX_SEGMENT_FIELD(CS),
1361         VMX_SEGMENT_FIELD(DS),
1362         VMX_SEGMENT_FIELD(ES),
1363         VMX_SEGMENT_FIELD(FS),
1364         VMX_SEGMENT_FIELD(GS),
1365         VMX_SEGMENT_FIELD(SS),
1366         VMX_SEGMENT_FIELD(TR),
1367         VMX_SEGMENT_FIELD(LDTR),
1368 };
1369
1370 static u64 host_efer;
1371
1372 static void ept_save_pdptrs(struct kvm_vcpu *vcpu);
1373
1374 /*
1375  * Keep MSR_STAR at the end, as setup_msrs() will try to optimize it
1376  * away by decrementing the array size.
1377  */
1378 static const u32 vmx_msr_index[] = {
1379 #ifdef CONFIG_X86_64
1380         MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR,
1381 #endif
1382         MSR_EFER, MSR_TSC_AUX, MSR_STAR,
1383 };
1384
1385 DEFINE_STATIC_KEY_FALSE(enable_evmcs);
1386
1387 #define current_evmcs ((struct hv_enlightened_vmcs *)this_cpu_read(current_vmcs))
1388
1389 #define KVM_EVMCS_VERSION 1
1390
1391 #if IS_ENABLED(CONFIG_HYPERV)
1392 static bool __read_mostly enlightened_vmcs = true;
1393 module_param(enlightened_vmcs, bool, 0444);
1394
1395 static inline void evmcs_write64(unsigned long field, u64 value)
1396 {
1397         u16 clean_field;
1398         int offset = get_evmcs_offset(field, &clean_field);
1399
1400         if (offset < 0)
1401                 return;
1402
1403         *(u64 *)((char *)current_evmcs + offset) = value;
1404
1405         current_evmcs->hv_clean_fields &= ~clean_field;
1406 }
1407
1408 static inline void evmcs_write32(unsigned long field, u32 value)
1409 {
1410         u16 clean_field;
1411         int offset = get_evmcs_offset(field, &clean_field);
1412
1413         if (offset < 0)
1414                 return;
1415
1416         *(u32 *)((char *)current_evmcs + offset) = value;
1417         current_evmcs->hv_clean_fields &= ~clean_field;
1418 }
1419
1420 static inline void evmcs_write16(unsigned long field, u16 value)
1421 {
1422         u16 clean_field;
1423         int offset = get_evmcs_offset(field, &clean_field);
1424
1425         if (offset < 0)
1426                 return;
1427
1428         *(u16 *)((char *)current_evmcs + offset) = value;
1429         current_evmcs->hv_clean_fields &= ~clean_field;
1430 }
1431
1432 static inline u64 evmcs_read64(unsigned long field)
1433 {
1434         int offset = get_evmcs_offset(field, NULL);
1435
1436         if (offset < 0)
1437                 return 0;
1438
1439         return *(u64 *)((char *)current_evmcs + offset);
1440 }
1441
1442 static inline u32 evmcs_read32(unsigned long field)
1443 {
1444         int offset = get_evmcs_offset(field, NULL);
1445
1446         if (offset < 0)
1447                 return 0;
1448
1449         return *(u32 *)((char *)current_evmcs + offset);
1450 }
1451
1452 static inline u16 evmcs_read16(unsigned long field)
1453 {
1454         int offset = get_evmcs_offset(field, NULL);
1455
1456         if (offset < 0)
1457                 return 0;
1458
1459         return *(u16 *)((char *)current_evmcs + offset);
1460 }
1461
1462 static inline void evmcs_touch_msr_bitmap(void)
1463 {
1464         if (unlikely(!current_evmcs))
1465                 return;
1466
1467         if (current_evmcs->hv_enlightenments_control.msr_bitmap)
1468                 current_evmcs->hv_clean_fields &=
1469                         ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP;
1470 }
1471
1472 static void evmcs_load(u64 phys_addr)
1473 {
1474         struct hv_vp_assist_page *vp_ap =
1475                 hv_get_vp_assist_page(smp_processor_id());
1476
1477         vp_ap->current_nested_vmcs = phys_addr;
1478         vp_ap->enlighten_vmentry = 1;
1479 }
1480
1481 static void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf)
1482 {
1483         /*
1484          * Enlightened VMCSv1 doesn't support these:
1485          *
1486          *      POSTED_INTR_NV                  = 0x00000002,
1487          *      GUEST_INTR_STATUS               = 0x00000810,
1488          *      APIC_ACCESS_ADDR                = 0x00002014,
1489          *      POSTED_INTR_DESC_ADDR           = 0x00002016,
1490          *      EOI_EXIT_BITMAP0                = 0x0000201c,
1491          *      EOI_EXIT_BITMAP1                = 0x0000201e,
1492          *      EOI_EXIT_BITMAP2                = 0x00002020,
1493          *      EOI_EXIT_BITMAP3                = 0x00002022,
1494          */
1495         vmcs_conf->pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
1496         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1497                 ~SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1498         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1499                 ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1500         vmcs_conf->cpu_based_2nd_exec_ctrl &=
1501                 ~SECONDARY_EXEC_APIC_REGISTER_VIRT;
1502
1503         /*
1504          *      GUEST_PML_INDEX                 = 0x00000812,
1505          *      PML_ADDRESS                     = 0x0000200e,
1506          */
1507         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_ENABLE_PML;
1508
1509         /*      VM_FUNCTION_CONTROL             = 0x00002018, */
1510         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_ENABLE_VMFUNC;
1511
1512         /*
1513          *      EPTP_LIST_ADDRESS               = 0x00002024,
1514          *      VMREAD_BITMAP                   = 0x00002026,
1515          *      VMWRITE_BITMAP                  = 0x00002028,
1516          */
1517         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_SHADOW_VMCS;
1518
1519         /*
1520          *      TSC_MULTIPLIER                  = 0x00002032,
1521          */
1522         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_TSC_SCALING;
1523
1524         /*
1525          *      PLE_GAP                         = 0x00004020,
1526          *      PLE_WINDOW                      = 0x00004022,
1527          */
1528         vmcs_conf->cpu_based_2nd_exec_ctrl &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1529
1530         /*
1531          *      VMX_PREEMPTION_TIMER_VALUE      = 0x0000482E,
1532          */
1533         vmcs_conf->pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
1534
1535         /*
1536          *      GUEST_IA32_PERF_GLOBAL_CTRL     = 0x00002808,
1537          *      HOST_IA32_PERF_GLOBAL_CTRL      = 0x00002c04,
1538          */
1539         vmcs_conf->vmexit_ctrl &= ~VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL;
1540         vmcs_conf->vmentry_ctrl &= ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL;
1541
1542         /*
1543          * Currently unsupported in KVM:
1544          *      GUEST_IA32_RTIT_CTL             = 0x00002814,
1545          */
1546 }
1547
1548 /* check_ept_pointer() should be under protection of ept_pointer_lock. */
1549 static void check_ept_pointer_match(struct kvm *kvm)
1550 {
1551         struct kvm_vcpu *vcpu;
1552         u64 tmp_eptp = INVALID_PAGE;
1553         int i;
1554
1555         kvm_for_each_vcpu(i, vcpu, kvm) {
1556                 if (!VALID_PAGE(tmp_eptp)) {
1557                         tmp_eptp = to_vmx(vcpu)->ept_pointer;
1558                 } else if (tmp_eptp != to_vmx(vcpu)->ept_pointer) {
1559                         to_kvm_vmx(kvm)->ept_pointers_match
1560                                 = EPT_POINTERS_MISMATCH;
1561                         return;
1562                 }
1563         }
1564
1565         to_kvm_vmx(kvm)->ept_pointers_match = EPT_POINTERS_MATCH;
1566 }
1567
1568 static int vmx_hv_remote_flush_tlb(struct kvm *kvm)
1569 {
1570         int ret;
1571
1572         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
1573
1574         if (to_kvm_vmx(kvm)->ept_pointers_match == EPT_POINTERS_CHECK)
1575                 check_ept_pointer_match(kvm);
1576
1577         if (to_kvm_vmx(kvm)->ept_pointers_match != EPT_POINTERS_MATCH) {
1578                 ret = -ENOTSUPP;
1579                 goto out;
1580         }
1581
1582         /*
1583          * FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE hypercall needs the address of the
1584          * base of EPT PML4 table, strip off EPT configuration information.
1585          */
1586         ret = hyperv_flush_guest_mapping(
1587                         to_vmx(kvm_get_vcpu(kvm, 0))->ept_pointer & PAGE_MASK);
1588
1589 out:
1590         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
1591         return ret;
1592 }
1593 #else /* !IS_ENABLED(CONFIG_HYPERV) */
1594 static inline void evmcs_write64(unsigned long field, u64 value) {}
1595 static inline void evmcs_write32(unsigned long field, u32 value) {}
1596 static inline void evmcs_write16(unsigned long field, u16 value) {}
1597 static inline u64 evmcs_read64(unsigned long field) { return 0; }
1598 static inline u32 evmcs_read32(unsigned long field) { return 0; }
1599 static inline u16 evmcs_read16(unsigned long field) { return 0; }
1600 static inline void evmcs_load(u64 phys_addr) {}
1601 static inline void evmcs_sanitize_exec_ctrls(struct vmcs_config *vmcs_conf) {}
1602 static inline void evmcs_touch_msr_bitmap(void) {}
1603 #endif /* IS_ENABLED(CONFIG_HYPERV) */
1604
1605 static inline bool is_exception_n(u32 intr_info, u8 vector)
1606 {
1607         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1608                              INTR_INFO_VALID_MASK)) ==
1609                 (INTR_TYPE_HARD_EXCEPTION | vector | INTR_INFO_VALID_MASK);
1610 }
1611
1612 static inline bool is_debug(u32 intr_info)
1613 {
1614         return is_exception_n(intr_info, DB_VECTOR);
1615 }
1616
1617 static inline bool is_breakpoint(u32 intr_info)
1618 {
1619         return is_exception_n(intr_info, BP_VECTOR);
1620 }
1621
1622 static inline bool is_page_fault(u32 intr_info)
1623 {
1624         return is_exception_n(intr_info, PF_VECTOR);
1625 }
1626
1627 static inline bool is_no_device(u32 intr_info)
1628 {
1629         return is_exception_n(intr_info, NM_VECTOR);
1630 }
1631
1632 static inline bool is_invalid_opcode(u32 intr_info)
1633 {
1634         return is_exception_n(intr_info, UD_VECTOR);
1635 }
1636
1637 static inline bool is_gp_fault(u32 intr_info)
1638 {
1639         return is_exception_n(intr_info, GP_VECTOR);
1640 }
1641
1642 static inline bool is_external_interrupt(u32 intr_info)
1643 {
1644         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1645                 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1646 }
1647
1648 static inline bool is_machine_check(u32 intr_info)
1649 {
1650         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
1651                              INTR_INFO_VALID_MASK)) ==
1652                 (INTR_TYPE_HARD_EXCEPTION | MC_VECTOR | INTR_INFO_VALID_MASK);
1653 }
1654
1655 /* Undocumented: icebp/int1 */
1656 static inline bool is_icebp(u32 intr_info)
1657 {
1658         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
1659                 == (INTR_TYPE_PRIV_SW_EXCEPTION | INTR_INFO_VALID_MASK);
1660 }
1661
1662 static inline bool cpu_has_vmx_msr_bitmap(void)
1663 {
1664         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_USE_MSR_BITMAPS;
1665 }
1666
1667 static inline bool cpu_has_vmx_tpr_shadow(void)
1668 {
1669         return vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW;
1670 }
1671
1672 static inline bool cpu_need_tpr_shadow(struct kvm_vcpu *vcpu)
1673 {
1674         return cpu_has_vmx_tpr_shadow() && lapic_in_kernel(vcpu);
1675 }
1676
1677 static inline bool cpu_has_secondary_exec_ctrls(void)
1678 {
1679         return vmcs_config.cpu_based_exec_ctrl &
1680                 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1681 }
1682
1683 static inline bool cpu_has_vmx_virtualize_apic_accesses(void)
1684 {
1685         return vmcs_config.cpu_based_2nd_exec_ctrl &
1686                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
1687 }
1688
1689 static inline bool cpu_has_vmx_virtualize_x2apic_mode(void)
1690 {
1691         return vmcs_config.cpu_based_2nd_exec_ctrl &
1692                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
1693 }
1694
1695 static inline bool cpu_has_vmx_apic_register_virt(void)
1696 {
1697         return vmcs_config.cpu_based_2nd_exec_ctrl &
1698                 SECONDARY_EXEC_APIC_REGISTER_VIRT;
1699 }
1700
1701 static inline bool cpu_has_vmx_virtual_intr_delivery(void)
1702 {
1703         return vmcs_config.cpu_based_2nd_exec_ctrl &
1704                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY;
1705 }
1706
1707 static inline bool cpu_has_vmx_encls_vmexit(void)
1708 {
1709         return vmcs_config.cpu_based_2nd_exec_ctrl &
1710                 SECONDARY_EXEC_ENCLS_EXITING;
1711 }
1712
1713 /*
1714  * Comment's format: document - errata name - stepping - processor name.
1715  * Refer from
1716  * https://www.virtualbox.org/svn/vbox/trunk/src/VBox/VMM/VMMR0/HMR0.cpp
1717  */
1718 static u32 vmx_preemption_cpu_tfms[] = {
1719 /* 323344.pdf - BA86   - D0 - Xeon 7500 Series */
1720 0x000206E6,
1721 /* 323056.pdf - AAX65  - C2 - Xeon L3406 */
1722 /* 322814.pdf - AAT59  - C2 - i7-600, i5-500, i5-400 and i3-300 Mobile */
1723 /* 322911.pdf - AAU65  - C2 - i5-600, i3-500 Desktop and Pentium G6950 */
1724 0x00020652,
1725 /* 322911.pdf - AAU65  - K0 - i5-600, i3-500 Desktop and Pentium G6950 */
1726 0x00020655,
1727 /* 322373.pdf - AAO95  - B1 - Xeon 3400 Series */
1728 /* 322166.pdf - AAN92  - B1 - i7-800 and i5-700 Desktop */
1729 /*
1730  * 320767.pdf - AAP86  - B1 -
1731  * i7-900 Mobile Extreme, i7-800 and i7-700 Mobile
1732  */
1733 0x000106E5,
1734 /* 321333.pdf - AAM126 - C0 - Xeon 3500 */
1735 0x000106A0,
1736 /* 321333.pdf - AAM126 - C1 - Xeon 3500 */
1737 0x000106A1,
1738 /* 320836.pdf - AAJ124 - C0 - i7-900 Desktop Extreme and i7-900 Desktop */
1739 0x000106A4,
1740  /* 321333.pdf - AAM126 - D0 - Xeon 3500 */
1741  /* 321324.pdf - AAK139 - D0 - Xeon 5500 */
1742  /* 320836.pdf - AAJ124 - D0 - i7-900 Extreme and i7-900 Desktop */
1743 0x000106A5,
1744 };
1745
1746 static inline bool cpu_has_broken_vmx_preemption_timer(void)
1747 {
1748         u32 eax = cpuid_eax(0x00000001), i;
1749
1750         /* Clear the reserved bits */
1751         eax &= ~(0x3U << 14 | 0xfU << 28);
1752         for (i = 0; i < ARRAY_SIZE(vmx_preemption_cpu_tfms); i++)
1753                 if (eax == vmx_preemption_cpu_tfms[i])
1754                         return true;
1755
1756         return false;
1757 }
1758
1759 static inline bool cpu_has_vmx_preemption_timer(void)
1760 {
1761         return vmcs_config.pin_based_exec_ctrl &
1762                 PIN_BASED_VMX_PREEMPTION_TIMER;
1763 }
1764
1765 static inline bool cpu_has_vmx_posted_intr(void)
1766 {
1767         return IS_ENABLED(CONFIG_X86_LOCAL_APIC) &&
1768                 vmcs_config.pin_based_exec_ctrl & PIN_BASED_POSTED_INTR;
1769 }
1770
1771 static inline bool cpu_has_vmx_apicv(void)
1772 {
1773         return cpu_has_vmx_apic_register_virt() &&
1774                 cpu_has_vmx_virtual_intr_delivery() &&
1775                 cpu_has_vmx_posted_intr();
1776 }
1777
1778 static inline bool cpu_has_vmx_flexpriority(void)
1779 {
1780         return cpu_has_vmx_tpr_shadow() &&
1781                 cpu_has_vmx_virtualize_apic_accesses();
1782 }
1783
1784 static inline bool cpu_has_vmx_ept_execute_only(void)
1785 {
1786         return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
1787 }
1788
1789 static inline bool cpu_has_vmx_ept_2m_page(void)
1790 {
1791         return vmx_capability.ept & VMX_EPT_2MB_PAGE_BIT;
1792 }
1793
1794 static inline bool cpu_has_vmx_ept_1g_page(void)
1795 {
1796         return vmx_capability.ept & VMX_EPT_1GB_PAGE_BIT;
1797 }
1798
1799 static inline bool cpu_has_vmx_ept_4levels(void)
1800 {
1801         return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
1802 }
1803
1804 static inline bool cpu_has_vmx_ept_mt_wb(void)
1805 {
1806         return vmx_capability.ept & VMX_EPTP_WB_BIT;
1807 }
1808
1809 static inline bool cpu_has_vmx_ept_5levels(void)
1810 {
1811         return vmx_capability.ept & VMX_EPT_PAGE_WALK_5_BIT;
1812 }
1813
1814 static inline bool cpu_has_vmx_ept_ad_bits(void)
1815 {
1816         return vmx_capability.ept & VMX_EPT_AD_BIT;
1817 }
1818
1819 static inline bool cpu_has_vmx_invept_context(void)
1820 {
1821         return vmx_capability.ept & VMX_EPT_EXTENT_CONTEXT_BIT;
1822 }
1823
1824 static inline bool cpu_has_vmx_invept_global(void)
1825 {
1826         return vmx_capability.ept & VMX_EPT_EXTENT_GLOBAL_BIT;
1827 }
1828
1829 static inline bool cpu_has_vmx_invvpid_individual_addr(void)
1830 {
1831         return vmx_capability.vpid & VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT;
1832 }
1833
1834 static inline bool cpu_has_vmx_invvpid_single(void)
1835 {
1836         return vmx_capability.vpid & VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT;
1837 }
1838
1839 static inline bool cpu_has_vmx_invvpid_global(void)
1840 {
1841         return vmx_capability.vpid & VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT;
1842 }
1843
1844 static inline bool cpu_has_vmx_invvpid(void)
1845 {
1846         return vmx_capability.vpid & VMX_VPID_INVVPID_BIT;
1847 }
1848
1849 static inline bool cpu_has_vmx_ept(void)
1850 {
1851         return vmcs_config.cpu_based_2nd_exec_ctrl &
1852                 SECONDARY_EXEC_ENABLE_EPT;
1853 }
1854
1855 static inline bool cpu_has_vmx_unrestricted_guest(void)
1856 {
1857         return vmcs_config.cpu_based_2nd_exec_ctrl &
1858                 SECONDARY_EXEC_UNRESTRICTED_GUEST;
1859 }
1860
1861 static inline bool cpu_has_vmx_ple(void)
1862 {
1863         return vmcs_config.cpu_based_2nd_exec_ctrl &
1864                 SECONDARY_EXEC_PAUSE_LOOP_EXITING;
1865 }
1866
1867 static inline bool cpu_has_vmx_basic_inout(void)
1868 {
1869         return  (((u64)vmcs_config.basic_cap << 32) & VMX_BASIC_INOUT);
1870 }
1871
1872 static inline bool cpu_need_virtualize_apic_accesses(struct kvm_vcpu *vcpu)
1873 {
1874         return flexpriority_enabled && lapic_in_kernel(vcpu);
1875 }
1876
1877 static inline bool cpu_has_vmx_vpid(void)
1878 {
1879         return vmcs_config.cpu_based_2nd_exec_ctrl &
1880                 SECONDARY_EXEC_ENABLE_VPID;
1881 }
1882
1883 static inline bool cpu_has_vmx_rdtscp(void)
1884 {
1885         return vmcs_config.cpu_based_2nd_exec_ctrl &
1886                 SECONDARY_EXEC_RDTSCP;
1887 }
1888
1889 static inline bool cpu_has_vmx_invpcid(void)
1890 {
1891         return vmcs_config.cpu_based_2nd_exec_ctrl &
1892                 SECONDARY_EXEC_ENABLE_INVPCID;
1893 }
1894
1895 static inline bool cpu_has_virtual_nmis(void)
1896 {
1897         return vmcs_config.pin_based_exec_ctrl & PIN_BASED_VIRTUAL_NMIS;
1898 }
1899
1900 static inline bool cpu_has_vmx_wbinvd_exit(void)
1901 {
1902         return vmcs_config.cpu_based_2nd_exec_ctrl &
1903                 SECONDARY_EXEC_WBINVD_EXITING;
1904 }
1905
1906 static inline bool cpu_has_vmx_shadow_vmcs(void)
1907 {
1908         u64 vmx_msr;
1909         rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
1910         /* check if the cpu supports writing r/o exit information fields */
1911         if (!(vmx_msr & MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS))
1912                 return false;
1913
1914         return vmcs_config.cpu_based_2nd_exec_ctrl &
1915                 SECONDARY_EXEC_SHADOW_VMCS;
1916 }
1917
1918 static inline bool cpu_has_vmx_pml(void)
1919 {
1920         return vmcs_config.cpu_based_2nd_exec_ctrl & SECONDARY_EXEC_ENABLE_PML;
1921 }
1922
1923 static inline bool cpu_has_vmx_tsc_scaling(void)
1924 {
1925         return vmcs_config.cpu_based_2nd_exec_ctrl &
1926                 SECONDARY_EXEC_TSC_SCALING;
1927 }
1928
1929 static inline bool cpu_has_vmx_vmfunc(void)
1930 {
1931         return vmcs_config.cpu_based_2nd_exec_ctrl &
1932                 SECONDARY_EXEC_ENABLE_VMFUNC;
1933 }
1934
1935 static bool vmx_umip_emulated(void)
1936 {
1937         return vmcs_config.cpu_based_2nd_exec_ctrl &
1938                 SECONDARY_EXEC_DESC;
1939 }
1940
1941 static inline bool report_flexpriority(void)
1942 {
1943         return flexpriority_enabled;
1944 }
1945
1946 static inline unsigned nested_cpu_vmx_misc_cr3_count(struct kvm_vcpu *vcpu)
1947 {
1948         return vmx_misc_cr3_count(to_vmx(vcpu)->nested.msrs.misc_low);
1949 }
1950
1951 /*
1952  * Do the virtual VMX capability MSRs specify that L1 can use VMWRITE
1953  * to modify any valid field of the VMCS, or are the VM-exit
1954  * information fields read-only?
1955  */
1956 static inline bool nested_cpu_has_vmwrite_any_field(struct kvm_vcpu *vcpu)
1957 {
1958         return to_vmx(vcpu)->nested.msrs.misc_low &
1959                 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS;
1960 }
1961
1962 static inline bool nested_cpu_has_zero_length_injection(struct kvm_vcpu *vcpu)
1963 {
1964         return to_vmx(vcpu)->nested.msrs.misc_low & VMX_MISC_ZERO_LEN_INS;
1965 }
1966
1967 static inline bool nested_cpu_supports_monitor_trap_flag(struct kvm_vcpu *vcpu)
1968 {
1969         return to_vmx(vcpu)->nested.msrs.procbased_ctls_high &
1970                         CPU_BASED_MONITOR_TRAP_FLAG;
1971 }
1972
1973 static inline bool nested_cpu_has_vmx_shadow_vmcs(struct kvm_vcpu *vcpu)
1974 {
1975         return to_vmx(vcpu)->nested.msrs.secondary_ctls_high &
1976                 SECONDARY_EXEC_SHADOW_VMCS;
1977 }
1978
1979 static inline bool nested_cpu_has(struct vmcs12 *vmcs12, u32 bit)
1980 {
1981         return vmcs12->cpu_based_vm_exec_control & bit;
1982 }
1983
1984 static inline bool nested_cpu_has2(struct vmcs12 *vmcs12, u32 bit)
1985 {
1986         return (vmcs12->cpu_based_vm_exec_control &
1987                         CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
1988                 (vmcs12->secondary_vm_exec_control & bit);
1989 }
1990
1991 static inline bool nested_cpu_has_preemption_timer(struct vmcs12 *vmcs12)
1992 {
1993         return vmcs12->pin_based_vm_exec_control &
1994                 PIN_BASED_VMX_PREEMPTION_TIMER;
1995 }
1996
1997 static inline bool nested_cpu_has_nmi_exiting(struct vmcs12 *vmcs12)
1998 {
1999         return vmcs12->pin_based_vm_exec_control & PIN_BASED_NMI_EXITING;
2000 }
2001
2002 static inline bool nested_cpu_has_virtual_nmis(struct vmcs12 *vmcs12)
2003 {
2004         return vmcs12->pin_based_vm_exec_control & PIN_BASED_VIRTUAL_NMIS;
2005 }
2006
2007 static inline int nested_cpu_has_ept(struct vmcs12 *vmcs12)
2008 {
2009         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_EPT);
2010 }
2011
2012 static inline bool nested_cpu_has_xsaves(struct vmcs12 *vmcs12)
2013 {
2014         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
2015 }
2016
2017 static inline bool nested_cpu_has_pml(struct vmcs12 *vmcs12)
2018 {
2019         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML);
2020 }
2021
2022 static inline bool nested_cpu_has_virt_x2apic_mode(struct vmcs12 *vmcs12)
2023 {
2024         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
2025 }
2026
2027 static inline bool nested_cpu_has_vpid(struct vmcs12 *vmcs12)
2028 {
2029         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VPID);
2030 }
2031
2032 static inline bool nested_cpu_has_apic_reg_virt(struct vmcs12 *vmcs12)
2033 {
2034         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_APIC_REGISTER_VIRT);
2035 }
2036
2037 static inline bool nested_cpu_has_vid(struct vmcs12 *vmcs12)
2038 {
2039         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
2040 }
2041
2042 static inline bool nested_cpu_has_posted_intr(struct vmcs12 *vmcs12)
2043 {
2044         return vmcs12->pin_based_vm_exec_control & PIN_BASED_POSTED_INTR;
2045 }
2046
2047 static inline bool nested_cpu_has_vmfunc(struct vmcs12 *vmcs12)
2048 {
2049         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_VMFUNC);
2050 }
2051
2052 static inline bool nested_cpu_has_eptp_switching(struct vmcs12 *vmcs12)
2053 {
2054         return nested_cpu_has_vmfunc(vmcs12) &&
2055                 (vmcs12->vm_function_control &
2056                  VMX_VMFUNC_EPTP_SWITCHING);
2057 }
2058
2059 static inline bool nested_cpu_has_shadow_vmcs(struct vmcs12 *vmcs12)
2060 {
2061         return nested_cpu_has2(vmcs12, SECONDARY_EXEC_SHADOW_VMCS);
2062 }
2063
2064 static inline bool is_nmi(u32 intr_info)
2065 {
2066         return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
2067                 == (INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK);
2068 }
2069
2070 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
2071                               u32 exit_intr_info,
2072                               unsigned long exit_qualification);
2073 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
2074                         struct vmcs12 *vmcs12,
2075                         u32 reason, unsigned long qualification);
2076
2077 static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr)
2078 {
2079         int i;
2080
2081         for (i = 0; i < vmx->nmsrs; ++i)
2082                 if (vmx_msr_index[vmx->guest_msrs[i].index] == msr)
2083                         return i;
2084         return -1;
2085 }
2086
2087 static inline void __invvpid(unsigned long ext, u16 vpid, gva_t gva)
2088 {
2089     struct {
2090         u64 vpid : 16;
2091         u64 rsvd : 48;
2092         u64 gva;
2093     } operand = { vpid, 0, gva };
2094     bool error;
2095
2096     asm volatile (__ex(ASM_VMX_INVVPID) CC_SET(na)
2097                   : CC_OUT(na) (error) : "a"(&operand), "c"(ext)
2098                   : "memory");
2099     BUG_ON(error);
2100 }
2101
2102 static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa)
2103 {
2104         struct {
2105                 u64 eptp, gpa;
2106         } operand = {eptp, gpa};
2107         bool error;
2108
2109         asm volatile (__ex(ASM_VMX_INVEPT) CC_SET(na)
2110                       : CC_OUT(na) (error) : "a" (&operand), "c" (ext)
2111                       : "memory");
2112         BUG_ON(error);
2113 }
2114
2115 static void vmx_setup_fb_clear_ctrl(void)
2116 {
2117         u64 msr;
2118
2119         if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES) &&
2120             !boot_cpu_has_bug(X86_BUG_MDS) &&
2121             !boot_cpu_has_bug(X86_BUG_TAA)) {
2122                 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, msr);
2123                 if (msr & ARCH_CAP_FB_CLEAR_CTRL)
2124                         vmx_fb_clear_ctrl_available = true;
2125         }
2126 }
2127
2128 static __always_inline void vmx_disable_fb_clear(struct vcpu_vmx *vmx)
2129 {
2130         u64 msr;
2131
2132         if (!vmx->disable_fb_clear)
2133                 return;
2134
2135         rdmsrl(MSR_IA32_MCU_OPT_CTRL, msr);
2136         msr |= FB_CLEAR_DIS;
2137         wrmsrl(MSR_IA32_MCU_OPT_CTRL, msr);
2138         /* Cache the MSR value to avoid reading it later */
2139         vmx->msr_ia32_mcu_opt_ctrl = msr;
2140 }
2141
2142 static __always_inline void vmx_enable_fb_clear(struct vcpu_vmx *vmx)
2143 {
2144         if (!vmx->disable_fb_clear)
2145                 return;
2146
2147         vmx->msr_ia32_mcu_opt_ctrl &= ~FB_CLEAR_DIS;
2148         wrmsrl(MSR_IA32_MCU_OPT_CTRL, vmx->msr_ia32_mcu_opt_ctrl);
2149 }
2150
2151 static void vmx_update_fb_clear_dis(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx)
2152 {
2153         vmx->disable_fb_clear = vmx_fb_clear_ctrl_available;
2154
2155         /*
2156          * If guest will not execute VERW, there is no need to set FB_CLEAR_DIS
2157          * at VMEntry. Skip the MSR read/write when a guest has no use case to
2158          * execute VERW.
2159          */
2160         if ((vcpu->arch.arch_capabilities & ARCH_CAP_FB_CLEAR) ||
2161            ((vcpu->arch.arch_capabilities & ARCH_CAP_MDS_NO) &&
2162             (vcpu->arch.arch_capabilities & ARCH_CAP_TAA_NO) &&
2163             (vcpu->arch.arch_capabilities & ARCH_CAP_PSDP_NO) &&
2164             (vcpu->arch.arch_capabilities & ARCH_CAP_FBSDP_NO) &&
2165             (vcpu->arch.arch_capabilities & ARCH_CAP_SBDR_SSDP_NO)))
2166                 vmx->disable_fb_clear = false;
2167 }
2168
2169 static struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr)
2170 {
2171         int i;
2172
2173         i = __find_msr_index(vmx, msr);
2174         if (i >= 0)
2175                 return &vmx->guest_msrs[i];
2176         return NULL;
2177 }
2178
2179 static void vmcs_clear(struct vmcs *vmcs)
2180 {
2181         u64 phys_addr = __pa(vmcs);
2182         bool error;
2183
2184         asm volatile (__ex(ASM_VMX_VMCLEAR_RAX) CC_SET(na)
2185                       : CC_OUT(na) (error) : "a"(&phys_addr), "m"(phys_addr)
2186                       : "memory");
2187         if (unlikely(error))
2188                 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
2189                        vmcs, phys_addr);
2190 }
2191
2192 static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs)
2193 {
2194         vmcs_clear(loaded_vmcs->vmcs);
2195         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
2196                 vmcs_clear(loaded_vmcs->shadow_vmcs);
2197         loaded_vmcs->cpu = -1;
2198         loaded_vmcs->launched = 0;
2199 }
2200
2201 static void vmcs_load(struct vmcs *vmcs)
2202 {
2203         u64 phys_addr = __pa(vmcs);
2204         bool error;
2205
2206         if (static_branch_unlikely(&enable_evmcs))
2207                 return evmcs_load(phys_addr);
2208
2209         asm volatile (__ex(ASM_VMX_VMPTRLD_RAX) CC_SET(na)
2210                       : CC_OUT(na) (error) : "a"(&phys_addr), "m"(phys_addr)
2211                       : "memory");
2212         if (unlikely(error))
2213                 printk(KERN_ERR "kvm: vmptrld %p/%llx failed\n",
2214                        vmcs, phys_addr);
2215 }
2216
2217 #ifdef CONFIG_KEXEC_CORE
2218 static void crash_vmclear_local_loaded_vmcss(void)
2219 {
2220         int cpu = raw_smp_processor_id();
2221         struct loaded_vmcs *v;
2222
2223         list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu),
2224                             loaded_vmcss_on_cpu_link)
2225                 vmcs_clear(v->vmcs);
2226 }
2227 #endif /* CONFIG_KEXEC_CORE */
2228
2229 static void __loaded_vmcs_clear(void *arg)
2230 {
2231         struct loaded_vmcs *loaded_vmcs = arg;
2232         int cpu = raw_smp_processor_id();
2233
2234         if (loaded_vmcs->cpu != cpu)
2235                 return; /* vcpu migration can race with cpu offline */
2236         if (per_cpu(current_vmcs, cpu) == loaded_vmcs->vmcs)
2237                 per_cpu(current_vmcs, cpu) = NULL;
2238
2239         vmcs_clear(loaded_vmcs->vmcs);
2240         if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched)
2241                 vmcs_clear(loaded_vmcs->shadow_vmcs);
2242
2243         list_del(&loaded_vmcs->loaded_vmcss_on_cpu_link);
2244
2245         /*
2246          * Ensure all writes to loaded_vmcs, including deleting it from its
2247          * current percpu list, complete before setting loaded_vmcs->vcpu to
2248          * -1, otherwise a different cpu can see vcpu == -1 first and add
2249          * loaded_vmcs to its percpu list before it's deleted from this cpu's
2250          * list. Pairs with the smp_rmb() in vmx_vcpu_load_vmcs().
2251          */
2252         smp_wmb();
2253
2254         loaded_vmcs->cpu = -1;
2255         loaded_vmcs->launched = 0;
2256 }
2257
2258 static void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs)
2259 {
2260         int cpu = loaded_vmcs->cpu;
2261
2262         if (cpu != -1)
2263                 smp_call_function_single(cpu,
2264                          __loaded_vmcs_clear, loaded_vmcs, 1);
2265 }
2266
2267 static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr)
2268 {
2269         if (vpid == 0)
2270                 return true;
2271
2272         if (cpu_has_vmx_invvpid_individual_addr()) {
2273                 __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr);
2274                 return true;
2275         }
2276
2277         return false;
2278 }
2279
2280 static inline void vpid_sync_vcpu_single(int vpid)
2281 {
2282         if (vpid == 0)
2283                 return;
2284
2285         if (cpu_has_vmx_invvpid_single())
2286                 __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0);
2287 }
2288
2289 static inline void vpid_sync_vcpu_global(void)
2290 {
2291         if (cpu_has_vmx_invvpid_global())
2292                 __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0);
2293 }
2294
2295 static inline void vpid_sync_context(int vpid)
2296 {
2297         if (cpu_has_vmx_invvpid_single())
2298                 vpid_sync_vcpu_single(vpid);
2299         else
2300                 vpid_sync_vcpu_global();
2301 }
2302
2303 static inline void ept_sync_global(void)
2304 {
2305         __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0);
2306 }
2307
2308 static inline void ept_sync_context(u64 eptp)
2309 {
2310         if (cpu_has_vmx_invept_context())
2311                 __invept(VMX_EPT_EXTENT_CONTEXT, eptp, 0);
2312         else
2313                 ept_sync_global();
2314 }
2315
2316 static __always_inline void vmcs_check16(unsigned long field)
2317 {
2318         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
2319                          "16-bit accessor invalid for 64-bit field");
2320         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2321                          "16-bit accessor invalid for 64-bit high field");
2322         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2323                          "16-bit accessor invalid for 32-bit high field");
2324         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2325                          "16-bit accessor invalid for natural width field");
2326 }
2327
2328 static __always_inline void vmcs_check32(unsigned long field)
2329 {
2330         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2331                          "32-bit accessor invalid for 16-bit field");
2332         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2333                          "32-bit accessor invalid for natural width field");
2334 }
2335
2336 static __always_inline void vmcs_check64(unsigned long field)
2337 {
2338         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2339                          "64-bit accessor invalid for 16-bit field");
2340         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2341                          "64-bit accessor invalid for 64-bit high field");
2342         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2343                          "64-bit accessor invalid for 32-bit field");
2344         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x6000,
2345                          "64-bit accessor invalid for natural width field");
2346 }
2347
2348 static __always_inline void vmcs_checkl(unsigned long field)
2349 {
2350         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0,
2351                          "Natural width accessor invalid for 16-bit field");
2352         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2000,
2353                          "Natural width accessor invalid for 64-bit field");
2354         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6001) == 0x2001,
2355                          "Natural width accessor invalid for 64-bit high field");
2356         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x4000,
2357                          "Natural width accessor invalid for 32-bit field");
2358 }
2359
2360 static __always_inline unsigned long __vmcs_readl(unsigned long field)
2361 {
2362         unsigned long value;
2363
2364         asm volatile (__ex_clear(ASM_VMX_VMREAD_RDX_RAX, "%0")
2365                       : "=a"(value) : "d"(field) : "cc");
2366         return value;
2367 }
2368
2369 static __always_inline u16 vmcs_read16(unsigned long field)
2370 {
2371         vmcs_check16(field);
2372         if (static_branch_unlikely(&enable_evmcs))
2373                 return evmcs_read16(field);
2374         return __vmcs_readl(field);
2375 }
2376
2377 static __always_inline u32 vmcs_read32(unsigned long field)
2378 {
2379         vmcs_check32(field);
2380         if (static_branch_unlikely(&enable_evmcs))
2381                 return evmcs_read32(field);
2382         return __vmcs_readl(field);
2383 }
2384
2385 static __always_inline u64 vmcs_read64(unsigned long field)
2386 {
2387         vmcs_check64(field);
2388         if (static_branch_unlikely(&enable_evmcs))
2389                 return evmcs_read64(field);
2390 #ifdef CONFIG_X86_64
2391         return __vmcs_readl(field);
2392 #else
2393         return __vmcs_readl(field) | ((u64)__vmcs_readl(field+1) << 32);
2394 #endif
2395 }
2396
2397 static __always_inline unsigned long vmcs_readl(unsigned long field)
2398 {
2399         vmcs_checkl(field);
2400         if (static_branch_unlikely(&enable_evmcs))
2401                 return evmcs_read64(field);
2402         return __vmcs_readl(field);
2403 }
2404
2405 static noinline void vmwrite_error(unsigned long field, unsigned long value)
2406 {
2407         printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
2408                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
2409         dump_stack();
2410 }
2411
2412 static __always_inline void __vmcs_writel(unsigned long field, unsigned long value)
2413 {
2414         bool error;
2415
2416         asm volatile (__ex(ASM_VMX_VMWRITE_RAX_RDX) CC_SET(na)
2417                       : CC_OUT(na) (error) : "a"(value), "d"(field));
2418         if (unlikely(error))
2419                 vmwrite_error(field, value);
2420 }
2421
2422 static __always_inline void vmcs_write16(unsigned long field, u16 value)
2423 {
2424         vmcs_check16(field);
2425         if (static_branch_unlikely(&enable_evmcs))
2426                 return evmcs_write16(field, value);
2427
2428         __vmcs_writel(field, value);
2429 }
2430
2431 static __always_inline void vmcs_write32(unsigned long field, u32 value)
2432 {
2433         vmcs_check32(field);
2434         if (static_branch_unlikely(&enable_evmcs))
2435                 return evmcs_write32(field, value);
2436
2437         __vmcs_writel(field, value);
2438 }
2439
2440 static __always_inline void vmcs_write64(unsigned long field, u64 value)
2441 {
2442         vmcs_check64(field);
2443         if (static_branch_unlikely(&enable_evmcs))
2444                 return evmcs_write64(field, value);
2445
2446         __vmcs_writel(field, value);
2447 #ifndef CONFIG_X86_64
2448         asm volatile ("");
2449         __vmcs_writel(field+1, value >> 32);
2450 #endif
2451 }
2452
2453 static __always_inline void vmcs_writel(unsigned long field, unsigned long value)
2454 {
2455         vmcs_checkl(field);
2456         if (static_branch_unlikely(&enable_evmcs))
2457                 return evmcs_write64(field, value);
2458
2459         __vmcs_writel(field, value);
2460 }
2461
2462 static __always_inline void vmcs_clear_bits(unsigned long field, u32 mask)
2463 {
2464         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
2465                          "vmcs_clear_bits does not support 64-bit fields");
2466         if (static_branch_unlikely(&enable_evmcs))
2467                 return evmcs_write32(field, evmcs_read32(field) & ~mask);
2468
2469         __vmcs_writel(field, __vmcs_readl(field) & ~mask);
2470 }
2471
2472 static __always_inline void vmcs_set_bits(unsigned long field, u32 mask)
2473 {
2474         BUILD_BUG_ON_MSG(__builtin_constant_p(field) && ((field) & 0x6000) == 0x2000,
2475                          "vmcs_set_bits does not support 64-bit fields");
2476         if (static_branch_unlikely(&enable_evmcs))
2477                 return evmcs_write32(field, evmcs_read32(field) | mask);
2478
2479         __vmcs_writel(field, __vmcs_readl(field) | mask);
2480 }
2481
2482 static inline void vm_entry_controls_reset_shadow(struct vcpu_vmx *vmx)
2483 {
2484         vmx->vm_entry_controls_shadow = vmcs_read32(VM_ENTRY_CONTROLS);
2485 }
2486
2487 static inline void vm_entry_controls_init(struct vcpu_vmx *vmx, u32 val)
2488 {
2489         vmcs_write32(VM_ENTRY_CONTROLS, val);
2490         vmx->vm_entry_controls_shadow = val;
2491 }
2492
2493 static inline void vm_entry_controls_set(struct vcpu_vmx *vmx, u32 val)
2494 {
2495         if (vmx->vm_entry_controls_shadow != val)
2496                 vm_entry_controls_init(vmx, val);
2497 }
2498
2499 static inline u32 vm_entry_controls_get(struct vcpu_vmx *vmx)
2500 {
2501         return vmx->vm_entry_controls_shadow;
2502 }
2503
2504
2505 static inline void vm_entry_controls_setbit(struct vcpu_vmx *vmx, u32 val)
2506 {
2507         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) | val);
2508 }
2509
2510 static inline void vm_entry_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
2511 {
2512         vm_entry_controls_set(vmx, vm_entry_controls_get(vmx) & ~val);
2513 }
2514
2515 static inline void vm_exit_controls_reset_shadow(struct vcpu_vmx *vmx)
2516 {
2517         vmx->vm_exit_controls_shadow = vmcs_read32(VM_EXIT_CONTROLS);
2518 }
2519
2520 static inline void vm_exit_controls_init(struct vcpu_vmx *vmx, u32 val)
2521 {
2522         vmcs_write32(VM_EXIT_CONTROLS, val);
2523         vmx->vm_exit_controls_shadow = val;
2524 }
2525
2526 static inline void vm_exit_controls_set(struct vcpu_vmx *vmx, u32 val)
2527 {
2528         if (vmx->vm_exit_controls_shadow != val)
2529                 vm_exit_controls_init(vmx, val);
2530 }
2531
2532 static inline u32 vm_exit_controls_get(struct vcpu_vmx *vmx)
2533 {
2534         return vmx->vm_exit_controls_shadow;
2535 }
2536
2537
2538 static inline void vm_exit_controls_setbit(struct vcpu_vmx *vmx, u32 val)
2539 {
2540         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) | val);
2541 }
2542
2543 static inline void vm_exit_controls_clearbit(struct vcpu_vmx *vmx, u32 val)
2544 {
2545         vm_exit_controls_set(vmx, vm_exit_controls_get(vmx) & ~val);
2546 }
2547
2548 static void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
2549 {
2550         vmx->segment_cache.bitmask = 0;
2551 }
2552
2553 static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
2554                                        unsigned field)
2555 {
2556         bool ret;
2557         u32 mask = 1 << (seg * SEG_FIELD_NR + field);
2558
2559         if (!(vmx->vcpu.arch.regs_avail & (1 << VCPU_EXREG_SEGMENTS))) {
2560                 vmx->vcpu.arch.regs_avail |= (1 << VCPU_EXREG_SEGMENTS);
2561                 vmx->segment_cache.bitmask = 0;
2562         }
2563         ret = vmx->segment_cache.bitmask & mask;
2564         vmx->segment_cache.bitmask |= mask;
2565         return ret;
2566 }
2567
2568 static u16 vmx_read_guest_seg_selector(struct vcpu_vmx *vmx, unsigned seg)
2569 {
2570         u16 *p = &vmx->segment_cache.seg[seg].selector;
2571
2572         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_SEL))
2573                 *p = vmcs_read16(kvm_vmx_segment_fields[seg].selector);
2574         return *p;
2575 }
2576
2577 static ulong vmx_read_guest_seg_base(struct vcpu_vmx *vmx, unsigned seg)
2578 {
2579         ulong *p = &vmx->segment_cache.seg[seg].base;
2580
2581         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_BASE))
2582                 *p = vmcs_readl(kvm_vmx_segment_fields[seg].base);
2583         return *p;
2584 }
2585
2586 static u32 vmx_read_guest_seg_limit(struct vcpu_vmx *vmx, unsigned seg)
2587 {
2588         u32 *p = &vmx->segment_cache.seg[seg].limit;
2589
2590         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_LIMIT))
2591                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].limit);
2592         return *p;
2593 }
2594
2595 static u32 vmx_read_guest_seg_ar(struct vcpu_vmx *vmx, unsigned seg)
2596 {
2597         u32 *p = &vmx->segment_cache.seg[seg].ar;
2598
2599         if (!vmx_segment_cache_test_set(vmx, seg, SEG_FIELD_AR))
2600                 *p = vmcs_read32(kvm_vmx_segment_fields[seg].ar_bytes);
2601         return *p;
2602 }
2603
2604 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
2605 {
2606         u32 eb;
2607
2608         eb = (1u << PF_VECTOR) | (1u << UD_VECTOR) | (1u << MC_VECTOR) |
2609              (1u << DB_VECTOR) | (1u << AC_VECTOR);
2610         /*
2611          * Guest access to VMware backdoor ports could legitimately
2612          * trigger #GP because of TSS I/O permission bitmap.
2613          * We intercept those #GP and allow access to them anyway
2614          * as VMware does.
2615          */
2616         if (enable_vmware_backdoor)
2617                 eb |= (1u << GP_VECTOR);
2618         if ((vcpu->guest_debug &
2619              (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP)) ==
2620             (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP))
2621                 eb |= 1u << BP_VECTOR;
2622         if (to_vmx(vcpu)->rmode.vm86_active)
2623                 eb = ~0;
2624         if (enable_ept)
2625                 eb &= ~(1u << PF_VECTOR); /* bypass_guest_pf = 0 */
2626
2627         /* When we are running a nested L2 guest and L1 specified for it a
2628          * certain exception bitmap, we must trap the same exceptions and pass
2629          * them to L1. When running L2, we will only handle the exceptions
2630          * specified above if L1 did not want them.
2631          */
2632         if (is_guest_mode(vcpu))
2633                 eb |= get_vmcs12(vcpu)->exception_bitmap;
2634
2635         vmcs_write32(EXCEPTION_BITMAP, eb);
2636 }
2637
2638 /*
2639  * Check if MSR is intercepted for currently loaded MSR bitmap.
2640  */
2641 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
2642 {
2643         unsigned long *msr_bitmap;
2644         int f = sizeof(unsigned long);
2645
2646         if (!cpu_has_vmx_msr_bitmap())
2647                 return true;
2648
2649         msr_bitmap = to_vmx(vcpu)->loaded_vmcs->msr_bitmap;
2650
2651         if (msr <= 0x1fff) {
2652                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
2653         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2654                 msr &= 0x1fff;
2655                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
2656         }
2657
2658         return true;
2659 }
2660
2661 /*
2662  * Check if MSR is intercepted for L01 MSR bitmap.
2663  */
2664 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
2665 {
2666         unsigned long *msr_bitmap;
2667         int f = sizeof(unsigned long);
2668
2669         if (!cpu_has_vmx_msr_bitmap())
2670                 return true;
2671
2672         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
2673
2674         if (msr <= 0x1fff) {
2675                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
2676         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
2677                 msr &= 0x1fff;
2678                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
2679         }
2680
2681         return true;
2682 }
2683
2684 static void clear_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2685                 unsigned long entry, unsigned long exit)
2686 {
2687         vm_entry_controls_clearbit(vmx, entry);
2688         vm_exit_controls_clearbit(vmx, exit);
2689 }
2690
2691 static int find_msr(struct vmx_msrs *m, unsigned int msr)
2692 {
2693         unsigned int i;
2694
2695         for (i = 0; i < m->nr; ++i) {
2696                 if (m->val[i].index == msr)
2697                         return i;
2698         }
2699         return -ENOENT;
2700 }
2701
2702 static void clear_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr)
2703 {
2704         int i;
2705         struct msr_autoload *m = &vmx->msr_autoload;
2706
2707         switch (msr) {
2708         case MSR_EFER:
2709                 if (cpu_has_load_ia32_efer) {
2710                         clear_atomic_switch_msr_special(vmx,
2711                                         VM_ENTRY_LOAD_IA32_EFER,
2712                                         VM_EXIT_LOAD_IA32_EFER);
2713                         return;
2714                 }
2715                 break;
2716         case MSR_CORE_PERF_GLOBAL_CTRL:
2717                 if (cpu_has_load_perf_global_ctrl) {
2718                         clear_atomic_switch_msr_special(vmx,
2719                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2720                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
2721                         return;
2722                 }
2723                 break;
2724         }
2725         i = find_msr(&m->guest, msr);
2726         if (i < 0)
2727                 goto skip_guest;
2728         --m->guest.nr;
2729         m->guest.val[i] = m->guest.val[m->guest.nr];
2730         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
2731
2732 skip_guest:
2733         i = find_msr(&m->host, msr);
2734         if (i < 0)
2735                 return;
2736
2737         --m->host.nr;
2738         m->host.val[i] = m->host.val[m->host.nr];
2739         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
2740 }
2741
2742 static void add_atomic_switch_msr_special(struct vcpu_vmx *vmx,
2743                 unsigned long entry, unsigned long exit,
2744                 unsigned long guest_val_vmcs, unsigned long host_val_vmcs,
2745                 u64 guest_val, u64 host_val)
2746 {
2747         vmcs_write64(guest_val_vmcs, guest_val);
2748         vmcs_write64(host_val_vmcs, host_val);
2749         vm_entry_controls_setbit(vmx, entry);
2750         vm_exit_controls_setbit(vmx, exit);
2751 }
2752
2753 static void add_atomic_switch_msr(struct vcpu_vmx *vmx, unsigned msr,
2754                                   u64 guest_val, u64 host_val, bool entry_only)
2755 {
2756         int i, j = 0;
2757         struct msr_autoload *m = &vmx->msr_autoload;
2758
2759         switch (msr) {
2760         case MSR_EFER:
2761                 if (cpu_has_load_ia32_efer) {
2762                         add_atomic_switch_msr_special(vmx,
2763                                         VM_ENTRY_LOAD_IA32_EFER,
2764                                         VM_EXIT_LOAD_IA32_EFER,
2765                                         GUEST_IA32_EFER,
2766                                         HOST_IA32_EFER,
2767                                         guest_val, host_val);
2768                         return;
2769                 }
2770                 break;
2771         case MSR_CORE_PERF_GLOBAL_CTRL:
2772                 if (cpu_has_load_perf_global_ctrl) {
2773                         add_atomic_switch_msr_special(vmx,
2774                                         VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL,
2775                                         VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL,
2776                                         GUEST_IA32_PERF_GLOBAL_CTRL,
2777                                         HOST_IA32_PERF_GLOBAL_CTRL,
2778                                         guest_val, host_val);
2779                         return;
2780                 }
2781                 break;
2782         case MSR_IA32_PEBS_ENABLE:
2783                 /* PEBS needs a quiescent period after being disabled (to write
2784                  * a record).  Disabling PEBS through VMX MSR swapping doesn't
2785                  * provide that period, so a CPU could write host's record into
2786                  * guest's memory.
2787                  */
2788                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
2789         }
2790
2791         i = find_msr(&m->guest, msr);
2792         if (!entry_only)
2793                 j = find_msr(&m->host, msr);
2794
2795         if ((i < 0 && m->guest.nr == NR_AUTOLOAD_MSRS) ||
2796                 (j < 0 &&  m->host.nr == NR_AUTOLOAD_MSRS)) {
2797                 printk_once(KERN_WARNING "Not enough msr switch entries. "
2798                                 "Can't add msr %x\n", msr);
2799                 return;
2800         }
2801         if (i < 0) {
2802                 i = m->guest.nr++;
2803                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, m->guest.nr);
2804         }
2805         m->guest.val[i].index = msr;
2806         m->guest.val[i].value = guest_val;
2807
2808         if (entry_only)
2809                 return;
2810
2811         if (j < 0) {
2812                 j = m->host.nr++;
2813                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, m->host.nr);
2814         }
2815         m->host.val[j].index = msr;
2816         m->host.val[j].value = host_val;
2817 }
2818
2819 static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
2820 {
2821         u64 guest_efer = vmx->vcpu.arch.efer;
2822         u64 ignore_bits = 0;
2823
2824         /* Shadow paging assumes NX to be available.  */
2825         if (!enable_ept)
2826                 guest_efer |= EFER_NX;
2827
2828         /*
2829          * LMA and LME handled by hardware; SCE meaningless outside long mode.
2830          */
2831         ignore_bits |= EFER_SCE;
2832 #ifdef CONFIG_X86_64
2833         ignore_bits |= EFER_LMA | EFER_LME;
2834         /* SCE is meaningful only in long mode on Intel */
2835         if (guest_efer & EFER_LMA)
2836                 ignore_bits &= ~(u64)EFER_SCE;
2837 #endif
2838
2839         clear_atomic_switch_msr(vmx, MSR_EFER);
2840
2841         /*
2842          * On EPT, we can't emulate NX, so we must switch EFER atomically.
2843          * On CPUs that support "load IA32_EFER", always switch EFER
2844          * atomically, since it's faster than switching it manually.
2845          */
2846         if (cpu_has_load_ia32_efer ||
2847             (enable_ept && ((vmx->vcpu.arch.efer ^ host_efer) & EFER_NX))) {
2848                 if (!(guest_efer & EFER_LMA))
2849                         guest_efer &= ~EFER_LME;
2850                 if (guest_efer != host_efer)
2851                         add_atomic_switch_msr(vmx, MSR_EFER,
2852                                               guest_efer, host_efer, false);
2853                 return false;
2854         } else {
2855                 guest_efer &= ~ignore_bits;
2856                 guest_efer |= host_efer & ignore_bits;
2857
2858                 vmx->guest_msrs[efer_offset].data = guest_efer;
2859                 vmx->guest_msrs[efer_offset].mask = ~ignore_bits;
2860
2861                 return true;
2862         }
2863 }
2864
2865 #ifdef CONFIG_X86_32
2866 /*
2867  * On 32-bit kernels, VM exits still load the FS and GS bases from the
2868  * VMCS rather than the segment table.  KVM uses this helper to figure
2869  * out the current bases to poke them into the VMCS before entry.
2870  */
2871 static unsigned long segment_base(u16 selector)
2872 {
2873         struct desc_struct *table;
2874         unsigned long v;
2875
2876         if (!(selector & ~SEGMENT_RPL_MASK))
2877                 return 0;
2878
2879         table = get_current_gdt_ro();
2880
2881         if ((selector & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2882                 u16 ldt_selector = kvm_read_ldt();
2883
2884                 if (!(ldt_selector & ~SEGMENT_RPL_MASK))
2885                         return 0;
2886
2887                 table = (struct desc_struct *)segment_base(ldt_selector);
2888         }
2889         v = get_desc_base(&table[selector >> 3]);
2890         return v;
2891 }
2892 #endif
2893
2894 static void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
2895 {
2896         struct vcpu_vmx *vmx = to_vmx(vcpu);
2897         struct vmcs_host_state *host_state;
2898 #ifdef CONFIG_X86_64
2899         int cpu = raw_smp_processor_id();
2900 #endif
2901         unsigned long fs_base, gs_base;
2902         u16 fs_sel, gs_sel;
2903         int i;
2904
2905         vmx->req_immediate_exit = false;
2906
2907         /*
2908          * Note that guest MSRs to be saved/restored can also be changed
2909          * when guest state is loaded. This happens when guest transitions
2910          * to/from long-mode by setting MSR_EFER.LMA.
2911          */
2912         if (!vmx->loaded_cpu_state || vmx->guest_msrs_dirty) {
2913                 vmx->guest_msrs_dirty = false;
2914                 for (i = 0; i < vmx->save_nmsrs; ++i)
2915                         kvm_set_shared_msr(vmx->guest_msrs[i].index,
2916                                            vmx->guest_msrs[i].data,
2917                                            vmx->guest_msrs[i].mask);
2918
2919         }
2920
2921         if (vmx->loaded_cpu_state)
2922                 return;
2923
2924         vmx->loaded_cpu_state = vmx->loaded_vmcs;
2925         host_state = &vmx->loaded_cpu_state->host_state;
2926
2927         /*
2928          * Set host fs and gs selectors.  Unfortunately, 22.2.3 does not
2929          * allow segment selectors with cpl > 0 or ti == 1.
2930          */
2931         host_state->ldt_sel = kvm_read_ldt();
2932
2933 #ifdef CONFIG_X86_64
2934         savesegment(ds, host_state->ds_sel);
2935         savesegment(es, host_state->es_sel);
2936
2937         gs_base = cpu_kernelmode_gs_base(cpu);
2938         if (likely(is_64bit_mm(current->mm))) {
2939                 save_fsgs_for_kvm();
2940                 fs_sel = current->thread.fsindex;
2941                 gs_sel = current->thread.gsindex;
2942                 fs_base = current->thread.fsbase;
2943                 vmx->msr_host_kernel_gs_base = current->thread.gsbase;
2944         } else {
2945                 savesegment(fs, fs_sel);
2946                 savesegment(gs, gs_sel);
2947                 fs_base = read_msr(MSR_FS_BASE);
2948                 vmx->msr_host_kernel_gs_base = read_msr(MSR_KERNEL_GS_BASE);
2949         }
2950
2951         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2952 #else
2953         savesegment(fs, fs_sel);
2954         savesegment(gs, gs_sel);
2955         fs_base = segment_base(fs_sel);
2956         gs_base = segment_base(gs_sel);
2957 #endif
2958
2959         if (unlikely(fs_sel != host_state->fs_sel)) {
2960                 if (!(fs_sel & 7))
2961                         vmcs_write16(HOST_FS_SELECTOR, fs_sel);
2962                 else
2963                         vmcs_write16(HOST_FS_SELECTOR, 0);
2964                 host_state->fs_sel = fs_sel;
2965         }
2966         if (unlikely(gs_sel != host_state->gs_sel)) {
2967                 if (!(gs_sel & 7))
2968                         vmcs_write16(HOST_GS_SELECTOR, gs_sel);
2969                 else
2970                         vmcs_write16(HOST_GS_SELECTOR, 0);
2971                 host_state->gs_sel = gs_sel;
2972         }
2973         if (unlikely(fs_base != host_state->fs_base)) {
2974                 vmcs_writel(HOST_FS_BASE, fs_base);
2975                 host_state->fs_base = fs_base;
2976         }
2977         if (unlikely(gs_base != host_state->gs_base)) {
2978                 vmcs_writel(HOST_GS_BASE, gs_base);
2979                 host_state->gs_base = gs_base;
2980         }
2981 }
2982
2983 static void vmx_prepare_switch_to_host(struct vcpu_vmx *vmx)
2984 {
2985         struct vmcs_host_state *host_state;
2986
2987         if (!vmx->loaded_cpu_state)
2988                 return;
2989
2990         WARN_ON_ONCE(vmx->loaded_cpu_state != vmx->loaded_vmcs);
2991         host_state = &vmx->loaded_cpu_state->host_state;
2992
2993         ++vmx->vcpu.stat.host_state_reload;
2994         vmx->loaded_cpu_state = NULL;
2995
2996 #ifdef CONFIG_X86_64
2997         rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
2998 #endif
2999         if (host_state->ldt_sel || (host_state->gs_sel & 7)) {
3000                 kvm_load_ldt(host_state->ldt_sel);
3001 #ifdef CONFIG_X86_64
3002                 load_gs_index(host_state->gs_sel);
3003 #else
3004                 loadsegment(gs, host_state->gs_sel);
3005 #endif
3006         }
3007         if (host_state->fs_sel & 7)
3008                 loadsegment(fs, host_state->fs_sel);
3009 #ifdef CONFIG_X86_64
3010         if (unlikely(host_state->ds_sel | host_state->es_sel)) {
3011                 loadsegment(ds, host_state->ds_sel);
3012                 loadsegment(es, host_state->es_sel);
3013         }
3014 #endif
3015         invalidate_tss_limit();
3016 #ifdef CONFIG_X86_64
3017         wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
3018 #endif
3019         load_fixmap_gdt(raw_smp_processor_id());
3020 }
3021
3022 #ifdef CONFIG_X86_64
3023 static u64 vmx_read_guest_kernel_gs_base(struct vcpu_vmx *vmx)
3024 {
3025         preempt_disable();
3026         if (vmx->loaded_cpu_state)
3027                 rdmsrl(MSR_KERNEL_GS_BASE, vmx->msr_guest_kernel_gs_base);
3028         preempt_enable();
3029         return vmx->msr_guest_kernel_gs_base;
3030 }
3031
3032 static void vmx_write_guest_kernel_gs_base(struct vcpu_vmx *vmx, u64 data)
3033 {
3034         preempt_disable();
3035         if (vmx->loaded_cpu_state)
3036                 wrmsrl(MSR_KERNEL_GS_BASE, data);
3037         preempt_enable();
3038         vmx->msr_guest_kernel_gs_base = data;
3039 }
3040 #endif
3041
3042 static void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu)
3043 {
3044         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
3045         struct pi_desc old, new;
3046         unsigned int dest;
3047
3048         /*
3049          * In case of hot-plug or hot-unplug, we may have to undo
3050          * vmx_vcpu_pi_put even if there is no assigned device.  And we
3051          * always keep PI.NDST up to date for simplicity: it makes the
3052          * code easier, and CPU migration is not a fast path.
3053          */
3054         if (!pi_test_sn(pi_desc) && vcpu->cpu == cpu)
3055                 return;
3056
3057         /*
3058          * First handle the simple case where no cmpxchg is necessary; just
3059          * allow posting non-urgent interrupts.
3060          *
3061          * If the 'nv' field is POSTED_INTR_WAKEUP_VECTOR, do not change
3062          * PI.NDST: pi_post_block will do it for us and the wakeup_handler
3063          * expects the VCPU to be on the blocked_vcpu_list that matches
3064          * PI.NDST.
3065          */
3066         if (pi_desc->nv == POSTED_INTR_WAKEUP_VECTOR ||
3067             vcpu->cpu == cpu) {
3068                 pi_clear_sn(pi_desc);
3069                 return;
3070         }
3071
3072         /* The full case.  */
3073         do {
3074                 old.control = new.control = pi_desc->control;
3075
3076                 dest = cpu_physical_id(cpu);
3077
3078                 if (x2apic_enabled())
3079                         new.ndst = dest;
3080                 else
3081                         new.ndst = (dest << 8) & 0xFF00;
3082
3083                 new.sn = 0;
3084         } while (cmpxchg64(&pi_desc->control, old.control,
3085                            new.control) != old.control);
3086 }
3087
3088 static void decache_tsc_multiplier(struct vcpu_vmx *vmx)
3089 {
3090         vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio;
3091         vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio);
3092 }
3093
3094 /*
3095  * Switches to specified vcpu, until a matching vcpu_put(), but assumes
3096  * vcpu mutex is already taken.
3097  */
3098 static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3099 {
3100         struct vcpu_vmx *vmx = to_vmx(vcpu);
3101         bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
3102
3103         if (!already_loaded) {
3104                 loaded_vmcs_clear(vmx->loaded_vmcs);
3105                 local_irq_disable();
3106
3107                 /*
3108                  * Ensure loaded_vmcs->cpu is read before adding loaded_vmcs to
3109                  * this cpu's percpu list, otherwise it may not yet be deleted
3110                  * from its previous cpu's percpu list.  Pairs with the
3111                  * smb_wmb() in __loaded_vmcs_clear().
3112                  */
3113                 smp_rmb();
3114
3115                 list_add(&vmx->loaded_vmcs->loaded_vmcss_on_cpu_link,
3116                          &per_cpu(loaded_vmcss_on_cpu, cpu));
3117                 local_irq_enable();
3118         }
3119
3120         if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
3121                 per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
3122                 vmcs_load(vmx->loaded_vmcs->vmcs);
3123                 indirect_branch_prediction_barrier();
3124         }
3125
3126         if (!already_loaded) {
3127                 void *gdt = get_current_gdt_ro();
3128                 unsigned long sysenter_esp;
3129
3130                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3131
3132                 /*
3133                  * Linux uses per-cpu TSS and GDT, so set these when switching
3134                  * processors.  See 22.2.4.
3135                  */
3136                 vmcs_writel(HOST_TR_BASE,
3137                             (unsigned long)&get_cpu_entry_area(cpu)->tss.x86_tss);
3138                 vmcs_writel(HOST_GDTR_BASE, (unsigned long)gdt);   /* 22.2.4 */
3139
3140                 /*
3141                  * VM exits change the host TR limit to 0x67 after a VM
3142                  * exit.  This is okay, since 0x67 covers everything except
3143                  * the IO bitmap and have have code to handle the IO bitmap
3144                  * being lost after a VM exit.
3145                  */
3146                 BUILD_BUG_ON(IO_BITMAP_OFFSET - 1 != 0x67);
3147
3148                 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
3149                 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
3150
3151                 vmx->loaded_vmcs->cpu = cpu;
3152         }
3153
3154         /* Setup TSC multiplier */
3155         if (kvm_has_tsc_control &&
3156             vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio)
3157                 decache_tsc_multiplier(vmx);
3158
3159         vmx_vcpu_pi_load(vcpu, cpu);
3160         vmx->host_pkru = read_pkru();
3161         vmx->host_debugctlmsr = get_debugctlmsr();
3162 }
3163
3164 static void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu)
3165 {
3166         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
3167
3168         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
3169                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
3170                 !kvm_vcpu_apicv_active(vcpu))
3171                 return;
3172
3173         /* Set SN when the vCPU is preempted */
3174         if (vcpu->preempted)
3175                 pi_set_sn(pi_desc);
3176 }
3177
3178 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
3179 {
3180         vmx_vcpu_pi_put(vcpu);
3181
3182         vmx_prepare_switch_to_host(to_vmx(vcpu));
3183 }
3184
3185 static bool emulation_required(struct kvm_vcpu *vcpu)
3186 {
3187         return emulate_invalid_guest_state && !guest_state_valid(vcpu);
3188 }
3189
3190 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu);
3191
3192 /*
3193  * Return the cr0 value that a nested guest would read. This is a combination
3194  * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by
3195  * its hypervisor (cr0_read_shadow).
3196  */
3197 static inline unsigned long nested_read_cr0(struct vmcs12 *fields)
3198 {
3199         return (fields->guest_cr0 & ~fields->cr0_guest_host_mask) |
3200                 (fields->cr0_read_shadow & fields->cr0_guest_host_mask);
3201 }
3202 static inline unsigned long nested_read_cr4(struct vmcs12 *fields)
3203 {
3204         return (fields->guest_cr4 & ~fields->cr4_guest_host_mask) |
3205                 (fields->cr4_read_shadow & fields->cr4_guest_host_mask);
3206 }
3207
3208 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
3209 {
3210         unsigned long rflags, save_rflags;
3211
3212         if (!test_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail)) {
3213                 __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
3214                 rflags = vmcs_readl(GUEST_RFLAGS);
3215                 if (to_vmx(vcpu)->rmode.vm86_active) {
3216                         rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
3217                         save_rflags = to_vmx(vcpu)->rmode.save_rflags;
3218                         rflags |= save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
3219                 }
3220                 to_vmx(vcpu)->rflags = rflags;
3221         }
3222         return to_vmx(vcpu)->rflags;
3223 }
3224
3225 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
3226 {
3227         unsigned long old_rflags = vmx_get_rflags(vcpu);
3228
3229         __set_bit(VCPU_EXREG_RFLAGS, (ulong *)&vcpu->arch.regs_avail);
3230         to_vmx(vcpu)->rflags = rflags;
3231         if (to_vmx(vcpu)->rmode.vm86_active) {
3232                 to_vmx(vcpu)->rmode.save_rflags = rflags;
3233                 rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
3234         }
3235         vmcs_writel(GUEST_RFLAGS, rflags);
3236
3237         if ((old_rflags ^ to_vmx(vcpu)->rflags) & X86_EFLAGS_VM)
3238                 to_vmx(vcpu)->emulation_required = emulation_required(vcpu);
3239 }
3240
3241 static u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu)
3242 {
3243         u32 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3244         int ret = 0;
3245
3246         if (interruptibility & GUEST_INTR_STATE_STI)
3247                 ret |= KVM_X86_SHADOW_INT_STI;
3248         if (interruptibility & GUEST_INTR_STATE_MOV_SS)
3249                 ret |= KVM_X86_SHADOW_INT_MOV_SS;
3250
3251         return ret;
3252 }
3253
3254 static void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
3255 {
3256         u32 interruptibility_old = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3257         u32 interruptibility = interruptibility_old;
3258
3259         interruptibility &= ~(GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS);
3260
3261         if (mask & KVM_X86_SHADOW_INT_MOV_SS)
3262                 interruptibility |= GUEST_INTR_STATE_MOV_SS;
3263         else if (mask & KVM_X86_SHADOW_INT_STI)
3264                 interruptibility |= GUEST_INTR_STATE_STI;
3265
3266         if ((interruptibility != interruptibility_old))
3267                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility);
3268 }
3269
3270 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
3271 {
3272         unsigned long rip;
3273
3274         rip = kvm_rip_read(vcpu);
3275         rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3276         kvm_rip_write(vcpu, rip);
3277
3278         /* skipping an emulated instruction also counts */
3279         vmx_set_interrupt_shadow(vcpu, 0);
3280 }
3281
3282 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
3283                                                unsigned long exit_qual)
3284 {
3285         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3286         unsigned int nr = vcpu->arch.exception.nr;
3287         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3288
3289         if (vcpu->arch.exception.has_error_code) {
3290                 vmcs12->vm_exit_intr_error_code = vcpu->arch.exception.error_code;
3291                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3292         }
3293
3294         if (kvm_exception_is_soft(nr))
3295                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3296         else
3297                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3298
3299         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
3300             vmx_get_nmi_mask(vcpu))
3301                 intr_info |= INTR_INFO_UNBLOCK_NMI;
3302
3303         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
3304 }
3305
3306 /*
3307  * KVM wants to inject page-faults which it got to the guest. This function
3308  * checks whether in a nested guest, we need to inject them to L1 or L2.
3309  */
3310 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
3311 {
3312         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3313         unsigned int nr = vcpu->arch.exception.nr;
3314
3315         if (nr == PF_VECTOR) {
3316                 if (vcpu->arch.exception.nested_apf) {
3317                         *exit_qual = vcpu->arch.apf.nested_apf_token;
3318                         return 1;
3319                 }
3320                 /*
3321                  * FIXME: we must not write CR2 when L1 intercepts an L2 #PF exception.
3322                  * The fix is to add the ancillary datum (CR2 or DR6) to structs
3323                  * kvm_queued_exception and kvm_vcpu_events, so that CR2 and DR6
3324                  * can be written only when inject_pending_event runs.  This should be
3325                  * conditional on a new capability---if the capability is disabled,
3326                  * kvm_multiple_exception would write the ancillary information to
3327                  * CR2 or DR6, for backwards ABI-compatibility.
3328                  */
3329                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
3330                                                     vcpu->arch.exception.error_code)) {
3331                         *exit_qual = vcpu->arch.cr2;
3332                         return 1;
3333                 }
3334         } else {
3335                 if (vmcs12->exception_bitmap & (1u << nr)) {
3336                         if (nr == DB_VECTOR) {
3337                                 *exit_qual = vcpu->arch.dr6;
3338                                 *exit_qual &= ~(DR6_FIXED_1 | DR6_BT);
3339                                 *exit_qual ^= DR6_RTM;
3340                         } else {
3341                                 *exit_qual = 0;
3342                         }
3343                         return 1;
3344                 }
3345         }
3346
3347         return 0;
3348 }
3349
3350 static void vmx_clear_hlt(struct kvm_vcpu *vcpu)
3351 {
3352         /*
3353          * Ensure that we clear the HLT state in the VMCS.  We don't need to
3354          * explicitly skip the instruction because if the HLT state is set,
3355          * then the instruction is already executing and RIP has already been
3356          * advanced.
3357          */
3358         if (kvm_hlt_in_guest(vcpu->kvm) &&
3359                         vmcs_read32(GUEST_ACTIVITY_STATE) == GUEST_ACTIVITY_HLT)
3360                 vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
3361 }
3362
3363 static void vmx_queue_exception(struct kvm_vcpu *vcpu)
3364 {
3365         struct vcpu_vmx *vmx = to_vmx(vcpu);
3366         unsigned nr = vcpu->arch.exception.nr;
3367         bool has_error_code = vcpu->arch.exception.has_error_code;
3368         u32 error_code = vcpu->arch.exception.error_code;
3369         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3370
3371         if (has_error_code) {
3372                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
3373                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3374         }
3375
3376         if (vmx->rmode.vm86_active) {
3377                 int inc_eip = 0;
3378                 if (kvm_exception_is_soft(nr))
3379                         inc_eip = vcpu->arch.event_exit_inst_len;
3380                 if (kvm_inject_realmode_interrupt(vcpu, nr, inc_eip) != EMULATE_DONE)
3381                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3382                 return;
3383         }
3384
3385         WARN_ON_ONCE(vmx->emulation_required);
3386
3387         if (kvm_exception_is_soft(nr)) {
3388                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
3389                              vmx->vcpu.arch.event_exit_inst_len);
3390                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3391         } else
3392                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3393
3394         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info);
3395
3396         vmx_clear_hlt(vcpu);
3397 }
3398
3399 static bool vmx_rdtscp_supported(void)
3400 {
3401         return cpu_has_vmx_rdtscp();
3402 }
3403
3404 static bool vmx_invpcid_supported(void)
3405 {
3406         return cpu_has_vmx_invpcid();
3407 }
3408
3409 /*
3410  * Swap MSR entry in host/guest MSR entry array.
3411  */
3412 static void move_msr_up(struct vcpu_vmx *vmx, int from, int to)
3413 {
3414         struct shared_msr_entry tmp;
3415
3416         tmp = vmx->guest_msrs[to];
3417         vmx->guest_msrs[to] = vmx->guest_msrs[from];
3418         vmx->guest_msrs[from] = tmp;
3419 }
3420
3421 /*
3422  * Set up the vmcs to automatically save and restore system
3423  * msrs.  Don't touch the 64-bit msrs if the guest is in legacy
3424  * mode, as fiddling with msrs is very expensive.
3425  */
3426 static void setup_msrs(struct vcpu_vmx *vmx)
3427 {
3428         int save_nmsrs, index;
3429
3430         save_nmsrs = 0;
3431 #ifdef CONFIG_X86_64
3432         if (is_long_mode(&vmx->vcpu)) {
3433                 index = __find_msr_index(vmx, MSR_SYSCALL_MASK);
3434                 if (index >= 0)
3435                         move_msr_up(vmx, index, save_nmsrs++);
3436                 index = __find_msr_index(vmx, MSR_LSTAR);
3437                 if (index >= 0)
3438                         move_msr_up(vmx, index, save_nmsrs++);
3439                 index = __find_msr_index(vmx, MSR_CSTAR);
3440                 if (index >= 0)
3441                         move_msr_up(vmx, index, save_nmsrs++);
3442                 /*
3443                  * MSR_STAR is only needed on long mode guests, and only
3444                  * if efer.sce is enabled.
3445                  */
3446                 index = __find_msr_index(vmx, MSR_STAR);
3447                 if ((index >= 0) && (vmx->vcpu.arch.efer & EFER_SCE))
3448                         move_msr_up(vmx, index, save_nmsrs++);
3449         }
3450 #endif
3451         index = __find_msr_index(vmx, MSR_EFER);
3452         if (index >= 0 && update_transition_efer(vmx, index))
3453                 move_msr_up(vmx, index, save_nmsrs++);
3454         index = __find_msr_index(vmx, MSR_TSC_AUX);
3455         if (index >= 0 && guest_cpuid_has(&vmx->vcpu, X86_FEATURE_RDTSCP))
3456                 move_msr_up(vmx, index, save_nmsrs++);
3457
3458         vmx->save_nmsrs = save_nmsrs;
3459         vmx->guest_msrs_dirty = true;
3460
3461         if (cpu_has_vmx_msr_bitmap())
3462                 vmx_update_msr_bitmap(&vmx->vcpu);
3463 }
3464
3465 static u64 vmx_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
3466 {
3467         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3468
3469         if (is_guest_mode(vcpu) &&
3470             (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING))
3471                 return vcpu->arch.tsc_offset - vmcs12->tsc_offset;
3472
3473         return vcpu->arch.tsc_offset;
3474 }
3475
3476 static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
3477 {
3478         u64 active_offset = offset;
3479         if (is_guest_mode(vcpu)) {
3480                 /*
3481                  * We're here if L1 chose not to trap WRMSR to TSC. According
3482                  * to the spec, this should set L1's TSC; The offset that L1
3483                  * set for L2 remains unchanged, and still needs to be added
3484                  * to the newly set TSC to get L2's TSC.
3485                  */
3486                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3487                 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETING))
3488                         active_offset += vmcs12->tsc_offset;
3489         } else {
3490                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
3491                                            vmcs_read64(TSC_OFFSET), offset);
3492         }
3493
3494         vmcs_write64(TSC_OFFSET, active_offset);
3495         return active_offset;
3496 }
3497
3498 /*
3499  * nested_vmx_allowed() checks whether a guest should be allowed to use VMX
3500  * instructions and MSRs (i.e., nested VMX). Nested VMX is disabled for
3501  * all guests if the "nested" module option is off, and can also be disabled
3502  * for a single guest by disabling its VMX cpuid bit.
3503  */
3504 static inline bool nested_vmx_allowed(struct kvm_vcpu *vcpu)
3505 {
3506         return nested && guest_cpuid_has(vcpu, X86_FEATURE_VMX);
3507 }
3508
3509 /*
3510  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
3511  * returned for the various VMX controls MSRs when nested VMX is enabled.
3512  * The same values should also be used to verify that vmcs12 control fields are
3513  * valid during nested entry from L1 to L2.
3514  * Each of these control msrs has a low and high 32-bit half: A low bit is on
3515  * if the corresponding bit in the (32-bit) control field *must* be on, and a
3516  * bit in the high half is on if the corresponding bit in the control field
3517  * may be on. See also vmx_control_verify().
3518  */
3519 static void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, bool apicv)
3520 {
3521         if (!nested) {
3522                 memset(msrs, 0, sizeof(*msrs));
3523                 return;
3524         }
3525
3526         /*
3527          * Note that as a general rule, the high half of the MSRs (bits in
3528          * the control fields which may be 1) should be initialized by the
3529          * intersection of the underlying hardware's MSR (i.e., features which
3530          * can be supported) and the list of features we want to expose -
3531          * because they are known to be properly supported in our code.
3532          * Also, usually, the low half of the MSRs (bits which must be 1) can
3533          * be set to 0, meaning that L1 may turn off any of these bits. The
3534          * reason is that if one of these bits is necessary, it will appear
3535          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
3536          * fields of vmcs01 and vmcs02, will turn these bits off - and
3537          * nested_vmx_exit_reflected() will not pass related exits to L1.
3538          * These rules have exceptions below.
3539          */
3540
3541         /* pin-based controls */
3542         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
3543                 msrs->pinbased_ctls_low,
3544                 msrs->pinbased_ctls_high);
3545         msrs->pinbased_ctls_low |=
3546                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3547         msrs->pinbased_ctls_high &=
3548                 PIN_BASED_EXT_INTR_MASK |
3549                 PIN_BASED_NMI_EXITING |
3550                 PIN_BASED_VIRTUAL_NMIS |
3551                 (apicv ? PIN_BASED_POSTED_INTR : 0);
3552         msrs->pinbased_ctls_high |=
3553                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
3554                 PIN_BASED_VMX_PREEMPTION_TIMER;
3555
3556         /* exit controls */
3557         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
3558                 msrs->exit_ctls_low,
3559                 msrs->exit_ctls_high);
3560         msrs->exit_ctls_low =
3561                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
3562
3563         msrs->exit_ctls_high &=
3564 #ifdef CONFIG_X86_64
3565                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
3566 #endif
3567                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
3568         msrs->exit_ctls_high |=
3569                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
3570                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
3571                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
3572
3573         /* We support free control of debug control saving. */
3574         msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
3575
3576         /* entry controls */
3577         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
3578                 msrs->entry_ctls_low,
3579                 msrs->entry_ctls_high);
3580         msrs->entry_ctls_low =
3581                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
3582         msrs->entry_ctls_high &=
3583 #ifdef CONFIG_X86_64
3584                 VM_ENTRY_IA32E_MODE |
3585 #endif
3586                 VM_ENTRY_LOAD_IA32_PAT;
3587         msrs->entry_ctls_high |=
3588                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
3589
3590         /* We support free control of debug control loading. */
3591         msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
3592
3593         /* cpu-based controls */
3594         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
3595                 msrs->procbased_ctls_low,
3596                 msrs->procbased_ctls_high);
3597         msrs->procbased_ctls_low =
3598                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
3599         msrs->procbased_ctls_high &=
3600                 CPU_BASED_VIRTUAL_INTR_PENDING |
3601                 CPU_BASED_VIRTUAL_NMI_PENDING | CPU_BASED_USE_TSC_OFFSETING |
3602                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
3603                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
3604                 CPU_BASED_CR3_STORE_EXITING |
3605 #ifdef CONFIG_X86_64
3606                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
3607 #endif
3608                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
3609                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
3610                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
3611                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
3612                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
3613         /*
3614          * We can allow some features even when not supported by the
3615          * hardware. For example, L1 can specify an MSR bitmap - and we
3616          * can use it to avoid exits to L1 - even when L0 runs L2
3617          * without MSR bitmaps.
3618          */
3619         msrs->procbased_ctls_high |=
3620                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
3621                 CPU_BASED_USE_MSR_BITMAPS;
3622
3623         /* We support free control of CR3 access interception. */
3624         msrs->procbased_ctls_low &=
3625                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
3626
3627         /*
3628          * secondary cpu-based controls.  Do not include those that
3629          * depend on CPUID bits, they are added later by vmx_cpuid_update.
3630          */
3631         if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
3632                 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
3633                       msrs->secondary_ctls_low,
3634                       msrs->secondary_ctls_high);
3635
3636         msrs->secondary_ctls_low = 0;
3637         msrs->secondary_ctls_high &=
3638                 SECONDARY_EXEC_DESC |
3639                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
3640                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
3641                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
3642                 SECONDARY_EXEC_WBINVD_EXITING;
3643
3644         /*
3645          * We can emulate "VMCS shadowing," even if the hardware
3646          * doesn't support it.
3647          */
3648         msrs->secondary_ctls_high |=
3649                 SECONDARY_EXEC_SHADOW_VMCS;
3650
3651         if (enable_ept) {
3652                 /* nested EPT: emulate EPT also to L1 */
3653                 msrs->secondary_ctls_high |=
3654                         SECONDARY_EXEC_ENABLE_EPT;
3655                 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
3656                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
3657                 if (cpu_has_vmx_ept_execute_only())
3658                         msrs->ept_caps |=
3659                                 VMX_EPT_EXECUTE_ONLY_BIT;
3660                 msrs->ept_caps &= vmx_capability.ept;
3661                 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
3662                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
3663                         VMX_EPT_1GB_PAGE_BIT;
3664                 if (enable_ept_ad_bits) {
3665                         msrs->secondary_ctls_high |=
3666                                 SECONDARY_EXEC_ENABLE_PML;
3667                         msrs->ept_caps |= VMX_EPT_AD_BIT;
3668                 }
3669         }
3670
3671         if (cpu_has_vmx_vmfunc()) {
3672                 msrs->secondary_ctls_high |=
3673                         SECONDARY_EXEC_ENABLE_VMFUNC;
3674                 /*
3675                  * Advertise EPTP switching unconditionally
3676                  * since we emulate it
3677                  */
3678                 if (enable_ept)
3679                         msrs->vmfunc_controls =
3680                                 VMX_VMFUNC_EPTP_SWITCHING;
3681         }
3682
3683         /*
3684          * Old versions of KVM use the single-context version without
3685          * checking for support, so declare that it is supported even
3686          * though it is treated as global context.  The alternative is
3687          * not failing the single-context invvpid, and it is worse.
3688          */
3689         if (enable_vpid) {
3690                 msrs->secondary_ctls_high |=
3691                         SECONDARY_EXEC_ENABLE_VPID;
3692                 msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
3693                         VMX_VPID_EXTENT_SUPPORTED_MASK;
3694         }
3695
3696         if (enable_unrestricted_guest)
3697                 msrs->secondary_ctls_high |=
3698                         SECONDARY_EXEC_UNRESTRICTED_GUEST;
3699
3700         if (flexpriority_enabled)
3701                 msrs->secondary_ctls_high |=
3702                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
3703
3704         /* miscellaneous data */
3705         rdmsr(MSR_IA32_VMX_MISC,
3706                 msrs->misc_low,
3707                 msrs->misc_high);
3708         msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
3709         msrs->misc_low |=
3710                 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
3711                 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
3712                 VMX_MISC_ACTIVITY_HLT;
3713         msrs->misc_high = 0;
3714
3715         /*
3716          * This MSR reports some information about VMX support. We
3717          * should return information about the VMX we emulate for the
3718          * guest, and the VMCS structure we give it - not about the
3719          * VMX support of the underlying hardware.
3720          */
3721         msrs->basic =
3722                 VMCS12_REVISION |
3723                 VMX_BASIC_TRUE_CTLS |
3724                 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
3725                 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
3726
3727         if (cpu_has_vmx_basic_inout())
3728                 msrs->basic |= VMX_BASIC_INOUT;
3729
3730         /*
3731          * These MSRs specify bits which the guest must keep fixed on
3732          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
3733          * We picked the standard core2 setting.
3734          */
3735 #define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
3736 #define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
3737         msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
3738         msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
3739
3740         /* These MSRs specify bits which the guest must keep fixed off. */
3741         rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
3742         rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
3743
3744         /* highest index: VMX_PREEMPTION_TIMER_VALUE */
3745         msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
3746 }
3747
3748 /*
3749  * if fixed0[i] == 1: val[i] must be 1
3750  * if fixed1[i] == 0: val[i] must be 0
3751  */
3752 static inline bool fixed_bits_valid(u64 val, u64 fixed0, u64 fixed1)
3753 {
3754         return ((val & fixed1) | fixed0) == val;
3755 }
3756
3757 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
3758 {
3759         return fixed_bits_valid(control, low, high);
3760 }
3761
3762 static inline u64 vmx_control_msr(u32 low, u32 high)
3763 {
3764         return low | ((u64)high << 32);
3765 }
3766
3767 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
3768 {
3769         superset &= mask;
3770         subset &= mask;
3771
3772         return (superset | subset) == superset;
3773 }
3774
3775 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
3776 {
3777         const u64 feature_and_reserved =
3778                 /* feature (except bit 48; see below) */
3779                 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
3780                 /* reserved */
3781                 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
3782         u64 vmx_basic = vmx->nested.msrs.basic;
3783
3784         if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
3785                 return -EINVAL;
3786
3787         /*
3788          * KVM does not emulate a version of VMX that constrains physical
3789          * addresses of VMX structures (e.g. VMCS) to 32-bits.
3790          */
3791         if (data & BIT_ULL(48))
3792                 return -EINVAL;
3793
3794         if (vmx_basic_vmcs_revision_id(vmx_basic) !=
3795             vmx_basic_vmcs_revision_id(data))
3796                 return -EINVAL;
3797
3798         if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
3799                 return -EINVAL;
3800
3801         vmx->nested.msrs.basic = data;
3802         return 0;
3803 }
3804
3805 static int
3806 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
3807 {
3808         u64 supported;
3809         u32 *lowp, *highp;
3810
3811         switch (msr_index) {
3812         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3813                 lowp = &vmx->nested.msrs.pinbased_ctls_low;
3814                 highp = &vmx->nested.msrs.pinbased_ctls_high;
3815                 break;
3816         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3817                 lowp = &vmx->nested.msrs.procbased_ctls_low;
3818                 highp = &vmx->nested.msrs.procbased_ctls_high;
3819                 break;
3820         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3821                 lowp = &vmx->nested.msrs.exit_ctls_low;
3822                 highp = &vmx->nested.msrs.exit_ctls_high;
3823                 break;
3824         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3825                 lowp = &vmx->nested.msrs.entry_ctls_low;
3826                 highp = &vmx->nested.msrs.entry_ctls_high;
3827                 break;
3828         case MSR_IA32_VMX_PROCBASED_CTLS2:
3829                 lowp = &vmx->nested.msrs.secondary_ctls_low;
3830                 highp = &vmx->nested.msrs.secondary_ctls_high;
3831                 break;
3832         default:
3833                 BUG();
3834         }
3835
3836         supported = vmx_control_msr(*lowp, *highp);
3837
3838         /* Check must-be-1 bits are still 1. */
3839         if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
3840                 return -EINVAL;
3841
3842         /* Check must-be-0 bits are still 0. */
3843         if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
3844                 return -EINVAL;
3845
3846         *lowp = data;
3847         *highp = data >> 32;
3848         return 0;
3849 }
3850
3851 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
3852 {
3853         const u64 feature_and_reserved_bits =
3854                 /* feature */
3855                 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
3856                 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
3857                 /* reserved */
3858                 GENMASK_ULL(13, 9) | BIT_ULL(31);
3859         u64 vmx_misc;
3860
3861         vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
3862                                    vmx->nested.msrs.misc_high);
3863
3864         if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
3865                 return -EINVAL;
3866
3867         if ((vmx->nested.msrs.pinbased_ctls_high &
3868              PIN_BASED_VMX_PREEMPTION_TIMER) &&
3869             vmx_misc_preemption_timer_rate(data) !=
3870             vmx_misc_preemption_timer_rate(vmx_misc))
3871                 return -EINVAL;
3872
3873         if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
3874                 return -EINVAL;
3875
3876         if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
3877                 return -EINVAL;
3878
3879         if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
3880                 return -EINVAL;
3881
3882         vmx->nested.msrs.misc_low = data;
3883         vmx->nested.msrs.misc_high = data >> 32;
3884
3885         /*
3886          * If L1 has read-only VM-exit information fields, use the
3887          * less permissive vmx_vmwrite_bitmap to specify write
3888          * permissions for the shadow VMCS.
3889          */
3890         if (enable_shadow_vmcs && !nested_cpu_has_vmwrite_any_field(&vmx->vcpu))
3891                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
3892
3893         return 0;
3894 }
3895
3896 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
3897 {
3898         u64 vmx_ept_vpid_cap;
3899
3900         vmx_ept_vpid_cap = vmx_control_msr(vmx->nested.msrs.ept_caps,
3901                                            vmx->nested.msrs.vpid_caps);
3902
3903         /* Every bit is either reserved or a feature bit. */
3904         if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
3905                 return -EINVAL;
3906
3907         vmx->nested.msrs.ept_caps = data;
3908         vmx->nested.msrs.vpid_caps = data >> 32;
3909         return 0;
3910 }
3911
3912 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
3913 {
3914         u64 *msr;
3915
3916         switch (msr_index) {
3917         case MSR_IA32_VMX_CR0_FIXED0:
3918                 msr = &vmx->nested.msrs.cr0_fixed0;
3919                 break;
3920         case MSR_IA32_VMX_CR4_FIXED0:
3921                 msr = &vmx->nested.msrs.cr4_fixed0;
3922                 break;
3923         default:
3924                 BUG();
3925         }
3926
3927         /*
3928          * 1 bits (which indicates bits which "must-be-1" during VMX operation)
3929          * must be 1 in the restored value.
3930          */
3931         if (!is_bitwise_subset(data, *msr, -1ULL))
3932                 return -EINVAL;
3933
3934         *msr = data;
3935         return 0;
3936 }
3937
3938 /*
3939  * Called when userspace is restoring VMX MSRs.
3940  *
3941  * Returns 0 on success, non-0 otherwise.
3942  */
3943 static int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
3944 {
3945         struct vcpu_vmx *vmx = to_vmx(vcpu);
3946
3947         /*
3948          * Don't allow changes to the VMX capability MSRs while the vCPU
3949          * is in VMX operation.
3950          */
3951         if (vmx->nested.vmxon)
3952                 return -EBUSY;
3953
3954         switch (msr_index) {
3955         case MSR_IA32_VMX_BASIC:
3956                 return vmx_restore_vmx_basic(vmx, data);
3957         case MSR_IA32_VMX_PINBASED_CTLS:
3958         case MSR_IA32_VMX_PROCBASED_CTLS:
3959         case MSR_IA32_VMX_EXIT_CTLS:
3960         case MSR_IA32_VMX_ENTRY_CTLS:
3961                 /*
3962                  * The "non-true" VMX capability MSRs are generated from the
3963                  * "true" MSRs, so we do not support restoring them directly.
3964                  *
3965                  * If userspace wants to emulate VMX_BASIC[55]=0, userspace
3966                  * should restore the "true" MSRs with the must-be-1 bits
3967                  * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
3968                  * DEFAULT SETTINGS".
3969                  */
3970                 return -EINVAL;
3971         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
3972         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
3973         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
3974         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
3975         case MSR_IA32_VMX_PROCBASED_CTLS2:
3976                 return vmx_restore_control_msr(vmx, msr_index, data);
3977         case MSR_IA32_VMX_MISC:
3978                 return vmx_restore_vmx_misc(vmx, data);
3979         case MSR_IA32_VMX_CR0_FIXED0:
3980         case MSR_IA32_VMX_CR4_FIXED0:
3981                 return vmx_restore_fixed0_msr(vmx, msr_index, data);
3982         case MSR_IA32_VMX_CR0_FIXED1:
3983         case MSR_IA32_VMX_CR4_FIXED1:
3984                 /*
3985                  * These MSRs are generated based on the vCPU's CPUID, so we
3986                  * do not support restoring them directly.
3987                  */
3988                 return -EINVAL;
3989         case MSR_IA32_VMX_EPT_VPID_CAP:
3990                 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
3991         case MSR_IA32_VMX_VMCS_ENUM:
3992                 vmx->nested.msrs.vmcs_enum = data;
3993                 return 0;
3994         default:
3995                 /*
3996                  * The rest of the VMX capability MSRs do not support restore.
3997                  */
3998                 return -EINVAL;
3999         }
4000 }
4001
4002 /* Returns 0 on success, non-0 otherwise. */
4003 static int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
4004 {
4005         switch (msr_index) {
4006         case MSR_IA32_VMX_BASIC:
4007                 *pdata = msrs->basic;
4008                 break;
4009         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
4010         case MSR_IA32_VMX_PINBASED_CTLS:
4011                 *pdata = vmx_control_msr(
4012                         msrs->pinbased_ctls_low,
4013                         msrs->pinbased_ctls_high);
4014                 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
4015                         *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
4016                 break;
4017         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
4018         case MSR_IA32_VMX_PROCBASED_CTLS:
4019                 *pdata = vmx_control_msr(
4020                         msrs->procbased_ctls_low,
4021                         msrs->procbased_ctls_high);
4022                 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
4023                         *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
4024                 break;
4025         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
4026         case MSR_IA32_VMX_EXIT_CTLS:
4027                 *pdata = vmx_control_msr(
4028                         msrs->exit_ctls_low,
4029                         msrs->exit_ctls_high);
4030                 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
4031                         *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
4032                 break;
4033         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
4034         case MSR_IA32_VMX_ENTRY_CTLS:
4035                 *pdata = vmx_control_msr(
4036                         msrs->entry_ctls_low,
4037                         msrs->entry_ctls_high);
4038                 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
4039                         *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
4040                 break;
4041         case MSR_IA32_VMX_MISC:
4042                 *pdata = vmx_control_msr(
4043                         msrs->misc_low,
4044                         msrs->misc_high);
4045                 break;
4046         case MSR_IA32_VMX_CR0_FIXED0:
4047                 *pdata = msrs->cr0_fixed0;
4048                 break;
4049         case MSR_IA32_VMX_CR0_FIXED1:
4050                 *pdata = msrs->cr0_fixed1;
4051                 break;
4052         case MSR_IA32_VMX_CR4_FIXED0:
4053                 *pdata = msrs->cr4_fixed0;
4054                 break;
4055         case MSR_IA32_VMX_CR4_FIXED1:
4056                 *pdata = msrs->cr4_fixed1;
4057                 break;
4058         case MSR_IA32_VMX_VMCS_ENUM:
4059                 *pdata = msrs->vmcs_enum;
4060                 break;
4061         case MSR_IA32_VMX_PROCBASED_CTLS2:
4062                 *pdata = vmx_control_msr(
4063                         msrs->secondary_ctls_low,
4064                         msrs->secondary_ctls_high);
4065                 break;
4066         case MSR_IA32_VMX_EPT_VPID_CAP:
4067                 *pdata = msrs->ept_caps |
4068                         ((u64)msrs->vpid_caps << 32);
4069                 break;
4070         case MSR_IA32_VMX_VMFUNC:
4071                 *pdata = msrs->vmfunc_controls;
4072                 break;
4073         default:
4074                 return 1;
4075         }
4076
4077         return 0;
4078 }
4079
4080 static inline bool vmx_feature_control_msr_valid(struct kvm_vcpu *vcpu,
4081                                                  uint64_t val)
4082 {
4083         uint64_t valid_bits = to_vmx(vcpu)->msr_ia32_feature_control_valid_bits;
4084
4085         return !(val & ~valid_bits);
4086 }
4087
4088 static int vmx_get_msr_feature(struct kvm_msr_entry *msr)
4089 {
4090         switch (msr->index) {
4091         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
4092                 if (!nested)
4093                         return 1;
4094                 return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data);
4095         default:
4096                 return 1;
4097         }
4098
4099         return 0;
4100 }
4101
4102 /*
4103  * Reads an msr value (of 'msr_index') into 'pdata'.
4104  * Returns 0 on success, non-0 otherwise.
4105  * Assumes vcpu_load() was already called.
4106  */
4107 static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4108 {
4109         struct vcpu_vmx *vmx = to_vmx(vcpu);
4110         struct shared_msr_entry *msr;
4111
4112         switch (msr_info->index) {
4113 #ifdef CONFIG_X86_64
4114         case MSR_FS_BASE:
4115                 msr_info->data = vmcs_readl(GUEST_FS_BASE);
4116                 break;
4117         case MSR_GS_BASE:
4118                 msr_info->data = vmcs_readl(GUEST_GS_BASE);
4119                 break;
4120         case MSR_KERNEL_GS_BASE:
4121                 msr_info->data = vmx_read_guest_kernel_gs_base(vmx);
4122                 break;
4123 #endif
4124         case MSR_EFER:
4125                 return kvm_get_msr_common(vcpu, msr_info);
4126         case MSR_IA32_SPEC_CTRL:
4127                 if (!msr_info->host_initiated &&
4128                     !guest_has_spec_ctrl_msr(vcpu))
4129                         return 1;
4130
4131                 msr_info->data = to_vmx(vcpu)->spec_ctrl;
4132                 break;
4133         case MSR_IA32_SYSENTER_CS:
4134                 msr_info->data = vmcs_read32(GUEST_SYSENTER_CS);
4135                 break;
4136         case MSR_IA32_SYSENTER_EIP:
4137                 msr_info->data = vmcs_readl(GUEST_SYSENTER_EIP);
4138                 break;
4139         case MSR_IA32_SYSENTER_ESP:
4140                 msr_info->data = vmcs_readl(GUEST_SYSENTER_ESP);
4141                 break;
4142         case MSR_IA32_BNDCFGS:
4143                 if (!kvm_mpx_supported() ||
4144                     (!msr_info->host_initiated &&
4145                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
4146                         return 1;
4147                 msr_info->data = vmcs_read64(GUEST_BNDCFGS);
4148                 break;
4149         case MSR_IA32_MCG_EXT_CTL:
4150                 if (!msr_info->host_initiated &&
4151                     !(vmx->msr_ia32_feature_control &
4152                       FEATURE_CONTROL_LMCE))
4153                         return 1;
4154                 msr_info->data = vcpu->arch.mcg_ext_ctl;
4155                 break;
4156         case MSR_IA32_FEATURE_CONTROL:
4157                 msr_info->data = vmx->msr_ia32_feature_control;
4158                 break;
4159         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
4160                 if (!nested_vmx_allowed(vcpu))
4161                         return 1;
4162                 return vmx_get_vmx_msr(&vmx->nested.msrs, msr_info->index,
4163                                        &msr_info->data);
4164         case MSR_IA32_XSS:
4165                 if (!vmx_xsaves_supported() ||
4166                     (!msr_info->host_initiated &&
4167                      !(guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4168                        guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))))
4169                         return 1;
4170                 msr_info->data = vcpu->arch.ia32_xss;
4171                 break;
4172         case MSR_TSC_AUX:
4173                 if (!msr_info->host_initiated &&
4174                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
4175                         return 1;
4176                 /* Otherwise falls through */
4177         default:
4178                 msr = find_msr_entry(vmx, msr_info->index);
4179                 if (msr) {
4180                         msr_info->data = msr->data;
4181                         break;
4182                 }
4183                 return kvm_get_msr_common(vcpu, msr_info);
4184         }
4185
4186         return 0;
4187 }
4188
4189 static void vmx_leave_nested(struct kvm_vcpu *vcpu);
4190
4191 /*
4192  * Writes msr value into into the appropriate "register".
4193  * Returns 0 on success, non-0 otherwise.
4194  * Assumes vcpu_load() was already called.
4195  */
4196 static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4197 {
4198         struct vcpu_vmx *vmx = to_vmx(vcpu);
4199         struct shared_msr_entry *msr;
4200         int ret = 0;
4201         u32 msr_index = msr_info->index;
4202         u64 data = msr_info->data;
4203
4204         switch (msr_index) {
4205         case MSR_EFER:
4206                 ret = kvm_set_msr_common(vcpu, msr_info);
4207                 break;
4208 #ifdef CONFIG_X86_64
4209         case MSR_FS_BASE:
4210                 vmx_segment_cache_clear(vmx);
4211                 vmcs_writel(GUEST_FS_BASE, data);
4212                 break;
4213         case MSR_GS_BASE:
4214                 vmx_segment_cache_clear(vmx);
4215                 vmcs_writel(GUEST_GS_BASE, data);
4216                 break;
4217         case MSR_KERNEL_GS_BASE:
4218                 vmx_write_guest_kernel_gs_base(vmx, data);
4219                 break;
4220 #endif
4221         case MSR_IA32_SYSENTER_CS:
4222                 vmcs_write32(GUEST_SYSENTER_CS, data);
4223                 break;
4224         case MSR_IA32_SYSENTER_EIP:
4225                 vmcs_writel(GUEST_SYSENTER_EIP, data);
4226                 break;
4227         case MSR_IA32_SYSENTER_ESP:
4228                 vmcs_writel(GUEST_SYSENTER_ESP, data);
4229                 break;
4230         case MSR_IA32_BNDCFGS:
4231                 if (!kvm_mpx_supported() ||
4232                     (!msr_info->host_initiated &&
4233                      !guest_cpuid_has(vcpu, X86_FEATURE_MPX)))
4234                         return 1;
4235                 if (is_noncanonical_address(data & PAGE_MASK, vcpu) ||
4236                     (data & MSR_IA32_BNDCFGS_RSVD))
4237                         return 1;
4238                 vmcs_write64(GUEST_BNDCFGS, data);
4239                 break;
4240         case MSR_IA32_SPEC_CTRL:
4241                 if (!msr_info->host_initiated &&
4242                     !guest_has_spec_ctrl_msr(vcpu))
4243                         return 1;
4244
4245                 /* The STIBP bit doesn't fault even if it's not advertised */
4246                 if (data & ~(SPEC_CTRL_IBRS | SPEC_CTRL_STIBP | SPEC_CTRL_SSBD))
4247                         return 1;
4248
4249                 vmx->spec_ctrl = data;
4250
4251                 if (!data)
4252                         break;
4253
4254                 /*
4255                  * For non-nested:
4256                  * When it's written (to non-zero) for the first time, pass
4257                  * it through.
4258                  *
4259                  * For nested:
4260                  * The handling of the MSR bitmap for L2 guests is done in
4261                  * nested_vmx_merge_msr_bitmap. We should not touch the
4262                  * vmcs02.msr_bitmap here since it gets completely overwritten
4263                  * in the merging. We update the vmcs01 here for L1 as well
4264                  * since it will end up touching the MSR anyway now.
4265                  */
4266                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap,
4267                                               MSR_IA32_SPEC_CTRL,
4268                                               MSR_TYPE_RW);
4269                 break;
4270         case MSR_IA32_PRED_CMD:
4271                 if (!msr_info->host_initiated &&
4272                     !guest_has_pred_cmd_msr(vcpu))
4273                         return 1;
4274
4275                 if (data & ~PRED_CMD_IBPB)
4276                         return 1;
4277
4278                 if (!data)
4279                         break;
4280
4281                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4282
4283                 /*
4284                  * For non-nested:
4285                  * When it's written (to non-zero) for the first time, pass
4286                  * it through.
4287                  *
4288                  * For nested:
4289                  * The handling of the MSR bitmap for L2 guests is done in
4290                  * nested_vmx_merge_msr_bitmap. We should not touch the
4291                  * vmcs02.msr_bitmap here since it gets completely overwritten
4292                  * in the merging.
4293                  */
4294                 vmx_disable_intercept_for_msr(vmx->vmcs01.msr_bitmap, MSR_IA32_PRED_CMD,
4295                                               MSR_TYPE_W);
4296                 break;
4297         case MSR_IA32_CR_PAT:
4298                 if (!kvm_pat_valid(data))
4299                         return 1;
4300
4301                 if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
4302                         vmcs_write64(GUEST_IA32_PAT, data);
4303                         vcpu->arch.pat = data;
4304                         break;
4305                 }
4306                 ret = kvm_set_msr_common(vcpu, msr_info);
4307                 break;
4308         case MSR_IA32_TSC_ADJUST:
4309                 ret = kvm_set_msr_common(vcpu, msr_info);
4310                 break;
4311         case MSR_IA32_MCG_EXT_CTL:
4312                 if ((!msr_info->host_initiated &&
4313                      !(to_vmx(vcpu)->msr_ia32_feature_control &
4314                        FEATURE_CONTROL_LMCE)) ||
4315                     (data & ~MCG_EXT_CTL_LMCE_EN))
4316                         return 1;
4317                 vcpu->arch.mcg_ext_ctl = data;
4318                 break;
4319         case MSR_IA32_FEATURE_CONTROL:
4320                 if (!vmx_feature_control_msr_valid(vcpu, data) ||
4321                     (to_vmx(vcpu)->msr_ia32_feature_control &
4322                      FEATURE_CONTROL_LOCKED && !msr_info->host_initiated))
4323                         return 1;
4324                 vmx->msr_ia32_feature_control = data;
4325                 if (msr_info->host_initiated && data == 0)
4326                         vmx_leave_nested(vcpu);
4327                 break;
4328         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
4329                 if (!msr_info->host_initiated)
4330                         return 1; /* they are read-only */
4331                 if (!nested_vmx_allowed(vcpu))
4332                         return 1;
4333                 return vmx_set_vmx_msr(vcpu, msr_index, data);
4334         case MSR_IA32_XSS:
4335                 if (!vmx_xsaves_supported() ||
4336                     (!msr_info->host_initiated &&
4337                      !(guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
4338                        guest_cpuid_has(vcpu, X86_FEATURE_XSAVES))))
4339                         return 1;
4340                 /*
4341                  * The only supported bit as of Skylake is bit 8, but
4342                  * it is not supported on KVM.
4343                  */
4344                 if (data != 0)
4345                         return 1;
4346                 vcpu->arch.ia32_xss = data;
4347                 if (vcpu->arch.ia32_xss != host_xss)
4348                         add_atomic_switch_msr(vmx, MSR_IA32_XSS,
4349                                 vcpu->arch.ia32_xss, host_xss, false);
4350                 else
4351                         clear_atomic_switch_msr(vmx, MSR_IA32_XSS);
4352                 break;
4353         case MSR_TSC_AUX:
4354                 if (!msr_info->host_initiated &&
4355                     !guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP))
4356                         return 1;
4357                 /* Check reserved bit, higher 32 bits should be zero */
4358                 if ((data >> 32) != 0)
4359                         return 1;
4360                 /* Otherwise falls through */
4361         default:
4362                 msr = find_msr_entry(vmx, msr_index);
4363                 if (msr) {
4364                         u64 old_msr_data = msr->data;
4365                         msr->data = data;
4366                         if (msr - vmx->guest_msrs < vmx->save_nmsrs) {
4367                                 preempt_disable();
4368                                 ret = kvm_set_shared_msr(msr->index, msr->data,
4369                                                          msr->mask);
4370                                 preempt_enable();
4371                                 if (ret)
4372                                         msr->data = old_msr_data;
4373                         }
4374                         break;
4375                 }
4376                         ret = kvm_set_msr_common(vcpu, msr_info);
4377         }
4378
4379         /* FB_CLEAR may have changed, also update the FB_CLEAR_DIS behavior */
4380         if (msr_index == MSR_IA32_ARCH_CAPABILITIES)
4381                 vmx_update_fb_clear_dis(vcpu, vmx);
4382
4383         return ret;
4384 }
4385
4386 static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
4387 {
4388         __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
4389         switch (reg) {
4390         case VCPU_REGS_RSP:
4391                 vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
4392                 break;
4393         case VCPU_REGS_RIP:
4394                 vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
4395                 break;
4396         case VCPU_EXREG_PDPTR:
4397                 if (enable_ept)
4398                         ept_save_pdptrs(vcpu);
4399                 break;
4400         default:
4401                 break;
4402         }
4403 }
4404
4405 static __init int cpu_has_kvm_support(void)
4406 {
4407         return cpu_has_vmx();
4408 }
4409
4410 static __init int vmx_disabled_by_bios(void)
4411 {
4412         u64 msr;
4413
4414         rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
4415         if (msr & FEATURE_CONTROL_LOCKED) {
4416                 /* launched w/ TXT and VMX disabled */
4417                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
4418                         && tboot_enabled())
4419                         return 1;
4420                 /* launched w/o TXT and VMX only enabled w/ TXT */
4421                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
4422                         && (msr & FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX)
4423                         && !tboot_enabled()) {
4424                         printk(KERN_WARNING "kvm: disable TXT in the BIOS or "
4425                                 "activate TXT before enabling KVM\n");
4426                         return 1;
4427                 }
4428                 /* launched w/o TXT and VMX disabled */
4429                 if (!(msr & FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX)
4430                         && !tboot_enabled())
4431                         return 1;
4432         }
4433
4434         return 0;
4435 }
4436
4437 static void kvm_cpu_vmxon(u64 addr)
4438 {
4439         cr4_set_bits(X86_CR4_VMXE);
4440         intel_pt_handle_vmx(1);
4441
4442         asm volatile (ASM_VMX_VMXON_RAX
4443                         : : "a"(&addr), "m"(addr)
4444                         : "memory", "cc");
4445 }
4446
4447 static int hardware_enable(void)
4448 {
4449         int cpu = raw_smp_processor_id();
4450         u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
4451         u64 old, test_bits;
4452
4453         if (cr4_read_shadow() & X86_CR4_VMXE)
4454                 return -EBUSY;
4455
4456         /*
4457          * This can happen if we hot-added a CPU but failed to allocate
4458          * VP assist page for it.
4459          */
4460         if (static_branch_unlikely(&enable_evmcs) &&
4461             !hv_get_vp_assist_page(cpu))
4462                 return -EFAULT;
4463
4464         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
4465
4466         test_bits = FEATURE_CONTROL_LOCKED;
4467         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
4468         if (tboot_enabled())
4469                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
4470
4471         if ((old & test_bits) != test_bits) {
4472                 /* enable and lock */
4473                 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
4474         }
4475         kvm_cpu_vmxon(phys_addr);
4476         if (enable_ept)
4477                 ept_sync_global();
4478
4479         return 0;
4480 }
4481
4482 static void vmclear_local_loaded_vmcss(void)
4483 {
4484         int cpu = raw_smp_processor_id();
4485         struct loaded_vmcs *v, *n;
4486
4487         list_for_each_entry_safe(v, n, &per_cpu(loaded_vmcss_on_cpu, cpu),
4488                                  loaded_vmcss_on_cpu_link)
4489                 __loaded_vmcs_clear(v);
4490 }
4491
4492
4493 /* Just like cpu_vmxoff(), but with the __kvm_handle_fault_on_reboot()
4494  * tricks.
4495  */
4496 static void kvm_cpu_vmxoff(void)
4497 {
4498         asm volatile (__ex(ASM_VMX_VMXOFF) : : : "cc");
4499
4500         intel_pt_handle_vmx(0);
4501         cr4_clear_bits(X86_CR4_VMXE);
4502 }
4503
4504 static void hardware_disable(void)
4505 {
4506         vmclear_local_loaded_vmcss();
4507         kvm_cpu_vmxoff();
4508 }
4509
4510 static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,
4511                                       u32 msr, u32 *result)
4512 {
4513         u32 vmx_msr_low, vmx_msr_high;
4514         u32 ctl = ctl_min | ctl_opt;
4515
4516         rdmsr(msr, vmx_msr_low, vmx_msr_high);
4517
4518         ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */
4519         ctl |= vmx_msr_low;  /* bit == 1 in low word  ==> must be one  */
4520
4521         /* Ensure minimum (required) set of control bits are supported. */
4522         if (ctl_min & ~ctl)
4523                 return -EIO;
4524
4525         *result = ctl;
4526         return 0;
4527 }
4528
4529 static __init bool allow_1_setting(u32 msr, u32 ctl)
4530 {
4531         u32 vmx_msr_low, vmx_msr_high;
4532
4533         rdmsr(msr, vmx_msr_low, vmx_msr_high);
4534         return vmx_msr_high & ctl;
4535 }
4536
4537 static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
4538 {
4539         u32 vmx_msr_low, vmx_msr_high;
4540         u32 min, opt, min2, opt2;
4541         u32 _pin_based_exec_control = 0;
4542         u32 _cpu_based_exec_control = 0;
4543         u32 _cpu_based_2nd_exec_control = 0;
4544         u32 _vmexit_control = 0;
4545         u32 _vmentry_control = 0;
4546
4547         memset(vmcs_conf, 0, sizeof(*vmcs_conf));
4548         min = CPU_BASED_HLT_EXITING |
4549 #ifdef CONFIG_X86_64
4550               CPU_BASED_CR8_LOAD_EXITING |
4551               CPU_BASED_CR8_STORE_EXITING |
4552 #endif
4553               CPU_BASED_CR3_LOAD_EXITING |
4554               CPU_BASED_CR3_STORE_EXITING |
4555               CPU_BASED_UNCOND_IO_EXITING |
4556               CPU_BASED_MOV_DR_EXITING |
4557               CPU_BASED_USE_TSC_OFFSETING |
4558               CPU_BASED_MWAIT_EXITING |
4559               CPU_BASED_MONITOR_EXITING |
4560               CPU_BASED_INVLPG_EXITING |
4561               CPU_BASED_RDPMC_EXITING;
4562
4563         opt = CPU_BASED_TPR_SHADOW |
4564               CPU_BASED_USE_MSR_BITMAPS |
4565               CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
4566         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS,
4567                                 &_cpu_based_exec_control) < 0)
4568                 return -EIO;
4569 #ifdef CONFIG_X86_64
4570         if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
4571                 _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING &
4572                                            ~CPU_BASED_CR8_STORE_EXITING;
4573 #endif
4574         if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) {
4575                 min2 = 0;
4576                 opt2 = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
4577                         SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
4578                         SECONDARY_EXEC_WBINVD_EXITING |
4579                         SECONDARY_EXEC_ENABLE_VPID |
4580                         SECONDARY_EXEC_ENABLE_EPT |
4581                         SECONDARY_EXEC_UNRESTRICTED_GUEST |
4582                         SECONDARY_EXEC_PAUSE_LOOP_EXITING |
4583                         SECONDARY_EXEC_DESC |
4584                         SECONDARY_EXEC_RDTSCP |
4585                         SECONDARY_EXEC_ENABLE_INVPCID |
4586                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
4587                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
4588                         SECONDARY_EXEC_SHADOW_VMCS |
4589                         SECONDARY_EXEC_XSAVES |
4590                         SECONDARY_EXEC_RDSEED_EXITING |
4591                         SECONDARY_EXEC_RDRAND_EXITING |
4592                         SECONDARY_EXEC_ENABLE_PML |
4593                         SECONDARY_EXEC_TSC_SCALING |
4594                         SECONDARY_EXEC_ENABLE_VMFUNC |
4595                         SECONDARY_EXEC_ENCLS_EXITING;
4596                 if (adjust_vmx_controls(min2, opt2,
4597                                         MSR_IA32_VMX_PROCBASED_CTLS2,
4598                                         &_cpu_based_2nd_exec_control) < 0)
4599                         return -EIO;
4600         }
4601 #ifndef CONFIG_X86_64
4602         if (!(_cpu_based_2nd_exec_control &
4603                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
4604                 _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW;
4605 #endif
4606
4607         if (!(_cpu_based_exec_control & CPU_BASED_TPR_SHADOW))
4608                 _cpu_based_2nd_exec_control &= ~(
4609                                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
4610                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
4611                                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
4612
4613         rdmsr_safe(MSR_IA32_VMX_EPT_VPID_CAP,
4614                 &vmx_capability.ept, &vmx_capability.vpid);
4615
4616         if (_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_EPT) {
4617                 /* CR3 accesses and invlpg don't need to cause VM Exits when EPT
4618                    enabled */
4619                 _cpu_based_exec_control &= ~(CPU_BASED_CR3_LOAD_EXITING |
4620                                              CPU_BASED_CR3_STORE_EXITING |
4621                                              CPU_BASED_INVLPG_EXITING);
4622         } else if (vmx_capability.ept) {
4623                 vmx_capability.ept = 0;
4624                 pr_warn_once("EPT CAP should not exist if not support "
4625                                 "1-setting enable EPT VM-execution control\n");
4626         }
4627         if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
4628                 vmx_capability.vpid) {
4629                 vmx_capability.vpid = 0;
4630                 pr_warn_once("VPID CAP should not exist if not support "
4631                                 "1-setting enable VPID VM-execution control\n");
4632         }
4633
4634         min = VM_EXIT_SAVE_DEBUG_CONTROLS | VM_EXIT_ACK_INTR_ON_EXIT;
4635 #ifdef CONFIG_X86_64
4636         min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
4637 #endif
4638         opt = VM_EXIT_SAVE_IA32_PAT | VM_EXIT_LOAD_IA32_PAT |
4639                 VM_EXIT_CLEAR_BNDCFGS;
4640         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS,
4641                                 &_vmexit_control) < 0)
4642                 return -EIO;
4643
4644         min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
4645         opt = PIN_BASED_VIRTUAL_NMIS | PIN_BASED_POSTED_INTR |
4646                  PIN_BASED_VMX_PREEMPTION_TIMER;
4647         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS,
4648                                 &_pin_based_exec_control) < 0)
4649                 return -EIO;
4650
4651         if (cpu_has_broken_vmx_preemption_timer())
4652                 _pin_based_exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
4653         if (!(_cpu_based_2nd_exec_control &
4654                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY))
4655                 _pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
4656
4657         min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
4658         opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
4659         if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
4660                                 &_vmentry_control) < 0)
4661                 return -EIO;
4662
4663         rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
4664
4665         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
4666         if ((vmx_msr_high & 0x1fff) > PAGE_SIZE)
4667                 return -EIO;
4668
4669 #ifdef CONFIG_X86_64
4670         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
4671         if (vmx_msr_high & (1u<<16))
4672                 return -EIO;
4673 #endif
4674
4675         /* Require Write-Back (WB) memory type for VMCS accesses. */
4676         if (((vmx_msr_high >> 18) & 15) != 6)
4677                 return -EIO;
4678
4679         vmcs_conf->size = vmx_msr_high & 0x1fff;
4680         vmcs_conf->order = get_order(vmcs_conf->size);
4681         vmcs_conf->basic_cap = vmx_msr_high & ~0x1fff;
4682
4683         vmcs_conf->revision_id = vmx_msr_low;
4684
4685         vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control;
4686         vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control;
4687         vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control;
4688         vmcs_conf->vmexit_ctrl         = _vmexit_control;
4689         vmcs_conf->vmentry_ctrl        = _vmentry_control;
4690
4691         if (static_branch_unlikely(&enable_evmcs))
4692                 evmcs_sanitize_exec_ctrls(vmcs_conf);
4693
4694         cpu_has_load_ia32_efer =
4695                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
4696                                 VM_ENTRY_LOAD_IA32_EFER)
4697                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
4698                                    VM_EXIT_LOAD_IA32_EFER);
4699
4700         cpu_has_load_perf_global_ctrl =
4701                 allow_1_setting(MSR_IA32_VMX_ENTRY_CTLS,
4702                                 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
4703                 && allow_1_setting(MSR_IA32_VMX_EXIT_CTLS,
4704                                    VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL);
4705
4706         /*
4707          * Some cpus support VM_ENTRY_(LOAD|SAVE)_IA32_PERF_GLOBAL_CTRL
4708          * but due to errata below it can't be used. Workaround is to use
4709          * msr load mechanism to switch IA32_PERF_GLOBAL_CTRL.
4710          *
4711          * VM Exit May Incorrectly Clear IA32_PERF_GLOBAL_CTRL [34:32]
4712          *
4713          * AAK155             (model 26)
4714          * AAP115             (model 30)
4715          * AAT100             (model 37)
4716          * BC86,AAY89,BD102   (model 44)
4717          * BA97               (model 46)
4718          *
4719          */
4720         if (cpu_has_load_perf_global_ctrl && boot_cpu_data.x86 == 0x6) {
4721                 switch (boot_cpu_data.x86_model) {
4722                 case 26:
4723                 case 30:
4724                 case 37:
4725                 case 44:
4726                 case 46:
4727                         cpu_has_load_perf_global_ctrl = false;
4728                         printk_once(KERN_WARNING"kvm: VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL "
4729                                         "does not work properly. Using workaround\n");
4730                         break;
4731                 default:
4732                         break;
4733                 }
4734         }
4735
4736         if (boot_cpu_has(X86_FEATURE_XSAVES))
4737                 rdmsrl(MSR_IA32_XSS, host_xss);
4738
4739         return 0;
4740 }
4741
4742 static struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu)
4743 {
4744         int node = cpu_to_node(cpu);
4745         struct page *pages;
4746         struct vmcs *vmcs;
4747
4748         pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order);
4749         if (!pages)
4750                 return NULL;
4751         vmcs = page_address(pages);
4752         memset(vmcs, 0, vmcs_config.size);
4753
4754         /* KVM supports Enlightened VMCS v1 only */
4755         if (static_branch_unlikely(&enable_evmcs))
4756                 vmcs->hdr.revision_id = KVM_EVMCS_VERSION;
4757         else
4758                 vmcs->hdr.revision_id = vmcs_config.revision_id;
4759
4760         if (shadow)
4761                 vmcs->hdr.shadow_vmcs = 1;
4762         return vmcs;
4763 }
4764
4765 static void free_vmcs(struct vmcs *vmcs)
4766 {
4767         free_pages((unsigned long)vmcs, vmcs_config.order);
4768 }
4769
4770 /*
4771  * Free a VMCS, but before that VMCLEAR it on the CPU where it was last loaded
4772  */
4773 static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
4774 {
4775         if (!loaded_vmcs->vmcs)
4776                 return;
4777         loaded_vmcs_clear(loaded_vmcs);
4778         free_vmcs(loaded_vmcs->vmcs);
4779         loaded_vmcs->vmcs = NULL;
4780         if (loaded_vmcs->msr_bitmap)
4781                 free_page((unsigned long)loaded_vmcs->msr_bitmap);
4782         WARN_ON(loaded_vmcs->shadow_vmcs != NULL);
4783 }
4784
4785 static struct vmcs *alloc_vmcs(bool shadow)
4786 {
4787         return alloc_vmcs_cpu(shadow, raw_smp_processor_id());
4788 }
4789
4790 static int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs)
4791 {
4792         loaded_vmcs->vmcs = alloc_vmcs(false);
4793         if (!loaded_vmcs->vmcs)
4794                 return -ENOMEM;
4795
4796         loaded_vmcs->shadow_vmcs = NULL;
4797         loaded_vmcs_init(loaded_vmcs);
4798
4799         if (cpu_has_vmx_msr_bitmap()) {
4800                 loaded_vmcs->msr_bitmap = (unsigned long *)__get_free_page(GFP_KERNEL);
4801                 if (!loaded_vmcs->msr_bitmap)
4802                         goto out_vmcs;
4803                 memset(loaded_vmcs->msr_bitmap, 0xff, PAGE_SIZE);
4804
4805                 if (IS_ENABLED(CONFIG_HYPERV) &&
4806                     static_branch_unlikely(&enable_evmcs) &&
4807                     (ms_hyperv.nested_features & HV_X64_NESTED_MSR_BITMAP)) {
4808                         struct hv_enlightened_vmcs *evmcs =
4809                                 (struct hv_enlightened_vmcs *)loaded_vmcs->vmcs;
4810
4811                         evmcs->hv_enlightenments_control.msr_bitmap = 1;
4812                 }
4813         }
4814
4815         memset(&loaded_vmcs->host_state, 0, sizeof(struct vmcs_host_state));
4816
4817         return 0;
4818
4819 out_vmcs:
4820         free_loaded_vmcs(loaded_vmcs);
4821         return -ENOMEM;
4822 }
4823
4824 static void free_kvm_area(void)
4825 {
4826         int cpu;
4827
4828         for_each_possible_cpu(cpu) {
4829                 free_vmcs(per_cpu(vmxarea, cpu));
4830                 per_cpu(vmxarea, cpu) = NULL;
4831         }
4832 }
4833
4834 enum vmcs_field_width {
4835         VMCS_FIELD_WIDTH_U16 = 0,
4836         VMCS_FIELD_WIDTH_U64 = 1,
4837         VMCS_FIELD_WIDTH_U32 = 2,
4838         VMCS_FIELD_WIDTH_NATURAL_WIDTH = 3
4839 };
4840
4841 static inline int vmcs_field_width(unsigned long field)
4842 {
4843         if (0x1 & field)        /* the *_HIGH fields are all 32 bit */
4844                 return VMCS_FIELD_WIDTH_U32;
4845         return (field >> 13) & 0x3 ;
4846 }
4847
4848 static inline int vmcs_field_readonly(unsigned long field)
4849 {
4850         return (((field >> 10) & 0x3) == 1);
4851 }
4852
4853 static void init_vmcs_shadow_fields(void)
4854 {
4855         int i, j;
4856
4857         for (i = j = 0; i < max_shadow_read_only_fields; i++) {
4858                 u16 field = shadow_read_only_fields[i];
4859                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
4860                     (i + 1 == max_shadow_read_only_fields ||
4861                      shadow_read_only_fields[i + 1] != field + 1))
4862                         pr_err("Missing field from shadow_read_only_field %x\n",
4863                                field + 1);
4864
4865                 clear_bit(field, vmx_vmread_bitmap);
4866 #ifdef CONFIG_X86_64
4867                 if (field & 1)
4868                         continue;
4869 #endif
4870                 if (j < i)
4871                         shadow_read_only_fields[j] = field;
4872                 j++;
4873         }
4874         max_shadow_read_only_fields = j;
4875
4876         for (i = j = 0; i < max_shadow_read_write_fields; i++) {
4877                 u16 field = shadow_read_write_fields[i];
4878                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
4879                     (i + 1 == max_shadow_read_write_fields ||
4880                      shadow_read_write_fields[i + 1] != field + 1))
4881                         pr_err("Missing field from shadow_read_write_field %x\n",
4882                                field + 1);
4883
4884                 /*
4885                  * PML and the preemption timer can be emulated, but the
4886                  * processor cannot vmwrite to fields that don't exist
4887                  * on bare metal.
4888                  */
4889                 switch (field) {
4890                 case GUEST_PML_INDEX:
4891                         if (!cpu_has_vmx_pml())
4892                                 continue;
4893                         break;
4894                 case VMX_PREEMPTION_TIMER_VALUE:
4895                         if (!cpu_has_vmx_preemption_timer())
4896                                 continue;
4897                         break;
4898                 case GUEST_INTR_STATUS:
4899                         if (!cpu_has_vmx_apicv())
4900                                 continue;
4901                         break;
4902                 default:
4903                         break;
4904                 }
4905
4906                 clear_bit(field, vmx_vmwrite_bitmap);
4907                 clear_bit(field, vmx_vmread_bitmap);
4908 #ifdef CONFIG_X86_64
4909                 if (field & 1)
4910                         continue;
4911 #endif
4912                 if (j < i)
4913                         shadow_read_write_fields[j] = field;
4914                 j++;
4915         }
4916         max_shadow_read_write_fields = j;
4917 }
4918
4919 static __init int alloc_kvm_area(void)
4920 {
4921         int cpu;
4922
4923         for_each_possible_cpu(cpu) {
4924                 struct vmcs *vmcs;
4925
4926                 vmcs = alloc_vmcs_cpu(false, cpu);
4927                 if (!vmcs) {
4928                         free_kvm_area();
4929                         return -ENOMEM;
4930                 }
4931
4932                 /*
4933                  * When eVMCS is enabled, alloc_vmcs_cpu() sets
4934                  * vmcs->revision_id to KVM_EVMCS_VERSION instead of
4935                  * revision_id reported by MSR_IA32_VMX_BASIC.
4936                  *
4937                  * However, even though not explictly documented by
4938                  * TLFS, VMXArea passed as VMXON argument should
4939                  * still be marked with revision_id reported by
4940                  * physical CPU.
4941                  */
4942                 if (static_branch_unlikely(&enable_evmcs))
4943                         vmcs->hdr.revision_id = vmcs_config.revision_id;
4944
4945                 per_cpu(vmxarea, cpu) = vmcs;
4946         }
4947         return 0;
4948 }
4949
4950 static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg,
4951                 struct kvm_segment *save)
4952 {
4953         if (!emulate_invalid_guest_state) {
4954                 /*
4955                  * CS and SS RPL should be equal during guest entry according
4956                  * to VMX spec, but in reality it is not always so. Since vcpu
4957                  * is in the middle of the transition from real mode to
4958                  * protected mode it is safe to assume that RPL 0 is a good
4959                  * default value.
4960                  */
4961                 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_SS)
4962                         save->selector &= ~SEGMENT_RPL_MASK;
4963                 save->dpl = save->selector & SEGMENT_RPL_MASK;
4964                 save->s = 1;
4965         }
4966         vmx_set_segment(vcpu, save, seg);
4967 }
4968
4969 static void enter_pmode(struct kvm_vcpu *vcpu)
4970 {
4971         unsigned long flags;
4972         struct vcpu_vmx *vmx = to_vmx(vcpu);
4973
4974         /*
4975          * Update real mode segment cache. It may be not up-to-date if sement
4976          * register was written while vcpu was in a guest mode.
4977          */
4978         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
4979         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
4980         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
4981         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
4982         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
4983         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
4984
4985         vmx->rmode.vm86_active = 0;
4986
4987         vmx_segment_cache_clear(vmx);
4988
4989         vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
4990
4991         flags = vmcs_readl(GUEST_RFLAGS);
4992         flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
4993         flags |= vmx->rmode.save_rflags & ~RMODE_GUEST_OWNED_EFLAGS_BITS;
4994         vmcs_writel(GUEST_RFLAGS, flags);
4995
4996         vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) |
4997                         (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME));
4998
4999         update_exception_bitmap(vcpu);
5000
5001         fix_pmode_seg(vcpu, VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
5002         fix_pmode_seg(vcpu, VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
5003         fix_pmode_seg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
5004         fix_pmode_seg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
5005         fix_pmode_seg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
5006         fix_pmode_seg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
5007 }
5008
5009 static void fix_rmode_seg(int seg, struct kvm_segment *save)
5010 {
5011         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
5012         struct kvm_segment var = *save;
5013
5014         var.dpl = 0x3;
5015         if (seg == VCPU_SREG_CS)
5016                 var.type = 0x3;
5017
5018         if (!emulate_invalid_guest_state) {
5019                 var.selector = var.base >> 4;
5020                 var.base = var.base & 0xffff0;
5021                 var.limit = 0xffff;
5022                 var.g = 0;
5023                 var.db = 0;
5024                 var.present = 1;
5025                 var.s = 1;
5026                 var.l = 0;
5027                 var.unusable = 0;
5028                 var.type = 0x3;
5029                 var.avl = 0;
5030                 if (save->base & 0xf)
5031                         printk_once(KERN_WARNING "kvm: segment base is not "
5032                                         "paragraph aligned when entering "
5033                                         "protected mode (seg=%d)", seg);
5034         }
5035
5036         vmcs_write16(sf->selector, var.selector);
5037         vmcs_writel(sf->base, var.base);
5038         vmcs_write32(sf->limit, var.limit);
5039         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(&var));
5040 }
5041
5042 static void enter_rmode(struct kvm_vcpu *vcpu)
5043 {
5044         unsigned long flags;
5045         struct vcpu_vmx *vmx = to_vmx(vcpu);
5046         struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm);
5047
5048         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
5049         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
5050         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
5051         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
5052         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
5053         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_SS], VCPU_SREG_SS);
5054         vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_CS], VCPU_SREG_CS);
5055
5056         vmx->rmode.vm86_active = 1;
5057
5058         /*
5059          * Very old userspace does not call KVM_SET_TSS_ADDR before entering
5060          * vcpu. Warn the user that an update is overdue.
5061          */
5062         if (!kvm_vmx->tss_addr)
5063                 printk_once(KERN_WARNING "kvm: KVM_SET_TSS_ADDR need to be "
5064                              "called before entering vcpu\n");
5065
5066         vmx_segment_cache_clear(vmx);
5067
5068         vmcs_writel(GUEST_TR_BASE, kvm_vmx->tss_addr);
5069         vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
5070         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
5071
5072         flags = vmcs_readl(GUEST_RFLAGS);
5073         vmx->rmode.save_rflags = flags;
5074
5075         flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM;
5076
5077         vmcs_writel(GUEST_RFLAGS, flags);
5078         vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME);
5079         update_exception_bitmap(vcpu);
5080
5081         fix_rmode_seg(VCPU_SREG_SS, &vmx->rmode.segs[VCPU_SREG_SS]);
5082         fix_rmode_seg(VCPU_SREG_CS, &vmx->rmode.segs[VCPU_SREG_CS]);
5083         fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
5084         fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
5085         fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
5086         fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
5087
5088         kvm_mmu_reset_context(vcpu);
5089 }
5090
5091 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
5092 {
5093         struct vcpu_vmx *vmx = to_vmx(vcpu);
5094         struct shared_msr_entry *msr = find_msr_entry(vmx, MSR_EFER);
5095
5096         if (!msr)
5097                 return;
5098
5099         vcpu->arch.efer = efer;
5100         if (efer & EFER_LMA) {
5101                 vm_entry_controls_setbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
5102                 msr->data = efer;
5103         } else {
5104                 vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
5105
5106                 msr->data = efer & ~EFER_LME;
5107         }
5108         setup_msrs(vmx);
5109 }
5110
5111 #ifdef CONFIG_X86_64
5112
5113 static void enter_lmode(struct kvm_vcpu *vcpu)
5114 {
5115         u32 guest_tr_ar;
5116
5117         vmx_segment_cache_clear(to_vmx(vcpu));
5118
5119         guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
5120         if ((guest_tr_ar & VMX_AR_TYPE_MASK) != VMX_AR_TYPE_BUSY_64_TSS) {
5121                 pr_debug_ratelimited("%s: tss fixup for long mode. \n",
5122                                      __func__);
5123                 vmcs_write32(GUEST_TR_AR_BYTES,
5124                              (guest_tr_ar & ~VMX_AR_TYPE_MASK)
5125                              | VMX_AR_TYPE_BUSY_64_TSS);
5126         }
5127         vmx_set_efer(vcpu, vcpu->arch.efer | EFER_LMA);
5128 }
5129
5130 static void exit_lmode(struct kvm_vcpu *vcpu)
5131 {
5132         vm_entry_controls_clearbit(to_vmx(vcpu), VM_ENTRY_IA32E_MODE);
5133         vmx_set_efer(vcpu, vcpu->arch.efer & ~EFER_LMA);
5134 }
5135
5136 #endif
5137
5138 static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
5139                                 bool invalidate_gpa)
5140 {
5141         if (enable_ept && (invalidate_gpa || !enable_vpid)) {
5142                 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
5143                         return;
5144                 ept_sync_context(construct_eptp(vcpu, vcpu->arch.mmu.root_hpa));
5145         } else {
5146                 vpid_sync_context(vpid);
5147         }
5148 }
5149
5150 static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5151 {
5152         __vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
5153 }
5154
5155 static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
5156 {
5157         int vpid = to_vmx(vcpu)->vpid;
5158
5159         if (!vpid_sync_vcpu_addr(vpid, addr))
5160                 vpid_sync_context(vpid);
5161
5162         /*
5163          * If VPIDs are not supported or enabled, then the above is a no-op.
5164          * But we don't really need a TLB flush in that case anyway, because
5165          * each VM entry/exit includes an implicit flush when VPID is 0.
5166          */
5167 }
5168
5169 static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
5170 {
5171         ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
5172
5173         vcpu->arch.cr0 &= ~cr0_guest_owned_bits;
5174         vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits;
5175 }
5176
5177 static void vmx_decache_cr3(struct kvm_vcpu *vcpu)
5178 {
5179         if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu)))
5180                 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
5181         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
5182 }
5183
5184 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
5185 {
5186         ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
5187
5188         vcpu->arch.cr4 &= ~cr4_guest_owned_bits;
5189         vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits;
5190 }
5191
5192 static void ept_load_pdptrs(struct kvm_vcpu *vcpu)
5193 {
5194         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
5195
5196         if (!test_bit(VCPU_EXREG_PDPTR,
5197                       (unsigned long *)&vcpu->arch.regs_dirty))
5198                 return;
5199
5200         if (is_pae_paging(vcpu)) {
5201                 vmcs_write64(GUEST_PDPTR0, mmu->pdptrs[0]);
5202                 vmcs_write64(GUEST_PDPTR1, mmu->pdptrs[1]);
5203                 vmcs_write64(GUEST_PDPTR2, mmu->pdptrs[2]);
5204                 vmcs_write64(GUEST_PDPTR3, mmu->pdptrs[3]);
5205         }
5206 }
5207
5208 static void ept_save_pdptrs(struct kvm_vcpu *vcpu)
5209 {
5210         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
5211
5212         if (is_pae_paging(vcpu)) {
5213                 mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0);
5214                 mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1);
5215                 mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
5216                 mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
5217         }
5218
5219         __set_bit(VCPU_EXREG_PDPTR,
5220                   (unsigned long *)&vcpu->arch.regs_avail);
5221         __set_bit(VCPU_EXREG_PDPTR,
5222                   (unsigned long *)&vcpu->arch.regs_dirty);
5223 }
5224
5225 static bool nested_guest_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
5226 {
5227         u64 fixed0 = to_vmx(vcpu)->nested.msrs.cr0_fixed0;
5228         u64 fixed1 = to_vmx(vcpu)->nested.msrs.cr0_fixed1;
5229         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5230
5231         if (to_vmx(vcpu)->nested.msrs.secondary_ctls_high &
5232                 SECONDARY_EXEC_UNRESTRICTED_GUEST &&
5233             nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST))
5234                 fixed0 &= ~(X86_CR0_PE | X86_CR0_PG);
5235
5236         return fixed_bits_valid(val, fixed0, fixed1);
5237 }
5238
5239 static bool nested_host_cr0_valid(struct kvm_vcpu *vcpu, unsigned long val)
5240 {
5241         u64 fixed0 = to_vmx(vcpu)->nested.msrs.cr0_fixed0;
5242         u64 fixed1 = to_vmx(vcpu)->nested.msrs.cr0_fixed1;
5243
5244         return fixed_bits_valid(val, fixed0, fixed1);
5245 }
5246
5247 static bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val)
5248 {
5249         u64 fixed0 = to_vmx(vcpu)->nested.msrs.cr4_fixed0;
5250         u64 fixed1 = to_vmx(vcpu)->nested.msrs.cr4_fixed1;
5251
5252         return fixed_bits_valid(val, fixed0, fixed1);
5253 }
5254
5255 /* No difference in the restrictions on guest and host CR4 in VMX operation. */
5256 #define nested_guest_cr4_valid  nested_cr4_valid
5257 #define nested_host_cr4_valid   nested_cr4_valid
5258
5259 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
5260
5261 static void ept_update_paging_mode_cr0(unsigned long *hw_cr0,
5262                                         unsigned long cr0,
5263                                         struct kvm_vcpu *vcpu)
5264 {
5265         if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail))
5266                 vmx_decache_cr3(vcpu);
5267         if (!(cr0 & X86_CR0_PG)) {
5268                 /* From paging/starting to nonpaging */
5269                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
5270                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) |
5271                              (CPU_BASED_CR3_LOAD_EXITING |
5272                               CPU_BASED_CR3_STORE_EXITING));
5273                 vcpu->arch.cr0 = cr0;
5274                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
5275         } else if (!is_paging(vcpu)) {
5276                 /* From nonpaging to paging */
5277                 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
5278                              vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
5279                              ~(CPU_BASED_CR3_LOAD_EXITING |
5280                                CPU_BASED_CR3_STORE_EXITING));
5281                 vcpu->arch.cr0 = cr0;
5282                 vmx_set_cr4(vcpu, kvm_read_cr4(vcpu));
5283         }
5284
5285         if (!(cr0 & X86_CR0_WP))
5286                 *hw_cr0 &= ~X86_CR0_WP;
5287 }
5288
5289 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
5290 {
5291         struct vcpu_vmx *vmx = to_vmx(vcpu);
5292         unsigned long hw_cr0;
5293
5294         hw_cr0 = (cr0 & ~KVM_GUEST_CR0_MASK);
5295         if (enable_unrestricted_guest)
5296                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON_UNRESTRICTED_GUEST;
5297         else {
5298                 hw_cr0 |= KVM_VM_CR0_ALWAYS_ON;
5299
5300                 if (vmx->rmode.vm86_active && (cr0 & X86_CR0_PE))
5301                         enter_pmode(vcpu);
5302
5303                 if (!vmx->rmode.vm86_active && !(cr0 & X86_CR0_PE))
5304                         enter_rmode(vcpu);
5305         }
5306
5307 #ifdef CONFIG_X86_64
5308         if (vcpu->arch.efer & EFER_LME) {
5309                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG))
5310                         enter_lmode(vcpu);
5311                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG))
5312                         exit_lmode(vcpu);
5313         }
5314 #endif
5315
5316         if (enable_ept && !enable_unrestricted_guest)
5317                 ept_update_paging_mode_cr0(&hw_cr0, cr0, vcpu);
5318
5319         vmcs_writel(CR0_READ_SHADOW, cr0);
5320         vmcs_writel(GUEST_CR0, hw_cr0);
5321         vcpu->arch.cr0 = cr0;
5322
5323         /* depends on vcpu->arch.cr0 to be set to a new value */
5324         vmx->emulation_required = emulation_required(vcpu);
5325 }
5326
5327 static int get_ept_level(struct kvm_vcpu *vcpu)
5328 {
5329         /* Nested EPT currently only supports 4-level walks. */
5330         if (is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu)))
5331                 return 4;
5332         if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48))
5333                 return 5;
5334         return 4;
5335 }
5336
5337 static u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa)
5338 {
5339         u64 eptp = VMX_EPTP_MT_WB;
5340
5341         eptp |= (get_ept_level(vcpu) == 5) ? VMX_EPTP_PWL_5 : VMX_EPTP_PWL_4;
5342
5343         if (enable_ept_ad_bits &&
5344             (!is_guest_mode(vcpu) || nested_ept_ad_enabled(vcpu)))
5345                 eptp |= VMX_EPTP_AD_ENABLE_BIT;
5346         eptp |= (root_hpa & PAGE_MASK);
5347
5348         return eptp;
5349 }
5350
5351 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
5352 {
5353         struct kvm *kvm = vcpu->kvm;
5354         unsigned long guest_cr3;
5355         u64 eptp;
5356
5357         guest_cr3 = cr3;
5358         if (enable_ept) {
5359                 eptp = construct_eptp(vcpu, cr3);
5360                 vmcs_write64(EPT_POINTER, eptp);
5361
5362                 if (kvm_x86_ops->tlb_remote_flush) {
5363                         spin_lock(&to_kvm_vmx(kvm)->ept_pointer_lock);
5364                         to_vmx(vcpu)->ept_pointer = eptp;
5365                         to_kvm_vmx(kvm)->ept_pointers_match
5366                                 = EPT_POINTERS_CHECK;
5367                         spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock);
5368                 }
5369
5370                 if (enable_unrestricted_guest || is_paging(vcpu) ||
5371                     is_guest_mode(vcpu))
5372                         guest_cr3 = kvm_read_cr3(vcpu);
5373                 else
5374                         guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
5375                 ept_load_pdptrs(vcpu);
5376         }
5377
5378         vmcs_writel(GUEST_CR3, guest_cr3);
5379 }
5380
5381 static int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
5382 {
5383         /*
5384          * Pass through host's Machine Check Enable value to hw_cr4, which
5385          * is in force while we are in guest mode.  Do not let guests control
5386          * this bit, even if host CR4.MCE == 0.
5387          */
5388         unsigned long hw_cr4;
5389
5390         hw_cr4 = (cr4_read_shadow() & X86_CR4_MCE) | (cr4 & ~X86_CR4_MCE);
5391         if (enable_unrestricted_guest)
5392                 hw_cr4 |= KVM_VM_CR4_ALWAYS_ON_UNRESTRICTED_GUEST;
5393         else if (to_vmx(vcpu)->rmode.vm86_active)
5394                 hw_cr4 |= KVM_RMODE_VM_CR4_ALWAYS_ON;
5395         else
5396                 hw_cr4 |= KVM_PMODE_VM_CR4_ALWAYS_ON;
5397
5398         if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated()) {
5399                 if (cr4 & X86_CR4_UMIP) {
5400                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
5401                                 SECONDARY_EXEC_DESC);
5402                         hw_cr4 &= ~X86_CR4_UMIP;
5403                 } else if (!is_guest_mode(vcpu) ||
5404                         !nested_cpu_has2(get_vmcs12(vcpu), SECONDARY_EXEC_DESC))
5405                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
5406                                         SECONDARY_EXEC_DESC);
5407         }
5408
5409         if (cr4 & X86_CR4_VMXE) {
5410                 /*
5411                  * To use VMXON (and later other VMX instructions), a guest
5412                  * must first be able to turn on cr4.VMXE (see handle_vmon()).
5413                  * So basically the check on whether to allow nested VMX
5414                  * is here.  We operate under the default treatment of SMM,
5415                  * so VMX cannot be enabled under SMM.
5416                  */
5417                 if (!nested_vmx_allowed(vcpu) || is_smm(vcpu))
5418                         return 1;
5419         }
5420
5421         if (to_vmx(vcpu)->nested.vmxon && !nested_cr4_valid(vcpu, cr4))
5422                 return 1;
5423
5424         vcpu->arch.cr4 = cr4;
5425
5426         if (!enable_unrestricted_guest) {
5427                 if (enable_ept) {
5428                         if (!is_paging(vcpu)) {
5429                                 hw_cr4 &= ~X86_CR4_PAE;
5430                                 hw_cr4 |= X86_CR4_PSE;
5431                         } else if (!(cr4 & X86_CR4_PAE)) {
5432                                 hw_cr4 &= ~X86_CR4_PAE;
5433                         }
5434                 }
5435
5436                 /*
5437                  * SMEP/SMAP/PKU is disabled if CPU is in non-paging mode in
5438                  * hardware.  To emulate this behavior, SMEP/SMAP/PKU needs
5439                  * to be manually disabled when guest switches to non-paging
5440                  * mode.
5441                  *
5442                  * If !enable_unrestricted_guest, the CPU is always running
5443                  * with CR0.PG=1 and CR4 needs to be modified.
5444                  * If enable_unrestricted_guest, the CPU automatically
5445                  * disables SMEP/SMAP/PKU when the guest sets CR0.PG=0.
5446                  */
5447                 if (!is_paging(vcpu))
5448                         hw_cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
5449         }
5450
5451         vmcs_writel(CR4_READ_SHADOW, cr4);
5452         vmcs_writel(GUEST_CR4, hw_cr4);
5453         return 0;
5454 }
5455
5456 static void vmx_get_segment(struct kvm_vcpu *vcpu,
5457                             struct kvm_segment *var, int seg)
5458 {
5459         struct vcpu_vmx *vmx = to_vmx(vcpu);
5460         u32 ar;
5461
5462         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
5463                 *var = vmx->rmode.segs[seg];
5464                 if (seg == VCPU_SREG_TR
5465                     || var->selector == vmx_read_guest_seg_selector(vmx, seg))
5466                         return;
5467                 var->base = vmx_read_guest_seg_base(vmx, seg);
5468                 var->selector = vmx_read_guest_seg_selector(vmx, seg);
5469                 return;
5470         }
5471         var->base = vmx_read_guest_seg_base(vmx, seg);
5472         var->limit = vmx_read_guest_seg_limit(vmx, seg);
5473         var->selector = vmx_read_guest_seg_selector(vmx, seg);
5474         ar = vmx_read_guest_seg_ar(vmx, seg);
5475         var->unusable = (ar >> 16) & 1;
5476         var->type = ar & 15;
5477         var->s = (ar >> 4) & 1;
5478         var->dpl = (ar >> 5) & 3;
5479         /*
5480          * Some userspaces do not preserve unusable property. Since usable
5481          * segment has to be present according to VMX spec we can use present
5482          * property to amend userspace bug by making unusable segment always
5483          * nonpresent. vmx_segment_access_rights() already marks nonpresent
5484          * segment as unusable.
5485          */
5486         var->present = !var->unusable;
5487         var->avl = (ar >> 12) & 1;
5488         var->l = (ar >> 13) & 1;
5489         var->db = (ar >> 14) & 1;
5490         var->g = (ar >> 15) & 1;
5491 }
5492
5493 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
5494 {
5495         struct kvm_segment s;
5496
5497         if (to_vmx(vcpu)->rmode.vm86_active) {
5498                 vmx_get_segment(vcpu, &s, seg);
5499                 return s.base;
5500         }
5501         return vmx_read_guest_seg_base(to_vmx(vcpu), seg);
5502 }
5503
5504 static int vmx_get_cpl(struct kvm_vcpu *vcpu)
5505 {
5506         struct vcpu_vmx *vmx = to_vmx(vcpu);
5507
5508         if (unlikely(vmx->rmode.vm86_active))
5509                 return 0;
5510         else {
5511                 int ar = vmx_read_guest_seg_ar(vmx, VCPU_SREG_SS);
5512                 return VMX_AR_DPL(ar);
5513         }
5514 }
5515
5516 static u32 vmx_segment_access_rights(struct kvm_segment *var)
5517 {
5518         u32 ar;
5519
5520         if (var->unusable || !var->present)
5521                 ar = 1 << 16;
5522         else {
5523                 ar = var->type & 15;
5524                 ar |= (var->s & 1) << 4;
5525                 ar |= (var->dpl & 3) << 5;
5526                 ar |= (var->present & 1) << 7;
5527                 ar |= (var->avl & 1) << 12;
5528                 ar |= (var->l & 1) << 13;
5529                 ar |= (var->db & 1) << 14;
5530                 ar |= (var->g & 1) << 15;
5531         }
5532
5533         return ar;
5534 }
5535
5536 static void vmx_set_segment(struct kvm_vcpu *vcpu,
5537                             struct kvm_segment *var, int seg)
5538 {
5539         struct vcpu_vmx *vmx = to_vmx(vcpu);
5540         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
5541
5542         vmx_segment_cache_clear(vmx);
5543
5544         if (vmx->rmode.vm86_active && seg != VCPU_SREG_LDTR) {
5545                 vmx->rmode.segs[seg] = *var;
5546                 if (seg == VCPU_SREG_TR)
5547                         vmcs_write16(sf->selector, var->selector);
5548                 else if (var->s)
5549                         fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
5550                 goto out;
5551         }
5552
5553         vmcs_writel(sf->base, var->base);
5554         vmcs_write32(sf->limit, var->limit);
5555         vmcs_write16(sf->selector, var->selector);
5556
5557         /*
5558          *   Fix the "Accessed" bit in AR field of segment registers for older
5559          * qemu binaries.
5560          *   IA32 arch specifies that at the time of processor reset the
5561          * "Accessed" bit in the AR field of segment registers is 1. And qemu
5562          * is setting it to 0 in the userland code. This causes invalid guest
5563          * state vmexit when "unrestricted guest" mode is turned on.
5564          *    Fix for this setup issue in cpu_reset is being pushed in the qemu
5565          * tree. Newer qemu binaries with that qemu fix would not need this
5566          * kvm hack.
5567          */
5568         if (enable_unrestricted_guest && (seg != VCPU_SREG_LDTR))
5569                 var->type |= 0x1; /* Accessed */
5570
5571         vmcs_write32(sf->ar_bytes, vmx_segment_access_rights(var));
5572
5573 out:
5574         vmx->emulation_required = emulation_required(vcpu);
5575 }
5576
5577 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
5578 {
5579         u32 ar = vmx_read_guest_seg_ar(to_vmx(vcpu), VCPU_SREG_CS);
5580
5581         *db = (ar >> 14) & 1;
5582         *l = (ar >> 13) & 1;
5583 }
5584
5585 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
5586 {
5587         dt->size = vmcs_read32(GUEST_IDTR_LIMIT);
5588         dt->address = vmcs_readl(GUEST_IDTR_BASE);
5589 }
5590
5591 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
5592 {
5593         vmcs_write32(GUEST_IDTR_LIMIT, dt->size);
5594         vmcs_writel(GUEST_IDTR_BASE, dt->address);
5595 }
5596
5597 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
5598 {
5599         dt->size = vmcs_read32(GUEST_GDTR_LIMIT);
5600         dt->address = vmcs_readl(GUEST_GDTR_BASE);
5601 }
5602
5603 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
5604 {
5605         vmcs_write32(GUEST_GDTR_LIMIT, dt->size);
5606         vmcs_writel(GUEST_GDTR_BASE, dt->address);
5607 }
5608
5609 static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
5610 {
5611         struct kvm_segment var;
5612         u32 ar;
5613
5614         vmx_get_segment(vcpu, &var, seg);
5615         var.dpl = 0x3;
5616         if (seg == VCPU_SREG_CS)
5617                 var.type = 0x3;
5618         ar = vmx_segment_access_rights(&var);
5619
5620         if (var.base != (var.selector << 4))
5621                 return false;
5622         if (var.limit != 0xffff)
5623                 return false;
5624         if (ar != 0xf3)
5625                 return false;
5626
5627         return true;
5628 }
5629
5630 static bool code_segment_valid(struct kvm_vcpu *vcpu)
5631 {
5632         struct kvm_segment cs;
5633         unsigned int cs_rpl;
5634
5635         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
5636         cs_rpl = cs.selector & SEGMENT_RPL_MASK;
5637
5638         if (cs.unusable)
5639                 return false;
5640         if (~cs.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_ACCESSES_MASK))
5641                 return false;
5642         if (!cs.s)
5643                 return false;
5644         if (cs.type & VMX_AR_TYPE_WRITEABLE_MASK) {
5645                 if (cs.dpl > cs_rpl)
5646                         return false;
5647         } else {
5648                 if (cs.dpl != cs_rpl)
5649                         return false;
5650         }
5651         if (!cs.present)
5652                 return false;
5653
5654         /* TODO: Add Reserved field check, this'll require a new member in the kvm_segment_field structure */
5655         return true;
5656 }
5657
5658 static bool stack_segment_valid(struct kvm_vcpu *vcpu)
5659 {
5660         struct kvm_segment ss;
5661         unsigned int ss_rpl;
5662
5663         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
5664         ss_rpl = ss.selector & SEGMENT_RPL_MASK;
5665
5666         if (ss.unusable)
5667                 return true;
5668         if (ss.type != 3 && ss.type != 7)
5669                 return false;
5670         if (!ss.s)
5671                 return false;
5672         if (ss.dpl != ss_rpl) /* DPL != RPL */
5673                 return false;
5674         if (!ss.present)
5675                 return false;
5676
5677         return true;
5678 }
5679
5680 static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg)
5681 {
5682         struct kvm_segment var;
5683         unsigned int rpl;
5684
5685         vmx_get_segment(vcpu, &var, seg);
5686         rpl = var.selector & SEGMENT_RPL_MASK;
5687
5688         if (var.unusable)
5689                 return true;
5690         if (!var.s)
5691                 return false;
5692         if (!var.present)
5693                 return false;
5694         if (~var.type & (VMX_AR_TYPE_CODE_MASK|VMX_AR_TYPE_WRITEABLE_MASK)) {
5695                 if (var.dpl < rpl) /* DPL < RPL */
5696                         return false;
5697         }
5698
5699         /* TODO: Add other members to kvm_segment_field to allow checking for other access
5700          * rights flags
5701          */
5702         return true;
5703 }
5704
5705 static bool tr_valid(struct kvm_vcpu *vcpu)
5706 {
5707         struct kvm_segment tr;
5708
5709         vmx_get_segment(vcpu, &tr, VCPU_SREG_TR);
5710
5711         if (tr.unusable)
5712                 return false;
5713         if (tr.selector & SEGMENT_TI_MASK)      /* TI = 1 */
5714                 return false;
5715         if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */
5716                 return false;
5717         if (!tr.present)
5718                 return false;
5719
5720         return true;
5721 }
5722
5723 static bool ldtr_valid(struct kvm_vcpu *vcpu)
5724 {
5725         struct kvm_segment ldtr;
5726
5727         vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR);
5728
5729         if (ldtr.unusable)
5730                 return true;
5731         if (ldtr.selector & SEGMENT_TI_MASK)    /* TI = 1 */
5732                 return false;
5733         if (ldtr.type != 2)
5734                 return false;
5735         if (!ldtr.present)
5736                 return false;
5737
5738         return true;
5739 }
5740
5741 static bool cs_ss_rpl_check(struct kvm_vcpu *vcpu)
5742 {
5743         struct kvm_segment cs, ss;
5744
5745         vmx_get_segment(vcpu, &cs, VCPU_SREG_CS);
5746         vmx_get_segment(vcpu, &ss, VCPU_SREG_SS);
5747
5748         return ((cs.selector & SEGMENT_RPL_MASK) ==
5749                  (ss.selector & SEGMENT_RPL_MASK));
5750 }
5751
5752 static bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu,
5753                                         unsigned int port, int size);
5754 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
5755                                        struct vmcs12 *vmcs12)
5756 {
5757         unsigned long exit_qualification;
5758         unsigned short port;
5759         int size;
5760
5761         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
5762                 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
5763
5764         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5765
5766         port = exit_qualification >> 16;
5767         size = (exit_qualification & 7) + 1;
5768
5769         return nested_vmx_check_io_bitmaps(vcpu, port, size);
5770 }
5771
5772 /*
5773  * Check if guest state is valid. Returns true if valid, false if
5774  * not.
5775  * We assume that registers are always usable
5776  */
5777 static bool guest_state_valid(struct kvm_vcpu *vcpu)
5778 {
5779         if (enable_unrestricted_guest)
5780                 return true;
5781
5782         /* real mode guest state checks */
5783         if (!is_protmode(vcpu) || (vmx_get_rflags(vcpu) & X86_EFLAGS_VM)) {
5784                 if (!rmode_segment_valid(vcpu, VCPU_SREG_CS))
5785                         return false;
5786                 if (!rmode_segment_valid(vcpu, VCPU_SREG_SS))
5787                         return false;
5788                 if (!rmode_segment_valid(vcpu, VCPU_SREG_DS))
5789                         return false;
5790                 if (!rmode_segment_valid(vcpu, VCPU_SREG_ES))
5791                         return false;
5792                 if (!rmode_segment_valid(vcpu, VCPU_SREG_FS))
5793                         return false;
5794                 if (!rmode_segment_valid(vcpu, VCPU_SREG_GS))
5795                         return false;
5796         } else {
5797         /* protected mode guest state checks */
5798                 if (!cs_ss_rpl_check(vcpu))
5799                         return false;
5800                 if (!code_segment_valid(vcpu))
5801                         return false;
5802                 if (!stack_segment_valid(vcpu))
5803                         return false;
5804                 if (!data_segment_valid(vcpu, VCPU_SREG_DS))
5805                         return false;
5806                 if (!data_segment_valid(vcpu, VCPU_SREG_ES))
5807                         return false;
5808                 if (!data_segment_valid(vcpu, VCPU_SREG_FS))
5809                         return false;
5810                 if (!data_segment_valid(vcpu, VCPU_SREG_GS))
5811                         return false;
5812                 if (!tr_valid(vcpu))
5813                         return false;
5814                 if (!ldtr_valid(vcpu))
5815                         return false;
5816         }
5817         /* TODO:
5818          * - Add checks on RIP
5819          * - Add checks on RFLAGS
5820          */
5821
5822         return true;
5823 }
5824
5825 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
5826 {
5827         return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
5828 }
5829
5830 static int init_rmode_tss(struct kvm *kvm)
5831 {
5832         gfn_t fn;
5833         u16 data = 0;
5834         int idx, r;
5835
5836         idx = srcu_read_lock(&kvm->srcu);
5837         fn = to_kvm_vmx(kvm)->tss_addr >> PAGE_SHIFT;
5838         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
5839         if (r < 0)
5840                 goto out;
5841         data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
5842         r = kvm_write_guest_page(kvm, fn++, &data,
5843                         TSS_IOPB_BASE_OFFSET, sizeof(u16));
5844         if (r < 0)
5845                 goto out;
5846         r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE);
5847         if (r < 0)
5848                 goto out;
5849         r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE);
5850         if (r < 0)
5851                 goto out;
5852         data = ~0;
5853         r = kvm_write_guest_page(kvm, fn, &data,
5854                                  RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1,
5855                                  sizeof(u8));
5856 out:
5857         srcu_read_unlock(&kvm->srcu, idx);
5858         return r;
5859 }
5860
5861 static int init_rmode_identity_map(struct kvm *kvm)
5862 {
5863         struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm);
5864         int i, idx, r = 0;
5865         kvm_pfn_t identity_map_pfn;
5866         u32 tmp;
5867
5868         /* Protect kvm_vmx->ept_identity_pagetable_done. */
5869         mutex_lock(&kvm->slots_lock);
5870
5871         if (likely(kvm_vmx->ept_identity_pagetable_done))
5872                 goto out2;
5873
5874         if (!kvm_vmx->ept_identity_map_addr)
5875                 kvm_vmx->ept_identity_map_addr = VMX_EPT_IDENTITY_PAGETABLE_ADDR;
5876         identity_map_pfn = kvm_vmx->ept_identity_map_addr >> PAGE_SHIFT;
5877
5878         r = __x86_set_memory_region(kvm, IDENTITY_PAGETABLE_PRIVATE_MEMSLOT,
5879                                     kvm_vmx->ept_identity_map_addr, PAGE_SIZE);
5880         if (r < 0)
5881                 goto out2;
5882
5883         idx = srcu_read_lock(&kvm->srcu);
5884         r = kvm_clear_guest_page(kvm, identity_map_pfn, 0, PAGE_SIZE);
5885         if (r < 0)
5886                 goto out;
5887         /* Set up identity-mapping pagetable for EPT in real mode */
5888         for (i = 0; i < PT32_ENT_PER_PAGE; i++) {
5889                 tmp = (i << 22) + (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |
5890                         _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_PSE);
5891                 r = kvm_write_guest_page(kvm, identity_map_pfn,
5892                                 &tmp, i * sizeof(tmp), sizeof(tmp));
5893                 if (r < 0)
5894                         goto out;
5895         }
5896         kvm_vmx->ept_identity_pagetable_done = true;
5897
5898 out:
5899         srcu_read_unlock(&kvm->srcu, idx);
5900
5901 out2:
5902         mutex_unlock(&kvm->slots_lock);
5903         return r;
5904 }
5905
5906 static void seg_setup(int seg)
5907 {
5908         const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
5909         unsigned int ar;
5910
5911         vmcs_write16(sf->selector, 0);
5912         vmcs_writel(sf->base, 0);
5913         vmcs_write32(sf->limit, 0xffff);
5914         ar = 0x93;
5915         if (seg == VCPU_SREG_CS)
5916                 ar |= 0x08; /* code segment */
5917
5918         vmcs_write32(sf->ar_bytes, ar);
5919 }
5920
5921 static int alloc_apic_access_page(struct kvm *kvm)
5922 {
5923         struct page *page;
5924         int r = 0;
5925
5926         mutex_lock(&kvm->slots_lock);
5927         if (kvm->arch.apic_access_page_done)
5928                 goto out;
5929         r = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
5930                                     APIC_DEFAULT_PHYS_BASE, PAGE_SIZE);
5931         if (r)
5932                 goto out;
5933
5934         page = gfn_to_page(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT);
5935         if (is_error_page(page)) {
5936                 r = -EFAULT;
5937                 goto out;
5938         }
5939
5940         /*
5941          * Do not pin the page in memory, so that memory hot-unplug
5942          * is able to migrate it.
5943          */
5944         put_page(page);
5945         kvm->arch.apic_access_page_done = true;
5946 out:
5947         mutex_unlock(&kvm->slots_lock);
5948         return r;
5949 }
5950
5951 static int allocate_vpid(void)
5952 {
5953         int vpid;
5954
5955         if (!enable_vpid)
5956                 return 0;
5957         spin_lock(&vmx_vpid_lock);
5958         vpid = find_first_zero_bit(vmx_vpid_bitmap, VMX_NR_VPIDS);
5959         if (vpid < VMX_NR_VPIDS)
5960                 __set_bit(vpid, vmx_vpid_bitmap);
5961         else
5962                 vpid = 0;
5963         spin_unlock(&vmx_vpid_lock);
5964         return vpid;
5965 }
5966
5967 static void free_vpid(int vpid)
5968 {
5969         if (!enable_vpid || vpid == 0)
5970                 return;
5971         spin_lock(&vmx_vpid_lock);
5972         __clear_bit(vpid, vmx_vpid_bitmap);
5973         spin_unlock(&vmx_vpid_lock);
5974 }
5975
5976 static __always_inline void vmx_disable_intercept_for_msr(unsigned long *msr_bitmap,
5977                                                           u32 msr, int type)
5978 {
5979         int f = sizeof(unsigned long);
5980
5981         if (!cpu_has_vmx_msr_bitmap())
5982                 return;
5983
5984         if (static_branch_unlikely(&enable_evmcs))
5985                 evmcs_touch_msr_bitmap();
5986
5987         /*
5988          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
5989          * have the write-low and read-high bitmap offsets the wrong way round.
5990          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
5991          */
5992         if (msr <= 0x1fff) {
5993                 if (type & MSR_TYPE_R)
5994                         /* read-low */
5995                         __clear_bit(msr, msr_bitmap + 0x000 / f);
5996
5997                 if (type & MSR_TYPE_W)
5998                         /* write-low */
5999                         __clear_bit(msr, msr_bitmap + 0x800 / f);
6000
6001         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
6002                 msr &= 0x1fff;
6003                 if (type & MSR_TYPE_R)
6004                         /* read-high */
6005                         __clear_bit(msr, msr_bitmap + 0x400 / f);
6006
6007                 if (type & MSR_TYPE_W)
6008                         /* write-high */
6009                         __clear_bit(msr, msr_bitmap + 0xc00 / f);
6010
6011         }
6012 }
6013
6014 static __always_inline void vmx_enable_intercept_for_msr(unsigned long *msr_bitmap,
6015                                                          u32 msr, int type)
6016 {
6017         int f = sizeof(unsigned long);
6018
6019         if (!cpu_has_vmx_msr_bitmap())
6020                 return;
6021
6022         if (static_branch_unlikely(&enable_evmcs))
6023                 evmcs_touch_msr_bitmap();
6024
6025         /*
6026          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
6027          * have the write-low and read-high bitmap offsets the wrong way round.
6028          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
6029          */
6030         if (msr <= 0x1fff) {
6031                 if (type & MSR_TYPE_R)
6032                         /* read-low */
6033                         __set_bit(msr, msr_bitmap + 0x000 / f);
6034
6035                 if (type & MSR_TYPE_W)
6036                         /* write-low */
6037                         __set_bit(msr, msr_bitmap + 0x800 / f);
6038
6039         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
6040                 msr &= 0x1fff;
6041                 if (type & MSR_TYPE_R)
6042                         /* read-high */
6043                         __set_bit(msr, msr_bitmap + 0x400 / f);
6044
6045                 if (type & MSR_TYPE_W)
6046                         /* write-high */
6047                         __set_bit(msr, msr_bitmap + 0xc00 / f);
6048
6049         }
6050 }
6051
6052 static __always_inline void vmx_set_intercept_for_msr(unsigned long *msr_bitmap,
6053                                                       u32 msr, int type, bool value)
6054 {
6055         if (value)
6056                 vmx_enable_intercept_for_msr(msr_bitmap, msr, type);
6057         else
6058                 vmx_disable_intercept_for_msr(msr_bitmap, msr, type);
6059 }
6060
6061 /*
6062  * If a msr is allowed by L0, we should check whether it is allowed by L1.
6063  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
6064  */
6065 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
6066                                                unsigned long *msr_bitmap_nested,
6067                                                u32 msr, int type)
6068 {
6069         int f = sizeof(unsigned long);
6070
6071         /*
6072          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
6073          * have the write-low and read-high bitmap offsets the wrong way round.
6074          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
6075          */
6076         if (msr <= 0x1fff) {
6077                 if (type & MSR_TYPE_R &&
6078                    !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
6079                         /* read-low */
6080                         __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
6081
6082                 if (type & MSR_TYPE_W &&
6083                    !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
6084                         /* write-low */
6085                         __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
6086
6087         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
6088                 msr &= 0x1fff;
6089                 if (type & MSR_TYPE_R &&
6090                    !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
6091                         /* read-high */
6092                         __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
6093
6094                 if (type & MSR_TYPE_W &&
6095                    !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
6096                         /* write-high */
6097                         __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
6098
6099         }
6100 }
6101
6102 static u8 vmx_msr_bitmap_mode(struct kvm_vcpu *vcpu)
6103 {
6104         u8 mode = 0;
6105
6106         if (cpu_has_secondary_exec_ctrls() &&
6107             (vmcs_read32(SECONDARY_VM_EXEC_CONTROL) &
6108              SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE)) {
6109                 mode |= MSR_BITMAP_MODE_X2APIC;
6110                 if (enable_apicv && kvm_vcpu_apicv_active(vcpu))
6111                         mode |= MSR_BITMAP_MODE_X2APIC_APICV;
6112         }
6113
6114         return mode;
6115 }
6116
6117 #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4))
6118
6119 static void vmx_update_msr_bitmap_x2apic(unsigned long *msr_bitmap,
6120                                          u8 mode)
6121 {
6122         int msr;
6123
6124         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
6125                 unsigned word = msr / BITS_PER_LONG;
6126                 msr_bitmap[word] = (mode & MSR_BITMAP_MODE_X2APIC_APICV) ? 0 : ~0;
6127                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
6128         }
6129
6130         if (mode & MSR_BITMAP_MODE_X2APIC) {
6131                 /*
6132                  * TPR reads and writes can be virtualized even if virtual interrupt
6133                  * delivery is not in use.
6134                  */
6135                 vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TASKPRI), MSR_TYPE_RW);
6136                 if (mode & MSR_BITMAP_MODE_X2APIC_APICV) {
6137                         vmx_enable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_TMCCT), MSR_TYPE_R);
6138                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_EOI), MSR_TYPE_W);
6139                         vmx_disable_intercept_for_msr(msr_bitmap, X2APIC_MSR(APIC_SELF_IPI), MSR_TYPE_W);
6140                 }
6141         }
6142 }
6143
6144 static void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu)
6145 {
6146         struct vcpu_vmx *vmx = to_vmx(vcpu);
6147         unsigned long *msr_bitmap = vmx->vmcs01.msr_bitmap;
6148         u8 mode = vmx_msr_bitmap_mode(vcpu);
6149         u8 changed = mode ^ vmx->msr_bitmap_mode;
6150
6151         if (!changed)
6152                 return;
6153
6154         if (changed & (MSR_BITMAP_MODE_X2APIC | MSR_BITMAP_MODE_X2APIC_APICV))
6155                 vmx_update_msr_bitmap_x2apic(msr_bitmap, mode);
6156
6157         vmx->msr_bitmap_mode = mode;
6158 }
6159
6160 static bool vmx_get_enable_apicv(struct kvm_vcpu *vcpu)
6161 {
6162         return enable_apicv;
6163 }
6164
6165 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
6166 {
6167         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
6168         gfn_t gfn;
6169
6170         /*
6171          * Don't need to mark the APIC access page dirty; it is never
6172          * written to by the CPU during APIC virtualization.
6173          */
6174
6175         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
6176                 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
6177                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
6178         }
6179
6180         if (nested_cpu_has_posted_intr(vmcs12)) {
6181                 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
6182                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
6183         }
6184 }
6185
6186
6187 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
6188 {
6189         struct vcpu_vmx *vmx = to_vmx(vcpu);
6190         int max_irr;
6191         void *vapic_page;
6192         u16 status;
6193
6194         if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
6195                 return;
6196
6197         vmx->nested.pi_pending = false;
6198         if (!pi_test_and_clear_on(vmx->nested.pi_desc))
6199                 return;
6200
6201         max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
6202         if (max_irr != 256) {
6203                 vapic_page = kmap(vmx->nested.virtual_apic_page);
6204                 __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
6205                         vapic_page, &max_irr);
6206                 kunmap(vmx->nested.virtual_apic_page);
6207
6208                 status = vmcs_read16(GUEST_INTR_STATUS);
6209                 if ((u8)max_irr > ((u8)status & 0xff)) {
6210                         status &= ~0xff;
6211                         status |= (u8)max_irr;
6212                         vmcs_write16(GUEST_INTR_STATUS, status);
6213                 }
6214         }
6215
6216         nested_mark_vmcs12_pages_dirty(vcpu);
6217 }
6218
6219 static u8 vmx_get_rvi(void)
6220 {
6221         return vmcs_read16(GUEST_INTR_STATUS) & 0xff;
6222 }
6223
6224 static bool vmx_guest_apic_has_interrupt(struct kvm_vcpu *vcpu)
6225 {
6226         struct vcpu_vmx *vmx = to_vmx(vcpu);
6227         void *vapic_page;
6228         u32 vppr;
6229         int rvi;
6230
6231         if (WARN_ON_ONCE(!is_guest_mode(vcpu)) ||
6232                 !nested_cpu_has_vid(get_vmcs12(vcpu)) ||
6233                 WARN_ON_ONCE(!vmx->nested.virtual_apic_page))
6234                 return false;
6235
6236         rvi = vmx_get_rvi();
6237
6238         vapic_page = kmap(vmx->nested.virtual_apic_page);
6239         vppr = *((u32 *)(vapic_page + APIC_PROCPRI));
6240         kunmap(vmx->nested.virtual_apic_page);
6241
6242         return ((rvi & 0xf0) > (vppr & 0xf0));
6243 }
6244
6245 static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
6246                                                      bool nested)
6247 {
6248 #ifdef CONFIG_SMP
6249         int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
6250
6251         if (vcpu->mode == IN_GUEST_MODE) {
6252                 /*
6253                  * The vector of interrupt to be delivered to vcpu had
6254                  * been set in PIR before this function.
6255                  *
6256                  * Following cases will be reached in this block, and
6257                  * we always send a notification event in all cases as
6258                  * explained below.
6259                  *
6260                  * Case 1: vcpu keeps in non-root mode. Sending a
6261                  * notification event posts the interrupt to vcpu.
6262                  *
6263                  * Case 2: vcpu exits to root mode and is still
6264                  * runnable. PIR will be synced to vIRR before the
6265                  * next vcpu entry. Sending a notification event in
6266                  * this case has no effect, as vcpu is not in root
6267                  * mode.
6268                  *
6269                  * Case 3: vcpu exits to root mode and is blocked.
6270                  * vcpu_block() has already synced PIR to vIRR and
6271                  * never blocks vcpu if vIRR is not cleared. Therefore,
6272                  * a blocked vcpu here does not wait for any requested
6273                  * interrupts in PIR, and sending a notification event
6274                  * which has no effect is safe here.
6275                  */
6276
6277                 apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
6278                 return true;
6279         }
6280 #endif
6281         return false;
6282 }
6283
6284 static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
6285                                                 int vector)
6286 {
6287         struct vcpu_vmx *vmx = to_vmx(vcpu);
6288
6289         if (is_guest_mode(vcpu) &&
6290             vector == vmx->nested.posted_intr_nv) {
6291                 /*
6292                  * If a posted intr is not recognized by hardware,
6293                  * we will accomplish it in the next vmentry.
6294                  */
6295                 vmx->nested.pi_pending = true;
6296                 kvm_make_request(KVM_REQ_EVENT, vcpu);
6297                 /* the PIR and ON have been set by L1. */
6298                 if (!kvm_vcpu_trigger_posted_interrupt(vcpu, true))
6299                         kvm_vcpu_kick(vcpu);
6300                 return 0;
6301         }
6302         return -1;
6303 }
6304 /*
6305  * Send interrupt to vcpu via posted interrupt way.
6306  * 1. If target vcpu is running(non-root mode), send posted interrupt
6307  * notification to vcpu and hardware will sync PIR to vIRR atomically.
6308  * 2. If target vcpu isn't running(root mode), kick it to pick up the
6309  * interrupt from PIR in next vmentry.
6310  */
6311 static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
6312 {
6313         struct vcpu_vmx *vmx = to_vmx(vcpu);
6314         int r;
6315
6316         r = vmx_deliver_nested_posted_interrupt(vcpu, vector);
6317         if (!r)
6318                 return 0;
6319
6320         if (!vcpu->arch.apicv_active)
6321                 return -1;
6322
6323         if (pi_test_and_set_pir(vector, &vmx->pi_desc))
6324                 return 0;
6325
6326         /* If a previous notification has sent the IPI, nothing to do.  */
6327         if (pi_test_and_set_on(&vmx->pi_desc))
6328                 return 0;
6329
6330         if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
6331                 kvm_vcpu_kick(vcpu);
6332
6333         return 0;
6334 }
6335
6336 /*
6337  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
6338  * will not change in the lifetime of the guest.
6339  * Note that host-state that does change is set elsewhere. E.g., host-state
6340  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
6341  */
6342 static void vmx_set_constant_host_state(struct vcpu_vmx *vmx)
6343 {
6344         u32 low32, high32;
6345         unsigned long tmpl;
6346         struct desc_ptr dt;
6347         unsigned long cr0, cr3, cr4;
6348
6349         cr0 = read_cr0();
6350         WARN_ON(cr0 & X86_CR0_TS);
6351         vmcs_writel(HOST_CR0, cr0);  /* 22.2.3 */
6352
6353         /*
6354          * Save the most likely value for this task's CR3 in the VMCS.
6355          * We can't use __get_current_cr3_fast() because we're not atomic.
6356          */
6357         cr3 = __read_cr3();
6358         vmcs_writel(HOST_CR3, cr3);             /* 22.2.3  FIXME: shadow tables */
6359         vmx->loaded_vmcs->host_state.cr3 = cr3;
6360
6361         /* Save the most likely value for this task's CR4 in the VMCS. */
6362         cr4 = cr4_read_shadow();
6363         vmcs_writel(HOST_CR4, cr4);                     /* 22.2.3, 22.2.5 */
6364         vmx->loaded_vmcs->host_state.cr4 = cr4;
6365
6366         vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS);  /* 22.2.4 */
6367 #ifdef CONFIG_X86_64
6368         /*
6369          * Load null selectors, so we can avoid reloading them in
6370          * vmx_prepare_switch_to_host(), in case userspace uses
6371          * the null selectors too (the expected case).
6372          */
6373         vmcs_write16(HOST_DS_SELECTOR, 0);
6374         vmcs_write16(HOST_ES_SELECTOR, 0);
6375 #else
6376         vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
6377         vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
6378 #endif
6379         vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS);  /* 22.2.4 */
6380         vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8);  /* 22.2.4 */
6381
6382         store_idt(&dt);
6383         vmcs_writel(HOST_IDTR_BASE, dt.address);   /* 22.2.4 */
6384         vmx->host_idt_base = dt.address;
6385
6386         vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
6387
6388         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
6389         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
6390         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
6391         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
6392
6393         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
6394                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
6395                 vmcs_write64(HOST_IA32_PAT, low32 | ((u64) high32 << 32));
6396         }
6397 }
6398
6399 static void set_cr4_guest_host_mask(struct vcpu_vmx *vmx)
6400 {
6401         BUILD_BUG_ON(KVM_CR4_GUEST_OWNED_BITS & ~KVM_POSSIBLE_CR4_GUEST_BITS);
6402
6403         vmx->vcpu.arch.cr4_guest_owned_bits = KVM_CR4_GUEST_OWNED_BITS;
6404         if (enable_ept)
6405                 vmx->vcpu.arch.cr4_guest_owned_bits |= X86_CR4_PGE;
6406         if (is_guest_mode(&vmx->vcpu))
6407                 vmx->vcpu.arch.cr4_guest_owned_bits &=
6408                         ~get_vmcs12(&vmx->vcpu)->cr4_guest_host_mask;
6409         vmcs_writel(CR4_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr4_guest_owned_bits);
6410 }
6411
6412 static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx)
6413 {
6414         u32 pin_based_exec_ctrl = vmcs_config.pin_based_exec_ctrl;
6415
6416         if (!kvm_vcpu_apicv_active(&vmx->vcpu))
6417                 pin_based_exec_ctrl &= ~PIN_BASED_POSTED_INTR;
6418
6419         if (!enable_vnmi)
6420                 pin_based_exec_ctrl &= ~PIN_BASED_VIRTUAL_NMIS;
6421
6422         /* Enable the preemption timer dynamically */
6423         pin_based_exec_ctrl &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
6424         return pin_based_exec_ctrl;
6425 }
6426
6427 static void vmx_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
6428 {
6429         struct vcpu_vmx *vmx = to_vmx(vcpu);
6430
6431         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
6432         if (cpu_has_secondary_exec_ctrls()) {
6433                 if (kvm_vcpu_apicv_active(vcpu))
6434                         vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
6435                                       SECONDARY_EXEC_APIC_REGISTER_VIRT |
6436                                       SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
6437                 else
6438                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
6439                                         SECONDARY_EXEC_APIC_REGISTER_VIRT |
6440                                         SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
6441         }
6442
6443         if (cpu_has_vmx_msr_bitmap())
6444                 vmx_update_msr_bitmap(vcpu);
6445 }
6446
6447 static u32 vmx_exec_control(struct vcpu_vmx *vmx)
6448 {
6449         u32 exec_control = vmcs_config.cpu_based_exec_ctrl;
6450
6451         if (vmx->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)
6452                 exec_control &= ~CPU_BASED_MOV_DR_EXITING;
6453
6454         if (!cpu_need_tpr_shadow(&vmx->vcpu)) {
6455                 exec_control &= ~CPU_BASED_TPR_SHADOW;
6456 #ifdef CONFIG_X86_64
6457                 exec_control |= CPU_BASED_CR8_STORE_EXITING |
6458                                 CPU_BASED_CR8_LOAD_EXITING;
6459 #endif
6460         }
6461         if (!enable_ept)
6462                 exec_control |= CPU_BASED_CR3_STORE_EXITING |
6463                                 CPU_BASED_CR3_LOAD_EXITING  |
6464                                 CPU_BASED_INVLPG_EXITING;
6465         if (kvm_mwait_in_guest(vmx->vcpu.kvm))
6466                 exec_control &= ~(CPU_BASED_MWAIT_EXITING |
6467                                 CPU_BASED_MONITOR_EXITING);
6468         if (kvm_hlt_in_guest(vmx->vcpu.kvm))
6469                 exec_control &= ~CPU_BASED_HLT_EXITING;
6470         return exec_control;
6471 }
6472
6473 static bool vmx_rdrand_supported(void)
6474 {
6475         return vmcs_config.cpu_based_2nd_exec_ctrl &
6476                 SECONDARY_EXEC_RDRAND_EXITING;
6477 }
6478
6479 static bool vmx_rdseed_supported(void)
6480 {
6481         return vmcs_config.cpu_based_2nd_exec_ctrl &
6482                 SECONDARY_EXEC_RDSEED_EXITING;
6483 }
6484
6485 static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx)
6486 {
6487         struct kvm_vcpu *vcpu = &vmx->vcpu;
6488
6489         u32 exec_control = vmcs_config.cpu_based_2nd_exec_ctrl;
6490
6491         if (!cpu_need_virtualize_apic_accesses(vcpu))
6492                 exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
6493         if (vmx->vpid == 0)
6494                 exec_control &= ~SECONDARY_EXEC_ENABLE_VPID;
6495         if (!enable_ept) {
6496                 exec_control &= ~SECONDARY_EXEC_ENABLE_EPT;
6497                 enable_unrestricted_guest = 0;
6498         }
6499         if (!enable_unrestricted_guest)
6500                 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST;
6501         if (kvm_pause_in_guest(vmx->vcpu.kvm))
6502                 exec_control &= ~SECONDARY_EXEC_PAUSE_LOOP_EXITING;
6503         if (!kvm_vcpu_apicv_active(vcpu))
6504                 exec_control &= ~(SECONDARY_EXEC_APIC_REGISTER_VIRT |
6505                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY);
6506         exec_control &= ~SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
6507
6508         /* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
6509          * in vmx_set_cr4.  */
6510         exec_control &= ~SECONDARY_EXEC_DESC;
6511
6512         /* SECONDARY_EXEC_SHADOW_VMCS is enabled when L1 executes VMPTRLD
6513            (handle_vmptrld).
6514            We can NOT enable shadow_vmcs here because we don't have yet
6515            a current VMCS12
6516         */
6517         exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
6518
6519         if (!enable_pml)
6520                 exec_control &= ~SECONDARY_EXEC_ENABLE_PML;
6521
6522         if (vmx_xsaves_supported()) {
6523                 /* Exposing XSAVES only when XSAVE is exposed */
6524                 bool xsaves_enabled =
6525                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
6526                         guest_cpuid_has(vcpu, X86_FEATURE_XSAVES);
6527
6528                 if (!xsaves_enabled)
6529                         exec_control &= ~SECONDARY_EXEC_XSAVES;
6530
6531                 if (nested) {
6532                         if (xsaves_enabled)
6533                                 vmx->nested.msrs.secondary_ctls_high |=
6534                                         SECONDARY_EXEC_XSAVES;
6535                         else
6536                                 vmx->nested.msrs.secondary_ctls_high &=
6537                                         ~SECONDARY_EXEC_XSAVES;
6538                 }
6539         }
6540
6541         if (vmx_rdtscp_supported()) {
6542                 bool rdtscp_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDTSCP);
6543                 if (!rdtscp_enabled)
6544                         exec_control &= ~SECONDARY_EXEC_RDTSCP;
6545
6546                 if (nested) {
6547                         if (rdtscp_enabled)
6548                                 vmx->nested.msrs.secondary_ctls_high |=
6549                                         SECONDARY_EXEC_RDTSCP;
6550                         else
6551                                 vmx->nested.msrs.secondary_ctls_high &=
6552                                         ~SECONDARY_EXEC_RDTSCP;
6553                 }
6554         }
6555
6556         if (vmx_invpcid_supported()) {
6557                 /* Exposing INVPCID only when PCID is exposed */
6558                 bool invpcid_enabled =
6559                         guest_cpuid_has(vcpu, X86_FEATURE_INVPCID) &&
6560                         guest_cpuid_has(vcpu, X86_FEATURE_PCID);
6561
6562                 if (!invpcid_enabled) {
6563                         exec_control &= ~SECONDARY_EXEC_ENABLE_INVPCID;
6564                         guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID);
6565                 }
6566
6567                 if (nested) {
6568                         if (invpcid_enabled)
6569                                 vmx->nested.msrs.secondary_ctls_high |=
6570                                         SECONDARY_EXEC_ENABLE_INVPCID;
6571                         else
6572                                 vmx->nested.msrs.secondary_ctls_high &=
6573                                         ~SECONDARY_EXEC_ENABLE_INVPCID;
6574                 }
6575         }
6576
6577         if (vmx_rdrand_supported()) {
6578                 bool rdrand_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDRAND);
6579                 if (rdrand_enabled)
6580                         exec_control &= ~SECONDARY_EXEC_RDRAND_EXITING;
6581
6582                 if (nested) {
6583                         if (rdrand_enabled)
6584                                 vmx->nested.msrs.secondary_ctls_high |=
6585                                         SECONDARY_EXEC_RDRAND_EXITING;
6586                         else
6587                                 vmx->nested.msrs.secondary_ctls_high &=
6588                                         ~SECONDARY_EXEC_RDRAND_EXITING;
6589                 }
6590         }
6591
6592         if (vmx_rdseed_supported()) {
6593                 bool rdseed_enabled = guest_cpuid_has(vcpu, X86_FEATURE_RDSEED);
6594                 if (rdseed_enabled)
6595                         exec_control &= ~SECONDARY_EXEC_RDSEED_EXITING;
6596
6597                 if (nested) {
6598                         if (rdseed_enabled)
6599                                 vmx->nested.msrs.secondary_ctls_high |=
6600                                         SECONDARY_EXEC_RDSEED_EXITING;
6601                         else
6602                                 vmx->nested.msrs.secondary_ctls_high &=
6603                                         ~SECONDARY_EXEC_RDSEED_EXITING;
6604                 }
6605         }
6606
6607         vmx->secondary_exec_control = exec_control;
6608 }
6609
6610 static void ept_set_mmio_spte_mask(void)
6611 {
6612         /*
6613          * EPT Misconfigurations can be generated if the value of bits 2:0
6614          * of an EPT paging-structure entry is 110b (write/execute).
6615          */
6616         kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK,
6617                                    VMX_EPT_MISCONFIG_WX_VALUE);
6618 }
6619
6620 #define VMX_XSS_EXIT_BITMAP 0
6621 /*
6622  * Sets up the vmcs for emulated real mode.
6623  */
6624 static void vmx_vcpu_setup(struct vcpu_vmx *vmx)
6625 {
6626         int i;
6627
6628         if (enable_shadow_vmcs) {
6629                 /*
6630                  * At vCPU creation, "VMWRITE to any supported field
6631                  * in the VMCS" is supported, so use the more
6632                  * permissive vmx_vmread_bitmap to specify both read
6633                  * and write permissions for the shadow VMCS.
6634                  */
6635                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
6636                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmread_bitmap));
6637         }
6638         if (cpu_has_vmx_msr_bitmap())
6639                 vmcs_write64(MSR_BITMAP, __pa(vmx->vmcs01.msr_bitmap));
6640
6641         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
6642
6643         /* Control */
6644         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmx_pin_based_exec_ctrl(vmx));
6645         vmx->hv_deadline_tsc = -1;
6646
6647         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, vmx_exec_control(vmx));
6648
6649         if (cpu_has_secondary_exec_ctrls()) {
6650                 vmx_compute_secondary_exec_control(vmx);
6651                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
6652                              vmx->secondary_exec_control);
6653         }
6654
6655         if (kvm_vcpu_apicv_active(&vmx->vcpu)) {
6656                 vmcs_write64(EOI_EXIT_BITMAP0, 0);
6657                 vmcs_write64(EOI_EXIT_BITMAP1, 0);
6658                 vmcs_write64(EOI_EXIT_BITMAP2, 0);
6659                 vmcs_write64(EOI_EXIT_BITMAP3, 0);
6660
6661                 vmcs_write16(GUEST_INTR_STATUS, 0);
6662
6663                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
6664                 vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->pi_desc)));
6665         }
6666
6667         if (!kvm_pause_in_guest(vmx->vcpu.kvm)) {
6668                 vmcs_write32(PLE_GAP, ple_gap);
6669                 vmx->ple_window = ple_window;
6670                 vmx->ple_window_dirty = true;
6671         }
6672
6673         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
6674         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
6675         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
6676
6677         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
6678         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
6679         vmx_set_constant_host_state(vmx);
6680         vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
6681         vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
6682
6683         if (cpu_has_vmx_vmfunc())
6684                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
6685
6686         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
6687         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
6688         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
6689         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
6690         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
6691
6692         if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT)
6693                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
6694
6695         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i) {
6696                 u32 index = vmx_msr_index[i];
6697                 u32 data_low, data_high;
6698                 int j = vmx->nmsrs;
6699
6700                 if (rdmsr_safe(index, &data_low, &data_high) < 0)
6701                         continue;
6702                 if (wrmsr_safe(index, data_low, data_high) < 0)
6703                         continue;
6704                 vmx->guest_msrs[j].index = i;
6705                 vmx->guest_msrs[j].data = 0;
6706                 vmx->guest_msrs[j].mask = -1ull;
6707                 ++vmx->nmsrs;
6708         }
6709
6710         vm_exit_controls_init(vmx, vmcs_config.vmexit_ctrl);
6711
6712         /* 22.2.1, 20.8.1 */
6713         vm_entry_controls_init(vmx, vmcs_config.vmentry_ctrl);
6714
6715         vmx->vcpu.arch.cr0_guest_owned_bits = X86_CR0_TS;
6716         vmcs_writel(CR0_GUEST_HOST_MASK, ~X86_CR0_TS);
6717
6718         set_cr4_guest_host_mask(vmx);
6719
6720         if (vmx_xsaves_supported())
6721                 vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP);
6722
6723         if (enable_pml) {
6724                 ASSERT(vmx->pml_pg);
6725                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
6726                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
6727         }
6728
6729         if (cpu_has_vmx_encls_vmexit())
6730                 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
6731 }
6732
6733 static void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
6734 {
6735         struct vcpu_vmx *vmx = to_vmx(vcpu);
6736         struct msr_data apic_base_msr;
6737         u64 cr0;
6738
6739         vmx->rmode.vm86_active = 0;
6740         vmx->spec_ctrl = 0;
6741
6742         vcpu->arch.microcode_version = 0x100000000ULL;
6743         vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val();
6744         kvm_set_cr8(vcpu, 0);
6745
6746         if (!init_event) {
6747                 apic_base_msr.data = APIC_DEFAULT_PHYS_BASE |
6748                                      MSR_IA32_APICBASE_ENABLE;
6749                 if (kvm_vcpu_is_reset_bsp(vcpu))
6750                         apic_base_msr.data |= MSR_IA32_APICBASE_BSP;
6751                 apic_base_msr.host_initiated = true;
6752                 kvm_set_apic_base(vcpu, &apic_base_msr);
6753         }
6754
6755         vmx_segment_cache_clear(vmx);
6756
6757         seg_setup(VCPU_SREG_CS);
6758         vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
6759         vmcs_writel(GUEST_CS_BASE, 0xffff0000ul);
6760
6761         seg_setup(VCPU_SREG_DS);
6762         seg_setup(VCPU_SREG_ES);
6763         seg_setup(VCPU_SREG_FS);
6764         seg_setup(VCPU_SREG_GS);
6765         seg_setup(VCPU_SREG_SS);
6766
6767         vmcs_write16(GUEST_TR_SELECTOR, 0);
6768         vmcs_writel(GUEST_TR_BASE, 0);
6769         vmcs_write32(GUEST_TR_LIMIT, 0xffff);
6770         vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
6771
6772         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
6773         vmcs_writel(GUEST_LDTR_BASE, 0);
6774         vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
6775         vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
6776
6777         if (!init_event) {
6778                 vmcs_write32(GUEST_SYSENTER_CS, 0);
6779                 vmcs_writel(GUEST_SYSENTER_ESP, 0);
6780                 vmcs_writel(GUEST_SYSENTER_EIP, 0);
6781                 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
6782         }
6783
6784         kvm_set_rflags(vcpu, X86_EFLAGS_FIXED);
6785         kvm_rip_write(vcpu, 0xfff0);
6786
6787         vmcs_writel(GUEST_GDTR_BASE, 0);
6788         vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
6789
6790         vmcs_writel(GUEST_IDTR_BASE, 0);
6791         vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
6792
6793         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
6794         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
6795         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 0);
6796         if (kvm_mpx_supported())
6797                 vmcs_write64(GUEST_BNDCFGS, 0);
6798
6799         setup_msrs(vmx);
6800
6801         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
6802
6803         if (cpu_has_vmx_tpr_shadow() && !init_event) {
6804                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0);
6805                 if (cpu_need_tpr_shadow(vcpu))
6806                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR,
6807                                      __pa(vcpu->arch.apic->regs));
6808                 vmcs_write32(TPR_THRESHOLD, 0);
6809         }
6810
6811         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
6812
6813         if (vmx->vpid != 0)
6814                 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
6815
6816         cr0 = X86_CR0_NW | X86_CR0_CD | X86_CR0_ET;
6817         vmx->vcpu.arch.cr0 = cr0;
6818         vmx_set_cr0(vcpu, cr0); /* enter rmode */
6819         vmx_set_cr4(vcpu, 0);
6820         vmx_set_efer(vcpu, 0);
6821
6822         update_exception_bitmap(vcpu);
6823
6824         vpid_sync_context(vmx->vpid);
6825         if (init_event)
6826                 vmx_clear_hlt(vcpu);
6827
6828         vmx_update_fb_clear_dis(vcpu, vmx);
6829 }
6830
6831 /*
6832  * In nested virtualization, check if L1 asked to exit on external interrupts.
6833  * For most existing hypervisors, this will always return true.
6834  */
6835 static bool nested_exit_on_intr(struct kvm_vcpu *vcpu)
6836 {
6837         return get_vmcs12(vcpu)->pin_based_vm_exec_control &
6838                 PIN_BASED_EXT_INTR_MASK;
6839 }
6840
6841 /*
6842  * In nested virtualization, check if L1 has set
6843  * VM_EXIT_ACK_INTR_ON_EXIT
6844  */
6845 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
6846 {
6847         return get_vmcs12(vcpu)->vm_exit_controls &
6848                 VM_EXIT_ACK_INTR_ON_EXIT;
6849 }
6850
6851 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
6852 {
6853         return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu));
6854 }
6855
6856 static void enable_irq_window(struct kvm_vcpu *vcpu)
6857 {
6858         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
6859                       CPU_BASED_VIRTUAL_INTR_PENDING);
6860 }
6861
6862 static void enable_nmi_window(struct kvm_vcpu *vcpu)
6863 {
6864         if (!enable_vnmi ||
6865             vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_STI) {
6866                 enable_irq_window(vcpu);
6867                 return;
6868         }
6869
6870         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
6871                       CPU_BASED_VIRTUAL_NMI_PENDING);
6872 }
6873
6874 static void vmx_inject_irq(struct kvm_vcpu *vcpu)
6875 {
6876         struct vcpu_vmx *vmx = to_vmx(vcpu);
6877         uint32_t intr;
6878         int irq = vcpu->arch.interrupt.nr;
6879
6880         trace_kvm_inj_virq(irq);
6881
6882         ++vcpu->stat.irq_injections;
6883         if (vmx->rmode.vm86_active) {
6884                 int inc_eip = 0;
6885                 if (vcpu->arch.interrupt.soft)
6886                         inc_eip = vcpu->arch.event_exit_inst_len;
6887                 if (kvm_inject_realmode_interrupt(vcpu, irq, inc_eip) != EMULATE_DONE)
6888                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
6889                 return;
6890         }
6891         intr = irq | INTR_INFO_VALID_MASK;
6892         if (vcpu->arch.interrupt.soft) {
6893                 intr |= INTR_TYPE_SOFT_INTR;
6894                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
6895                              vmx->vcpu.arch.event_exit_inst_len);
6896         } else
6897                 intr |= INTR_TYPE_EXT_INTR;
6898         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr);
6899
6900         vmx_clear_hlt(vcpu);
6901 }
6902
6903 static void vmx_inject_nmi(struct kvm_vcpu *vcpu)
6904 {
6905         struct vcpu_vmx *vmx = to_vmx(vcpu);
6906
6907         if (!enable_vnmi) {
6908                 /*
6909                  * Tracking the NMI-blocked state in software is built upon
6910                  * finding the next open IRQ window. This, in turn, depends on
6911                  * well-behaving guests: They have to keep IRQs disabled at
6912                  * least as long as the NMI handler runs. Otherwise we may
6913                  * cause NMI nesting, maybe breaking the guest. But as this is
6914                  * highly unlikely, we can live with the residual risk.
6915                  */
6916                 vmx->loaded_vmcs->soft_vnmi_blocked = 1;
6917                 vmx->loaded_vmcs->vnmi_blocked_time = 0;
6918         }
6919
6920         ++vcpu->stat.nmi_injections;
6921         vmx->loaded_vmcs->nmi_known_unmasked = false;
6922
6923         if (vmx->rmode.vm86_active) {
6924                 if (kvm_inject_realmode_interrupt(vcpu, NMI_VECTOR, 0) != EMULATE_DONE)
6925                         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
6926                 return;
6927         }
6928
6929         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
6930                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR);
6931
6932         vmx_clear_hlt(vcpu);
6933 }
6934
6935 static bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu)
6936 {
6937         struct vcpu_vmx *vmx = to_vmx(vcpu);
6938         bool masked;
6939
6940         if (!enable_vnmi)
6941                 return vmx->loaded_vmcs->soft_vnmi_blocked;
6942         if (vmx->loaded_vmcs->nmi_known_unmasked)
6943                 return false;
6944         masked = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & GUEST_INTR_STATE_NMI;
6945         vmx->loaded_vmcs->nmi_known_unmasked = !masked;
6946         return masked;
6947 }
6948
6949 static void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
6950 {
6951         struct vcpu_vmx *vmx = to_vmx(vcpu);
6952
6953         if (!enable_vnmi) {
6954                 if (vmx->loaded_vmcs->soft_vnmi_blocked != masked) {
6955                         vmx->loaded_vmcs->soft_vnmi_blocked = masked;
6956                         vmx->loaded_vmcs->vnmi_blocked_time = 0;
6957                 }
6958         } else {
6959                 vmx->loaded_vmcs->nmi_known_unmasked = !masked;
6960                 if (masked)
6961                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
6962                                       GUEST_INTR_STATE_NMI);
6963                 else
6964                         vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO,
6965                                         GUEST_INTR_STATE_NMI);
6966         }
6967 }
6968
6969 static int vmx_nmi_allowed(struct kvm_vcpu *vcpu)
6970 {
6971         if (to_vmx(vcpu)->nested.nested_run_pending)
6972                 return 0;
6973
6974         if (!enable_vnmi &&
6975             to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked)
6976                 return 0;
6977
6978         return  !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
6979                   (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI
6980                    | GUEST_INTR_STATE_NMI));
6981 }
6982
6983 static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu)
6984 {
6985         if (to_vmx(vcpu)->nested.nested_run_pending)
6986                 return false;
6987
6988         if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu))
6989                 return true;
6990
6991         return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
6992                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) &
6993                         (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS));
6994 }
6995
6996 static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr)
6997 {
6998         int ret;
6999
7000         if (enable_unrestricted_guest)
7001                 return 0;
7002
7003         ret = x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, addr,
7004                                     PAGE_SIZE * 3);
7005         if (ret)
7006                 return ret;
7007         to_kvm_vmx(kvm)->tss_addr = addr;
7008         return init_rmode_tss(kvm);
7009 }
7010
7011 static int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
7012 {
7013         to_kvm_vmx(kvm)->ept_identity_map_addr = ident_addr;
7014         return 0;
7015 }
7016
7017 static bool rmode_exception(struct kvm_vcpu *vcpu, int vec)
7018 {
7019         switch (vec) {
7020         case BP_VECTOR:
7021                 /*
7022                  * Update instruction length as we may reinject the exception
7023                  * from user space while in guest debugging mode.
7024                  */
7025                 to_vmx(vcpu)->vcpu.arch.event_exit_inst_len =
7026                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
7027                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
7028                         return false;
7029                 /* fall through */
7030         case DB_VECTOR:
7031                 if (vcpu->guest_debug &
7032                         (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
7033                         return false;
7034                 /* fall through */
7035         case DE_VECTOR:
7036         case OF_VECTOR:
7037         case BR_VECTOR:
7038         case UD_VECTOR:
7039         case DF_VECTOR:
7040         case SS_VECTOR:
7041         case GP_VECTOR:
7042         case MF_VECTOR:
7043                 return true;
7044         break;
7045         }
7046         return false;
7047 }
7048
7049 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
7050                                   int vec, u32 err_code)
7051 {
7052         /*
7053          * Instruction with address size override prefix opcode 0x67
7054          * Cause the #SS fault with 0 error code in VM86 mode.
7055          */
7056         if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) {
7057                 if (kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE) {
7058                         if (vcpu->arch.halt_request) {
7059                                 vcpu->arch.halt_request = 0;
7060                                 return kvm_vcpu_halt(vcpu);
7061                         }
7062                         return 1;
7063                 }
7064                 return 0;
7065         }
7066
7067         /*
7068          * Forward all other exceptions that are valid in real mode.
7069          * FIXME: Breaks guest debugging in real mode, needs to be fixed with
7070          *        the required debugging infrastructure rework.
7071          */
7072         kvm_queue_exception(vcpu, vec);
7073         return 1;
7074 }
7075
7076 /*
7077  * Trigger machine check on the host. We assume all the MSRs are already set up
7078  * by the CPU and that we still run on the same CPU as the MCE occurred on.
7079  * We pass a fake environment to the machine check handler because we want
7080  * the guest to be always treated like user space, no matter what context
7081  * it used internally.
7082  */
7083 static void kvm_machine_check(void)
7084 {
7085 #if defined(CONFIG_X86_MCE)
7086         struct pt_regs regs = {
7087                 .cs = 3, /* Fake ring 3 no matter what the guest ran on */
7088                 .flags = X86_EFLAGS_IF,
7089         };
7090
7091         do_machine_check(&regs, 0);
7092 #endif
7093 }
7094
7095 static int handle_machine_check(struct kvm_vcpu *vcpu)
7096 {
7097         /* already handled by vcpu_run */
7098         return 1;
7099 }
7100
7101 static int handle_exception(struct kvm_vcpu *vcpu)
7102 {
7103         struct vcpu_vmx *vmx = to_vmx(vcpu);
7104         struct kvm_run *kvm_run = vcpu->run;
7105         u32 intr_info, ex_no, error_code;
7106         unsigned long cr2, rip, dr6;
7107         u32 vect_info;
7108         enum emulation_result er;
7109
7110         vect_info = vmx->idt_vectoring_info;
7111         intr_info = vmx->exit_intr_info;
7112
7113         if (is_machine_check(intr_info))
7114                 return handle_machine_check(vcpu);
7115
7116         if (is_nmi(intr_info))
7117                 return 1;  /* already handled by vmx_vcpu_run() */
7118
7119         if (is_invalid_opcode(intr_info))
7120                 return handle_ud(vcpu);
7121
7122         error_code = 0;
7123         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
7124                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
7125
7126         if (!vmx->rmode.vm86_active && is_gp_fault(intr_info)) {
7127                 WARN_ON_ONCE(!enable_vmware_backdoor);
7128                 er = kvm_emulate_instruction(vcpu,
7129                         EMULTYPE_VMWARE | EMULTYPE_NO_UD_ON_FAIL);
7130                 if (er == EMULATE_USER_EXIT)
7131                         return 0;
7132                 else if (er != EMULATE_DONE)
7133                         kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
7134                 return 1;
7135         }
7136
7137         /*
7138          * The #PF with PFEC.RSVD = 1 indicates the guest is accessing
7139          * MMIO, it is better to report an internal error.
7140          * See the comments in vmx_handle_exit.
7141          */
7142         if ((vect_info & VECTORING_INFO_VALID_MASK) &&
7143             !(is_page_fault(intr_info) && !(error_code & PFERR_RSVD_MASK))) {
7144                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7145                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_SIMUL_EX;
7146                 vcpu->run->internal.ndata = 3;
7147                 vcpu->run->internal.data[0] = vect_info;
7148                 vcpu->run->internal.data[1] = intr_info;
7149                 vcpu->run->internal.data[2] = error_code;
7150                 return 0;
7151         }
7152
7153         if (is_page_fault(intr_info)) {
7154                 cr2 = vmcs_readl(EXIT_QUALIFICATION);
7155                 /* EPT won't cause page fault directly */
7156                 WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept);
7157                 return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0);
7158         }
7159
7160         ex_no = intr_info & INTR_INFO_VECTOR_MASK;
7161
7162         if (vmx->rmode.vm86_active && rmode_exception(vcpu, ex_no))
7163                 return handle_rmode_exception(vcpu, ex_no, error_code);
7164
7165         switch (ex_no) {
7166         case AC_VECTOR:
7167                 kvm_queue_exception_e(vcpu, AC_VECTOR, error_code);
7168                 return 1;
7169         case DB_VECTOR:
7170                 dr6 = vmcs_readl(EXIT_QUALIFICATION);
7171                 if (!(vcpu->guest_debug &
7172                       (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) {
7173                         vcpu->arch.dr6 &= ~15;
7174                         vcpu->arch.dr6 |= dr6 | DR6_RTM;
7175                         if (is_icebp(intr_info))
7176                                 skip_emulated_instruction(vcpu);
7177
7178                         kvm_queue_exception(vcpu, DB_VECTOR);
7179                         return 1;
7180                 }
7181                 kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1;
7182                 kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7);
7183                 /* fall through */
7184         case BP_VECTOR:
7185                 /*
7186                  * Update instruction length as we may reinject #BP from
7187                  * user space while in guest debugging mode. Reading it for
7188                  * #DB as well causes no harm, it is not used in that case.
7189                  */
7190                 vmx->vcpu.arch.event_exit_inst_len =
7191                         vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
7192                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
7193                 rip = kvm_rip_read(vcpu);
7194                 kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip;
7195                 kvm_run->debug.arch.exception = ex_no;
7196                 break;
7197         default:
7198                 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
7199                 kvm_run->ex.exception = ex_no;
7200                 kvm_run->ex.error_code = error_code;
7201                 break;
7202         }
7203         return 0;
7204 }
7205
7206 static int handle_external_interrupt(struct kvm_vcpu *vcpu)
7207 {
7208         ++vcpu->stat.irq_exits;
7209         return 1;
7210 }
7211
7212 static int handle_triple_fault(struct kvm_vcpu *vcpu)
7213 {
7214         vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN;
7215         vcpu->mmio_needed = 0;
7216         return 0;
7217 }
7218
7219 static int handle_io(struct kvm_vcpu *vcpu)
7220 {
7221         unsigned long exit_qualification;
7222         int size, in, string;
7223         unsigned port;
7224
7225         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7226         string = (exit_qualification & 16) != 0;
7227
7228         ++vcpu->stat.io_exits;
7229
7230         if (string)
7231                 return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7232
7233         port = exit_qualification >> 16;
7234         size = (exit_qualification & 7) + 1;
7235         in = (exit_qualification & 8) != 0;
7236
7237         return kvm_fast_pio(vcpu, size, port, in);
7238 }
7239
7240 static void
7241 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
7242 {
7243         /*
7244          * Patch in the VMCALL instruction:
7245          */
7246         hypercall[0] = 0x0f;
7247         hypercall[1] = 0x01;
7248         hypercall[2] = 0xc1;
7249 }
7250
7251 /* called to set cr0 as appropriate for a mov-to-cr0 exit. */
7252 static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
7253 {
7254         if (is_guest_mode(vcpu)) {
7255                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7256                 unsigned long orig_val = val;
7257
7258                 /*
7259                  * We get here when L2 changed cr0 in a way that did not change
7260                  * any of L1's shadowed bits (see nested_vmx_exit_handled_cr),
7261                  * but did change L0 shadowed bits. So we first calculate the
7262                  * effective cr0 value that L1 would like to write into the
7263                  * hardware. It consists of the L2-owned bits from the new
7264                  * value combined with the L1-owned bits from L1's guest_cr0.
7265                  */
7266                 val = (val & ~vmcs12->cr0_guest_host_mask) |
7267                         (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask);
7268
7269                 if (!nested_guest_cr0_valid(vcpu, val))
7270                         return 1;
7271
7272                 if (kvm_set_cr0(vcpu, val))
7273                         return 1;
7274                 vmcs_writel(CR0_READ_SHADOW, orig_val);
7275                 return 0;
7276         } else {
7277                 if (to_vmx(vcpu)->nested.vmxon &&
7278                     !nested_host_cr0_valid(vcpu, val))
7279                         return 1;
7280
7281                 return kvm_set_cr0(vcpu, val);
7282         }
7283 }
7284
7285 static int handle_set_cr4(struct kvm_vcpu *vcpu, unsigned long val)
7286 {
7287         if (is_guest_mode(vcpu)) {
7288                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
7289                 unsigned long orig_val = val;
7290
7291                 /* analogously to handle_set_cr0 */
7292                 val = (val & ~vmcs12->cr4_guest_host_mask) |
7293                         (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask);
7294                 if (kvm_set_cr4(vcpu, val))
7295                         return 1;
7296                 vmcs_writel(CR4_READ_SHADOW, orig_val);
7297                 return 0;
7298         } else
7299                 return kvm_set_cr4(vcpu, val);
7300 }
7301
7302 static int handle_desc(struct kvm_vcpu *vcpu)
7303 {
7304         WARN_ON(!(vcpu->arch.cr4 & X86_CR4_UMIP));
7305         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7306 }
7307
7308 static int handle_cr(struct kvm_vcpu *vcpu)
7309 {
7310         unsigned long exit_qualification, val;
7311         int cr;
7312         int reg;
7313         int err;
7314         int ret;
7315
7316         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7317         cr = exit_qualification & 15;
7318         reg = (exit_qualification >> 8) & 15;
7319         switch ((exit_qualification >> 4) & 3) {
7320         case 0: /* mov to cr */
7321                 val = kvm_register_readl(vcpu, reg);
7322                 trace_kvm_cr_write(cr, val);
7323                 switch (cr) {
7324                 case 0:
7325                         err = handle_set_cr0(vcpu, val);
7326                         return kvm_complete_insn_gp(vcpu, err);
7327                 case 3:
7328                         WARN_ON_ONCE(enable_unrestricted_guest);
7329                         err = kvm_set_cr3(vcpu, val);
7330                         return kvm_complete_insn_gp(vcpu, err);
7331                 case 4:
7332                         err = handle_set_cr4(vcpu, val);
7333                         return kvm_complete_insn_gp(vcpu, err);
7334                 case 8: {
7335                                 u8 cr8_prev = kvm_get_cr8(vcpu);
7336                                 u8 cr8 = (u8)val;
7337                                 err = kvm_set_cr8(vcpu, cr8);
7338                                 ret = kvm_complete_insn_gp(vcpu, err);
7339                                 if (lapic_in_kernel(vcpu))
7340                                         return ret;
7341                                 if (cr8_prev <= cr8)
7342                                         return ret;
7343                                 /*
7344                                  * TODO: we might be squashing a
7345                                  * KVM_GUESTDBG_SINGLESTEP-triggered
7346                                  * KVM_EXIT_DEBUG here.
7347                                  */
7348                                 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
7349                                 return 0;
7350                         }
7351                 }
7352                 break;
7353         case 2: /* clts */
7354                 WARN_ONCE(1, "Guest should always own CR0.TS");
7355                 vmx_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~X86_CR0_TS));
7356                 trace_kvm_cr_write(0, kvm_read_cr0(vcpu));
7357                 return kvm_skip_emulated_instruction(vcpu);
7358         case 1: /*mov from cr*/
7359                 switch (cr) {
7360                 case 3:
7361                         WARN_ON_ONCE(enable_unrestricted_guest);
7362                         val = kvm_read_cr3(vcpu);
7363                         kvm_register_write(vcpu, reg, val);
7364                         trace_kvm_cr_read(cr, val);
7365                         return kvm_skip_emulated_instruction(vcpu);
7366                 case 8:
7367                         val = kvm_get_cr8(vcpu);
7368                         kvm_register_write(vcpu, reg, val);
7369                         trace_kvm_cr_read(cr, val);
7370                         return kvm_skip_emulated_instruction(vcpu);
7371                 }
7372                 break;
7373         case 3: /* lmsw */
7374                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
7375                 trace_kvm_cr_write(0, (kvm_read_cr0(vcpu) & ~0xful) | val);
7376                 kvm_lmsw(vcpu, val);
7377
7378                 return kvm_skip_emulated_instruction(vcpu);
7379         default:
7380                 break;
7381         }
7382         vcpu->run->exit_reason = 0;
7383         vcpu_unimpl(vcpu, "unhandled control register: op %d cr %d\n",
7384                (int)(exit_qualification >> 4) & 3, cr);
7385         return 0;
7386 }
7387
7388 static int handle_dr(struct kvm_vcpu *vcpu)
7389 {
7390         unsigned long exit_qualification;
7391         int dr, dr7, reg;
7392
7393         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7394         dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
7395
7396         /* First, if DR does not exist, trigger UD */
7397         if (!kvm_require_dr(vcpu, dr))
7398                 return 1;
7399
7400         /* Do not handle if the CPL > 0, will trigger GP on re-entry */
7401         if (!kvm_require_cpl(vcpu, 0))
7402                 return 1;
7403         dr7 = vmcs_readl(GUEST_DR7);
7404         if (dr7 & DR7_GD) {
7405                 /*
7406                  * As the vm-exit takes precedence over the debug trap, we
7407                  * need to emulate the latter, either for the host or the
7408                  * guest debugging itself.
7409                  */
7410                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
7411                         vcpu->run->debug.arch.dr6 = vcpu->arch.dr6;
7412                         vcpu->run->debug.arch.dr7 = dr7;
7413                         vcpu->run->debug.arch.pc = kvm_get_linear_rip(vcpu);
7414                         vcpu->run->debug.arch.exception = DB_VECTOR;
7415                         vcpu->run->exit_reason = KVM_EXIT_DEBUG;
7416                         return 0;
7417                 } else {
7418                         vcpu->arch.dr6 &= ~15;
7419                         vcpu->arch.dr6 |= DR6_BD | DR6_RTM;
7420                         kvm_queue_exception(vcpu, DB_VECTOR);
7421                         return 1;
7422                 }
7423         }
7424
7425         if (vcpu->guest_debug == 0) {
7426                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
7427                                 CPU_BASED_MOV_DR_EXITING);
7428
7429                 /*
7430                  * No more DR vmexits; force a reload of the debug registers
7431                  * and reenter on this instruction.  The next vmexit will
7432                  * retrieve the full state of the debug registers.
7433                  */
7434                 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
7435                 return 1;
7436         }
7437
7438         reg = DEBUG_REG_ACCESS_REG(exit_qualification);
7439         if (exit_qualification & TYPE_MOV_FROM_DR) {
7440                 unsigned long val;
7441
7442                 if (kvm_get_dr(vcpu, dr, &val))
7443                         return 1;
7444                 kvm_register_write(vcpu, reg, val);
7445         } else
7446                 if (kvm_set_dr(vcpu, dr, kvm_register_readl(vcpu, reg)))
7447                         return 1;
7448
7449         return kvm_skip_emulated_instruction(vcpu);
7450 }
7451
7452 static u64 vmx_get_dr6(struct kvm_vcpu *vcpu)
7453 {
7454         return vcpu->arch.dr6;
7455 }
7456
7457 static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
7458 {
7459 }
7460
7461 static void vmx_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
7462 {
7463         get_debugreg(vcpu->arch.db[0], 0);
7464         get_debugreg(vcpu->arch.db[1], 1);
7465         get_debugreg(vcpu->arch.db[2], 2);
7466         get_debugreg(vcpu->arch.db[3], 3);
7467         get_debugreg(vcpu->arch.dr6, 6);
7468         vcpu->arch.dr7 = vmcs_readl(GUEST_DR7);
7469
7470         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
7471         vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL, CPU_BASED_MOV_DR_EXITING);
7472 }
7473
7474 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
7475 {
7476         vmcs_writel(GUEST_DR7, val);
7477 }
7478
7479 static int handle_cpuid(struct kvm_vcpu *vcpu)
7480 {
7481         return kvm_emulate_cpuid(vcpu);
7482 }
7483
7484 static int handle_rdmsr(struct kvm_vcpu *vcpu)
7485 {
7486         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
7487         struct msr_data msr_info;
7488
7489         msr_info.index = ecx;
7490         msr_info.host_initiated = false;
7491         if (vmx_get_msr(vcpu, &msr_info)) {
7492                 trace_kvm_msr_read_ex(ecx);
7493                 kvm_inject_gp(vcpu, 0);
7494                 return 1;
7495         }
7496
7497         trace_kvm_msr_read(ecx, msr_info.data);
7498
7499         /* FIXME: handling of bits 32:63 of rax, rdx */
7500         vcpu->arch.regs[VCPU_REGS_RAX] = msr_info.data & -1u;
7501         vcpu->arch.regs[VCPU_REGS_RDX] = (msr_info.data >> 32) & -1u;
7502         return kvm_skip_emulated_instruction(vcpu);
7503 }
7504
7505 static int handle_wrmsr(struct kvm_vcpu *vcpu)
7506 {
7507         struct msr_data msr;
7508         u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX];
7509         u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u)
7510                 | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32);
7511
7512         msr.data = data;
7513         msr.index = ecx;
7514         msr.host_initiated = false;
7515         if (kvm_set_msr(vcpu, &msr) != 0) {
7516                 trace_kvm_msr_write_ex(ecx, data);
7517                 kvm_inject_gp(vcpu, 0);
7518                 return 1;
7519         }
7520
7521         trace_kvm_msr_write(ecx, data);
7522         return kvm_skip_emulated_instruction(vcpu);
7523 }
7524
7525 static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu)
7526 {
7527         kvm_apic_update_ppr(vcpu);
7528         return 1;
7529 }
7530
7531 static int handle_interrupt_window(struct kvm_vcpu *vcpu)
7532 {
7533         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
7534                         CPU_BASED_VIRTUAL_INTR_PENDING);
7535
7536         kvm_make_request(KVM_REQ_EVENT, vcpu);
7537
7538         ++vcpu->stat.irq_window_exits;
7539         return 1;
7540 }
7541
7542 static int handle_halt(struct kvm_vcpu *vcpu)
7543 {
7544         return kvm_emulate_halt(vcpu);
7545 }
7546
7547 static int handle_vmcall(struct kvm_vcpu *vcpu)
7548 {
7549         return kvm_emulate_hypercall(vcpu);
7550 }
7551
7552 static int handle_invd(struct kvm_vcpu *vcpu)
7553 {
7554         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7555 }
7556
7557 static int handle_invlpg(struct kvm_vcpu *vcpu)
7558 {
7559         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7560
7561         kvm_mmu_invlpg(vcpu, exit_qualification);
7562         return kvm_skip_emulated_instruction(vcpu);
7563 }
7564
7565 static int handle_rdpmc(struct kvm_vcpu *vcpu)
7566 {
7567         int err;
7568
7569         err = kvm_rdpmc(vcpu);
7570         return kvm_complete_insn_gp(vcpu, err);
7571 }
7572
7573 static int handle_wbinvd(struct kvm_vcpu *vcpu)
7574 {
7575         return kvm_emulate_wbinvd(vcpu);
7576 }
7577
7578 static int handle_xsetbv(struct kvm_vcpu *vcpu)
7579 {
7580         u64 new_bv = kvm_read_edx_eax(vcpu);
7581         u32 index = kvm_register_read(vcpu, VCPU_REGS_RCX);
7582
7583         if (kvm_set_xcr(vcpu, index, new_bv) == 0)
7584                 return kvm_skip_emulated_instruction(vcpu);
7585         return 1;
7586 }
7587
7588 static int handle_xsaves(struct kvm_vcpu *vcpu)
7589 {
7590         kvm_skip_emulated_instruction(vcpu);
7591         WARN(1, "this should never happen\n");
7592         return 1;
7593 }
7594
7595 static int handle_xrstors(struct kvm_vcpu *vcpu)
7596 {
7597         kvm_skip_emulated_instruction(vcpu);
7598         WARN(1, "this should never happen\n");
7599         return 1;
7600 }
7601
7602 static int handle_apic_access(struct kvm_vcpu *vcpu)
7603 {
7604         if (likely(fasteoi)) {
7605                 unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7606                 int access_type, offset;
7607
7608                 access_type = exit_qualification & APIC_ACCESS_TYPE;
7609                 offset = exit_qualification & APIC_ACCESS_OFFSET;
7610                 /*
7611                  * Sane guest uses MOV to write EOI, with written value
7612                  * not cared. So make a short-circuit here by avoiding
7613                  * heavy instruction emulation.
7614                  */
7615                 if ((access_type == TYPE_LINEAR_APIC_INST_WRITE) &&
7616                     (offset == APIC_EOI)) {
7617                         kvm_lapic_set_eoi(vcpu);
7618                         return kvm_skip_emulated_instruction(vcpu);
7619                 }
7620         }
7621         return kvm_emulate_instruction(vcpu, 0) == EMULATE_DONE;
7622 }
7623
7624 static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu)
7625 {
7626         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7627         int vector = exit_qualification & 0xff;
7628
7629         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
7630         kvm_apic_set_eoi_accelerated(vcpu, vector);
7631         return 1;
7632 }
7633
7634 static int handle_apic_write(struct kvm_vcpu *vcpu)
7635 {
7636         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7637         u32 offset = exit_qualification & 0xfff;
7638
7639         /* APIC-write VM exit is trap-like and thus no need to adjust IP */
7640         kvm_apic_write_nodecode(vcpu, offset);
7641         return 1;
7642 }
7643
7644 static int handle_task_switch(struct kvm_vcpu *vcpu)
7645 {
7646         struct vcpu_vmx *vmx = to_vmx(vcpu);
7647         unsigned long exit_qualification;
7648         bool has_error_code = false;
7649         u32 error_code = 0;
7650         u16 tss_selector;
7651         int reason, type, idt_v, idt_index;
7652
7653         idt_v = (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK);
7654         idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK);
7655         type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK);
7656
7657         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7658
7659         reason = (u32)exit_qualification >> 30;
7660         if (reason == TASK_SWITCH_GATE && idt_v) {
7661                 switch (type) {
7662                 case INTR_TYPE_NMI_INTR:
7663                         vcpu->arch.nmi_injected = false;
7664                         vmx_set_nmi_mask(vcpu, true);
7665                         break;
7666                 case INTR_TYPE_EXT_INTR:
7667                 case INTR_TYPE_SOFT_INTR:
7668                         kvm_clear_interrupt_queue(vcpu);
7669                         break;
7670                 case INTR_TYPE_HARD_EXCEPTION:
7671                         if (vmx->idt_vectoring_info &
7672                             VECTORING_INFO_DELIVER_CODE_MASK) {
7673                                 has_error_code = true;
7674                                 error_code =
7675                                         vmcs_read32(IDT_VECTORING_ERROR_CODE);
7676                         }
7677                         /* fall through */
7678                 case INTR_TYPE_SOFT_EXCEPTION:
7679                         kvm_clear_exception_queue(vcpu);
7680                         break;
7681                 default:
7682                         break;
7683                 }
7684         }
7685         tss_selector = exit_qualification;
7686
7687         if (!idt_v || (type != INTR_TYPE_HARD_EXCEPTION &&
7688                        type != INTR_TYPE_EXT_INTR &&
7689                        type != INTR_TYPE_NMI_INTR))
7690                 skip_emulated_instruction(vcpu);
7691
7692         if (kvm_task_switch(vcpu, tss_selector,
7693                             type == INTR_TYPE_SOFT_INTR ? idt_index : -1, reason,
7694                             has_error_code, error_code) == EMULATE_FAIL) {
7695                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7696                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7697                 vcpu->run->internal.ndata = 0;
7698                 return 0;
7699         }
7700
7701         /*
7702          * TODO: What about debug traps on tss switch?
7703          *       Are we supposed to inject them and update dr6?
7704          */
7705
7706         return 1;
7707 }
7708
7709 static int handle_ept_violation(struct kvm_vcpu *vcpu)
7710 {
7711         unsigned long exit_qualification;
7712         gpa_t gpa;
7713         u64 error_code;
7714
7715         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
7716
7717         /*
7718          * EPT violation happened while executing iret from NMI,
7719          * "blocked by NMI" bit has to be set before next VM entry.
7720          * There are errata that may cause this bit to not be set:
7721          * AAK134, BY25.
7722          */
7723         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
7724                         enable_vnmi &&
7725                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
7726                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI);
7727
7728         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
7729         trace_kvm_page_fault(gpa, exit_qualification);
7730
7731         /* Is it a read fault? */
7732         error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
7733                      ? PFERR_USER_MASK : 0;
7734         /* Is it a write fault? */
7735         error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
7736                       ? PFERR_WRITE_MASK : 0;
7737         /* Is it a fetch fault? */
7738         error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
7739                       ? PFERR_FETCH_MASK : 0;
7740         /* ept page table entry is present? */
7741         error_code |= (exit_qualification &
7742                        (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE |
7743                         EPT_VIOLATION_EXECUTABLE))
7744                       ? PFERR_PRESENT_MASK : 0;
7745
7746         error_code |= (exit_qualification & 0x100) != 0 ?
7747                PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
7748
7749         vcpu->arch.exit_qualification = exit_qualification;
7750         return kvm_mmu_page_fault(vcpu, gpa, error_code, NULL, 0);
7751 }
7752
7753 static int handle_ept_misconfig(struct kvm_vcpu *vcpu)
7754 {
7755         gpa_t gpa;
7756
7757         /*
7758          * A nested guest cannot optimize MMIO vmexits, because we have an
7759          * nGPA here instead of the required GPA.
7760          */
7761         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
7762         if (!is_guest_mode(vcpu) &&
7763             !kvm_io_bus_write(vcpu, KVM_FAST_MMIO_BUS, gpa, 0, NULL)) {
7764                 trace_kvm_fast_mmio(gpa);
7765                 /*
7766                  * Doing kvm_skip_emulated_instruction() depends on undefined
7767                  * behavior: Intel's manual doesn't mandate
7768                  * VM_EXIT_INSTRUCTION_LEN to be set in VMCS when EPT MISCONFIG
7769                  * occurs and while on real hardware it was observed to be set,
7770                  * other hypervisors (namely Hyper-V) don't set it, we end up
7771                  * advancing IP with some random value. Disable fast mmio when
7772                  * running nested and keep it for real hardware in hope that
7773                  * VM_EXIT_INSTRUCTION_LEN will always be set correctly.
7774                  */
7775                 if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
7776                         return kvm_skip_emulated_instruction(vcpu);
7777                 else
7778                         return kvm_emulate_instruction(vcpu, EMULTYPE_SKIP) ==
7779                                                                 EMULATE_DONE;
7780         }
7781
7782         return kvm_mmu_page_fault(vcpu, gpa, PFERR_RSVD_MASK, NULL, 0);
7783 }
7784
7785 static int handle_nmi_window(struct kvm_vcpu *vcpu)
7786 {
7787         WARN_ON_ONCE(!enable_vnmi);
7788         vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
7789                         CPU_BASED_VIRTUAL_NMI_PENDING);
7790         ++vcpu->stat.nmi_window_exits;
7791         kvm_make_request(KVM_REQ_EVENT, vcpu);
7792
7793         return 1;
7794 }
7795
7796 static int handle_invalid_guest_state(struct kvm_vcpu *vcpu)
7797 {
7798         struct vcpu_vmx *vmx = to_vmx(vcpu);
7799         enum emulation_result err = EMULATE_DONE;
7800         int ret = 1;
7801         u32 cpu_exec_ctrl;
7802         bool intr_window_requested;
7803         unsigned count = 130;
7804
7805         /*
7806          * We should never reach the point where we are emulating L2
7807          * due to invalid guest state as that means we incorrectly
7808          * allowed a nested VMEntry with an invalid vmcs12.
7809          */
7810         WARN_ON_ONCE(vmx->emulation_required && vmx->nested.nested_run_pending);
7811
7812         cpu_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
7813         intr_window_requested = cpu_exec_ctrl & CPU_BASED_VIRTUAL_INTR_PENDING;
7814
7815         while (vmx->emulation_required && count-- != 0) {
7816                 if (intr_window_requested && vmx_interrupt_allowed(vcpu))
7817                         return handle_interrupt_window(&vmx->vcpu);
7818
7819                 if (kvm_test_request(KVM_REQ_EVENT, vcpu))
7820                         return 1;
7821
7822                 err = kvm_emulate_instruction(vcpu, 0);
7823
7824                 if (err == EMULATE_USER_EXIT) {
7825                         ++vcpu->stat.mmio_exits;
7826                         ret = 0;
7827                         goto out;
7828                 }
7829
7830                 if (err != EMULATE_DONE)
7831                         goto emulation_error;
7832
7833                 if (vmx->emulation_required && !vmx->rmode.vm86_active &&
7834                     vcpu->arch.exception.pending)
7835                         goto emulation_error;
7836
7837                 if (vcpu->arch.halt_request) {
7838                         vcpu->arch.halt_request = 0;
7839                         ret = kvm_vcpu_halt(vcpu);
7840                         goto out;
7841                 }
7842
7843                 if (signal_pending(current))
7844                         goto out;
7845                 if (need_resched())
7846                         schedule();
7847         }
7848
7849 out:
7850         return ret;
7851
7852 emulation_error:
7853         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
7854         vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
7855         vcpu->run->internal.ndata = 0;
7856         return 0;
7857 }
7858
7859 static void grow_ple_window(struct kvm_vcpu *vcpu)
7860 {
7861         struct vcpu_vmx *vmx = to_vmx(vcpu);
7862         int old = vmx->ple_window;
7863
7864         vmx->ple_window = __grow_ple_window(old, ple_window,
7865                                             ple_window_grow,
7866                                             ple_window_max);
7867
7868         if (vmx->ple_window != old)
7869                 vmx->ple_window_dirty = true;
7870
7871         trace_kvm_ple_window_grow(vcpu->vcpu_id, vmx->ple_window, old);
7872 }
7873
7874 static void shrink_ple_window(struct kvm_vcpu *vcpu)
7875 {
7876         struct vcpu_vmx *vmx = to_vmx(vcpu);
7877         int old = vmx->ple_window;
7878
7879         vmx->ple_window = __shrink_ple_window(old, ple_window,
7880                                               ple_window_shrink,
7881                                               ple_window);
7882
7883         if (vmx->ple_window != old)
7884                 vmx->ple_window_dirty = true;
7885
7886         trace_kvm_ple_window_shrink(vcpu->vcpu_id, vmx->ple_window, old);
7887 }
7888
7889 /*
7890  * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR.
7891  */
7892 static void wakeup_handler(void)
7893 {
7894         struct kvm_vcpu *vcpu;
7895         int cpu = smp_processor_id();
7896
7897         spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
7898         list_for_each_entry(vcpu, &per_cpu(blocked_vcpu_on_cpu, cpu),
7899                         blocked_vcpu_list) {
7900                 struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
7901
7902                 if (pi_test_on(pi_desc) == 1)
7903                         kvm_vcpu_kick(vcpu);
7904         }
7905         spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
7906 }
7907
7908 static void vmx_enable_tdp(void)
7909 {
7910         kvm_mmu_set_mask_ptes(VMX_EPT_READABLE_MASK,
7911                 enable_ept_ad_bits ? VMX_EPT_ACCESS_BIT : 0ull,
7912                 enable_ept_ad_bits ? VMX_EPT_DIRTY_BIT : 0ull,
7913                 0ull, VMX_EPT_EXECUTABLE_MASK,
7914                 cpu_has_vmx_ept_execute_only() ? 0ull : VMX_EPT_READABLE_MASK,
7915                 VMX_EPT_RWX_MASK, 0ull);
7916
7917         ept_set_mmio_spte_mask();
7918         kvm_enable_tdp();
7919 }
7920
7921 static __init int hardware_setup(void)
7922 {
7923         unsigned long host_bndcfgs;
7924         int r = -ENOMEM, i;
7925
7926         rdmsrl_safe(MSR_EFER, &host_efer);
7927
7928         for (i = 0; i < ARRAY_SIZE(vmx_msr_index); ++i)
7929                 kvm_define_shared_msr(i, vmx_msr_index[i]);
7930
7931         for (i = 0; i < VMX_BITMAP_NR; i++) {
7932                 vmx_bitmap[i] = (unsigned long *)__get_free_page(GFP_KERNEL);
7933                 if (!vmx_bitmap[i])
7934                         goto out;
7935         }
7936
7937         memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
7938         memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
7939
7940         if (setup_vmcs_config(&vmcs_config) < 0) {
7941                 r = -EIO;
7942                 goto out;
7943         }
7944
7945         if (boot_cpu_has(X86_FEATURE_NX))
7946                 kvm_enable_efer_bits(EFER_NX);
7947
7948         if (boot_cpu_has(X86_FEATURE_MPX)) {
7949                 rdmsrl(MSR_IA32_BNDCFGS, host_bndcfgs);
7950                 WARN_ONCE(host_bndcfgs, "KVM: BNDCFGS in host will be lost");
7951         }
7952
7953         if (!cpu_has_vmx_vpid() || !cpu_has_vmx_invvpid() ||
7954                 !(cpu_has_vmx_invvpid_single() || cpu_has_vmx_invvpid_global()))
7955                 enable_vpid = 0;
7956
7957         if (!cpu_has_vmx_ept() ||
7958             !cpu_has_vmx_ept_4levels() ||
7959             !cpu_has_vmx_ept_mt_wb() ||
7960             !cpu_has_vmx_invept_global())
7961                 enable_ept = 0;
7962
7963         if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
7964                 enable_ept_ad_bits = 0;
7965
7966         if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
7967                 enable_unrestricted_guest = 0;
7968
7969         if (!cpu_has_vmx_flexpriority())
7970                 flexpriority_enabled = 0;
7971
7972         if (!cpu_has_virtual_nmis())
7973                 enable_vnmi = 0;
7974
7975         /*
7976          * set_apic_access_page_addr() is used to reload apic access
7977          * page upon invalidation.  No need to do anything if not
7978          * using the APIC_ACCESS_ADDR VMCS field.
7979          */
7980         if (!flexpriority_enabled)
7981                 kvm_x86_ops->set_apic_access_page_addr = NULL;
7982
7983         if (!cpu_has_vmx_tpr_shadow())
7984                 kvm_x86_ops->update_cr8_intercept = NULL;
7985
7986         if (enable_ept && !cpu_has_vmx_ept_2m_page())
7987                 kvm_disable_largepages();
7988
7989 #if IS_ENABLED(CONFIG_HYPERV)
7990         if (ms_hyperv.nested_features & HV_X64_NESTED_GUEST_MAPPING_FLUSH
7991             && enable_ept)
7992                 kvm_x86_ops->tlb_remote_flush = vmx_hv_remote_flush_tlb;
7993 #endif
7994
7995         if (!cpu_has_vmx_ple()) {
7996                 ple_gap = 0;
7997                 ple_window = 0;
7998                 ple_window_grow = 0;
7999                 ple_window_max = 0;
8000                 ple_window_shrink = 0;
8001         }
8002
8003         if (!cpu_has_vmx_apicv()) {
8004                 enable_apicv = 0;
8005                 kvm_x86_ops->sync_pir_to_irr = NULL;
8006         }
8007
8008         if (cpu_has_vmx_tsc_scaling()) {
8009                 kvm_has_tsc_control = true;
8010                 kvm_max_tsc_scaling_ratio = KVM_VMX_TSC_MULTIPLIER_MAX;
8011                 kvm_tsc_scaling_ratio_frac_bits = 48;
8012         }
8013
8014         set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
8015
8016         if (enable_ept)
8017                 vmx_enable_tdp();
8018         else
8019                 kvm_disable_tdp();
8020
8021         if (!nested) {
8022                 kvm_x86_ops->get_nested_state = NULL;
8023                 kvm_x86_ops->set_nested_state = NULL;
8024         }
8025
8026         /*
8027          * Only enable PML when hardware supports PML feature, and both EPT
8028          * and EPT A/D bit features are enabled -- PML depends on them to work.
8029          */
8030         if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml())
8031                 enable_pml = 0;
8032
8033         if (!enable_pml) {
8034                 kvm_x86_ops->slot_enable_log_dirty = NULL;
8035                 kvm_x86_ops->slot_disable_log_dirty = NULL;
8036                 kvm_x86_ops->flush_log_dirty = NULL;
8037                 kvm_x86_ops->enable_log_dirty_pt_masked = NULL;
8038         }
8039
8040         if (!cpu_has_vmx_preemption_timer())
8041                 kvm_x86_ops->request_immediate_exit = __kvm_request_immediate_exit;
8042
8043         if (cpu_has_vmx_preemption_timer() && enable_preemption_timer) {
8044                 u64 vmx_msr;
8045
8046                 rdmsrl(MSR_IA32_VMX_MISC, vmx_msr);
8047                 cpu_preemption_timer_multi =
8048                          vmx_msr & VMX_MISC_PREEMPTION_TIMER_RATE_MASK;
8049         } else {
8050                 kvm_x86_ops->set_hv_timer = NULL;
8051                 kvm_x86_ops->cancel_hv_timer = NULL;
8052         }
8053
8054         if (!cpu_has_vmx_shadow_vmcs())
8055                 enable_shadow_vmcs = 0;
8056         if (enable_shadow_vmcs)
8057                 init_vmcs_shadow_fields();
8058
8059         kvm_set_posted_intr_wakeup_handler(wakeup_handler);
8060         nested_vmx_setup_ctls_msrs(&vmcs_config.nested, enable_apicv);
8061
8062         kvm_mce_cap_supported |= MCG_LMCE_P;
8063
8064         r = alloc_kvm_area();
8065         if (r)
8066                 goto out;
8067         return 0;
8068
8069 out:
8070         for (i = 0; i < VMX_BITMAP_NR; i++)
8071                 free_page((unsigned long)vmx_bitmap[i]);
8072
8073         return r;
8074 }
8075
8076 static __exit void hardware_unsetup(void)
8077 {
8078         int i;
8079
8080         for (i = 0; i < VMX_BITMAP_NR; i++)
8081                 free_page((unsigned long)vmx_bitmap[i]);
8082
8083         free_kvm_area();
8084 }
8085
8086 /*
8087  * Indicate a busy-waiting vcpu in spinlock. We do not enable the PAUSE
8088  * exiting, so only get here on cpu with PAUSE-Loop-Exiting.
8089  */
8090 static int handle_pause(struct kvm_vcpu *vcpu)
8091 {
8092         if (!kvm_pause_in_guest(vcpu->kvm))
8093                 grow_ple_window(vcpu);
8094
8095         /*
8096          * Intel sdm vol3 ch-25.1.3 says: The "PAUSE-loop exiting"
8097          * VM-execution control is ignored if CPL > 0. OTOH, KVM
8098          * never set PAUSE_EXITING and just set PLE if supported,
8099          * so the vcpu must be CPL=0 if it gets a PAUSE exit.
8100          */
8101         kvm_vcpu_on_spin(vcpu, true);
8102         return kvm_skip_emulated_instruction(vcpu);
8103 }
8104
8105 static int handle_nop(struct kvm_vcpu *vcpu)
8106 {
8107         return kvm_skip_emulated_instruction(vcpu);
8108 }
8109
8110 static int handle_mwait(struct kvm_vcpu *vcpu)
8111 {
8112         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
8113         return handle_nop(vcpu);
8114 }
8115
8116 static int handle_invalid_op(struct kvm_vcpu *vcpu)
8117 {
8118         kvm_queue_exception(vcpu, UD_VECTOR);
8119         return 1;
8120 }
8121
8122 static int handle_monitor_trap(struct kvm_vcpu *vcpu)
8123 {
8124         return 1;
8125 }
8126
8127 static int handle_monitor(struct kvm_vcpu *vcpu)
8128 {
8129         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
8130         return handle_nop(vcpu);
8131 }
8132
8133 /*
8134  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
8135  * set the success or error code of an emulated VMX instruction, as specified
8136  * by Vol 2B, VMX Instruction Reference, "Conventions".
8137  */
8138 static void nested_vmx_succeed(struct kvm_vcpu *vcpu)
8139 {
8140         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
8141                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
8142                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
8143 }
8144
8145 static void nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
8146 {
8147         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
8148                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
8149                             X86_EFLAGS_SF | X86_EFLAGS_OF))
8150                         | X86_EFLAGS_CF);
8151 }
8152
8153 static void nested_vmx_failValid(struct kvm_vcpu *vcpu,
8154                                         u32 vm_instruction_error)
8155 {
8156         if (to_vmx(vcpu)->nested.current_vmptr == -1ull) {
8157                 /*
8158                  * failValid writes the error number to the current VMCS, which
8159                  * can't be done there isn't a current VMCS.
8160                  */
8161                 nested_vmx_failInvalid(vcpu);
8162                 return;
8163         }
8164         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
8165                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
8166                             X86_EFLAGS_SF | X86_EFLAGS_OF))
8167                         | X86_EFLAGS_ZF);
8168         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
8169         /*
8170          * We don't need to force a shadow sync because
8171          * VM_INSTRUCTION_ERROR is not shadowed
8172          */
8173 }
8174
8175 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
8176 {
8177         /* TODO: not to reset guest simply here. */
8178         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
8179         pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
8180 }
8181
8182 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
8183 {
8184         struct vcpu_vmx *vmx =
8185                 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
8186
8187         vmx->nested.preemption_timer_expired = true;
8188         kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
8189         kvm_vcpu_kick(&vmx->vcpu);
8190
8191         return HRTIMER_NORESTART;
8192 }
8193
8194 /*
8195  * Decode the memory-address operand of a vmx instruction, as recorded on an
8196  * exit caused by such an instruction (run by a guest hypervisor).
8197  * On success, returns 0. When the operand is invalid, returns 1 and throws
8198  * #UD or #GP.
8199  */
8200 static int get_vmx_mem_address(struct kvm_vcpu *vcpu,
8201                                  unsigned long exit_qualification,
8202                                  u32 vmx_instruction_info, bool wr, gva_t *ret)
8203 {
8204         gva_t off;
8205         bool exn;
8206         struct kvm_segment s;
8207
8208         /*
8209          * According to Vol. 3B, "Information for VM Exits Due to Instruction
8210          * Execution", on an exit, vmx_instruction_info holds most of the
8211          * addressing components of the operand. Only the displacement part
8212          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
8213          * For how an actual address is calculated from all these components,
8214          * refer to Vol. 1, "Operand Addressing".
8215          */
8216         int  scaling = vmx_instruction_info & 3;
8217         int  addr_size = (vmx_instruction_info >> 7) & 7;
8218         bool is_reg = vmx_instruction_info & (1u << 10);
8219         int  seg_reg = (vmx_instruction_info >> 15) & 7;
8220         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
8221         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
8222         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
8223         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
8224
8225         if (is_reg) {
8226                 kvm_queue_exception(vcpu, UD_VECTOR);
8227                 return 1;
8228         }
8229
8230         /* Addr = segment_base + offset */
8231         /* offset = base + [index * scale] + displacement */
8232         off = exit_qualification; /* holds the displacement */
8233         if (addr_size == 1)
8234                 off = (gva_t)sign_extend64(off, 31);
8235         else if (addr_size == 0)
8236                 off = (gva_t)sign_extend64(off, 15);
8237         if (base_is_valid)
8238                 off += kvm_register_read(vcpu, base_reg);
8239         if (index_is_valid)
8240                 off += kvm_register_read(vcpu, index_reg)<<scaling;
8241         vmx_get_segment(vcpu, &s, seg_reg);
8242
8243         /*
8244          * The effective address, i.e. @off, of a memory operand is truncated
8245          * based on the address size of the instruction.  Note that this is
8246          * the *effective address*, i.e. the address prior to accounting for
8247          * the segment's base.
8248          */
8249         if (addr_size == 1) /* 32 bit */
8250                 off &= 0xffffffff;
8251         else if (addr_size == 0) /* 16 bit */
8252                 off &= 0xffff;
8253
8254         /* Checks for #GP/#SS exceptions. */
8255         exn = false;
8256         if (is_long_mode(vcpu)) {
8257                 /*
8258                  * The virtual/linear address is never truncated in 64-bit
8259                  * mode, e.g. a 32-bit address size can yield a 64-bit virtual
8260                  * address when using FS/GS with a non-zero base.
8261                  */
8262                 *ret = s.base + off;
8263
8264                 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
8265                  * non-canonical form. This is the only check on the memory
8266                  * destination for long mode!
8267                  */
8268                 exn = is_noncanonical_address(*ret, vcpu);
8269         } else if (is_protmode(vcpu)) {
8270                 /*
8271                  * When not in long mode, the virtual/linear address is
8272                  * unconditionally truncated to 32 bits regardless of the
8273                  * address size.
8274                  */
8275                 *ret = (s.base + off) & 0xffffffff;
8276
8277                 /* Protected mode: apply checks for segment validity in the
8278                  * following order:
8279                  * - segment type check (#GP(0) may be thrown)
8280                  * - usability check (#GP(0)/#SS(0))
8281                  * - limit check (#GP(0)/#SS(0))
8282                  */
8283                 if (wr)
8284                         /* #GP(0) if the destination operand is located in a
8285                          * read-only data segment or any code segment.
8286                          */
8287                         exn = ((s.type & 0xa) == 0 || (s.type & 8));
8288                 else
8289                         /* #GP(0) if the source operand is located in an
8290                          * execute-only code segment
8291                          */
8292                         exn = ((s.type & 0xa) == 8);
8293                 if (exn) {
8294                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
8295                         return 1;
8296                 }
8297                 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
8298                  */
8299                 exn = (s.unusable != 0);
8300
8301                 /*
8302                  * Protected mode: #GP(0)/#SS(0) if the memory operand is
8303                  * outside the segment limit.  All CPUs that support VMX ignore
8304                  * limit checks for flat segments, i.e. segments with base==0,
8305                  * limit==0xffffffff and of type expand-up data or code.
8306                  */
8307                 if (!(s.base == 0 && s.limit == 0xffffffff &&
8308                      ((s.type & 8) || !(s.type & 4))))
8309                         exn = exn || (off + sizeof(u64) > s.limit);
8310         }
8311         if (exn) {
8312                 kvm_queue_exception_e(vcpu,
8313                                       seg_reg == VCPU_SREG_SS ?
8314                                                 SS_VECTOR : GP_VECTOR,
8315                                       0);
8316                 return 1;
8317         }
8318
8319         return 0;
8320 }
8321
8322 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
8323 {
8324         gva_t gva;
8325         struct x86_exception e;
8326
8327         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
8328                         vmcs_read32(VMX_INSTRUCTION_INFO), false, &gva))
8329                 return 1;
8330
8331         if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
8332                 kvm_inject_page_fault(vcpu, &e);
8333                 return 1;
8334         }
8335
8336         return 0;
8337 }
8338
8339 /*
8340  * Allocate a shadow VMCS and associate it with the currently loaded
8341  * VMCS, unless such a shadow VMCS already exists. The newly allocated
8342  * VMCS is also VMCLEARed, so that it is ready for use.
8343  */
8344 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
8345 {
8346         struct vcpu_vmx *vmx = to_vmx(vcpu);
8347         struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
8348
8349         /*
8350          * We should allocate a shadow vmcs for vmcs01 only when L1
8351          * executes VMXON and free it when L1 executes VMXOFF.
8352          * As it is invalid to execute VMXON twice, we shouldn't reach
8353          * here when vmcs01 already have an allocated shadow vmcs.
8354          */
8355         WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
8356
8357         if (!loaded_vmcs->shadow_vmcs) {
8358                 loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
8359                 if (loaded_vmcs->shadow_vmcs)
8360                         vmcs_clear(loaded_vmcs->shadow_vmcs);
8361         }
8362         return loaded_vmcs->shadow_vmcs;
8363 }
8364
8365 static int enter_vmx_operation(struct kvm_vcpu *vcpu)
8366 {
8367         struct vcpu_vmx *vmx = to_vmx(vcpu);
8368         int r;
8369
8370         r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
8371         if (r < 0)
8372                 goto out_vmcs02;
8373
8374         vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL);
8375         if (!vmx->nested.cached_vmcs12)
8376                 goto out_cached_vmcs12;
8377
8378         vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL);
8379         if (!vmx->nested.cached_shadow_vmcs12)
8380                 goto out_cached_shadow_vmcs12;
8381
8382         if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
8383                 goto out_shadow_vmcs;
8384
8385         hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
8386                      HRTIMER_MODE_REL_PINNED);
8387         vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
8388
8389         vmx->nested.vpid02 = allocate_vpid();
8390
8391         vmx->nested.vmxon = true;
8392         return 0;
8393
8394 out_shadow_vmcs:
8395         kfree(vmx->nested.cached_shadow_vmcs12);
8396
8397 out_cached_shadow_vmcs12:
8398         kfree(vmx->nested.cached_vmcs12);
8399
8400 out_cached_vmcs12:
8401         free_loaded_vmcs(&vmx->nested.vmcs02);
8402
8403 out_vmcs02:
8404         return -ENOMEM;
8405 }
8406
8407 /*
8408  * Emulate the VMXON instruction.
8409  * Currently, we just remember that VMX is active, and do not save or even
8410  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
8411  * do not currently need to store anything in that guest-allocated memory
8412  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
8413  * argument is different from the VMXON pointer (which the spec says they do).
8414  */
8415 static int handle_vmon(struct kvm_vcpu *vcpu)
8416 {
8417         int ret;
8418         gpa_t vmptr;
8419         struct page *page;
8420         struct vcpu_vmx *vmx = to_vmx(vcpu);
8421         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
8422                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
8423
8424         /*
8425          * The Intel VMX Instruction Reference lists a bunch of bits that are
8426          * prerequisite to running VMXON, most notably cr4.VMXE must be set to
8427          * 1 (see vmx_set_cr4() for when we allow the guest to set this).
8428          * Otherwise, we should fail with #UD.  But most faulting conditions
8429          * have already been checked by hardware, prior to the VM-exit for
8430          * VMXON.  We do test guest cr4.VMXE because processor CR4 always has
8431          * that bit set to 1 in non-root mode.
8432          */
8433         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
8434                 kvm_queue_exception(vcpu, UD_VECTOR);
8435                 return 1;
8436         }
8437
8438         /* CPL=0 must be checked manually. */
8439         if (vmx_get_cpl(vcpu)) {
8440                 kvm_inject_gp(vcpu, 0);
8441                 return 1;
8442         }
8443
8444         if (vmx->nested.vmxon) {
8445                 nested_vmx_failValid(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
8446                 return kvm_skip_emulated_instruction(vcpu);
8447         }
8448
8449         if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
8450                         != VMXON_NEEDED_FEATURES) {
8451                 kvm_inject_gp(vcpu, 0);
8452                 return 1;
8453         }
8454
8455         if (nested_vmx_get_vmptr(vcpu, &vmptr))
8456                 return 1;
8457
8458         /*
8459          * SDM 3: 24.11.5
8460          * The first 4 bytes of VMXON region contain the supported
8461          * VMCS revision identifier
8462          *
8463          * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
8464          * which replaces physical address width with 32
8465          */
8466         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
8467                 nested_vmx_failInvalid(vcpu);
8468                 return kvm_skip_emulated_instruction(vcpu);
8469         }
8470
8471         page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
8472         if (is_error_page(page)) {
8473                 nested_vmx_failInvalid(vcpu);
8474                 return kvm_skip_emulated_instruction(vcpu);
8475         }
8476         if (*(u32 *)kmap(page) != VMCS12_REVISION) {
8477                 kunmap(page);
8478                 kvm_release_page_clean(page);
8479                 nested_vmx_failInvalid(vcpu);
8480                 return kvm_skip_emulated_instruction(vcpu);
8481         }
8482         kunmap(page);
8483         kvm_release_page_clean(page);
8484
8485         vmx->nested.vmxon_ptr = vmptr;
8486         ret = enter_vmx_operation(vcpu);
8487         if (ret)
8488                 return ret;
8489
8490         nested_vmx_succeed(vcpu);
8491         return kvm_skip_emulated_instruction(vcpu);
8492 }
8493
8494 /*
8495  * Intel's VMX Instruction Reference specifies a common set of prerequisites
8496  * for running VMX instructions (except VMXON, whose prerequisites are
8497  * slightly different). It also specifies what exception to inject otherwise.
8498  * Note that many of these exceptions have priority over VM exits, so they
8499  * don't have to be checked again here.
8500  */
8501 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
8502 {
8503         if (!to_vmx(vcpu)->nested.vmxon) {
8504                 kvm_queue_exception(vcpu, UD_VECTOR);
8505                 return 0;
8506         }
8507
8508         if (vmx_get_cpl(vcpu)) {
8509                 kvm_inject_gp(vcpu, 0);
8510                 return 0;
8511         }
8512
8513         return 1;
8514 }
8515
8516 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
8517 {
8518         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS);
8519         vmcs_write64(VMCS_LINK_POINTER, -1ull);
8520         vmx->nested.sync_shadow_vmcs = false;
8521 }
8522
8523 static inline void nested_release_vmcs12(struct vcpu_vmx *vmx)
8524 {
8525         if (vmx->nested.current_vmptr == -1ull)
8526                 return;
8527
8528         if (enable_shadow_vmcs) {
8529                 /* copy to memory all shadowed fields in case
8530                    they were modified */
8531                 copy_shadow_to_vmcs12(vmx);
8532                 vmx_disable_shadow_vmcs(vmx);
8533         }
8534         vmx->nested.posted_intr_nv = -1;
8535
8536         /* Flush VMCS12 to guest memory */
8537         kvm_vcpu_write_guest_page(&vmx->vcpu,
8538                                   vmx->nested.current_vmptr >> PAGE_SHIFT,
8539                                   vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
8540
8541         vmx->nested.current_vmptr = -1ull;
8542 }
8543
8544 /*
8545  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
8546  * just stops using VMX.
8547  */
8548 static void free_nested(struct vcpu_vmx *vmx)
8549 {
8550         if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
8551                 return;
8552
8553         kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, &vmx->vcpu);
8554
8555         hrtimer_cancel(&vmx->nested.preemption_timer);
8556         vmx->nested.vmxon = false;
8557         vmx->nested.smm.vmxon = false;
8558         free_vpid(vmx->nested.vpid02);
8559         vmx->nested.posted_intr_nv = -1;
8560         vmx->nested.current_vmptr = -1ull;
8561         if (enable_shadow_vmcs) {
8562                 vmx_disable_shadow_vmcs(vmx);
8563                 vmcs_clear(vmx->vmcs01.shadow_vmcs);
8564                 free_vmcs(vmx->vmcs01.shadow_vmcs);
8565                 vmx->vmcs01.shadow_vmcs = NULL;
8566         }
8567         kfree(vmx->nested.cached_vmcs12);
8568         kfree(vmx->nested.cached_shadow_vmcs12);
8569         /* Unpin physical memory we referred to in the vmcs02 */
8570         if (vmx->nested.apic_access_page) {
8571                 kvm_release_page_dirty(vmx->nested.apic_access_page);
8572                 vmx->nested.apic_access_page = NULL;
8573         }
8574         if (vmx->nested.virtual_apic_page) {
8575                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
8576                 vmx->nested.virtual_apic_page = NULL;
8577         }
8578         if (vmx->nested.pi_desc_page) {
8579                 kunmap(vmx->nested.pi_desc_page);
8580                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
8581                 vmx->nested.pi_desc_page = NULL;
8582                 vmx->nested.pi_desc = NULL;
8583         }
8584
8585         free_loaded_vmcs(&vmx->nested.vmcs02);
8586 }
8587
8588 /* Emulate the VMXOFF instruction */
8589 static int handle_vmoff(struct kvm_vcpu *vcpu)
8590 {
8591         if (!nested_vmx_check_permission(vcpu))
8592                 return 1;
8593         free_nested(to_vmx(vcpu));
8594         nested_vmx_succeed(vcpu);
8595         return kvm_skip_emulated_instruction(vcpu);
8596 }
8597
8598 /* Emulate the VMCLEAR instruction */
8599 static int handle_vmclear(struct kvm_vcpu *vcpu)
8600 {
8601         struct vcpu_vmx *vmx = to_vmx(vcpu);
8602         u32 zero = 0;
8603         gpa_t vmptr;
8604
8605         if (!nested_vmx_check_permission(vcpu))
8606                 return 1;
8607
8608         if (nested_vmx_get_vmptr(vcpu, &vmptr))
8609                 return 1;
8610
8611         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
8612                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS);
8613                 return kvm_skip_emulated_instruction(vcpu);
8614         }
8615
8616         if (vmptr == vmx->nested.vmxon_ptr) {
8617                 nested_vmx_failValid(vcpu, VMXERR_VMCLEAR_VMXON_POINTER);
8618                 return kvm_skip_emulated_instruction(vcpu);
8619         }
8620
8621         if (vmptr == vmx->nested.current_vmptr)
8622                 nested_release_vmcs12(vmx);
8623
8624         kvm_vcpu_write_guest(vcpu,
8625                         vmptr + offsetof(struct vmcs12, launch_state),
8626                         &zero, sizeof(zero));
8627
8628         nested_vmx_succeed(vcpu);
8629         return kvm_skip_emulated_instruction(vcpu);
8630 }
8631
8632 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
8633
8634 /* Emulate the VMLAUNCH instruction */
8635 static int handle_vmlaunch(struct kvm_vcpu *vcpu)
8636 {
8637         return nested_vmx_run(vcpu, true);
8638 }
8639
8640 /* Emulate the VMRESUME instruction */
8641 static int handle_vmresume(struct kvm_vcpu *vcpu)
8642 {
8643
8644         return nested_vmx_run(vcpu, false);
8645 }
8646
8647 /*
8648  * Read a vmcs12 field. Since these can have varying lengths and we return
8649  * one type, we chose the biggest type (u64) and zero-extend the return value
8650  * to that size. Note that the caller, handle_vmread, might need to use only
8651  * some of the bits we return here (e.g., on 32-bit guests, only 32 bits of
8652  * 64-bit fields are to be returned).
8653  */
8654 static inline int vmcs12_read_any(struct vmcs12 *vmcs12,
8655                                   unsigned long field, u64 *ret)
8656 {
8657         short offset = vmcs_field_to_offset(field);
8658         char *p;
8659
8660         if (offset < 0)
8661                 return offset;
8662
8663         p = (char *)vmcs12 + offset;
8664
8665         switch (vmcs_field_width(field)) {
8666         case VMCS_FIELD_WIDTH_NATURAL_WIDTH:
8667                 *ret = *((natural_width *)p);
8668                 return 0;
8669         case VMCS_FIELD_WIDTH_U16:
8670                 *ret = *((u16 *)p);
8671                 return 0;
8672         case VMCS_FIELD_WIDTH_U32:
8673                 *ret = *((u32 *)p);
8674                 return 0;
8675         case VMCS_FIELD_WIDTH_U64:
8676                 *ret = *((u64 *)p);
8677                 return 0;
8678         default:
8679                 WARN_ON(1);
8680                 return -ENOENT;
8681         }
8682 }
8683
8684
8685 static inline int vmcs12_write_any(struct vmcs12 *vmcs12,
8686                                    unsigned long field, u64 field_value){
8687         short offset = vmcs_field_to_offset(field);
8688         char *p = (char *)vmcs12 + offset;
8689         if (offset < 0)
8690                 return offset;
8691
8692         switch (vmcs_field_width(field)) {
8693         case VMCS_FIELD_WIDTH_U16:
8694                 *(u16 *)p = field_value;
8695                 return 0;
8696         case VMCS_FIELD_WIDTH_U32:
8697                 *(u32 *)p = field_value;
8698                 return 0;
8699         case VMCS_FIELD_WIDTH_U64:
8700                 *(u64 *)p = field_value;
8701                 return 0;
8702         case VMCS_FIELD_WIDTH_NATURAL_WIDTH:
8703                 *(natural_width *)p = field_value;
8704                 return 0;
8705         default:
8706                 WARN_ON(1);
8707                 return -ENOENT;
8708         }
8709
8710 }
8711
8712 /*
8713  * Copy the writable VMCS shadow fields back to the VMCS12, in case
8714  * they have been modified by the L1 guest. Note that the "read-only"
8715  * VM-exit information fields are actually writable if the vCPU is
8716  * configured to support "VMWRITE to any supported field in the VMCS."
8717  */
8718 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
8719 {
8720         const u16 *fields[] = {
8721                 shadow_read_write_fields,
8722                 shadow_read_only_fields
8723         };
8724         const int max_fields[] = {
8725                 max_shadow_read_write_fields,
8726                 max_shadow_read_only_fields
8727         };
8728         int i, q;
8729         unsigned long field;
8730         u64 field_value;
8731         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
8732
8733         if (WARN_ON(!shadow_vmcs))
8734                 return;
8735
8736         preempt_disable();
8737
8738         vmcs_load(shadow_vmcs);
8739
8740         for (q = 0; q < ARRAY_SIZE(fields); q++) {
8741                 for (i = 0; i < max_fields[q]; i++) {
8742                         field = fields[q][i];
8743                         field_value = __vmcs_readl(field);
8744                         vmcs12_write_any(get_vmcs12(&vmx->vcpu), field, field_value);
8745                 }
8746                 /*
8747                  * Skip the VM-exit information fields if they are read-only.
8748                  */
8749                 if (!nested_cpu_has_vmwrite_any_field(&vmx->vcpu))
8750                         break;
8751         }
8752
8753         vmcs_clear(shadow_vmcs);
8754         vmcs_load(vmx->loaded_vmcs->vmcs);
8755
8756         preempt_enable();
8757 }
8758
8759 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
8760 {
8761         const u16 *fields[] = {
8762                 shadow_read_write_fields,
8763                 shadow_read_only_fields
8764         };
8765         const int max_fields[] = {
8766                 max_shadow_read_write_fields,
8767                 max_shadow_read_only_fields
8768         };
8769         int i, q;
8770         unsigned long field;
8771         u64 field_value = 0;
8772         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
8773
8774         if (WARN_ON(!shadow_vmcs))
8775                 return;
8776
8777         vmcs_load(shadow_vmcs);
8778
8779         for (q = 0; q < ARRAY_SIZE(fields); q++) {
8780                 for (i = 0; i < max_fields[q]; i++) {
8781                         field = fields[q][i];
8782                         vmcs12_read_any(get_vmcs12(&vmx->vcpu), field, &field_value);
8783                         __vmcs_writel(field, field_value);
8784                 }
8785         }
8786
8787         vmcs_clear(shadow_vmcs);
8788         vmcs_load(vmx->loaded_vmcs->vmcs);
8789 }
8790
8791 /*
8792  * VMX instructions which assume a current vmcs12 (i.e., that VMPTRLD was
8793  * used before) all generate the same failure when it is missing.
8794  */
8795 static int nested_vmx_check_vmcs12(struct kvm_vcpu *vcpu)
8796 {
8797         struct vcpu_vmx *vmx = to_vmx(vcpu);
8798         if (vmx->nested.current_vmptr == -1ull) {
8799                 nested_vmx_failInvalid(vcpu);
8800                 return 0;
8801         }
8802         return 1;
8803 }
8804
8805 static int handle_vmread(struct kvm_vcpu *vcpu)
8806 {
8807         unsigned long field;
8808         u64 field_value;
8809         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8810         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
8811         gva_t gva = 0;
8812         struct vmcs12 *vmcs12;
8813         struct x86_exception e;
8814
8815         if (!nested_vmx_check_permission(vcpu))
8816                 return 1;
8817
8818         if (!nested_vmx_check_vmcs12(vcpu))
8819                 return kvm_skip_emulated_instruction(vcpu);
8820
8821         if (!is_guest_mode(vcpu))
8822                 vmcs12 = get_vmcs12(vcpu);
8823         else {
8824                 /*
8825                  * When vmcs->vmcs_link_pointer is -1ull, any VMREAD
8826                  * to shadowed-field sets the ALU flags for VMfailInvalid.
8827                  */
8828                 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) {
8829                         nested_vmx_failInvalid(vcpu);
8830                         return kvm_skip_emulated_instruction(vcpu);
8831                 }
8832                 vmcs12 = get_shadow_vmcs12(vcpu);
8833         }
8834
8835         /* Decode instruction info and find the field to read */
8836         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
8837         /* Read the field, zero-extended to a u64 field_value */
8838         if (vmcs12_read_any(vmcs12, field, &field_value) < 0) {
8839                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
8840                 return kvm_skip_emulated_instruction(vcpu);
8841         }
8842         /*
8843          * Now copy part of this value to register or memory, as requested.
8844          * Note that the number of bits actually copied is 32 or 64 depending
8845          * on the guest's mode (32 or 64 bit), not on the given field's length.
8846          */
8847         if (vmx_instruction_info & (1u << 10)) {
8848                 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
8849                         field_value);
8850         } else {
8851                 if (get_vmx_mem_address(vcpu, exit_qualification,
8852                                 vmx_instruction_info, true, &gva))
8853                         return 1;
8854                 /* _system ok, nested_vmx_check_permission has verified cpl=0 */
8855                 if (kvm_write_guest_virt_system(vcpu, gva, &field_value,
8856                                                 (is_long_mode(vcpu) ? 8 : 4),
8857                                                 &e)) {
8858                         kvm_inject_page_fault(vcpu, &e);
8859                         return 1;
8860                 }
8861         }
8862
8863         nested_vmx_succeed(vcpu);
8864         return kvm_skip_emulated_instruction(vcpu);
8865 }
8866
8867
8868 static int handle_vmwrite(struct kvm_vcpu *vcpu)
8869 {
8870         unsigned long field;
8871         gva_t gva;
8872         struct vcpu_vmx *vmx = to_vmx(vcpu);
8873         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
8874         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
8875
8876         /* The value to write might be 32 or 64 bits, depending on L1's long
8877          * mode, and eventually we need to write that into a field of several
8878          * possible lengths. The code below first zero-extends the value to 64
8879          * bit (field_value), and then copies only the appropriate number of
8880          * bits into the vmcs12 field.
8881          */
8882         u64 field_value = 0;
8883         struct x86_exception e;
8884         struct vmcs12 *vmcs12;
8885
8886         if (!nested_vmx_check_permission(vcpu))
8887                 return 1;
8888
8889         if (!nested_vmx_check_vmcs12(vcpu))
8890                 return kvm_skip_emulated_instruction(vcpu);
8891
8892         if (vmx_instruction_info & (1u << 10))
8893                 field_value = kvm_register_readl(vcpu,
8894                         (((vmx_instruction_info) >> 3) & 0xf));
8895         else {
8896                 if (get_vmx_mem_address(vcpu, exit_qualification,
8897                                 vmx_instruction_info, false, &gva))
8898                         return 1;
8899                 if (kvm_read_guest_virt(vcpu, gva, &field_value,
8900                                         (is_64_bit_mode(vcpu) ? 8 : 4), &e)) {
8901                         kvm_inject_page_fault(vcpu, &e);
8902                         return 1;
8903                 }
8904         }
8905
8906
8907         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
8908         /*
8909          * If the vCPU supports "VMWRITE to any supported field in the
8910          * VMCS," then the "read-only" fields are actually read/write.
8911          */
8912         if (vmcs_field_readonly(field) &&
8913             !nested_cpu_has_vmwrite_any_field(vcpu)) {
8914                 nested_vmx_failValid(vcpu,
8915                         VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
8916                 return kvm_skip_emulated_instruction(vcpu);
8917         }
8918
8919         if (!is_guest_mode(vcpu))
8920                 vmcs12 = get_vmcs12(vcpu);
8921         else {
8922                 /*
8923                  * When vmcs->vmcs_link_pointer is -1ull, any VMWRITE
8924                  * to shadowed-field sets the ALU flags for VMfailInvalid.
8925                  */
8926                 if (get_vmcs12(vcpu)->vmcs_link_pointer == -1ull) {
8927                         nested_vmx_failInvalid(vcpu);
8928                         return kvm_skip_emulated_instruction(vcpu);
8929                 }
8930                 vmcs12 = get_shadow_vmcs12(vcpu);
8931
8932         }
8933
8934         if (vmcs12_write_any(vmcs12, field, field_value) < 0) {
8935                 nested_vmx_failValid(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT);
8936                 return kvm_skip_emulated_instruction(vcpu);
8937         }
8938
8939         /*
8940          * Do not track vmcs12 dirty-state if in guest-mode
8941          * as we actually dirty shadow vmcs12 instead of vmcs12.
8942          */
8943         if (!is_guest_mode(vcpu)) {
8944                 switch (field) {
8945 #define SHADOW_FIELD_RW(x) case x:
8946 #include "vmx_shadow_fields.h"
8947                         /*
8948                          * The fields that can be updated by L1 without a vmexit are
8949                          * always updated in the vmcs02, the others go down the slow
8950                          * path of prepare_vmcs02.
8951                          */
8952                         break;
8953                 default:
8954                         vmx->nested.dirty_vmcs12 = true;
8955                         break;
8956                 }
8957         }
8958
8959         nested_vmx_succeed(vcpu);
8960         return kvm_skip_emulated_instruction(vcpu);
8961 }
8962
8963 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
8964 {
8965         vmx->nested.current_vmptr = vmptr;
8966         if (enable_shadow_vmcs) {
8967                 vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL,
8968                               SECONDARY_EXEC_SHADOW_VMCS);
8969                 vmcs_write64(VMCS_LINK_POINTER,
8970                              __pa(vmx->vmcs01.shadow_vmcs));
8971                 vmx->nested.sync_shadow_vmcs = true;
8972         }
8973         vmx->nested.dirty_vmcs12 = true;
8974 }
8975
8976 /* Emulate the VMPTRLD instruction */
8977 static int handle_vmptrld(struct kvm_vcpu *vcpu)
8978 {
8979         struct vcpu_vmx *vmx = to_vmx(vcpu);
8980         gpa_t vmptr;
8981
8982         if (!nested_vmx_check_permission(vcpu))
8983                 return 1;
8984
8985         if (nested_vmx_get_vmptr(vcpu, &vmptr))
8986                 return 1;
8987
8988         if (!PAGE_ALIGNED(vmptr) || (vmptr >> cpuid_maxphyaddr(vcpu))) {
8989                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS);
8990                 return kvm_skip_emulated_instruction(vcpu);
8991         }
8992
8993         if (vmptr == vmx->nested.vmxon_ptr) {
8994                 nested_vmx_failValid(vcpu, VMXERR_VMPTRLD_VMXON_POINTER);
8995                 return kvm_skip_emulated_instruction(vcpu);
8996         }
8997
8998         if (vmx->nested.current_vmptr != vmptr) {
8999                 struct vmcs12 *new_vmcs12;
9000                 struct page *page;
9001                 page = kvm_vcpu_gpa_to_page(vcpu, vmptr);
9002                 if (is_error_page(page)) {
9003                         nested_vmx_failInvalid(vcpu);
9004                         return kvm_skip_emulated_instruction(vcpu);
9005                 }
9006                 new_vmcs12 = kmap(page);
9007                 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION ||
9008                     (new_vmcs12->hdr.shadow_vmcs &&
9009                      !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
9010                         kunmap(page);
9011                         kvm_release_page_clean(page);
9012                         nested_vmx_failValid(vcpu,
9013                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
9014                         return kvm_skip_emulated_instruction(vcpu);
9015                 }
9016
9017                 nested_release_vmcs12(vmx);
9018                 /*
9019                  * Load VMCS12 from guest memory since it is not already
9020                  * cached.
9021                  */
9022                 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
9023                 kunmap(page);
9024                 kvm_release_page_clean(page);
9025
9026                 set_current_vmptr(vmx, vmptr);
9027         }
9028
9029         nested_vmx_succeed(vcpu);
9030         return kvm_skip_emulated_instruction(vcpu);
9031 }
9032
9033 /* Emulate the VMPTRST instruction */
9034 static int handle_vmptrst(struct kvm_vcpu *vcpu)
9035 {
9036         unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION);
9037         u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
9038         gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
9039         struct x86_exception e;
9040         gva_t gva;
9041
9042         if (!nested_vmx_check_permission(vcpu))
9043                 return 1;
9044
9045         if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, &gva))
9046                 return 1;
9047         /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
9048         if (kvm_write_guest_virt_system(vcpu, gva, (void *)&current_vmptr,
9049                                         sizeof(gpa_t), &e)) {
9050                 kvm_inject_page_fault(vcpu, &e);
9051                 return 1;
9052         }
9053         nested_vmx_succeed(vcpu);
9054         return kvm_skip_emulated_instruction(vcpu);
9055 }
9056
9057 /* Emulate the INVEPT instruction */
9058 static int handle_invept(struct kvm_vcpu *vcpu)
9059 {
9060         struct vcpu_vmx *vmx = to_vmx(vcpu);
9061         u32 vmx_instruction_info, types;
9062         unsigned long type;
9063         gva_t gva;
9064         struct x86_exception e;
9065         struct {
9066                 u64 eptp, gpa;
9067         } operand;
9068
9069         if (!(vmx->nested.msrs.secondary_ctls_high &
9070               SECONDARY_EXEC_ENABLE_EPT) ||
9071             !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
9072                 kvm_queue_exception(vcpu, UD_VECTOR);
9073                 return 1;
9074         }
9075
9076         if (!nested_vmx_check_permission(vcpu))
9077                 return 1;
9078
9079         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
9080         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
9081
9082         types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
9083
9084         if (type >= 32 || !(types & (1 << type))) {
9085                 nested_vmx_failValid(vcpu,
9086                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
9087                 return kvm_skip_emulated_instruction(vcpu);
9088         }
9089
9090         /* According to the Intel VMX instruction reference, the memory
9091          * operand is read even if it isn't needed (e.g., for type==global)
9092          */
9093         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
9094                         vmx_instruction_info, false, &gva))
9095                 return 1;
9096         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
9097                 kvm_inject_page_fault(vcpu, &e);
9098                 return 1;
9099         }
9100
9101         switch (type) {
9102         case VMX_EPT_EXTENT_GLOBAL:
9103         /*
9104          * TODO: track mappings and invalidate
9105          * single context requests appropriately
9106          */
9107         case VMX_EPT_EXTENT_CONTEXT:
9108                 kvm_mmu_sync_roots(vcpu);
9109                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
9110                 nested_vmx_succeed(vcpu);
9111                 break;
9112         default:
9113                 BUG_ON(1);
9114                 break;
9115         }
9116
9117         return kvm_skip_emulated_instruction(vcpu);
9118 }
9119
9120 static int handle_invvpid(struct kvm_vcpu *vcpu)
9121 {
9122         struct vcpu_vmx *vmx = to_vmx(vcpu);
9123         u32 vmx_instruction_info;
9124         unsigned long type, types;
9125         gva_t gva;
9126         struct x86_exception e;
9127         struct {
9128                 u64 vpid;
9129                 u64 gla;
9130         } operand;
9131
9132         if (!(vmx->nested.msrs.secondary_ctls_high &
9133               SECONDARY_EXEC_ENABLE_VPID) ||
9134                         !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
9135                 kvm_queue_exception(vcpu, UD_VECTOR);
9136                 return 1;
9137         }
9138
9139         if (!nested_vmx_check_permission(vcpu))
9140                 return 1;
9141
9142         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
9143         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
9144
9145         types = (vmx->nested.msrs.vpid_caps &
9146                         VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
9147
9148         if (type >= 32 || !(types & (1 << type))) {
9149                 nested_vmx_failValid(vcpu,
9150                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
9151                 return kvm_skip_emulated_instruction(vcpu);
9152         }
9153
9154         /* according to the intel vmx instruction reference, the memory
9155          * operand is read even if it isn't needed (e.g., for type==global)
9156          */
9157         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
9158                         vmx_instruction_info, false, &gva))
9159                 return 1;
9160         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
9161                 kvm_inject_page_fault(vcpu, &e);
9162                 return 1;
9163         }
9164         if (operand.vpid >> 16) {
9165                 nested_vmx_failValid(vcpu,
9166                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
9167                 return kvm_skip_emulated_instruction(vcpu);
9168         }
9169
9170         switch (type) {
9171         case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
9172                 if (!operand.vpid ||
9173                     is_noncanonical_address(operand.gla, vcpu)) {
9174                         nested_vmx_failValid(vcpu,
9175                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
9176                         return kvm_skip_emulated_instruction(vcpu);
9177                 }
9178                 if (cpu_has_vmx_invvpid_individual_addr() &&
9179                     vmx->nested.vpid02) {
9180                         __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
9181                                 vmx->nested.vpid02, operand.gla);
9182                 } else
9183                         __vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
9184                 break;
9185         case VMX_VPID_EXTENT_SINGLE_CONTEXT:
9186         case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
9187                 if (!operand.vpid) {
9188                         nested_vmx_failValid(vcpu,
9189                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
9190                         return kvm_skip_emulated_instruction(vcpu);
9191                 }
9192                 __vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
9193                 break;
9194         case VMX_VPID_EXTENT_ALL_CONTEXT:
9195                 __vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
9196                 break;
9197         default:
9198                 WARN_ON_ONCE(1);
9199                 return kvm_skip_emulated_instruction(vcpu);
9200         }
9201
9202         nested_vmx_succeed(vcpu);
9203
9204         return kvm_skip_emulated_instruction(vcpu);
9205 }
9206
9207 static int handle_invpcid(struct kvm_vcpu *vcpu)
9208 {
9209         u32 vmx_instruction_info;
9210         unsigned long type;
9211         bool pcid_enabled;
9212         gva_t gva;
9213         struct x86_exception e;
9214         unsigned i;
9215         unsigned long roots_to_free = 0;
9216         struct {
9217                 u64 pcid;
9218                 u64 gla;
9219         } operand;
9220
9221         if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) {
9222                 kvm_queue_exception(vcpu, UD_VECTOR);
9223                 return 1;
9224         }
9225
9226         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
9227         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
9228
9229         if (type > 3) {
9230                 kvm_inject_gp(vcpu, 0);
9231                 return 1;
9232         }
9233
9234         /* According to the Intel instruction reference, the memory operand
9235          * is read even if it isn't needed (e.g., for type==all)
9236          */
9237         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
9238                                 vmx_instruction_info, false, &gva))
9239                 return 1;
9240
9241         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
9242                 kvm_inject_page_fault(vcpu, &e);
9243                 return 1;
9244         }
9245
9246         if (operand.pcid >> 12 != 0) {
9247                 kvm_inject_gp(vcpu, 0);
9248                 return 1;
9249         }
9250
9251         pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE);
9252
9253         switch (type) {
9254         case INVPCID_TYPE_INDIV_ADDR:
9255                 if ((!pcid_enabled && (operand.pcid != 0)) ||
9256                     is_noncanonical_address(operand.gla, vcpu)) {
9257                         kvm_inject_gp(vcpu, 0);
9258                         return 1;
9259                 }
9260                 kvm_mmu_invpcid_gva(vcpu, operand.gla, operand.pcid);
9261                 return kvm_skip_emulated_instruction(vcpu);
9262
9263         case INVPCID_TYPE_SINGLE_CTXT:
9264                 if (!pcid_enabled && (operand.pcid != 0)) {
9265                         kvm_inject_gp(vcpu, 0);
9266                         return 1;
9267                 }
9268
9269                 if (kvm_get_active_pcid(vcpu) == operand.pcid) {
9270                         kvm_mmu_sync_roots(vcpu);
9271                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
9272                 }
9273
9274                 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
9275                         if (kvm_get_pcid(vcpu, vcpu->arch.mmu.prev_roots[i].cr3)
9276                             == operand.pcid)
9277                                 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
9278
9279                 kvm_mmu_free_roots(vcpu, roots_to_free);
9280                 /*
9281                  * If neither the current cr3 nor any of the prev_roots use the
9282                  * given PCID, then nothing needs to be done here because a
9283                  * resync will happen anyway before switching to any other CR3.
9284                  */
9285
9286                 return kvm_skip_emulated_instruction(vcpu);
9287
9288         case INVPCID_TYPE_ALL_NON_GLOBAL:
9289                 /*
9290                  * Currently, KVM doesn't mark global entries in the shadow
9291                  * page tables, so a non-global flush just degenerates to a
9292                  * global flush. If needed, we could optimize this later by
9293                  * keeping track of global entries in shadow page tables.
9294                  */
9295
9296                 /* fall-through */
9297         case INVPCID_TYPE_ALL_INCL_GLOBAL:
9298                 kvm_mmu_unload(vcpu);
9299                 return kvm_skip_emulated_instruction(vcpu);
9300
9301         default:
9302                 BUG(); /* We have already checked above that type <= 3 */
9303         }
9304 }
9305
9306 static int handle_pml_full(struct kvm_vcpu *vcpu)
9307 {
9308         unsigned long exit_qualification;
9309
9310         trace_kvm_pml_full(vcpu->vcpu_id);
9311
9312         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
9313
9314         /*
9315          * PML buffer FULL happened while executing iret from NMI,
9316          * "blocked by NMI" bit has to be set before next VM entry.
9317          */
9318         if (!(to_vmx(vcpu)->idt_vectoring_info & VECTORING_INFO_VALID_MASK) &&
9319                         enable_vnmi &&
9320                         (exit_qualification & INTR_INFO_UNBLOCK_NMI))
9321                 vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
9322                                 GUEST_INTR_STATE_NMI);
9323
9324         /*
9325          * PML buffer already flushed at beginning of VMEXIT. Nothing to do
9326          * here.., and there's no userspace involvement needed for PML.
9327          */
9328         return 1;
9329 }
9330
9331 static int handle_preemption_timer(struct kvm_vcpu *vcpu)
9332 {
9333         if (!to_vmx(vcpu)->req_immediate_exit)
9334                 kvm_lapic_expired_hv_timer(vcpu);
9335         return 1;
9336 }
9337
9338 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
9339 {
9340         struct vcpu_vmx *vmx = to_vmx(vcpu);
9341         int maxphyaddr = cpuid_maxphyaddr(vcpu);
9342
9343         /* Check for memory type validity */
9344         switch (address & VMX_EPTP_MT_MASK) {
9345         case VMX_EPTP_MT_UC:
9346                 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))
9347                         return false;
9348                 break;
9349         case VMX_EPTP_MT_WB:
9350                 if (!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))
9351                         return false;
9352                 break;
9353         default:
9354                 return false;
9355         }
9356
9357         /* only 4 levels page-walk length are valid */
9358         if ((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4)
9359                 return false;
9360
9361         /* Reserved bits should not be set */
9362         if (address >> maxphyaddr || ((address >> 7) & 0x1f))
9363                 return false;
9364
9365         /* AD, if set, should be supported */
9366         if (address & VMX_EPTP_AD_ENABLE_BIT) {
9367                 if (!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))
9368                         return false;
9369         }
9370
9371         return true;
9372 }
9373
9374 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
9375                                      struct vmcs12 *vmcs12)
9376 {
9377         u32 index = vcpu->arch.regs[VCPU_REGS_RCX];
9378         u64 address;
9379         bool accessed_dirty;
9380         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
9381
9382         if (!nested_cpu_has_eptp_switching(vmcs12) ||
9383             !nested_cpu_has_ept(vmcs12))
9384                 return 1;
9385
9386         if (index >= VMFUNC_EPTP_ENTRIES)
9387                 return 1;
9388
9389
9390         if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
9391                                      &address, index * 8, 8))
9392                 return 1;
9393
9394         accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
9395
9396         /*
9397          * If the (L2) guest does a vmfunc to the currently
9398          * active ept pointer, we don't have to do anything else
9399          */
9400         if (vmcs12->ept_pointer != address) {
9401                 if (!valid_ept_address(vcpu, address))
9402                         return 1;
9403
9404                 kvm_mmu_unload(vcpu);
9405                 mmu->ept_ad = accessed_dirty;
9406                 mmu->base_role.ad_disabled = !accessed_dirty;
9407                 vmcs12->ept_pointer = address;
9408                 /*
9409                  * TODO: Check what's the correct approach in case
9410                  * mmu reload fails. Currently, we just let the next
9411                  * reload potentially fail
9412                  */
9413                 kvm_mmu_reload(vcpu);
9414         }
9415
9416         return 0;
9417 }
9418
9419 static int handle_vmfunc(struct kvm_vcpu *vcpu)
9420 {
9421         struct vcpu_vmx *vmx = to_vmx(vcpu);
9422         struct vmcs12 *vmcs12;
9423         u32 function = vcpu->arch.regs[VCPU_REGS_RAX];
9424
9425         /*
9426          * VMFUNC is only supported for nested guests, but we always enable the
9427          * secondary control for simplicity; for non-nested mode, fake that we
9428          * didn't by injecting #UD.
9429          */
9430         if (!is_guest_mode(vcpu)) {
9431                 kvm_queue_exception(vcpu, UD_VECTOR);
9432                 return 1;
9433         }
9434
9435         vmcs12 = get_vmcs12(vcpu);
9436         if ((vmcs12->vm_function_control & (1 << function)) == 0)
9437                 goto fail;
9438
9439         switch (function) {
9440         case 0:
9441                 if (nested_vmx_eptp_switching(vcpu, vmcs12))
9442                         goto fail;
9443                 break;
9444         default:
9445                 goto fail;
9446         }
9447         return kvm_skip_emulated_instruction(vcpu);
9448
9449 fail:
9450         nested_vmx_vmexit(vcpu, vmx->exit_reason,
9451                           vmcs_read32(VM_EXIT_INTR_INFO),
9452                           vmcs_readl(EXIT_QUALIFICATION));
9453         return 1;
9454 }
9455
9456 static int handle_encls(struct kvm_vcpu *vcpu)
9457 {
9458         /*
9459          * SGX virtualization is not yet supported.  There is no software
9460          * enable bit for SGX, so we have to trap ENCLS and inject a #UD
9461          * to prevent the guest from executing ENCLS.
9462          */
9463         kvm_queue_exception(vcpu, UD_VECTOR);
9464         return 1;
9465 }
9466
9467 /*
9468  * The exit handlers return 1 if the exit was handled fully and guest execution
9469  * may resume.  Otherwise they set the kvm_run parameter to indicate what needs
9470  * to be done to userspace and return 0.
9471  */
9472 static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
9473         [EXIT_REASON_EXCEPTION_NMI]           = handle_exception,
9474         [EXIT_REASON_EXTERNAL_INTERRUPT]      = handle_external_interrupt,
9475         [EXIT_REASON_TRIPLE_FAULT]            = handle_triple_fault,
9476         [EXIT_REASON_NMI_WINDOW]              = handle_nmi_window,
9477         [EXIT_REASON_IO_INSTRUCTION]          = handle_io,
9478         [EXIT_REASON_CR_ACCESS]               = handle_cr,
9479         [EXIT_REASON_DR_ACCESS]               = handle_dr,
9480         [EXIT_REASON_CPUID]                   = handle_cpuid,
9481         [EXIT_REASON_MSR_READ]                = handle_rdmsr,
9482         [EXIT_REASON_MSR_WRITE]               = handle_wrmsr,
9483         [EXIT_REASON_PENDING_INTERRUPT]       = handle_interrupt_window,
9484         [EXIT_REASON_HLT]                     = handle_halt,
9485         [EXIT_REASON_INVD]                    = handle_invd,
9486         [EXIT_REASON_INVLPG]                  = handle_invlpg,
9487         [EXIT_REASON_RDPMC]                   = handle_rdpmc,
9488         [EXIT_REASON_VMCALL]                  = handle_vmcall,
9489         [EXIT_REASON_VMCLEAR]                 = handle_vmclear,
9490         [EXIT_REASON_VMLAUNCH]                = handle_vmlaunch,
9491         [EXIT_REASON_VMPTRLD]                 = handle_vmptrld,
9492         [EXIT_REASON_VMPTRST]                 = handle_vmptrst,
9493         [EXIT_REASON_VMREAD]                  = handle_vmread,
9494         [EXIT_REASON_VMRESUME]                = handle_vmresume,
9495         [EXIT_REASON_VMWRITE]                 = handle_vmwrite,
9496         [EXIT_REASON_VMOFF]                   = handle_vmoff,
9497         [EXIT_REASON_VMON]                    = handle_vmon,
9498         [EXIT_REASON_TPR_BELOW_THRESHOLD]     = handle_tpr_below_threshold,
9499         [EXIT_REASON_APIC_ACCESS]             = handle_apic_access,
9500         [EXIT_REASON_APIC_WRITE]              = handle_apic_write,
9501         [EXIT_REASON_EOI_INDUCED]             = handle_apic_eoi_induced,
9502         [EXIT_REASON_WBINVD]                  = handle_wbinvd,
9503         [EXIT_REASON_XSETBV]                  = handle_xsetbv,
9504         [EXIT_REASON_TASK_SWITCH]             = handle_task_switch,
9505         [EXIT_REASON_MCE_DURING_VMENTRY]      = handle_machine_check,
9506         [EXIT_REASON_GDTR_IDTR]               = handle_desc,
9507         [EXIT_REASON_LDTR_TR]                 = handle_desc,
9508         [EXIT_REASON_EPT_VIOLATION]           = handle_ept_violation,
9509         [EXIT_REASON_EPT_MISCONFIG]           = handle_ept_misconfig,
9510         [EXIT_REASON_PAUSE_INSTRUCTION]       = handle_pause,
9511         [EXIT_REASON_MWAIT_INSTRUCTION]       = handle_mwait,
9512         [EXIT_REASON_MONITOR_TRAP_FLAG]       = handle_monitor_trap,
9513         [EXIT_REASON_MONITOR_INSTRUCTION]     = handle_monitor,
9514         [EXIT_REASON_INVEPT]                  = handle_invept,
9515         [EXIT_REASON_INVVPID]                 = handle_invvpid,
9516         [EXIT_REASON_RDRAND]                  = handle_invalid_op,
9517         [EXIT_REASON_RDSEED]                  = handle_invalid_op,
9518         [EXIT_REASON_XSAVES]                  = handle_xsaves,
9519         [EXIT_REASON_XRSTORS]                 = handle_xrstors,
9520         [EXIT_REASON_PML_FULL]                = handle_pml_full,
9521         [EXIT_REASON_INVPCID]                 = handle_invpcid,
9522         [EXIT_REASON_VMFUNC]                  = handle_vmfunc,
9523         [EXIT_REASON_PREEMPTION_TIMER]        = handle_preemption_timer,
9524         [EXIT_REASON_ENCLS]                   = handle_encls,
9525 };
9526
9527 static const int kvm_vmx_max_exit_handlers =
9528         ARRAY_SIZE(kvm_vmx_exit_handlers);
9529
9530 /*
9531  * Return true if an IO instruction with the specified port and size should cause
9532  * a VM-exit into L1.
9533  */
9534 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
9535                                  int size)
9536 {
9537         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9538         gpa_t bitmap, last_bitmap;
9539         u8 b;
9540
9541         last_bitmap = (gpa_t)-1;
9542         b = -1;
9543
9544         while (size > 0) {
9545                 if (port < 0x8000)
9546                         bitmap = vmcs12->io_bitmap_a;
9547                 else if (port < 0x10000)
9548                         bitmap = vmcs12->io_bitmap_b;
9549                 else
9550                         return true;
9551                 bitmap += (port & 0x7fff) / 8;
9552
9553                 if (last_bitmap != bitmap)
9554                         if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
9555                                 return true;
9556                 if (b & (1 << (port & 7)))
9557                         return true;
9558
9559                 port++;
9560                 size--;
9561                 last_bitmap = bitmap;
9562         }
9563
9564         return false;
9565 }
9566
9567 /*
9568  * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
9569  * rather than handle it ourselves in L0. I.e., check whether L1 expressed
9570  * disinterest in the current event (read or write a specific MSR) by using an
9571  * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
9572  */
9573 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
9574         struct vmcs12 *vmcs12, u32 exit_reason)
9575 {
9576         u32 msr_index = vcpu->arch.regs[VCPU_REGS_RCX];
9577         gpa_t bitmap;
9578
9579         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
9580                 return true;
9581
9582         /*
9583          * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
9584          * for the four combinations of read/write and low/high MSR numbers.
9585          * First we need to figure out which of the four to use:
9586          */
9587         bitmap = vmcs12->msr_bitmap;
9588         if (exit_reason == EXIT_REASON_MSR_WRITE)
9589                 bitmap += 2048;
9590         if (msr_index >= 0xc0000000) {
9591                 msr_index -= 0xc0000000;
9592                 bitmap += 1024;
9593         }
9594
9595         /* Then read the msr_index'th bit from this bitmap: */
9596         if (msr_index < 1024*8) {
9597                 unsigned char b;
9598                 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
9599                         return true;
9600                 return 1 & (b >> (msr_index & 7));
9601         } else
9602                 return true; /* let L1 handle the wrong parameter */
9603 }
9604
9605 /*
9606  * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
9607  * rather than handle it ourselves in L0. I.e., check if L1 wanted to
9608  * intercept (via guest_host_mask etc.) the current event.
9609  */
9610 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
9611         struct vmcs12 *vmcs12)
9612 {
9613         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
9614         int cr = exit_qualification & 15;
9615         int reg;
9616         unsigned long val;
9617
9618         switch ((exit_qualification >> 4) & 3) {
9619         case 0: /* mov to cr */
9620                 reg = (exit_qualification >> 8) & 15;
9621                 val = kvm_register_readl(vcpu, reg);
9622                 switch (cr) {
9623                 case 0:
9624                         if (vmcs12->cr0_guest_host_mask &
9625                             (val ^ vmcs12->cr0_read_shadow))
9626                                 return true;
9627                         break;
9628                 case 3:
9629                         if ((vmcs12->cr3_target_count >= 1 &&
9630                                         vmcs12->cr3_target_value0 == val) ||
9631                                 (vmcs12->cr3_target_count >= 2 &&
9632                                         vmcs12->cr3_target_value1 == val) ||
9633                                 (vmcs12->cr3_target_count >= 3 &&
9634                                         vmcs12->cr3_target_value2 == val) ||
9635                                 (vmcs12->cr3_target_count >= 4 &&
9636                                         vmcs12->cr3_target_value3 == val))
9637                                 return false;
9638                         if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
9639                                 return true;
9640                         break;
9641                 case 4:
9642                         if (vmcs12->cr4_guest_host_mask &
9643                             (vmcs12->cr4_read_shadow ^ val))
9644                                 return true;
9645                         break;
9646                 case 8:
9647                         if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
9648                                 return true;
9649                         break;
9650                 }
9651                 break;
9652         case 2: /* clts */
9653                 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
9654                     (vmcs12->cr0_read_shadow & X86_CR0_TS))
9655                         return true;
9656                 break;
9657         case 1: /* mov from cr */
9658                 switch (cr) {
9659                 case 3:
9660                         if (vmcs12->cpu_based_vm_exec_control &
9661                             CPU_BASED_CR3_STORE_EXITING)
9662                                 return true;
9663                         break;
9664                 case 8:
9665                         if (vmcs12->cpu_based_vm_exec_control &
9666                             CPU_BASED_CR8_STORE_EXITING)
9667                                 return true;
9668                         break;
9669                 }
9670                 break;
9671         case 3: /* lmsw */
9672                 /*
9673                  * lmsw can change bits 1..3 of cr0, and only set bit 0 of
9674                  * cr0. Other attempted changes are ignored, with no exit.
9675                  */
9676                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
9677                 if (vmcs12->cr0_guest_host_mask & 0xe &
9678                     (val ^ vmcs12->cr0_read_shadow))
9679                         return true;
9680                 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
9681                     !(vmcs12->cr0_read_shadow & 0x1) &&
9682                     (val & 0x1))
9683                         return true;
9684                 break;
9685         }
9686         return false;
9687 }
9688
9689 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
9690         struct vmcs12 *vmcs12, gpa_t bitmap)
9691 {
9692         u32 vmx_instruction_info;
9693         unsigned long field;
9694         u8 b;
9695
9696         if (!nested_cpu_has_shadow_vmcs(vmcs12))
9697                 return true;
9698
9699         /* Decode instruction info and find the field to access */
9700         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
9701         field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
9702
9703         /* Out-of-range fields always cause a VM exit from L2 to L1 */
9704         if (field >> 15)
9705                 return true;
9706
9707         if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
9708                 return true;
9709
9710         return 1 & (b >> (field & 7));
9711 }
9712
9713 /*
9714  * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
9715  * should handle it ourselves in L0 (and then continue L2). Only call this
9716  * when in is_guest_mode (L2).
9717  */
9718 static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
9719 {
9720         u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9721         struct vcpu_vmx *vmx = to_vmx(vcpu);
9722         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9723
9724         if (vmx->nested.nested_run_pending)
9725                 return false;
9726
9727         if (unlikely(vmx->fail)) {
9728                 pr_info_ratelimited("%s failed vm entry %x\n", __func__,
9729                                     vmcs_read32(VM_INSTRUCTION_ERROR));
9730                 return true;
9731         }
9732
9733         /*
9734          * The host physical addresses of some pages of guest memory
9735          * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
9736          * Page). The CPU may write to these pages via their host
9737          * physical address while L2 is running, bypassing any
9738          * address-translation-based dirty tracking (e.g. EPT write
9739          * protection).
9740          *
9741          * Mark them dirty on every exit from L2 to prevent them from
9742          * getting out of sync with dirty tracking.
9743          */
9744         nested_mark_vmcs12_pages_dirty(vcpu);
9745
9746         trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
9747                                 vmcs_readl(EXIT_QUALIFICATION),
9748                                 vmx->idt_vectoring_info,
9749                                 intr_info,
9750                                 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
9751                                 KVM_ISA_VMX);
9752
9753         switch ((u16)exit_reason) {
9754         case EXIT_REASON_EXCEPTION_NMI:
9755                 if (is_nmi(intr_info))
9756                         return false;
9757                 else if (is_page_fault(intr_info))
9758                         return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
9759                 else if (is_no_device(intr_info) &&
9760                          !(vmcs12->guest_cr0 & X86_CR0_TS))
9761                         return false;
9762                 else if (is_debug(intr_info) &&
9763                          vcpu->guest_debug &
9764                          (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
9765                         return false;
9766                 else if (is_breakpoint(intr_info) &&
9767                          vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
9768                         return false;
9769                 return vmcs12->exception_bitmap &
9770                                 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
9771         case EXIT_REASON_EXTERNAL_INTERRUPT:
9772                 return false;
9773         case EXIT_REASON_TRIPLE_FAULT:
9774                 return true;
9775         case EXIT_REASON_PENDING_INTERRUPT:
9776                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_INTR_PENDING);
9777         case EXIT_REASON_NMI_WINDOW:
9778                 return nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING);
9779         case EXIT_REASON_TASK_SWITCH:
9780                 return true;
9781         case EXIT_REASON_CPUID:
9782                 return true;
9783         case EXIT_REASON_HLT:
9784                 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
9785         case EXIT_REASON_INVD:
9786                 return true;
9787         case EXIT_REASON_INVLPG:
9788                 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
9789         case EXIT_REASON_RDPMC:
9790                 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
9791         case EXIT_REASON_RDRAND:
9792                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
9793         case EXIT_REASON_RDSEED:
9794                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
9795         case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
9796                 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
9797         case EXIT_REASON_VMREAD:
9798                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
9799                         vmcs12->vmread_bitmap);
9800         case EXIT_REASON_VMWRITE:
9801                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
9802                         vmcs12->vmwrite_bitmap);
9803         case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
9804         case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
9805         case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
9806         case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
9807         case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
9808                 /*
9809                  * VMX instructions trap unconditionally. This allows L1 to
9810                  * emulate them for its L2 guest, i.e., allows 3-level nesting!
9811                  */
9812                 return true;
9813         case EXIT_REASON_CR_ACCESS:
9814                 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
9815         case EXIT_REASON_DR_ACCESS:
9816                 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
9817         case EXIT_REASON_IO_INSTRUCTION:
9818                 return nested_vmx_exit_handled_io(vcpu, vmcs12);
9819         case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
9820                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
9821         case EXIT_REASON_MSR_READ:
9822         case EXIT_REASON_MSR_WRITE:
9823                 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
9824         case EXIT_REASON_INVALID_STATE:
9825                 return true;
9826         case EXIT_REASON_MWAIT_INSTRUCTION:
9827                 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
9828         case EXIT_REASON_MONITOR_TRAP_FLAG:
9829                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
9830         case EXIT_REASON_MONITOR_INSTRUCTION:
9831                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
9832         case EXIT_REASON_PAUSE_INSTRUCTION:
9833                 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
9834                         nested_cpu_has2(vmcs12,
9835                                 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
9836         case EXIT_REASON_MCE_DURING_VMENTRY:
9837                 return false;
9838         case EXIT_REASON_TPR_BELOW_THRESHOLD:
9839                 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
9840         case EXIT_REASON_APIC_ACCESS:
9841         case EXIT_REASON_APIC_WRITE:
9842         case EXIT_REASON_EOI_INDUCED:
9843                 /*
9844                  * The controls for "virtualize APIC accesses," "APIC-
9845                  * register virtualization," and "virtual-interrupt
9846                  * delivery" only come from vmcs12.
9847                  */
9848                 return true;
9849         case EXIT_REASON_EPT_VIOLATION:
9850                 /*
9851                  * L0 always deals with the EPT violation. If nested EPT is
9852                  * used, and the nested mmu code discovers that the address is
9853                  * missing in the guest EPT table (EPT12), the EPT violation
9854                  * will be injected with nested_ept_inject_page_fault()
9855                  */
9856                 return false;
9857         case EXIT_REASON_EPT_MISCONFIG:
9858                 /*
9859                  * L2 never uses directly L1's EPT, but rather L0's own EPT
9860                  * table (shadow on EPT) or a merged EPT table that L0 built
9861                  * (EPT on EPT). So any problems with the structure of the
9862                  * table is L0's fault.
9863                  */
9864                 return false;
9865         case EXIT_REASON_INVPCID:
9866                 return
9867                         nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
9868                         nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
9869         case EXIT_REASON_WBINVD:
9870                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
9871         case EXIT_REASON_XSETBV:
9872                 return true;
9873         case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
9874                 /*
9875                  * This should never happen, since it is not possible to
9876                  * set XSS to a non-zero value---neither in L1 nor in L2.
9877                  * If if it were, XSS would have to be checked against
9878                  * the XSS exit bitmap in vmcs12.
9879                  */
9880                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
9881         case EXIT_REASON_PREEMPTION_TIMER:
9882                 return false;
9883         case EXIT_REASON_PML_FULL:
9884                 /* We emulate PML support to L1. */
9885                 return false;
9886         case EXIT_REASON_VMFUNC:
9887                 /* VM functions are emulated through L2->L0 vmexits. */
9888                 return false;
9889         case EXIT_REASON_ENCLS:
9890                 /* SGX is never exposed to L1 */
9891                 return false;
9892         default:
9893                 return true;
9894         }
9895 }
9896
9897 static int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason)
9898 {
9899         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
9900
9901         /*
9902          * At this point, the exit interruption info in exit_intr_info
9903          * is only valid for EXCEPTION_NMI exits.  For EXTERNAL_INTERRUPT
9904          * we need to query the in-kernel LAPIC.
9905          */
9906         WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT);
9907         if ((exit_intr_info &
9908              (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) ==
9909             (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) {
9910                 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
9911                 vmcs12->vm_exit_intr_error_code =
9912                         vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
9913         }
9914
9915         nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info,
9916                           vmcs_readl(EXIT_QUALIFICATION));
9917         return 1;
9918 }
9919
9920 static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
9921 {
9922         *info1 = vmcs_readl(EXIT_QUALIFICATION);
9923         *info2 = vmcs_read32(VM_EXIT_INTR_INFO);
9924 }
9925
9926 static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx)
9927 {
9928         if (vmx->pml_pg) {
9929                 __free_page(vmx->pml_pg);
9930                 vmx->pml_pg = NULL;
9931         }
9932 }
9933
9934 static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu)
9935 {
9936         struct vcpu_vmx *vmx = to_vmx(vcpu);
9937         u64 *pml_buf;
9938         u16 pml_idx;
9939
9940         pml_idx = vmcs_read16(GUEST_PML_INDEX);
9941
9942         /* Do nothing if PML buffer is empty */
9943         if (pml_idx == (PML_ENTITY_NUM - 1))
9944                 return;
9945
9946         /* PML index always points to next available PML buffer entity */
9947         if (pml_idx >= PML_ENTITY_NUM)
9948                 pml_idx = 0;
9949         else
9950                 pml_idx++;
9951
9952         pml_buf = page_address(vmx->pml_pg);
9953         for (; pml_idx < PML_ENTITY_NUM; pml_idx++) {
9954                 u64 gpa;
9955
9956                 gpa = pml_buf[pml_idx];
9957                 WARN_ON(gpa & (PAGE_SIZE - 1));
9958                 kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT);
9959         }
9960
9961         /* reset PML index */
9962         vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
9963 }
9964
9965 /*
9966  * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap.
9967  * Called before reporting dirty_bitmap to userspace.
9968  */
9969 static void kvm_flush_pml_buffers(struct kvm *kvm)
9970 {
9971         int i;
9972         struct kvm_vcpu *vcpu;
9973         /*
9974          * We only need to kick vcpu out of guest mode here, as PML buffer
9975          * is flushed at beginning of all VMEXITs, and it's obvious that only
9976          * vcpus running in guest are possible to have unflushed GPAs in PML
9977          * buffer.
9978          */
9979         kvm_for_each_vcpu(i, vcpu, kvm)
9980                 kvm_vcpu_kick(vcpu);
9981 }
9982
9983 static void vmx_dump_sel(char *name, uint32_t sel)
9984 {
9985         pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n",
9986                name, vmcs_read16(sel),
9987                vmcs_read32(sel + GUEST_ES_AR_BYTES - GUEST_ES_SELECTOR),
9988                vmcs_read32(sel + GUEST_ES_LIMIT - GUEST_ES_SELECTOR),
9989                vmcs_readl(sel + GUEST_ES_BASE - GUEST_ES_SELECTOR));
9990 }
9991
9992 static void vmx_dump_dtsel(char *name, uint32_t limit)
9993 {
9994         pr_err("%s                           limit=0x%08x, base=0x%016lx\n",
9995                name, vmcs_read32(limit),
9996                vmcs_readl(limit + GUEST_GDTR_BASE - GUEST_GDTR_LIMIT));
9997 }
9998
9999 static void dump_vmcs(void)
10000 {
10001         u32 vmentry_ctl = vmcs_read32(VM_ENTRY_CONTROLS);
10002         u32 vmexit_ctl = vmcs_read32(VM_EXIT_CONTROLS);
10003         u32 cpu_based_exec_ctrl = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
10004         u32 pin_based_exec_ctrl = vmcs_read32(PIN_BASED_VM_EXEC_CONTROL);
10005         u32 secondary_exec_control = 0;
10006         unsigned long cr4 = vmcs_readl(GUEST_CR4);
10007         u64 efer = vmcs_read64(GUEST_IA32_EFER);
10008         int i, n;
10009
10010         if (cpu_has_secondary_exec_ctrls())
10011                 secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
10012
10013         pr_err("*** Guest State ***\n");
10014         pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
10015                vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW),
10016                vmcs_readl(CR0_GUEST_HOST_MASK));
10017         pr_err("CR4: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n",
10018                cr4, vmcs_readl(CR4_READ_SHADOW), vmcs_readl(CR4_GUEST_HOST_MASK));
10019         pr_err("CR3 = 0x%016lx\n", vmcs_readl(GUEST_CR3));
10020         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT) &&
10021             (cr4 & X86_CR4_PAE) && !(efer & EFER_LMA))
10022         {
10023                 pr_err("PDPTR0 = 0x%016llx  PDPTR1 = 0x%016llx\n",
10024                        vmcs_read64(GUEST_PDPTR0), vmcs_read64(GUEST_PDPTR1));
10025                 pr_err("PDPTR2 = 0x%016llx  PDPTR3 = 0x%016llx\n",
10026                        vmcs_read64(GUEST_PDPTR2), vmcs_read64(GUEST_PDPTR3));
10027         }
10028         pr_err("RSP = 0x%016lx  RIP = 0x%016lx\n",
10029                vmcs_readl(GUEST_RSP), vmcs_readl(GUEST_RIP));
10030         pr_err("RFLAGS=0x%08lx         DR7 = 0x%016lx\n",
10031                vmcs_readl(GUEST_RFLAGS), vmcs_readl(GUEST_DR7));
10032         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
10033                vmcs_readl(GUEST_SYSENTER_ESP),
10034                vmcs_read32(GUEST_SYSENTER_CS), vmcs_readl(GUEST_SYSENTER_EIP));
10035         vmx_dump_sel("CS:  ", GUEST_CS_SELECTOR);
10036         vmx_dump_sel("DS:  ", GUEST_DS_SELECTOR);
10037         vmx_dump_sel("SS:  ", GUEST_SS_SELECTOR);
10038         vmx_dump_sel("ES:  ", GUEST_ES_SELECTOR);
10039         vmx_dump_sel("FS:  ", GUEST_FS_SELECTOR);
10040         vmx_dump_sel("GS:  ", GUEST_GS_SELECTOR);
10041         vmx_dump_dtsel("GDTR:", GUEST_GDTR_LIMIT);
10042         vmx_dump_sel("LDTR:", GUEST_LDTR_SELECTOR);
10043         vmx_dump_dtsel("IDTR:", GUEST_IDTR_LIMIT);
10044         vmx_dump_sel("TR:  ", GUEST_TR_SELECTOR);
10045         if ((vmexit_ctl & (VM_EXIT_SAVE_IA32_PAT | VM_EXIT_SAVE_IA32_EFER)) ||
10046             (vmentry_ctl & (VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_IA32_EFER)))
10047                 pr_err("EFER =     0x%016llx  PAT = 0x%016llx\n",
10048                        efer, vmcs_read64(GUEST_IA32_PAT));
10049         pr_err("DebugCtl = 0x%016llx  DebugExceptions = 0x%016lx\n",
10050                vmcs_read64(GUEST_IA32_DEBUGCTL),
10051                vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS));
10052         if (cpu_has_load_perf_global_ctrl &&
10053             vmentry_ctl & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL)
10054                 pr_err("PerfGlobCtl = 0x%016llx\n",
10055                        vmcs_read64(GUEST_IA32_PERF_GLOBAL_CTRL));
10056         if (vmentry_ctl & VM_ENTRY_LOAD_BNDCFGS)
10057                 pr_err("BndCfgS = 0x%016llx\n", vmcs_read64(GUEST_BNDCFGS));
10058         pr_err("Interruptibility = %08x  ActivityState = %08x\n",
10059                vmcs_read32(GUEST_INTERRUPTIBILITY_INFO),
10060                vmcs_read32(GUEST_ACTIVITY_STATE));
10061         if (secondary_exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
10062                 pr_err("InterruptStatus = %04x\n",
10063                        vmcs_read16(GUEST_INTR_STATUS));
10064
10065         pr_err("*** Host State ***\n");
10066         pr_err("RIP = 0x%016lx  RSP = 0x%016lx\n",
10067                vmcs_readl(HOST_RIP), vmcs_readl(HOST_RSP));
10068         pr_err("CS=%04x SS=%04x DS=%04x ES=%04x FS=%04x GS=%04x TR=%04x\n",
10069                vmcs_read16(HOST_CS_SELECTOR), vmcs_read16(HOST_SS_SELECTOR),
10070                vmcs_read16(HOST_DS_SELECTOR), vmcs_read16(HOST_ES_SELECTOR),
10071                vmcs_read16(HOST_FS_SELECTOR), vmcs_read16(HOST_GS_SELECTOR),
10072                vmcs_read16(HOST_TR_SELECTOR));
10073         pr_err("FSBase=%016lx GSBase=%016lx TRBase=%016lx\n",
10074                vmcs_readl(HOST_FS_BASE), vmcs_readl(HOST_GS_BASE),
10075                vmcs_readl(HOST_TR_BASE));
10076         pr_err("GDTBase=%016lx IDTBase=%016lx\n",
10077                vmcs_readl(HOST_GDTR_BASE), vmcs_readl(HOST_IDTR_BASE));
10078         pr_err("CR0=%016lx CR3=%016lx CR4=%016lx\n",
10079                vmcs_readl(HOST_CR0), vmcs_readl(HOST_CR3),
10080                vmcs_readl(HOST_CR4));
10081         pr_err("Sysenter RSP=%016lx CS:RIP=%04x:%016lx\n",
10082                vmcs_readl(HOST_IA32_SYSENTER_ESP),
10083                vmcs_read32(HOST_IA32_SYSENTER_CS),
10084                vmcs_readl(HOST_IA32_SYSENTER_EIP));
10085         if (vmexit_ctl & (VM_EXIT_LOAD_IA32_PAT | VM_EXIT_LOAD_IA32_EFER))
10086                 pr_err("EFER = 0x%016llx  PAT = 0x%016llx\n",
10087                        vmcs_read64(HOST_IA32_EFER),
10088                        vmcs_read64(HOST_IA32_PAT));
10089         if (cpu_has_load_perf_global_ctrl &&
10090             vmexit_ctl & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
10091                 pr_err("PerfGlobCtl = 0x%016llx\n",
10092                        vmcs_read64(HOST_IA32_PERF_GLOBAL_CTRL));
10093
10094         pr_err("*** Control State ***\n");
10095         pr_err("PinBased=%08x CPUBased=%08x SecondaryExec=%08x\n",
10096                pin_based_exec_ctrl, cpu_based_exec_ctrl, secondary_exec_control);
10097         pr_err("EntryControls=%08x ExitControls=%08x\n", vmentry_ctl, vmexit_ctl);
10098         pr_err("ExceptionBitmap=%08x PFECmask=%08x PFECmatch=%08x\n",
10099                vmcs_read32(EXCEPTION_BITMAP),
10100                vmcs_read32(PAGE_FAULT_ERROR_CODE_MASK),
10101                vmcs_read32(PAGE_FAULT_ERROR_CODE_MATCH));
10102         pr_err("VMEntry: intr_info=%08x errcode=%08x ilen=%08x\n",
10103                vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
10104                vmcs_read32(VM_ENTRY_EXCEPTION_ERROR_CODE),
10105                vmcs_read32(VM_ENTRY_INSTRUCTION_LEN));
10106         pr_err("VMExit: intr_info=%08x errcode=%08x ilen=%08x\n",
10107                vmcs_read32(VM_EXIT_INTR_INFO),
10108                vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
10109                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
10110         pr_err("        reason=%08x qualification=%016lx\n",
10111                vmcs_read32(VM_EXIT_REASON), vmcs_readl(EXIT_QUALIFICATION));
10112         pr_err("IDTVectoring: info=%08x errcode=%08x\n",
10113                vmcs_read32(IDT_VECTORING_INFO_FIELD),
10114                vmcs_read32(IDT_VECTORING_ERROR_CODE));
10115         pr_err("TSC Offset = 0x%016llx\n", vmcs_read64(TSC_OFFSET));
10116         if (secondary_exec_control & SECONDARY_EXEC_TSC_SCALING)
10117                 pr_err("TSC Multiplier = 0x%016llx\n",
10118                        vmcs_read64(TSC_MULTIPLIER));
10119         if (cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW)
10120                 pr_err("TPR Threshold = 0x%02x\n", vmcs_read32(TPR_THRESHOLD));
10121         if (pin_based_exec_ctrl & PIN_BASED_POSTED_INTR)
10122                 pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV));
10123         if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT))
10124                 pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER));
10125         n = vmcs_read32(CR3_TARGET_COUNT);
10126         for (i = 0; i + 1 < n; i += 4)
10127                 pr_err("CR3 target%u=%016lx target%u=%016lx\n",
10128                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2),
10129                        i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2));
10130         if (i < n)
10131                 pr_err("CR3 target%u=%016lx\n",
10132                        i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2));
10133         if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING)
10134                 pr_err("PLE Gap=%08x Window=%08x\n",
10135                        vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW));
10136         if (secondary_exec_control & SECONDARY_EXEC_ENABLE_VPID)
10137                 pr_err("Virtual processor ID = 0x%04x\n",
10138                        vmcs_read16(VIRTUAL_PROCESSOR_ID));
10139 }
10140
10141 /*
10142  * The guest has exited.  See if we can fix it or if we need userspace
10143  * assistance.
10144  */
10145 static int vmx_handle_exit(struct kvm_vcpu *vcpu)
10146 {
10147         struct vcpu_vmx *vmx = to_vmx(vcpu);
10148         u32 exit_reason = vmx->exit_reason;
10149         u32 vectoring_info = vmx->idt_vectoring_info;
10150
10151         trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX);
10152
10153         /*
10154          * Flush logged GPAs PML buffer, this will make dirty_bitmap more
10155          * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before
10156          * querying dirty_bitmap, we only need to kick all vcpus out of guest
10157          * mode as if vcpus is in root mode, the PML buffer must has been
10158          * flushed already.
10159          */
10160         if (enable_pml)
10161                 vmx_flush_pml_buffer(vcpu);
10162
10163         /* If guest state is invalid, start emulating */
10164         if (vmx->emulation_required)
10165                 return handle_invalid_guest_state(vcpu);
10166
10167         if (is_guest_mode(vcpu) && nested_vmx_exit_reflected(vcpu, exit_reason))
10168                 return nested_vmx_reflect_vmexit(vcpu, exit_reason);
10169
10170         if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) {
10171                 dump_vmcs();
10172                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
10173                 vcpu->run->fail_entry.hardware_entry_failure_reason
10174                         = exit_reason;
10175                 return 0;
10176         }
10177
10178         if (unlikely(vmx->fail)) {
10179                 vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
10180                 vcpu->run->fail_entry.hardware_entry_failure_reason
10181                         = vmcs_read32(VM_INSTRUCTION_ERROR);
10182                 return 0;
10183         }
10184
10185         /*
10186          * Note:
10187          * Do not try to fix EXIT_REASON_EPT_MISCONFIG if it caused by
10188          * delivery event since it indicates guest is accessing MMIO.
10189          * The vm-exit can be triggered again after return to guest that
10190          * will cause infinite loop.
10191          */
10192         if ((vectoring_info & VECTORING_INFO_VALID_MASK) &&
10193                         (exit_reason != EXIT_REASON_EXCEPTION_NMI &&
10194                         exit_reason != EXIT_REASON_EPT_VIOLATION &&
10195                         exit_reason != EXIT_REASON_PML_FULL &&
10196                         exit_reason != EXIT_REASON_APIC_ACCESS &&
10197                         exit_reason != EXIT_REASON_TASK_SWITCH)) {
10198                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
10199                 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_DELIVERY_EV;
10200                 vcpu->run->internal.ndata = 3;
10201                 vcpu->run->internal.data[0] = vectoring_info;
10202                 vcpu->run->internal.data[1] = exit_reason;
10203                 vcpu->run->internal.data[2] = vcpu->arch.exit_qualification;
10204                 if (exit_reason == EXIT_REASON_EPT_MISCONFIG) {
10205                         vcpu->run->internal.ndata++;
10206                         vcpu->run->internal.data[3] =
10207                                 vmcs_read64(GUEST_PHYSICAL_ADDRESS);
10208                 }
10209                 return 0;
10210         }
10211
10212         if (unlikely(!enable_vnmi &&
10213                      vmx->loaded_vmcs->soft_vnmi_blocked)) {
10214                 if (vmx_interrupt_allowed(vcpu)) {
10215                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
10216                 } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL &&
10217                            vcpu->arch.nmi_pending) {
10218                         /*
10219                          * This CPU don't support us in finding the end of an
10220                          * NMI-blocked window if the guest runs with IRQs
10221                          * disabled. So we pull the trigger after 1 s of
10222                          * futile waiting, but inform the user about this.
10223                          */
10224                         printk(KERN_WARNING "%s: Breaking out of NMI-blocked "
10225                                "state on VCPU %d after 1 s timeout\n",
10226                                __func__, vcpu->vcpu_id);
10227                         vmx->loaded_vmcs->soft_vnmi_blocked = 0;
10228                 }
10229         }
10230
10231         if (exit_reason < kvm_vmx_max_exit_handlers
10232             && kvm_vmx_exit_handlers[exit_reason])
10233                 return kvm_vmx_exit_handlers[exit_reason](vcpu);
10234         else {
10235                 vcpu_unimpl(vcpu, "vmx: unexpected exit reason 0x%x\n",
10236                                 exit_reason);
10237                 kvm_queue_exception(vcpu, UD_VECTOR);
10238                 return 1;
10239         }
10240 }
10241
10242 /*
10243  * Software based L1D cache flush which is used when microcode providing
10244  * the cache control MSR is not loaded.
10245  *
10246  * The L1D cache is 32 KiB on Nehalem and later microarchitectures, but to
10247  * flush it is required to read in 64 KiB because the replacement algorithm
10248  * is not exactly LRU. This could be sized at runtime via topology
10249  * information but as all relevant affected CPUs have 32KiB L1D cache size
10250  * there is no point in doing so.
10251  */
10252 static void vmx_l1d_flush(struct kvm_vcpu *vcpu)
10253 {
10254         int size = PAGE_SIZE << L1D_CACHE_ORDER;
10255
10256         /*
10257          * This code is only executed when the the flush mode is 'cond' or
10258          * 'always'
10259          */
10260         if (static_branch_likely(&vmx_l1d_flush_cond)) {
10261                 bool flush_l1d;
10262
10263                 /*
10264                  * Clear the per-vcpu flush bit, it gets set again
10265                  * either from vcpu_run() or from one of the unsafe
10266                  * VMEXIT handlers.
10267                  */
10268                 flush_l1d = vcpu->arch.l1tf_flush_l1d;
10269                 vcpu->arch.l1tf_flush_l1d = false;
10270
10271                 /*
10272                  * Clear the per-cpu flush bit, it gets set again from
10273                  * the interrupt handlers.
10274                  */
10275                 flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
10276                 kvm_clear_cpu_l1tf_flush_l1d();
10277
10278                 if (!flush_l1d)
10279                         return;
10280         }
10281
10282         vcpu->stat.l1d_flush++;
10283
10284         if (static_cpu_has(X86_FEATURE_FLUSH_L1D)) {
10285                 wrmsrl(MSR_IA32_FLUSH_CMD, L1D_FLUSH);
10286                 return;
10287         }
10288
10289         asm volatile(
10290                 /* First ensure the pages are in the TLB */
10291                 "xorl   %%eax, %%eax\n"
10292                 ".Lpopulate_tlb:\n\t"
10293                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
10294                 "addl   $4096, %%eax\n\t"
10295                 "cmpl   %%eax, %[size]\n\t"
10296                 "jne    .Lpopulate_tlb\n\t"
10297                 "xorl   %%eax, %%eax\n\t"
10298                 "cpuid\n\t"
10299                 /* Now fill the cache */
10300                 "xorl   %%eax, %%eax\n"
10301                 ".Lfill_cache:\n"
10302                 "movzbl (%[flush_pages], %%" _ASM_AX "), %%ecx\n\t"
10303                 "addl   $64, %%eax\n\t"
10304                 "cmpl   %%eax, %[size]\n\t"
10305                 "jne    .Lfill_cache\n\t"
10306                 "lfence\n"
10307                 :: [flush_pages] "r" (vmx_l1d_flush_pages),
10308                     [size] "r" (size)
10309                 : "eax", "ebx", "ecx", "edx");
10310 }
10311
10312 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
10313 {
10314         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
10315
10316         if (is_guest_mode(vcpu) &&
10317                 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
10318                 return;
10319
10320         if (irr == -1 || tpr < irr) {
10321                 vmcs_write32(TPR_THRESHOLD, 0);
10322                 return;
10323         }
10324
10325         vmcs_write32(TPR_THRESHOLD, irr);
10326 }
10327
10328 static void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
10329 {
10330         u32 sec_exec_control;
10331
10332         if (!lapic_in_kernel(vcpu))
10333                 return;
10334
10335         if (!flexpriority_enabled &&
10336             !cpu_has_vmx_virtualize_x2apic_mode())
10337                 return;
10338
10339         /* Postpone execution until vmcs01 is the current VMCS. */
10340         if (is_guest_mode(vcpu)) {
10341                 to_vmx(vcpu)->nested.change_vmcs01_virtual_apic_mode = true;
10342                 return;
10343         }
10344
10345         sec_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
10346         sec_exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
10347                               SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE);
10348
10349         switch (kvm_get_apic_mode(vcpu)) {
10350         case LAPIC_MODE_INVALID:
10351                 WARN_ONCE(true, "Invalid local APIC state");
10352         case LAPIC_MODE_DISABLED:
10353                 break;
10354         case LAPIC_MODE_XAPIC:
10355                 if (flexpriority_enabled) {
10356                         sec_exec_control |=
10357                                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
10358                         vmx_flush_tlb(vcpu, true);
10359                 }
10360                 break;
10361         case LAPIC_MODE_X2APIC:
10362                 if (cpu_has_vmx_virtualize_x2apic_mode())
10363                         sec_exec_control |=
10364                                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE;
10365                 break;
10366         }
10367         vmcs_write32(SECONDARY_VM_EXEC_CONTROL, sec_exec_control);
10368
10369         vmx_update_msr_bitmap(vcpu);
10370 }
10371
10372 static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa)
10373 {
10374         if (!is_guest_mode(vcpu)) {
10375                 vmcs_write64(APIC_ACCESS_ADDR, hpa);
10376                 vmx_flush_tlb(vcpu, true);
10377         }
10378 }
10379
10380 static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
10381 {
10382         u16 status;
10383         u8 old;
10384
10385         if (max_isr == -1)
10386                 max_isr = 0;
10387
10388         status = vmcs_read16(GUEST_INTR_STATUS);
10389         old = status >> 8;
10390         if (max_isr != old) {
10391                 status &= 0xff;
10392                 status |= max_isr << 8;
10393                 vmcs_write16(GUEST_INTR_STATUS, status);
10394         }
10395 }
10396
10397 static void vmx_set_rvi(int vector)
10398 {
10399         u16 status;
10400         u8 old;
10401
10402         if (vector == -1)
10403                 vector = 0;
10404
10405         status = vmcs_read16(GUEST_INTR_STATUS);
10406         old = (u8)status & 0xff;
10407         if ((u8)vector != old) {
10408                 status &= ~0xff;
10409                 status |= (u8)vector;
10410                 vmcs_write16(GUEST_INTR_STATUS, status);
10411         }
10412 }
10413
10414 static void vmx_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
10415 {
10416         /*
10417          * When running L2, updating RVI is only relevant when
10418          * vmcs12 virtual-interrupt-delivery enabled.
10419          * However, it can be enabled only when L1 also
10420          * intercepts external-interrupts and in that case
10421          * we should not update vmcs02 RVI but instead intercept
10422          * interrupt. Therefore, do nothing when running L2.
10423          */
10424         if (!is_guest_mode(vcpu))
10425                 vmx_set_rvi(max_irr);
10426 }
10427
10428 static int vmx_sync_pir_to_irr(struct kvm_vcpu *vcpu)
10429 {
10430         struct vcpu_vmx *vmx = to_vmx(vcpu);
10431         int max_irr;
10432         bool max_irr_updated;
10433
10434         WARN_ON(!vcpu->arch.apicv_active);
10435         if (pi_test_on(&vmx->pi_desc)) {
10436                 pi_clear_on(&vmx->pi_desc);
10437                 /*
10438                  * IOMMU can write to PIR.ON, so the barrier matters even on UP.
10439                  * But on x86 this is just a compiler barrier anyway.
10440                  */
10441                 smp_mb__after_atomic();
10442                 max_irr_updated =
10443                         kvm_apic_update_irr(vcpu, vmx->pi_desc.pir, &max_irr);
10444
10445                 /*
10446                  * If we are running L2 and L1 has a new pending interrupt
10447                  * which can be injected, we should re-evaluate
10448                  * what should be done with this new L1 interrupt.
10449                  * If L1 intercepts external-interrupts, we should
10450                  * exit from L2 to L1. Otherwise, interrupt should be
10451                  * delivered directly to L2.
10452                  */
10453                 if (is_guest_mode(vcpu) && max_irr_updated) {
10454                         if (nested_exit_on_intr(vcpu))
10455                                 kvm_vcpu_exiting_guest_mode(vcpu);
10456                         else
10457                                 kvm_make_request(KVM_REQ_EVENT, vcpu);
10458                 }
10459         } else {
10460                 max_irr = kvm_lapic_find_highest_irr(vcpu);
10461         }
10462         vmx_hwapic_irr_update(vcpu, max_irr);
10463         return max_irr;
10464 }
10465
10466 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
10467 {
10468         u8 rvi = vmx_get_rvi();
10469         u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
10470
10471         return ((rvi & 0xf0) > (vppr & 0xf0));
10472 }
10473
10474 static bool vmx_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
10475 {
10476         return pi_test_on(vcpu_to_pi_desc(vcpu));
10477 }
10478
10479 static void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
10480 {
10481         if (!kvm_vcpu_apicv_active(vcpu))
10482                 return;
10483
10484         vmcs_write64(EOI_EXIT_BITMAP0, eoi_exit_bitmap[0]);
10485         vmcs_write64(EOI_EXIT_BITMAP1, eoi_exit_bitmap[1]);
10486         vmcs_write64(EOI_EXIT_BITMAP2, eoi_exit_bitmap[2]);
10487         vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
10488 }
10489
10490 static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu)
10491 {
10492         struct vcpu_vmx *vmx = to_vmx(vcpu);
10493
10494         pi_clear_on(&vmx->pi_desc);
10495         memset(vmx->pi_desc.pir, 0, sizeof(vmx->pi_desc.pir));
10496 }
10497
10498 static void vmx_complete_atomic_exit(struct vcpu_vmx *vmx)
10499 {
10500         if (vmx->exit_reason != EXIT_REASON_EXCEPTION_NMI)
10501                 return;
10502
10503         vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
10504
10505         /* if exit due to PF check for async PF */
10506         if (is_page_fault(vmx->exit_intr_info))
10507                 vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
10508
10509         /* Handle machine checks before interrupts are enabled */
10510         if (is_machine_check(vmx->exit_intr_info))
10511                 kvm_machine_check();
10512
10513         /* We need to handle NMIs before interrupts are enabled */
10514         if (is_nmi(vmx->exit_intr_info)) {
10515                 kvm_before_interrupt(&vmx->vcpu);
10516                 asm("int $2");
10517                 kvm_after_interrupt(&vmx->vcpu);
10518         }
10519 }
10520
10521 static void vmx_handle_external_intr(struct kvm_vcpu *vcpu)
10522 {
10523         u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
10524
10525         if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_INTR_TYPE_MASK))
10526                         == (INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR)) {
10527                 unsigned int vector;
10528                 unsigned long entry;
10529                 gate_desc *desc;
10530                 struct vcpu_vmx *vmx = to_vmx(vcpu);
10531 #ifdef CONFIG_X86_64
10532                 unsigned long tmp;
10533 #endif
10534
10535                 vector =  exit_intr_info & INTR_INFO_VECTOR_MASK;
10536                 desc = (gate_desc *)vmx->host_idt_base + vector;
10537                 entry = gate_offset(desc);
10538                 asm volatile(
10539 #ifdef CONFIG_X86_64
10540                         "mov %%" _ASM_SP ", %[sp]\n\t"
10541                         "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t"
10542                         "push $%c[ss]\n\t"
10543                         "push %[sp]\n\t"
10544 #endif
10545                         "pushf\n\t"
10546                         __ASM_SIZE(push) " $%c[cs]\n\t"
10547                         CALL_NOSPEC
10548                         :
10549 #ifdef CONFIG_X86_64
10550                         [sp]"=&r"(tmp),
10551 #endif
10552                         ASM_CALL_CONSTRAINT
10553                         :
10554                         THUNK_TARGET(entry),
10555                         [ss]"i"(__KERNEL_DS),
10556                         [cs]"i"(__KERNEL_CS)
10557                         );
10558         }
10559 }
10560 STACK_FRAME_NON_STANDARD(vmx_handle_external_intr);
10561
10562 static bool vmx_has_emulated_msr(int index)
10563 {
10564         switch (index) {
10565         case MSR_IA32_SMBASE:
10566                 /*
10567                  * We cannot do SMM unless we can run the guest in big
10568                  * real mode.
10569                  */
10570                 return enable_unrestricted_guest || emulate_invalid_guest_state;
10571         case MSR_AMD64_VIRT_SPEC_CTRL:
10572                 /* This is AMD only.  */
10573                 return false;
10574         default:
10575                 return true;
10576         }
10577 }
10578
10579 static bool vmx_mpx_supported(void)
10580 {
10581         return (vmcs_config.vmexit_ctrl & VM_EXIT_CLEAR_BNDCFGS) &&
10582                 (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_BNDCFGS);
10583 }
10584
10585 static bool vmx_xsaves_supported(void)
10586 {
10587         return vmcs_config.cpu_based_2nd_exec_ctrl &
10588                 SECONDARY_EXEC_XSAVES;
10589 }
10590
10591 static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx)
10592 {
10593         u32 exit_intr_info;
10594         bool unblock_nmi;
10595         u8 vector;
10596         bool idtv_info_valid;
10597
10598         idtv_info_valid = vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK;
10599
10600         if (enable_vnmi) {
10601                 if (vmx->loaded_vmcs->nmi_known_unmasked)
10602                         return;
10603                 /*
10604                  * Can't use vmx->exit_intr_info since we're not sure what
10605                  * the exit reason is.
10606                  */
10607                 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
10608                 unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0;
10609                 vector = exit_intr_info & INTR_INFO_VECTOR_MASK;
10610                 /*
10611                  * SDM 3: 27.7.1.2 (September 2008)
10612                  * Re-set bit "block by NMI" before VM entry if vmexit caused by
10613                  * a guest IRET fault.
10614                  * SDM 3: 23.2.2 (September 2008)
10615                  * Bit 12 is undefined in any of the following cases:
10616                  *  If the VM exit sets the valid bit in the IDT-vectoring
10617                  *   information field.
10618                  *  If the VM exit is due to a double fault.
10619                  */
10620                 if ((exit_intr_info & INTR_INFO_VALID_MASK) && unblock_nmi &&
10621                     vector != DF_VECTOR && !idtv_info_valid)
10622                         vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO,
10623                                       GUEST_INTR_STATE_NMI);
10624                 else
10625                         vmx->loaded_vmcs->nmi_known_unmasked =
10626                                 !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO)
10627                                   & GUEST_INTR_STATE_NMI);
10628         } else if (unlikely(vmx->loaded_vmcs->soft_vnmi_blocked))
10629                 vmx->loaded_vmcs->vnmi_blocked_time +=
10630                         ktime_to_ns(ktime_sub(ktime_get(),
10631                                               vmx->loaded_vmcs->entry_time));
10632 }
10633
10634 static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
10635                                       u32 idt_vectoring_info,
10636                                       int instr_len_field,
10637                                       int error_code_field)
10638 {
10639         u8 vector;
10640         int type;
10641         bool idtv_info_valid;
10642
10643         idtv_info_valid = idt_vectoring_info & VECTORING_INFO_VALID_MASK;
10644
10645         vcpu->arch.nmi_injected = false;
10646         kvm_clear_exception_queue(vcpu);
10647         kvm_clear_interrupt_queue(vcpu);
10648
10649         if (!idtv_info_valid)
10650                 return;
10651
10652         kvm_make_request(KVM_REQ_EVENT, vcpu);
10653
10654         vector = idt_vectoring_info & VECTORING_INFO_VECTOR_MASK;
10655         type = idt_vectoring_info & VECTORING_INFO_TYPE_MASK;
10656
10657         switch (type) {
10658         case INTR_TYPE_NMI_INTR:
10659                 vcpu->arch.nmi_injected = true;
10660                 /*
10661                  * SDM 3: 27.7.1.2 (September 2008)
10662                  * Clear bit "block by NMI" before VM entry if a NMI
10663                  * delivery faulted.
10664                  */
10665                 vmx_set_nmi_mask(vcpu, false);
10666                 break;
10667         case INTR_TYPE_SOFT_EXCEPTION:
10668                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
10669                 /* fall through */
10670         case INTR_TYPE_HARD_EXCEPTION:
10671                 if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
10672                         u32 err = vmcs_read32(error_code_field);
10673                         kvm_requeue_exception_e(vcpu, vector, err);
10674                 } else
10675                         kvm_requeue_exception(vcpu, vector);
10676                 break;
10677         case INTR_TYPE_SOFT_INTR:
10678                 vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
10679                 /* fall through */
10680         case INTR_TYPE_EXT_INTR:
10681                 kvm_queue_interrupt(vcpu, vector, type == INTR_TYPE_SOFT_INTR);
10682                 break;
10683         default:
10684                 break;
10685         }
10686 }
10687
10688 static void vmx_complete_interrupts(struct vcpu_vmx *vmx)
10689 {
10690         __vmx_complete_interrupts(&vmx->vcpu, vmx->idt_vectoring_info,
10691                                   VM_EXIT_INSTRUCTION_LEN,
10692                                   IDT_VECTORING_ERROR_CODE);
10693 }
10694
10695 static void vmx_cancel_injection(struct kvm_vcpu *vcpu)
10696 {
10697         __vmx_complete_interrupts(vcpu,
10698                                   vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
10699                                   VM_ENTRY_INSTRUCTION_LEN,
10700                                   VM_ENTRY_EXCEPTION_ERROR_CODE);
10701
10702         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
10703 }
10704
10705 static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
10706 {
10707         int i, nr_msrs;
10708         struct perf_guest_switch_msr *msrs;
10709
10710         msrs = perf_guest_get_msrs(&nr_msrs);
10711
10712         if (!msrs)
10713                 return;
10714
10715         for (i = 0; i < nr_msrs; i++)
10716                 if (msrs[i].host == msrs[i].guest)
10717                         clear_atomic_switch_msr(vmx, msrs[i].msr);
10718                 else
10719                         add_atomic_switch_msr(vmx, msrs[i].msr, msrs[i].guest,
10720                                         msrs[i].host, false);
10721 }
10722
10723 static void vmx_arm_hv_timer(struct vcpu_vmx *vmx, u32 val)
10724 {
10725         vmcs_write32(VMX_PREEMPTION_TIMER_VALUE, val);
10726         if (!vmx->loaded_vmcs->hv_timer_armed)
10727                 vmcs_set_bits(PIN_BASED_VM_EXEC_CONTROL,
10728                               PIN_BASED_VMX_PREEMPTION_TIMER);
10729         vmx->loaded_vmcs->hv_timer_armed = true;
10730 }
10731
10732 static void vmx_update_hv_timer(struct kvm_vcpu *vcpu)
10733 {
10734         struct vcpu_vmx *vmx = to_vmx(vcpu);
10735         u64 tscl;
10736         u32 delta_tsc;
10737
10738         if (vmx->req_immediate_exit) {
10739                 vmx_arm_hv_timer(vmx, 0);
10740                 return;
10741         }
10742
10743         if (vmx->hv_deadline_tsc != -1) {
10744                 tscl = rdtsc();
10745                 if (vmx->hv_deadline_tsc > tscl)
10746                         /* set_hv_timer ensures the delta fits in 32-bits */
10747                         delta_tsc = (u32)((vmx->hv_deadline_tsc - tscl) >>
10748                                 cpu_preemption_timer_multi);
10749                 else
10750                         delta_tsc = 0;
10751
10752                 vmx_arm_hv_timer(vmx, delta_tsc);
10753                 return;
10754         }
10755
10756         if (vmx->loaded_vmcs->hv_timer_armed)
10757                 vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL,
10758                                 PIN_BASED_VMX_PREEMPTION_TIMER);
10759         vmx->loaded_vmcs->hv_timer_armed = false;
10760 }
10761
10762 static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
10763 {
10764         struct vcpu_vmx *vmx = to_vmx(vcpu);
10765         unsigned long cr3, cr4, evmcs_rsp;
10766
10767         /* Record the guest's net vcpu time for enforced NMI injections. */
10768         if (unlikely(!enable_vnmi &&
10769                      vmx->loaded_vmcs->soft_vnmi_blocked))
10770                 vmx->loaded_vmcs->entry_time = ktime_get();
10771
10772         /* Don't enter VMX if guest state is invalid, let the exit handler
10773            start emulation until we arrive back to a valid state */
10774         if (vmx->emulation_required)
10775                 return;
10776
10777         if (vmx->ple_window_dirty) {
10778                 vmx->ple_window_dirty = false;
10779                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
10780         }
10781
10782         if (vmx->nested.sync_shadow_vmcs) {
10783                 copy_vmcs12_to_shadow(vmx);
10784                 vmx->nested.sync_shadow_vmcs = false;
10785         }
10786
10787         if (test_bit(VCPU_REGS_RSP, (unsigned long *)&vcpu->arch.regs_dirty))
10788                 vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
10789         if (test_bit(VCPU_REGS_RIP, (unsigned long *)&vcpu->arch.regs_dirty))
10790                 vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
10791
10792         cr3 = __get_current_cr3_fast();
10793         if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
10794                 vmcs_writel(HOST_CR3, cr3);
10795                 vmx->loaded_vmcs->host_state.cr3 = cr3;
10796         }
10797
10798         cr4 = cr4_read_shadow();
10799         if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
10800                 vmcs_writel(HOST_CR4, cr4);
10801                 vmx->loaded_vmcs->host_state.cr4 = cr4;
10802         }
10803
10804         /* When single-stepping over STI and MOV SS, we must clear the
10805          * corresponding interruptibility bits in the guest state. Otherwise
10806          * vmentry fails as it then expects bit 14 (BS) in pending debug
10807          * exceptions being set, but that's not correct for the guest debugging
10808          * case. */
10809         if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
10810                 vmx_set_interrupt_shadow(vcpu, 0);
10811
10812         kvm_load_guest_xcr0(vcpu);
10813
10814         if (static_cpu_has(X86_FEATURE_PKU) &&
10815             kvm_read_cr4_bits(vcpu, X86_CR4_PKE) &&
10816             vcpu->arch.pkru != vmx->host_pkru)
10817                 __write_pkru(vcpu->arch.pkru);
10818
10819         atomic_switch_perf_msrs(vmx);
10820
10821         vmx_update_hv_timer(vcpu);
10822
10823         /*
10824          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
10825          * it's non-zero. Since vmentry is serialising on affected CPUs, there
10826          * is no need to worry about the conditional branch over the wrmsr
10827          * being speculatively taken.
10828          */
10829         x86_spec_ctrl_set_guest(vmx->spec_ctrl, 0);
10830
10831         vmx->__launched = vmx->loaded_vmcs->launched;
10832
10833         evmcs_rsp = static_branch_unlikely(&enable_evmcs) ?
10834                 (unsigned long)&current_evmcs->host_rsp : 0;
10835
10836         /* L1D Flush includes CPU buffer clear to mitigate MDS */
10837         if (static_branch_unlikely(&vmx_l1d_should_flush))
10838                 vmx_l1d_flush(vcpu);
10839         else if (static_branch_unlikely(&mds_user_clear))
10840                 mds_clear_cpu_buffers();
10841         else if (static_branch_unlikely(&mmio_stale_data_clear) &&
10842                  kvm_arch_has_assigned_device(vcpu->kvm))
10843                 mds_clear_cpu_buffers();
10844
10845         vmx_disable_fb_clear(vmx);
10846
10847         asm volatile (
10848                 /* Store host registers */
10849                 "push %%" _ASM_DX "; push %%" _ASM_BP ";"
10850                 "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
10851                 "push %%" _ASM_CX " \n\t"
10852                 "cmp %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t"
10853                 "je 1f \n\t"
10854                 "mov %%" _ASM_SP ", %c[host_rsp](%%" _ASM_CX ") \n\t"
10855                 /* Avoid VMWRITE when Enlightened VMCS is in use */
10856                 "test %%" _ASM_SI ", %%" _ASM_SI " \n\t"
10857                 "jz 2f \n\t"
10858                 "mov %%" _ASM_SP ", (%%" _ASM_SI ") \n\t"
10859                 "jmp 1f \n\t"
10860                 "2: \n\t"
10861                 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
10862                 "1: \n\t"
10863                 /* Reload cr2 if changed */
10864                 "mov %c[cr2](%%" _ASM_CX "), %%" _ASM_AX " \n\t"
10865                 "mov %%cr2, %%" _ASM_DX " \n\t"
10866                 "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
10867                 "je 3f \n\t"
10868                 "mov %%" _ASM_AX", %%cr2 \n\t"
10869                 "3: \n\t"
10870                 /* Check if vmlaunch of vmresume is needed */
10871                 "cmpb $0, %c[launched](%%" _ASM_CX ") \n\t"
10872                 /* Load guest registers.  Don't clobber flags. */
10873                 "mov %c[rax](%%" _ASM_CX "), %%" _ASM_AX " \n\t"
10874                 "mov %c[rbx](%%" _ASM_CX "), %%" _ASM_BX " \n\t"
10875                 "mov %c[rdx](%%" _ASM_CX "), %%" _ASM_DX " \n\t"
10876                 "mov %c[rsi](%%" _ASM_CX "), %%" _ASM_SI " \n\t"
10877                 "mov %c[rdi](%%" _ASM_CX "), %%" _ASM_DI " \n\t"
10878                 "mov %c[rbp](%%" _ASM_CX "), %%" _ASM_BP " \n\t"
10879 #ifdef CONFIG_X86_64
10880                 "mov %c[r8](%%" _ASM_CX "),  %%r8  \n\t"
10881                 "mov %c[r9](%%" _ASM_CX "),  %%r9  \n\t"
10882                 "mov %c[r10](%%" _ASM_CX "), %%r10 \n\t"
10883                 "mov %c[r11](%%" _ASM_CX "), %%r11 \n\t"
10884                 "mov %c[r12](%%" _ASM_CX "), %%r12 \n\t"
10885                 "mov %c[r13](%%" _ASM_CX "), %%r13 \n\t"
10886                 "mov %c[r14](%%" _ASM_CX "), %%r14 \n\t"
10887                 "mov %c[r15](%%" _ASM_CX "), %%r15 \n\t"
10888 #endif
10889                 /* Load guest RCX.  This kills the vmx_vcpu pointer! */
10890                 "mov %c[rcx](%%" _ASM_CX "), %%" _ASM_CX " \n\t"
10891
10892                 /* Enter guest mode */
10893                 "jne 1f \n\t"
10894                 __ex(ASM_VMX_VMLAUNCH) "\n\t"
10895                 "jmp 2f \n\t"
10896                 "1: " __ex(ASM_VMX_VMRESUME) "\n\t"
10897                 "2: "
10898
10899                 /* Save guest's RCX to the stack placeholder (see above) */
10900                 "mov %%" _ASM_CX ", %c[wordsize](%%" _ASM_SP ") \n\t"
10901
10902                 /* Load host's RCX, i.e. the vmx_vcpu pointer */
10903                 "pop %%" _ASM_CX " \n\t"
10904
10905                 /* Set vmx->fail based on EFLAGS.{CF,ZF} */
10906                 "setbe %c[fail](%%" _ASM_CX ")\n\t"
10907
10908                 /* Save all guest registers, including RCX from the stack */
10909                 "mov %%" _ASM_AX ", %c[rax](%%" _ASM_CX ") \n\t"
10910                 "mov %%" _ASM_BX ", %c[rbx](%%" _ASM_CX ") \n\t"
10911                 __ASM_SIZE(pop) " %c[rcx](%%" _ASM_CX ") \n\t"
10912                 "mov %%" _ASM_DX ", %c[rdx](%%" _ASM_CX ") \n\t"
10913                 "mov %%" _ASM_SI ", %c[rsi](%%" _ASM_CX ") \n\t"
10914                 "mov %%" _ASM_DI ", %c[rdi](%%" _ASM_CX ") \n\t"
10915                 "mov %%" _ASM_BP ", %c[rbp](%%" _ASM_CX ") \n\t"
10916 #ifdef CONFIG_X86_64
10917                 "mov %%r8,  %c[r8](%%" _ASM_CX ") \n\t"
10918                 "mov %%r9,  %c[r9](%%" _ASM_CX ") \n\t"
10919                 "mov %%r10, %c[r10](%%" _ASM_CX ") \n\t"
10920                 "mov %%r11, %c[r11](%%" _ASM_CX ") \n\t"
10921                 "mov %%r12, %c[r12](%%" _ASM_CX ") \n\t"
10922                 "mov %%r13, %c[r13](%%" _ASM_CX ") \n\t"
10923                 "mov %%r14, %c[r14](%%" _ASM_CX ") \n\t"
10924                 "mov %%r15, %c[r15](%%" _ASM_CX ") \n\t"
10925
10926                 /*
10927                  * Clear all general purpose registers (except RSP, which is loaded by
10928                  * the CPU during VM-Exit) to prevent speculative use of the guest's
10929                  * values, even those that are saved/loaded via the stack.  In theory,
10930                  * an L1 cache miss when restoring registers could lead to speculative
10931                  * execution with the guest's values.  Zeroing XORs are dirt cheap,
10932                  * i.e. the extra paranoia is essentially free.
10933                  */
10934                 "xor %%r8d,  %%r8d \n\t"
10935                 "xor %%r9d,  %%r9d \n\t"
10936                 "xor %%r10d, %%r10d \n\t"
10937                 "xor %%r11d, %%r11d \n\t"
10938                 "xor %%r12d, %%r12d \n\t"
10939                 "xor %%r13d, %%r13d \n\t"
10940                 "xor %%r14d, %%r14d \n\t"
10941                 "xor %%r15d, %%r15d \n\t"
10942 #endif
10943                 "mov %%cr2, %%" _ASM_AX "   \n\t"
10944                 "mov %%" _ASM_AX ", %c[cr2](%%" _ASM_CX ") \n\t"
10945
10946                 "xor %%eax, %%eax \n\t"
10947                 "xor %%ebx, %%ebx \n\t"
10948                 "xor %%ecx, %%ecx \n\t"
10949                 "xor %%edx, %%edx \n\t"
10950                 "xor %%esi, %%esi \n\t"
10951                 "xor %%edi, %%edi \n\t"
10952                 "xor %%ebp, %%ebp \n\t"
10953                 "pop  %%" _ASM_BP "; pop  %%" _ASM_DX " \n\t"
10954                 ".pushsection .rodata \n\t"
10955                 ".global vmx_return \n\t"
10956                 "vmx_return: " _ASM_PTR " 2b \n\t"
10957                 ".popsection"
10958               : "=c"((int){0}), "=d"((int){0}), "=S"((int){0})
10959               : "c"(vmx), "d"((unsigned long)HOST_RSP), "S"(evmcs_rsp),
10960                 [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
10961                 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
10962                 [host_rsp]"i"(offsetof(struct vcpu_vmx, host_rsp)),
10963                 [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])),
10964                 [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])),
10965                 [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])),
10966                 [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])),
10967                 [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])),
10968                 [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])),
10969                 [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])),
10970 #ifdef CONFIG_X86_64
10971                 [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])),
10972                 [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])),
10973                 [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])),
10974                 [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])),
10975                 [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])),
10976                 [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])),
10977                 [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])),
10978                 [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])),
10979 #endif
10980                 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
10981                 [wordsize]"i"(sizeof(ulong))
10982               : "cc", "memory"
10983 #ifdef CONFIG_X86_64
10984                 , "rax", "rbx", "rdi"
10985                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
10986 #else
10987                 , "eax", "ebx", "edi"
10988 #endif
10989               );
10990
10991         /* Eliminate branch target predictions from guest mode */
10992         vmexit_fill_RSB();
10993
10994         vmx_enable_fb_clear(vmx);
10995
10996         /*
10997          * We do not use IBRS in the kernel. If this vCPU has used the
10998          * SPEC_CTRL MSR it may have left it on; save the value and
10999          * turn it off. This is much more efficient than blindly adding
11000          * it to the atomic save/restore list. Especially as the former
11001          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
11002          *
11003          * For non-nested case:
11004          * If the L01 MSR bitmap does not intercept the MSR, then we need to
11005          * save it.
11006          *
11007          * For nested case:
11008          * If the L02 MSR bitmap does not intercept the MSR, then we need to
11009          * save it.
11010          */
11011         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
11012                 vmx->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
11013
11014         x86_spec_ctrl_restore_host(vmx->spec_ctrl, 0);
11015
11016         /* All fields are clean at this point */
11017         if (static_branch_unlikely(&enable_evmcs))
11018                 current_evmcs->hv_clean_fields |=
11019                         HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
11020
11021         /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
11022         if (vmx->host_debugctlmsr)
11023                 update_debugctlmsr(vmx->host_debugctlmsr);
11024
11025 #ifndef CONFIG_X86_64
11026         /*
11027          * The sysexit path does not restore ds/es, so we must set them to
11028          * a reasonable value ourselves.
11029          *
11030          * We can't defer this to vmx_prepare_switch_to_host() since that
11031          * function may be executed in interrupt context, which saves and
11032          * restore segments around it, nullifying its effect.
11033          */
11034         loadsegment(ds, __USER_DS);
11035         loadsegment(es, __USER_DS);
11036 #endif
11037
11038         vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)
11039                                   | (1 << VCPU_EXREG_RFLAGS)
11040                                   | (1 << VCPU_EXREG_PDPTR)
11041                                   | (1 << VCPU_EXREG_SEGMENTS)
11042                                   | (1 << VCPU_EXREG_CR3));
11043         vcpu->arch.regs_dirty = 0;
11044
11045         /*
11046          * eager fpu is enabled if PKEY is supported and CR4 is switched
11047          * back on host, so it is safe to read guest PKRU from current
11048          * XSAVE.
11049          */
11050         if (static_cpu_has(X86_FEATURE_PKU) &&
11051             kvm_read_cr4_bits(vcpu, X86_CR4_PKE)) {
11052                 vcpu->arch.pkru = __read_pkru();
11053                 if (vcpu->arch.pkru != vmx->host_pkru)
11054                         __write_pkru(vmx->host_pkru);
11055         }
11056
11057         kvm_put_guest_xcr0(vcpu);
11058
11059         vmx->nested.nested_run_pending = 0;
11060         vmx->idt_vectoring_info = 0;
11061
11062         vmx->exit_reason = vmx->fail ? 0xdead : vmcs_read32(VM_EXIT_REASON);
11063         if ((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)
11064                 kvm_machine_check();
11065
11066         if (vmx->fail || (vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY))
11067                 return;
11068
11069         vmx->loaded_vmcs->launched = 1;
11070         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
11071
11072         vmx_complete_atomic_exit(vmx);
11073         vmx_recover_nmi_blocking(vmx);
11074         vmx_complete_interrupts(vmx);
11075 }
11076 STACK_FRAME_NON_STANDARD(vmx_vcpu_run);
11077
11078 static struct kvm *vmx_vm_alloc(void)
11079 {
11080         struct kvm_vmx *kvm_vmx = vzalloc(sizeof(struct kvm_vmx));
11081
11082         if (!kvm_vmx)
11083                 return NULL;
11084
11085         return &kvm_vmx->kvm;
11086 }
11087
11088 static void vmx_vm_free(struct kvm *kvm)
11089 {
11090         vfree(to_kvm_vmx(kvm));
11091 }
11092
11093 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
11094 {
11095         struct vcpu_vmx *vmx = to_vmx(vcpu);
11096         int cpu;
11097
11098         if (vmx->loaded_vmcs == vmcs)
11099                 return;
11100
11101         cpu = get_cpu();
11102         vmx_vcpu_put(vcpu);
11103         vmx->loaded_vmcs = vmcs;
11104         vmx_vcpu_load(vcpu, cpu);
11105         put_cpu();
11106 }
11107
11108 /*
11109  * Ensure that the current vmcs of the logical processor is the
11110  * vmcs01 of the vcpu before calling free_nested().
11111  */
11112 static void vmx_free_vcpu_nested(struct kvm_vcpu *vcpu)
11113 {
11114        struct vcpu_vmx *vmx = to_vmx(vcpu);
11115
11116        vcpu_load(vcpu);
11117        vmx_switch_vmcs(vcpu, &vmx->vmcs01);
11118        free_nested(vmx);
11119        vcpu_put(vcpu);
11120 }
11121
11122 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
11123 {
11124         struct vcpu_vmx *vmx = to_vmx(vcpu);
11125
11126         if (enable_pml)
11127                 vmx_destroy_pml_buffer(vmx);
11128         free_vpid(vmx->vpid);
11129         leave_guest_mode(vcpu);
11130         vmx_free_vcpu_nested(vcpu);
11131         free_loaded_vmcs(vmx->loaded_vmcs);
11132         kfree(vmx->guest_msrs);
11133         kvm_vcpu_uninit(vcpu);
11134         kmem_cache_free(kvm_vcpu_cache, vmx);
11135 }
11136
11137 static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
11138 {
11139         int err;
11140         struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
11141         unsigned long *msr_bitmap;
11142         int cpu;
11143
11144         if (!vmx)
11145                 return ERR_PTR(-ENOMEM);
11146
11147         vmx->vpid = allocate_vpid();
11148
11149         err = kvm_vcpu_init(&vmx->vcpu, kvm, id);
11150         if (err)
11151                 goto free_vcpu;
11152
11153         err = -ENOMEM;
11154
11155         /*
11156          * If PML is turned on, failure on enabling PML just results in failure
11157          * of creating the vcpu, therefore we can simplify PML logic (by
11158          * avoiding dealing with cases, such as enabling PML partially on vcpus
11159          * for the guest, etc.
11160          */
11161         if (enable_pml) {
11162                 vmx->pml_pg = alloc_page(GFP_KERNEL | __GFP_ZERO);
11163                 if (!vmx->pml_pg)
11164                         goto uninit_vcpu;
11165         }
11166
11167         vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
11168         BUILD_BUG_ON(ARRAY_SIZE(vmx_msr_index) * sizeof(vmx->guest_msrs[0])
11169                      > PAGE_SIZE);
11170
11171         if (!vmx->guest_msrs)
11172                 goto free_pml;
11173
11174         err = alloc_loaded_vmcs(&vmx->vmcs01);
11175         if (err < 0)
11176                 goto free_msrs;
11177
11178         msr_bitmap = vmx->vmcs01.msr_bitmap;
11179         vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE, MSR_TYPE_RW);
11180         vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE, MSR_TYPE_RW);
11181         vmx_disable_intercept_for_msr(msr_bitmap, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
11182         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
11183         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW);
11184         vmx_disable_intercept_for_msr(msr_bitmap, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW);
11185         vmx->msr_bitmap_mode = 0;
11186
11187         vmx->loaded_vmcs = &vmx->vmcs01;
11188         cpu = get_cpu();
11189         vmx_vcpu_load(&vmx->vcpu, cpu);
11190         vmx->vcpu.cpu = cpu;
11191         vmx_vcpu_setup(vmx);
11192         vmx_vcpu_put(&vmx->vcpu);
11193         put_cpu();
11194         if (cpu_need_virtualize_apic_accesses(&vmx->vcpu)) {
11195                 err = alloc_apic_access_page(kvm);
11196                 if (err)
11197                         goto free_vmcs;
11198         }
11199
11200         if (enable_ept && !enable_unrestricted_guest) {
11201                 err = init_rmode_identity_map(kvm);
11202                 if (err)
11203                         goto free_vmcs;
11204         }
11205
11206         if (nested)
11207                 nested_vmx_setup_ctls_msrs(&vmx->nested.msrs,
11208                                            kvm_vcpu_apicv_active(&vmx->vcpu));
11209
11210         vmx->nested.posted_intr_nv = -1;
11211         vmx->nested.current_vmptr = -1ull;
11212
11213         vmx->msr_ia32_feature_control_valid_bits = FEATURE_CONTROL_LOCKED;
11214
11215         /*
11216          * Enforce invariant: pi_desc.nv is always either POSTED_INTR_VECTOR
11217          * or POSTED_INTR_WAKEUP_VECTOR.
11218          */
11219         vmx->pi_desc.nv = POSTED_INTR_VECTOR;
11220         vmx->pi_desc.sn = 1;
11221
11222         return &vmx->vcpu;
11223
11224 free_vmcs:
11225         free_loaded_vmcs(vmx->loaded_vmcs);
11226 free_msrs:
11227         kfree(vmx->guest_msrs);
11228 free_pml:
11229         vmx_destroy_pml_buffer(vmx);
11230 uninit_vcpu:
11231         kvm_vcpu_uninit(&vmx->vcpu);
11232 free_vcpu:
11233         free_vpid(vmx->vpid);
11234         kmem_cache_free(kvm_vcpu_cache, vmx);
11235         return ERR_PTR(err);
11236 }
11237
11238 #define L1TF_MSG_SMT "L1TF CPU bug present and SMT on, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
11239 #define L1TF_MSG_L1D "L1TF CPU bug present and virtualization mitigation disabled, data leak possible. See CVE-2018-3646 and https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html for details.\n"
11240
11241 static int vmx_vm_init(struct kvm *kvm)
11242 {
11243         spin_lock_init(&to_kvm_vmx(kvm)->ept_pointer_lock);
11244
11245         if (!ple_gap)
11246                 kvm->arch.pause_in_guest = true;
11247
11248         if (boot_cpu_has(X86_BUG_L1TF) && enable_ept) {
11249                 switch (l1tf_mitigation) {
11250                 case L1TF_MITIGATION_OFF:
11251                 case L1TF_MITIGATION_FLUSH_NOWARN:
11252                         /* 'I explicitly don't care' is set */
11253                         break;
11254                 case L1TF_MITIGATION_FLUSH:
11255                 case L1TF_MITIGATION_FLUSH_NOSMT:
11256                 case L1TF_MITIGATION_FULL:
11257                         /*
11258                          * Warn upon starting the first VM in a potentially
11259                          * insecure environment.
11260                          */
11261                         if (sched_smt_active())
11262                                 pr_warn_once(L1TF_MSG_SMT);
11263                         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER)
11264                                 pr_warn_once(L1TF_MSG_L1D);
11265                         break;
11266                 case L1TF_MITIGATION_FULL_FORCE:
11267                         /* Flush is enforced */
11268                         break;
11269                 }
11270         }
11271         return 0;
11272 }
11273
11274 static void __init vmx_check_processor_compat(void *rtn)
11275 {
11276         struct vmcs_config vmcs_conf;
11277
11278         *(int *)rtn = 0;
11279         if (setup_vmcs_config(&vmcs_conf) < 0)
11280                 *(int *)rtn = -EIO;
11281         nested_vmx_setup_ctls_msrs(&vmcs_conf.nested, enable_apicv);
11282         if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) {
11283                 printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n",
11284                                 smp_processor_id());
11285                 *(int *)rtn = -EIO;
11286         }
11287 }
11288
11289 static u64 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
11290 {
11291         u8 cache;
11292         u64 ipat = 0;
11293
11294         /* For VT-d and EPT combination
11295          * 1. MMIO: always map as UC
11296          * 2. EPT with VT-d:
11297          *   a. VT-d without snooping control feature: can't guarantee the
11298          *      result, try to trust guest.
11299          *   b. VT-d with snooping control feature: snooping control feature of
11300          *      VT-d engine can guarantee the cache correctness. Just set it
11301          *      to WB to keep consistent with host. So the same as item 3.
11302          * 3. EPT without VT-d: always map as WB and set IPAT=1 to keep
11303          *    consistent with host MTRR
11304          */
11305         if (is_mmio) {
11306                 cache = MTRR_TYPE_UNCACHABLE;
11307                 goto exit;
11308         }
11309
11310         if (!kvm_arch_has_noncoherent_dma(vcpu->kvm)) {
11311                 ipat = VMX_EPT_IPAT_BIT;
11312                 cache = MTRR_TYPE_WRBACK;
11313                 goto exit;
11314         }
11315
11316         if (kvm_read_cr0(vcpu) & X86_CR0_CD) {
11317                 ipat = VMX_EPT_IPAT_BIT;
11318                 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
11319                         cache = MTRR_TYPE_WRBACK;
11320                 else
11321                         cache = MTRR_TYPE_UNCACHABLE;
11322                 goto exit;
11323         }
11324
11325         cache = kvm_mtrr_get_guest_memory_type(vcpu, gfn);
11326
11327 exit:
11328         return (cache << VMX_EPT_MT_EPTE_SHIFT) | ipat;
11329 }
11330
11331 static int vmx_get_lpage_level(void)
11332 {
11333         if (enable_ept && !cpu_has_vmx_ept_1g_page())
11334                 return PT_DIRECTORY_LEVEL;
11335         else
11336                 /* For shadow and EPT supported 1GB page */
11337                 return PT_PDPE_LEVEL;
11338 }
11339
11340 static void vmcs_set_secondary_exec_control(u32 new_ctl)
11341 {
11342         /*
11343          * These bits in the secondary execution controls field
11344          * are dynamic, the others are mostly based on the hypervisor
11345          * architecture and the guest's CPUID.  Do not touch the
11346          * dynamic bits.
11347          */
11348         u32 mask =
11349                 SECONDARY_EXEC_SHADOW_VMCS |
11350                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
11351                 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
11352                 SECONDARY_EXEC_DESC;
11353
11354         u32 cur_ctl = vmcs_read32(SECONDARY_VM_EXEC_CONTROL);
11355
11356         vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
11357                      (new_ctl & ~mask) | (cur_ctl & mask));
11358 }
11359
11360 /*
11361  * Generate MSR_IA32_VMX_CR{0,4}_FIXED1 according to CPUID. Only set bits
11362  * (indicating "allowed-1") if they are supported in the guest's CPUID.
11363  */
11364 static void nested_vmx_cr_fixed1_bits_update(struct kvm_vcpu *vcpu)
11365 {
11366         struct vcpu_vmx *vmx = to_vmx(vcpu);
11367         struct kvm_cpuid_entry2 *entry;
11368
11369         vmx->nested.msrs.cr0_fixed1 = 0xffffffff;
11370         vmx->nested.msrs.cr4_fixed1 = X86_CR4_PCE;
11371
11372 #define cr4_fixed1_update(_cr4_mask, _reg, _cpuid_mask) do {            \
11373         if (entry && (entry->_reg & (_cpuid_mask)))                     \
11374                 vmx->nested.msrs.cr4_fixed1 |= (_cr4_mask);     \
11375 } while (0)
11376
11377         entry = kvm_find_cpuid_entry(vcpu, 0x1, 0);
11378         cr4_fixed1_update(X86_CR4_VME,        edx, bit(X86_FEATURE_VME));
11379         cr4_fixed1_update(X86_CR4_PVI,        edx, bit(X86_FEATURE_VME));
11380         cr4_fixed1_update(X86_CR4_TSD,        edx, bit(X86_FEATURE_TSC));
11381         cr4_fixed1_update(X86_CR4_DE,         edx, bit(X86_FEATURE_DE));
11382         cr4_fixed1_update(X86_CR4_PSE,        edx, bit(X86_FEATURE_PSE));
11383         cr4_fixed1_update(X86_CR4_PAE,        edx, bit(X86_FEATURE_PAE));
11384         cr4_fixed1_update(X86_CR4_MCE,        edx, bit(X86_FEATURE_MCE));
11385         cr4_fixed1_update(X86_CR4_PGE,        edx, bit(X86_FEATURE_PGE));
11386         cr4_fixed1_update(X86_CR4_OSFXSR,     edx, bit(X86_FEATURE_FXSR));
11387         cr4_fixed1_update(X86_CR4_OSXMMEXCPT, edx, bit(X86_FEATURE_XMM));
11388         cr4_fixed1_update(X86_CR4_VMXE,       ecx, bit(X86_FEATURE_VMX));
11389         cr4_fixed1_update(X86_CR4_SMXE,       ecx, bit(X86_FEATURE_SMX));
11390         cr4_fixed1_update(X86_CR4_PCIDE,      ecx, bit(X86_FEATURE_PCID));
11391         cr4_fixed1_update(X86_CR4_OSXSAVE,    ecx, bit(X86_FEATURE_XSAVE));
11392
11393         entry = kvm_find_cpuid_entry(vcpu, 0x7, 0);
11394         cr4_fixed1_update(X86_CR4_FSGSBASE,   ebx, bit(X86_FEATURE_FSGSBASE));
11395         cr4_fixed1_update(X86_CR4_SMEP,       ebx, bit(X86_FEATURE_SMEP));
11396         cr4_fixed1_update(X86_CR4_SMAP,       ebx, bit(X86_FEATURE_SMAP));
11397         cr4_fixed1_update(X86_CR4_PKE,        ecx, bit(X86_FEATURE_PKU));
11398         cr4_fixed1_update(X86_CR4_UMIP,       ecx, bit(X86_FEATURE_UMIP));
11399
11400 #undef cr4_fixed1_update
11401 }
11402
11403 static void nested_vmx_entry_exit_ctls_update(struct kvm_vcpu *vcpu)
11404 {
11405         struct vcpu_vmx *vmx = to_vmx(vcpu);
11406
11407         if (kvm_mpx_supported()) {
11408                 bool mpx_enabled = guest_cpuid_has(vcpu, X86_FEATURE_MPX);
11409
11410                 if (mpx_enabled) {
11411                         vmx->nested.msrs.entry_ctls_high |= VM_ENTRY_LOAD_BNDCFGS;
11412                         vmx->nested.msrs.exit_ctls_high |= VM_EXIT_CLEAR_BNDCFGS;
11413                 } else {
11414                         vmx->nested.msrs.entry_ctls_high &= ~VM_ENTRY_LOAD_BNDCFGS;
11415                         vmx->nested.msrs.exit_ctls_high &= ~VM_EXIT_CLEAR_BNDCFGS;
11416                 }
11417         }
11418 }
11419
11420 static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
11421 {
11422         struct vcpu_vmx *vmx = to_vmx(vcpu);
11423
11424         if (cpu_has_secondary_exec_ctrls()) {
11425                 vmx_compute_secondary_exec_control(vmx);
11426                 vmcs_set_secondary_exec_control(vmx->secondary_exec_control);
11427         }
11428
11429         if (nested_vmx_allowed(vcpu))
11430                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
11431                         FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
11432         else
11433                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
11434                         ~FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
11435
11436         if (nested_vmx_allowed(vcpu)) {
11437                 nested_vmx_cr_fixed1_bits_update(vcpu);
11438                 nested_vmx_entry_exit_ctls_update(vcpu);
11439         }
11440 }
11441
11442 static void vmx_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
11443 {
11444         if (func == 1 && nested)
11445                 entry->ecx |= bit(X86_FEATURE_VMX);
11446 }
11447
11448 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
11449                 struct x86_exception *fault)
11450 {
11451         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11452         struct vcpu_vmx *vmx = to_vmx(vcpu);
11453         u32 exit_reason;
11454         unsigned long exit_qualification = vcpu->arch.exit_qualification;
11455
11456         if (vmx->nested.pml_full) {
11457                 exit_reason = EXIT_REASON_PML_FULL;
11458                 vmx->nested.pml_full = false;
11459                 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
11460         } else if (fault->error_code & PFERR_RSVD_MASK)
11461                 exit_reason = EXIT_REASON_EPT_MISCONFIG;
11462         else
11463                 exit_reason = EXIT_REASON_EPT_VIOLATION;
11464
11465         nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
11466         vmcs12->guest_physical_address = fault->address;
11467 }
11468
11469 static bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu)
11470 {
11471         return nested_ept_get_cr3(vcpu) & VMX_EPTP_AD_ENABLE_BIT;
11472 }
11473
11474 /* Callbacks for nested_ept_init_mmu_context: */
11475
11476 static unsigned long nested_ept_get_cr3(struct kvm_vcpu *vcpu)
11477 {
11478         /* return the page table to be shadowed - in our case, EPT12 */
11479         return get_vmcs12(vcpu)->ept_pointer;
11480 }
11481
11482 static int nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
11483 {
11484         WARN_ON(mmu_is_nested(vcpu));
11485         if (!valid_ept_address(vcpu, nested_ept_get_cr3(vcpu)))
11486                 return 1;
11487
11488         kvm_init_shadow_ept_mmu(vcpu,
11489                         to_vmx(vcpu)->nested.msrs.ept_caps &
11490                         VMX_EPT_EXECUTE_ONLY_BIT,
11491                         nested_ept_ad_enabled(vcpu),
11492                         nested_ept_get_cr3(vcpu));
11493         vcpu->arch.mmu.set_cr3           = vmx_set_cr3;
11494         vcpu->arch.mmu.get_cr3           = nested_ept_get_cr3;
11495         vcpu->arch.mmu.inject_page_fault = nested_ept_inject_page_fault;
11496
11497         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
11498         return 0;
11499 }
11500
11501 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
11502 {
11503         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
11504 }
11505
11506 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
11507                                             u16 error_code)
11508 {
11509         bool inequality, bit;
11510
11511         bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
11512         inequality =
11513                 (error_code & vmcs12->page_fault_error_code_mask) !=
11514                  vmcs12->page_fault_error_code_match;
11515         return inequality ^ bit;
11516 }
11517
11518 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
11519                 struct x86_exception *fault)
11520 {
11521         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11522
11523         WARN_ON(!is_guest_mode(vcpu));
11524
11525         if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
11526                 !to_vmx(vcpu)->nested.nested_run_pending) {
11527                 vmcs12->vm_exit_intr_error_code = fault->error_code;
11528                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
11529                                   PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
11530                                   INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
11531                                   fault->address);
11532         } else {
11533                 kvm_inject_page_fault(vcpu, fault);
11534         }
11535 }
11536
11537 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
11538                                                  struct vmcs12 *vmcs12);
11539
11540 static void nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
11541 {
11542         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11543         struct vcpu_vmx *vmx = to_vmx(vcpu);
11544         struct page *page;
11545         u64 hpa;
11546
11547         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
11548                 /*
11549                  * Translate L1 physical address to host physical
11550                  * address for vmcs02. Keep the page pinned, so this
11551                  * physical address remains valid. We keep a reference
11552                  * to it so we can release it later.
11553                  */
11554                 if (vmx->nested.apic_access_page) { /* shouldn't happen */
11555                         kvm_release_page_dirty(vmx->nested.apic_access_page);
11556                         vmx->nested.apic_access_page = NULL;
11557                 }
11558                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
11559                 /*
11560                  * If translation failed, no matter: This feature asks
11561                  * to exit when accessing the given address, and if it
11562                  * can never be accessed, this feature won't do
11563                  * anything anyway.
11564                  */
11565                 if (!is_error_page(page)) {
11566                         vmx->nested.apic_access_page = page;
11567                         hpa = page_to_phys(vmx->nested.apic_access_page);
11568                         vmcs_write64(APIC_ACCESS_ADDR, hpa);
11569                 } else {
11570                         vmcs_clear_bits(SECONDARY_VM_EXEC_CONTROL,
11571                                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES);
11572                 }
11573         }
11574
11575         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
11576                 if (vmx->nested.virtual_apic_page) { /* shouldn't happen */
11577                         kvm_release_page_dirty(vmx->nested.virtual_apic_page);
11578                         vmx->nested.virtual_apic_page = NULL;
11579                 }
11580                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->virtual_apic_page_addr);
11581
11582                 /*
11583                  * If translation failed, VM entry will fail because
11584                  * prepare_vmcs02 set VIRTUAL_APIC_PAGE_ADDR to -1ull.
11585                  * Failing the vm entry is _not_ what the processor
11586                  * does but it's basically the only possibility we
11587                  * have.  We could still enter the guest if CR8 load
11588                  * exits are enabled, CR8 store exits are enabled, and
11589                  * virtualize APIC access is disabled; in this case
11590                  * the processor would never use the TPR shadow and we
11591                  * could simply clear the bit from the execution
11592                  * control.  But such a configuration is useless, so
11593                  * let's keep the code simple.
11594                  */
11595                 if (!is_error_page(page)) {
11596                         vmx->nested.virtual_apic_page = page;
11597                         hpa = page_to_phys(vmx->nested.virtual_apic_page);
11598                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, hpa);
11599                 }
11600         }
11601
11602         if (nested_cpu_has_posted_intr(vmcs12)) {
11603                 if (vmx->nested.pi_desc_page) { /* shouldn't happen */
11604                         kunmap(vmx->nested.pi_desc_page);
11605                         kvm_release_page_dirty(vmx->nested.pi_desc_page);
11606                         vmx->nested.pi_desc_page = NULL;
11607                         vmx->nested.pi_desc = NULL;
11608                         vmcs_write64(POSTED_INTR_DESC_ADDR, -1ull);
11609                 }
11610                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->posted_intr_desc_addr);
11611                 if (is_error_page(page))
11612                         return;
11613                 vmx->nested.pi_desc_page = page;
11614                 vmx->nested.pi_desc = kmap(vmx->nested.pi_desc_page);
11615                 vmx->nested.pi_desc =
11616                         (struct pi_desc *)((void *)vmx->nested.pi_desc +
11617                         (unsigned long)(vmcs12->posted_intr_desc_addr &
11618                         (PAGE_SIZE - 1)));
11619                 vmcs_write64(POSTED_INTR_DESC_ADDR,
11620                         page_to_phys(vmx->nested.pi_desc_page) +
11621                         (unsigned long)(vmcs12->posted_intr_desc_addr &
11622                         (PAGE_SIZE - 1)));
11623         }
11624         if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
11625                 vmcs_set_bits(CPU_BASED_VM_EXEC_CONTROL,
11626                               CPU_BASED_USE_MSR_BITMAPS);
11627         else
11628                 vmcs_clear_bits(CPU_BASED_VM_EXEC_CONTROL,
11629                                 CPU_BASED_USE_MSR_BITMAPS);
11630 }
11631
11632 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
11633 {
11634         u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
11635         struct vcpu_vmx *vmx = to_vmx(vcpu);
11636
11637         /*
11638          * A timer value of zero is architecturally guaranteed to cause
11639          * a VMExit prior to executing any instructions in the guest.
11640          */
11641         if (preemption_timeout == 0) {
11642                 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
11643                 return;
11644         }
11645
11646         if (vcpu->arch.virtual_tsc_khz == 0)
11647                 return;
11648
11649         preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
11650         preemption_timeout *= 1000000;
11651         do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
11652         hrtimer_start(&vmx->nested.preemption_timer,
11653                       ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
11654 }
11655
11656 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
11657                                                struct vmcs12 *vmcs12)
11658 {
11659         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
11660                 return 0;
11661
11662         if (!page_address_valid(vcpu, vmcs12->io_bitmap_a) ||
11663             !page_address_valid(vcpu, vmcs12->io_bitmap_b))
11664                 return -EINVAL;
11665
11666         return 0;
11667 }
11668
11669 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
11670                                                 struct vmcs12 *vmcs12)
11671 {
11672         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
11673                 return 0;
11674
11675         if (!page_address_valid(vcpu, vmcs12->msr_bitmap))
11676                 return -EINVAL;
11677
11678         return 0;
11679 }
11680
11681 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
11682                                                 struct vmcs12 *vmcs12)
11683 {
11684         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
11685                 return 0;
11686
11687         if (!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))
11688                 return -EINVAL;
11689
11690         return 0;
11691 }
11692
11693 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) {
11694         int msr;
11695
11696         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
11697                 unsigned word = msr / BITS_PER_LONG;
11698
11699                 msr_bitmap[word] = ~0;
11700                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
11701         }
11702 }
11703
11704 /*
11705  * Merge L0's and L1's MSR bitmap, return false to indicate that
11706  * we do not use the hardware.
11707  */
11708 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
11709                                                  struct vmcs12 *vmcs12)
11710 {
11711         int msr;
11712         struct page *page;
11713         unsigned long *msr_bitmap_l1;
11714         unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
11715         /*
11716          * pred_cmd & spec_ctrl are trying to verify two things:
11717          *
11718          * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
11719          *    ensures that we do not accidentally generate an L02 MSR bitmap
11720          *    from the L12 MSR bitmap that is too permissive.
11721          * 2. That L1 or L2s have actually used the MSR. This avoids
11722          *    unnecessarily merging of the bitmap if the MSR is unused. This
11723          *    works properly because we only update the L01 MSR bitmap lazily.
11724          *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
11725          *    updated to reflect this when L1 (or its L2s) actually write to
11726          *    the MSR.
11727          */
11728         bool pred_cmd = !msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD);
11729         bool spec_ctrl = !msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL);
11730
11731         /* Nothing to do if the MSR bitmap is not in use.  */
11732         if (!cpu_has_vmx_msr_bitmap() ||
11733             !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
11734                 return false;
11735
11736         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
11737             !pred_cmd && !spec_ctrl)
11738                 return false;
11739
11740         page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->msr_bitmap);
11741         if (is_error_page(page))
11742                 return false;
11743
11744         msr_bitmap_l1 = (unsigned long *)kmap(page);
11745
11746         /*
11747          * To keep the control flow simple, pay eight 8-byte writes (sixteen
11748          * 4-byte writes on 32-bit systems) up front to enable intercepts for
11749          * the x2APIC MSR range and selectively disable them below.
11750          */
11751         enable_x2apic_msr_intercepts(msr_bitmap_l0);
11752
11753         if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
11754                 if (nested_cpu_has_apic_reg_virt(vmcs12)) {
11755                         /*
11756                          * L0 need not intercept reads for MSRs between 0x800
11757                          * and 0x8ff, it just lets the processor take the value
11758                          * from the virtual-APIC page; take those 256 bits
11759                          * directly from the L1 bitmap.
11760                          */
11761                         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
11762                                 unsigned word = msr / BITS_PER_LONG;
11763
11764                                 msr_bitmap_l0[word] = msr_bitmap_l1[word];
11765                         }
11766                 }
11767
11768                 nested_vmx_disable_intercept_for_msr(
11769                         msr_bitmap_l1, msr_bitmap_l0,
11770                         X2APIC_MSR(APIC_TASKPRI),
11771                         MSR_TYPE_R | MSR_TYPE_W);
11772
11773                 if (nested_cpu_has_vid(vmcs12)) {
11774                         nested_vmx_disable_intercept_for_msr(
11775                                 msr_bitmap_l1, msr_bitmap_l0,
11776                                 X2APIC_MSR(APIC_EOI),
11777                                 MSR_TYPE_W);
11778                         nested_vmx_disable_intercept_for_msr(
11779                                 msr_bitmap_l1, msr_bitmap_l0,
11780                                 X2APIC_MSR(APIC_SELF_IPI),
11781                                 MSR_TYPE_W);
11782                 }
11783         }
11784
11785         if (spec_ctrl)
11786                 nested_vmx_disable_intercept_for_msr(
11787                                         msr_bitmap_l1, msr_bitmap_l0,
11788                                         MSR_IA32_SPEC_CTRL,
11789                                         MSR_TYPE_R | MSR_TYPE_W);
11790
11791         if (pred_cmd)
11792                 nested_vmx_disable_intercept_for_msr(
11793                                         msr_bitmap_l1, msr_bitmap_l0,
11794                                         MSR_IA32_PRED_CMD,
11795                                         MSR_TYPE_W);
11796
11797         kunmap(page);
11798         kvm_release_page_clean(page);
11799
11800         return true;
11801 }
11802
11803 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
11804                                        struct vmcs12 *vmcs12)
11805 {
11806         struct vmcs12 *shadow;
11807         struct page *page;
11808
11809         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
11810             vmcs12->vmcs_link_pointer == -1ull)
11811                 return;
11812
11813         shadow = get_shadow_vmcs12(vcpu);
11814         page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer);
11815
11816         memcpy(shadow, kmap(page), VMCS12_SIZE);
11817
11818         kunmap(page);
11819         kvm_release_page_clean(page);
11820 }
11821
11822 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
11823                                               struct vmcs12 *vmcs12)
11824 {
11825         struct vcpu_vmx *vmx = to_vmx(vcpu);
11826
11827         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
11828             vmcs12->vmcs_link_pointer == -1ull)
11829                 return;
11830
11831         kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
11832                         get_shadow_vmcs12(vcpu), VMCS12_SIZE);
11833 }
11834
11835 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
11836                                           struct vmcs12 *vmcs12)
11837 {
11838         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
11839             !page_address_valid(vcpu, vmcs12->apic_access_addr))
11840                 return -EINVAL;
11841         else
11842                 return 0;
11843 }
11844
11845 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
11846                                            struct vmcs12 *vmcs12)
11847 {
11848         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
11849             !nested_cpu_has_apic_reg_virt(vmcs12) &&
11850             !nested_cpu_has_vid(vmcs12) &&
11851             !nested_cpu_has_posted_intr(vmcs12))
11852                 return 0;
11853
11854         /*
11855          * If virtualize x2apic mode is enabled,
11856          * virtualize apic access must be disabled.
11857          */
11858         if (nested_cpu_has_virt_x2apic_mode(vmcs12) &&
11859             nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
11860                 return -EINVAL;
11861
11862         /*
11863          * If virtual interrupt delivery is enabled,
11864          * we must exit on external interrupts.
11865          */
11866         if (nested_cpu_has_vid(vmcs12) &&
11867            !nested_exit_on_intr(vcpu))
11868                 return -EINVAL;
11869
11870         /*
11871          * bits 15:8 should be zero in posted_intr_nv,
11872          * the descriptor address has been already checked
11873          * in nested_get_vmcs12_pages.
11874          *
11875          * bits 5:0 of posted_intr_desc_addr should be zero.
11876          */
11877         if (nested_cpu_has_posted_intr(vmcs12) &&
11878            (!nested_cpu_has_vid(vmcs12) ||
11879             !nested_exit_intr_ack_set(vcpu) ||
11880             (vmcs12->posted_intr_nv & 0xff00) ||
11881             (vmcs12->posted_intr_desc_addr & 0x3f) ||
11882             (vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu))))
11883                 return -EINVAL;
11884
11885         /* tpr shadow is needed by all apicv features. */
11886         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
11887                 return -EINVAL;
11888
11889         return 0;
11890 }
11891
11892 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
11893                                        unsigned long count_field,
11894                                        unsigned long addr_field)
11895 {
11896         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
11897         int maxphyaddr;
11898         u64 count, addr;
11899
11900         if (vmcs12_read_any(vmcs12, count_field, &count) ||
11901             vmcs12_read_any(vmcs12, addr_field, &addr)) {
11902                 WARN_ON(1);
11903                 return -EINVAL;
11904         }
11905         if (count == 0)
11906                 return 0;
11907         maxphyaddr = cpuid_maxphyaddr(vcpu);
11908         if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
11909             (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr) {
11910                 pr_debug_ratelimited(
11911                         "nVMX: invalid MSR switch (0x%lx, %d, %llu, 0x%08llx)",
11912                         addr_field, maxphyaddr, count, addr);
11913                 return -EINVAL;
11914         }
11915         return 0;
11916 }
11917
11918 static int nested_vmx_check_msr_switch_controls(struct kvm_vcpu *vcpu,
11919                                                 struct vmcs12 *vmcs12)
11920 {
11921         if (vmcs12->vm_exit_msr_load_count == 0 &&
11922             vmcs12->vm_exit_msr_store_count == 0 &&
11923             vmcs12->vm_entry_msr_load_count == 0)
11924                 return 0; /* Fast path */
11925         if (nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_LOAD_COUNT,
11926                                         VM_EXIT_MSR_LOAD_ADDR) ||
11927             nested_vmx_check_msr_switch(vcpu, VM_EXIT_MSR_STORE_COUNT,
11928                                         VM_EXIT_MSR_STORE_ADDR) ||
11929             nested_vmx_check_msr_switch(vcpu, VM_ENTRY_MSR_LOAD_COUNT,
11930                                         VM_ENTRY_MSR_LOAD_ADDR))
11931                 return -EINVAL;
11932         return 0;
11933 }
11934
11935 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
11936                                          struct vmcs12 *vmcs12)
11937 {
11938         u64 address = vmcs12->pml_address;
11939         int maxphyaddr = cpuid_maxphyaddr(vcpu);
11940
11941         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_PML)) {
11942                 if (!nested_cpu_has_ept(vmcs12) ||
11943                     !IS_ALIGNED(address, 4096)  ||
11944                     address >> maxphyaddr)
11945                         return -EINVAL;
11946         }
11947
11948         return 0;
11949 }
11950
11951 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
11952                                                  struct vmcs12 *vmcs12)
11953 {
11954         if (!nested_cpu_has_shadow_vmcs(vmcs12))
11955                 return 0;
11956
11957         if (!page_address_valid(vcpu, vmcs12->vmread_bitmap) ||
11958             !page_address_valid(vcpu, vmcs12->vmwrite_bitmap))
11959                 return -EINVAL;
11960
11961         return 0;
11962 }
11963
11964 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
11965                                        struct vmx_msr_entry *e)
11966 {
11967         /* x2APIC MSR accesses are not allowed */
11968         if (vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)
11969                 return -EINVAL;
11970         if (e->index == MSR_IA32_UCODE_WRITE || /* SDM Table 35-2 */
11971             e->index == MSR_IA32_UCODE_REV)
11972                 return -EINVAL;
11973         if (e->reserved != 0)
11974                 return -EINVAL;
11975         return 0;
11976 }
11977
11978 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
11979                                      struct vmx_msr_entry *e)
11980 {
11981         if (e->index == MSR_FS_BASE ||
11982             e->index == MSR_GS_BASE ||
11983             e->index == MSR_IA32_SMM_MONITOR_CTL || /* SMM is not supported */
11984             nested_vmx_msr_check_common(vcpu, e))
11985                 return -EINVAL;
11986         return 0;
11987 }
11988
11989 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
11990                                       struct vmx_msr_entry *e)
11991 {
11992         if (e->index == MSR_IA32_SMBASE || /* SMM is not supported */
11993             nested_vmx_msr_check_common(vcpu, e))
11994                 return -EINVAL;
11995         return 0;
11996 }
11997
11998 /*
11999  * Load guest's/host's msr at nested entry/exit.
12000  * return 0 for success, entry index for failure.
12001  */
12002 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
12003 {
12004         u32 i;
12005         struct vmx_msr_entry e;
12006         struct msr_data msr;
12007
12008         msr.host_initiated = false;
12009         for (i = 0; i < count; i++) {
12010                 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
12011                                         &e, sizeof(e))) {
12012                         pr_debug_ratelimited(
12013                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
12014                                 __func__, i, gpa + i * sizeof(e));
12015                         goto fail;
12016                 }
12017                 if (nested_vmx_load_msr_check(vcpu, &e)) {
12018                         pr_debug_ratelimited(
12019                                 "%s check failed (%u, 0x%x, 0x%x)\n",
12020                                 __func__, i, e.index, e.reserved);
12021                         goto fail;
12022                 }
12023                 msr.index = e.index;
12024                 msr.data = e.value;
12025                 if (kvm_set_msr(vcpu, &msr)) {
12026                         pr_debug_ratelimited(
12027                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
12028                                 __func__, i, e.index, e.value);
12029                         goto fail;
12030                 }
12031         }
12032         return 0;
12033 fail:
12034         return i + 1;
12035 }
12036
12037 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
12038 {
12039         u32 i;
12040         struct vmx_msr_entry e;
12041
12042         for (i = 0; i < count; i++) {
12043                 struct msr_data msr_info;
12044                 if (kvm_vcpu_read_guest(vcpu,
12045                                         gpa + i * sizeof(e),
12046                                         &e, 2 * sizeof(u32))) {
12047                         pr_debug_ratelimited(
12048                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
12049                                 __func__, i, gpa + i * sizeof(e));
12050                         return -EINVAL;
12051                 }
12052                 if (nested_vmx_store_msr_check(vcpu, &e)) {
12053                         pr_debug_ratelimited(
12054                                 "%s check failed (%u, 0x%x, 0x%x)\n",
12055                                 __func__, i, e.index, e.reserved);
12056                         return -EINVAL;
12057                 }
12058                 msr_info.host_initiated = false;
12059                 msr_info.index = e.index;
12060                 if (kvm_get_msr(vcpu, &msr_info)) {
12061                         pr_debug_ratelimited(
12062                                 "%s cannot read MSR (%u, 0x%x)\n",
12063                                 __func__, i, e.index);
12064                         return -EINVAL;
12065                 }
12066                 if (kvm_vcpu_write_guest(vcpu,
12067                                          gpa + i * sizeof(e) +
12068                                              offsetof(struct vmx_msr_entry, value),
12069                                          &msr_info.data, sizeof(msr_info.data))) {
12070                         pr_debug_ratelimited(
12071                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
12072                                 __func__, i, e.index, msr_info.data);
12073                         return -EINVAL;
12074                 }
12075         }
12076         return 0;
12077 }
12078
12079 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
12080 {
12081         unsigned long invalid_mask;
12082
12083         invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
12084         return (val & invalid_mask) == 0;
12085 }
12086
12087 /*
12088  * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
12089  * emulating VM entry into a guest with EPT enabled.
12090  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
12091  * is assigned to entry_failure_code on failure.
12092  */
12093 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
12094                                u32 *entry_failure_code)
12095 {
12096         if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
12097                 if (!nested_cr3_valid(vcpu, cr3)) {
12098                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
12099                         return 1;
12100                 }
12101
12102                 /*
12103                  * If PAE paging and EPT are both on, CR3 is not used by the CPU and
12104                  * must not be dereferenced.
12105                  */
12106                 if (is_pae_paging(vcpu) && !nested_ept) {
12107                         if (!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) {
12108                                 *entry_failure_code = ENTRY_FAIL_PDPTE;
12109                                 return 1;
12110                         }
12111                 }
12112         }
12113
12114         if (!nested_ept)
12115                 kvm_mmu_new_cr3(vcpu, cr3, false);
12116
12117         vcpu->arch.cr3 = cr3;
12118         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
12119
12120         kvm_init_mmu(vcpu, false);
12121
12122         return 0;
12123 }
12124
12125 static void prepare_vmcs02_full(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
12126 {
12127         struct vcpu_vmx *vmx = to_vmx(vcpu);
12128
12129         vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
12130         vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
12131         vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
12132         vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
12133         vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
12134         vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
12135         vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
12136         vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
12137         vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
12138         vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
12139         vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
12140         vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
12141         vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
12142         vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
12143         vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
12144         vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
12145         vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
12146         vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
12147         vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
12148         vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
12149         vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
12150         vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
12151         vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
12152         vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
12153         vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
12154         vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
12155         vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
12156         vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
12157         vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
12158         vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
12159         vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
12160
12161         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
12162         vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
12163                 vmcs12->guest_pending_dbg_exceptions);
12164         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
12165         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
12166
12167         if (nested_cpu_has_xsaves(vmcs12))
12168                 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
12169         vmcs_write64(VMCS_LINK_POINTER, -1ull);
12170
12171         if (cpu_has_vmx_posted_intr())
12172                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
12173
12174         /*
12175          * Whether page-faults are trapped is determined by a combination of
12176          * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
12177          * If enable_ept, L0 doesn't care about page faults and we should
12178          * set all of these to L1's desires. However, if !enable_ept, L0 does
12179          * care about (at least some) page faults, and because it is not easy
12180          * (if at all possible?) to merge L0 and L1's desires, we simply ask
12181          * to exit on each and every L2 page fault. This is done by setting
12182          * MASK=MATCH=0 and (see below) EB.PF=1.
12183          * Note that below we don't need special code to set EB.PF beyond the
12184          * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
12185          * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
12186          * !enable_ept, EB.PF is 1, so the "or" will always be 1.
12187          */
12188         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
12189                 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
12190         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
12191                 enable_ept ? vmcs12->page_fault_error_code_match : 0);
12192
12193         /* All VMFUNCs are currently emulated through L0 vmexits.  */
12194         if (cpu_has_vmx_vmfunc())
12195                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
12196
12197         if (cpu_has_vmx_apicv()) {
12198                 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
12199                 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
12200                 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
12201                 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
12202         }
12203
12204         /*
12205          * Set host-state according to L0's settings (vmcs12 is irrelevant here)
12206          * Some constant fields are set here by vmx_set_constant_host_state().
12207          * Other fields are different per CPU, and will be set later when
12208          * vmx_vcpu_load() is called, and when vmx_prepare_switch_to_guest()
12209          * is called.
12210          */
12211         vmx_set_constant_host_state(vmx);
12212
12213         /*
12214          * Set the MSR load/store lists to match L0's settings.
12215          */
12216         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
12217         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
12218         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
12219         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
12220         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
12221
12222         set_cr4_guest_host_mask(vmx);
12223
12224         if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
12225             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
12226                 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
12227
12228         if (enable_vpid) {
12229                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
12230                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
12231                 else
12232                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
12233         }
12234
12235         /*
12236          * L1 may access the L2's PDPTR, so save them to construct vmcs12
12237          */
12238         if (enable_ept) {
12239                 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
12240                 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
12241                 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
12242                 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
12243         }
12244
12245         if (cpu_has_vmx_msr_bitmap())
12246                 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
12247 }
12248
12249 /*
12250  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
12251  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
12252  * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
12253  * guest in a way that will both be appropriate to L1's requests, and our
12254  * needs. In addition to modifying the active vmcs (which is vmcs02), this
12255  * function also has additional necessary side-effects, like setting various
12256  * vcpu->arch fields.
12257  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
12258  * is assigned to entry_failure_code on failure.
12259  */
12260 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
12261                           u32 *entry_failure_code)
12262 {
12263         struct vcpu_vmx *vmx = to_vmx(vcpu);
12264         u32 exec_control, vmcs12_exec_ctrl;
12265
12266         if (vmx->nested.dirty_vmcs12) {
12267                 prepare_vmcs02_full(vcpu, vmcs12);
12268                 vmx->nested.dirty_vmcs12 = false;
12269         }
12270
12271         /*
12272          * First, the fields that are shadowed.  This must be kept in sync
12273          * with vmx_shadow_fields.h.
12274          */
12275
12276         vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
12277         vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
12278         vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
12279         vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
12280         vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
12281
12282         if (vmx->nested.nested_run_pending &&
12283             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
12284                 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
12285                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
12286         } else {
12287                 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
12288                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
12289         }
12290         if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
12291             !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
12292                 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
12293         if (vmx->nested.nested_run_pending) {
12294                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
12295                              vmcs12->vm_entry_intr_info_field);
12296                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
12297                              vmcs12->vm_entry_exception_error_code);
12298                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
12299                              vmcs12->vm_entry_instruction_len);
12300                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
12301                              vmcs12->guest_interruptibility_info);
12302                 vmx->loaded_vmcs->nmi_known_unmasked =
12303                         !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
12304         } else {
12305                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
12306         }
12307         vmx_set_rflags(vcpu, vmcs12->guest_rflags);
12308
12309         exec_control = vmcs12->pin_based_vm_exec_control;
12310
12311         /* Preemption timer setting is computed directly in vmx_vcpu_run.  */
12312         exec_control |= vmcs_config.pin_based_exec_ctrl;
12313         exec_control &= ~PIN_BASED_VMX_PREEMPTION_TIMER;
12314         vmx->loaded_vmcs->hv_timer_armed = false;
12315
12316         /* Posted interrupts setting is only taken from vmcs12.  */
12317         if (nested_cpu_has_posted_intr(vmcs12)) {
12318                 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
12319                 vmx->nested.pi_pending = false;
12320         } else {
12321                 exec_control &= ~PIN_BASED_POSTED_INTR;
12322         }
12323
12324         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, exec_control);
12325
12326         vmx->nested.preemption_timer_expired = false;
12327         if (nested_cpu_has_preemption_timer(vmcs12))
12328                 vmx_start_preemption_timer(vcpu);
12329
12330         if (cpu_has_secondary_exec_ctrls()) {
12331                 exec_control = vmx->secondary_exec_control;
12332
12333                 /* Take the following fields only from vmcs12 */
12334                 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
12335                                   SECONDARY_EXEC_ENABLE_INVPCID |
12336                                   SECONDARY_EXEC_RDTSCP |
12337                                   SECONDARY_EXEC_XSAVES |
12338                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
12339                                   SECONDARY_EXEC_APIC_REGISTER_VIRT |
12340                                   SECONDARY_EXEC_ENABLE_VMFUNC);
12341                 if (nested_cpu_has(vmcs12,
12342                                    CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
12343                         vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
12344                                 ~SECONDARY_EXEC_ENABLE_PML;
12345                         exec_control |= vmcs12_exec_ctrl;
12346                 }
12347
12348                 /* VMCS shadowing for L2 is emulated for now */
12349                 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
12350
12351                 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
12352                         vmcs_write16(GUEST_INTR_STATUS,
12353                                 vmcs12->guest_intr_status);
12354
12355                 /*
12356                  * Write an illegal value to APIC_ACCESS_ADDR. Later,
12357                  * nested_get_vmcs12_pages will either fix it up or
12358                  * remove the VM execution control.
12359                  */
12360                 if (exec_control & SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)
12361                         vmcs_write64(APIC_ACCESS_ADDR, -1ull);
12362
12363                 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING)
12364                         vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
12365
12366                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control);
12367         }
12368
12369         /*
12370          * HOST_RSP is normally set correctly in vmx_vcpu_run() just before
12371          * entry, but only if the current (host) sp changed from the value
12372          * we wrote last (vmx->host_rsp). This cache is no longer relevant
12373          * if we switch vmcs, and rather than hold a separate cache per vmcs,
12374          * here we just force the write to happen on entry.
12375          */
12376         vmx->host_rsp = 0;
12377
12378         exec_control = vmx_exec_control(vmx); /* L0's desires */
12379         exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
12380         exec_control &= ~CPU_BASED_VIRTUAL_NMI_PENDING;
12381         exec_control &= ~CPU_BASED_TPR_SHADOW;
12382         exec_control |= vmcs12->cpu_based_vm_exec_control;
12383
12384         /*
12385          * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR. Later, if
12386          * nested_get_vmcs12_pages can't fix it up, the illegal value
12387          * will result in a VM entry failure.
12388          */
12389         if (exec_control & CPU_BASED_TPR_SHADOW) {
12390                 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
12391                 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
12392         } else {
12393 #ifdef CONFIG_X86_64
12394                 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
12395                                 CPU_BASED_CR8_STORE_EXITING;
12396 #endif
12397         }
12398
12399         /*
12400          * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
12401          * for I/O port accesses.
12402          */
12403         exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
12404         exec_control |= CPU_BASED_UNCOND_IO_EXITING;
12405
12406         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control);
12407
12408         /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
12409          * bitwise-or of what L1 wants to trap for L2, and what we want to
12410          * trap. Note that CR0.TS also needs updating - we do this later.
12411          */
12412         update_exception_bitmap(vcpu);
12413         vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
12414         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
12415
12416         /* L2->L1 exit controls are emulated - the hardware exit is to L0 so
12417          * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
12418          * bits are further modified by vmx_set_efer() below.
12419          */
12420         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
12421
12422         /* vmcs12's VM_ENTRY_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE are
12423          * emulated by vmx_set_efer(), below.
12424          */
12425         vm_entry_controls_init(vmx,
12426                 (vmcs12->vm_entry_controls & ~VM_ENTRY_LOAD_IA32_EFER &
12427                         ~VM_ENTRY_IA32E_MODE) |
12428                 (vmcs_config.vmentry_ctrl & ~VM_ENTRY_IA32E_MODE));
12429
12430         if (vmx->nested.nested_run_pending &&
12431             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
12432                 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
12433                 vcpu->arch.pat = vmcs12->guest_ia32_pat;
12434         } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
12435                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
12436         }
12437
12438         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
12439
12440         if (kvm_has_tsc_control)
12441                 decache_tsc_multiplier(vmx);
12442
12443         if (enable_vpid) {
12444                 /*
12445                  * There is no direct mapping between vpid02 and vpid12, the
12446                  * vpid02 is per-vCPU for L0 and reused while the value of
12447                  * vpid12 is changed w/ one invvpid during nested vmentry.
12448                  * The vpid12 is allocated by L1 for L2, so it will not
12449                  * influence global bitmap(for vpid01 and vpid02 allocation)
12450                  * even if spawn a lot of nested vCPUs.
12451                  */
12452                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) {
12453                         if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
12454                                 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
12455                                 __vmx_flush_tlb(vcpu, vmx->nested.vpid02, true);
12456                         }
12457                 } else {
12458                         vmx_flush_tlb(vcpu, true);
12459                 }
12460         }
12461
12462         if (enable_pml) {
12463                 /*
12464                  * Conceptually we want to copy the PML address and index from
12465                  * vmcs01 here, and then back to vmcs01 on nested vmexit. But,
12466                  * since we always flush the log on each vmexit, this happens
12467                  * to be equivalent to simply resetting the fields in vmcs02.
12468                  */
12469                 ASSERT(vmx->pml_pg);
12470                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
12471                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
12472         }
12473
12474         if (nested_cpu_has_ept(vmcs12)) {
12475                 if (nested_ept_init_mmu_context(vcpu)) {
12476                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
12477                         return 1;
12478                 }
12479         } else if (nested_cpu_has2(vmcs12,
12480                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
12481                 vmx_flush_tlb(vcpu, true);
12482         }
12483
12484         /*
12485          * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
12486          * bits which we consider mandatory enabled.
12487          * The CR0_READ_SHADOW is what L2 should have expected to read given
12488          * the specifications by L1; It's not enough to take
12489          * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
12490          * have more bits than L1 expected.
12491          */
12492         vmx_set_cr0(vcpu, vmcs12->guest_cr0);
12493         vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
12494
12495         vmx_set_cr4(vcpu, vmcs12->guest_cr4);
12496         vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
12497
12498         if (vmx->nested.nested_run_pending &&
12499             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
12500                 vcpu->arch.efer = vmcs12->guest_ia32_efer;
12501         else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
12502                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
12503         else
12504                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
12505         /* Note: modifies VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
12506         vmx_set_efer(vcpu, vcpu->arch.efer);
12507
12508         /*
12509          * Guest state is invalid and unrestricted guest is disabled,
12510          * which means L1 attempted VMEntry to L2 with invalid state.
12511          * Fail the VMEntry.
12512          */
12513         if (vmx->emulation_required) {
12514                 *entry_failure_code = ENTRY_FAIL_DEFAULT;
12515                 return 1;
12516         }
12517
12518         /* Shadow page tables on either EPT or shadow page tables. */
12519         if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
12520                                 entry_failure_code))
12521                 return 1;
12522
12523         if (!enable_ept)
12524                 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
12525
12526         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->guest_rsp);
12527         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->guest_rip);
12528         return 0;
12529 }
12530
12531 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
12532 {
12533         if (!nested_cpu_has_nmi_exiting(vmcs12) &&
12534             nested_cpu_has_virtual_nmis(vmcs12))
12535                 return -EINVAL;
12536
12537         if (!nested_cpu_has_virtual_nmis(vmcs12) &&
12538             nested_cpu_has(vmcs12, CPU_BASED_VIRTUAL_NMI_PENDING))
12539                 return -EINVAL;
12540
12541         return 0;
12542 }
12543
12544 static int check_vmentry_prereqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
12545 {
12546         struct vcpu_vmx *vmx = to_vmx(vcpu);
12547
12548         if (vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
12549             vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT)
12550                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12551
12552         if (nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)
12553                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12554
12555         if (nested_vmx_check_io_bitmap_controls(vcpu, vmcs12))
12556                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12557
12558         if (nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12))
12559                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12560
12561         if (nested_vmx_check_apic_access_controls(vcpu, vmcs12))
12562                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12563
12564         if (nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12))
12565                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12566
12567         if (nested_vmx_check_apicv_controls(vcpu, vmcs12))
12568                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12569
12570         if (nested_vmx_check_msr_switch_controls(vcpu, vmcs12))
12571                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12572
12573         if (nested_vmx_check_pml_controls(vcpu, vmcs12))
12574                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12575
12576         if (nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12))
12577                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12578
12579         if (!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
12580                                 vmx->nested.msrs.procbased_ctls_low,
12581                                 vmx->nested.msrs.procbased_ctls_high) ||
12582             (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
12583              !vmx_control_verify(vmcs12->secondary_vm_exec_control,
12584                                  vmx->nested.msrs.secondary_ctls_low,
12585                                  vmx->nested.msrs.secondary_ctls_high)) ||
12586             !vmx_control_verify(vmcs12->pin_based_vm_exec_control,
12587                                 vmx->nested.msrs.pinbased_ctls_low,
12588                                 vmx->nested.msrs.pinbased_ctls_high) ||
12589             !vmx_control_verify(vmcs12->vm_exit_controls,
12590                                 vmx->nested.msrs.exit_ctls_low,
12591                                 vmx->nested.msrs.exit_ctls_high) ||
12592             !vmx_control_verify(vmcs12->vm_entry_controls,
12593                                 vmx->nested.msrs.entry_ctls_low,
12594                                 vmx->nested.msrs.entry_ctls_high))
12595                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12596
12597         if (nested_vmx_check_nmi_controls(vmcs12))
12598                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12599
12600         if (nested_cpu_has_vmfunc(vmcs12)) {
12601                 if (vmcs12->vm_function_control &
12602                     ~vmx->nested.msrs.vmfunc_controls)
12603                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12604
12605                 if (nested_cpu_has_eptp_switching(vmcs12)) {
12606                         if (!nested_cpu_has_ept(vmcs12) ||
12607                             !page_address_valid(vcpu, vmcs12->eptp_list_address))
12608                                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12609                 }
12610         }
12611
12612         if (vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu))
12613                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12614
12615         if (!nested_host_cr0_valid(vcpu, vmcs12->host_cr0) ||
12616             !nested_host_cr4_valid(vcpu, vmcs12->host_cr4) ||
12617             !nested_cr3_valid(vcpu, vmcs12->host_cr3))
12618                 return VMXERR_ENTRY_INVALID_HOST_STATE_FIELD;
12619
12620         /*
12621          * From the Intel SDM, volume 3:
12622          * Fields relevant to VM-entry event injection must be set properly.
12623          * These fields are the VM-entry interruption-information field, the
12624          * VM-entry exception error code, and the VM-entry instruction length.
12625          */
12626         if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
12627                 u32 intr_info = vmcs12->vm_entry_intr_info_field;
12628                 u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
12629                 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
12630                 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
12631                 bool should_have_error_code;
12632                 bool urg = nested_cpu_has2(vmcs12,
12633                                            SECONDARY_EXEC_UNRESTRICTED_GUEST);
12634                 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
12635
12636                 /* VM-entry interruption-info field: interruption type */
12637                 if (intr_type == INTR_TYPE_RESERVED ||
12638                     (intr_type == INTR_TYPE_OTHER_EVENT &&
12639                      !nested_cpu_supports_monitor_trap_flag(vcpu)))
12640                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12641
12642                 /* VM-entry interruption-info field: vector */
12643                 if ((intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
12644                     (intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
12645                     (intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
12646                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12647
12648                 /* VM-entry interruption-info field: deliver error code */
12649                 should_have_error_code =
12650                         intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
12651                         x86_exception_has_error_code(vector);
12652                 if (has_error_code != should_have_error_code)
12653                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12654
12655                 /* VM-entry exception error code */
12656                 if (has_error_code &&
12657                     vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))
12658                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12659
12660                 /* VM-entry interruption-info field: reserved bits */
12661                 if (intr_info & INTR_INFO_RESVD_BITS_MASK)
12662                         return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12663
12664                 /* VM-entry instruction length */
12665                 switch (intr_type) {
12666                 case INTR_TYPE_SOFT_EXCEPTION:
12667                 case INTR_TYPE_SOFT_INTR:
12668                 case INTR_TYPE_PRIV_SW_EXCEPTION:
12669                         if ((vmcs12->vm_entry_instruction_len > 15) ||
12670                             (vmcs12->vm_entry_instruction_len == 0 &&
12671                              !nested_cpu_has_zero_length_injection(vcpu)))
12672                                 return VMXERR_ENTRY_INVALID_CONTROL_FIELD;
12673                 }
12674         }
12675
12676         return 0;
12677 }
12678
12679 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
12680                                           struct vmcs12 *vmcs12)
12681 {
12682         int r;
12683         struct page *page;
12684         struct vmcs12 *shadow;
12685
12686         if (vmcs12->vmcs_link_pointer == -1ull)
12687                 return 0;
12688
12689         if (!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))
12690                 return -EINVAL;
12691
12692         page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->vmcs_link_pointer);
12693         if (is_error_page(page))
12694                 return -EINVAL;
12695
12696         r = 0;
12697         shadow = kmap(page);
12698         if (shadow->hdr.revision_id != VMCS12_REVISION ||
12699             shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))
12700                 r = -EINVAL;
12701         kunmap(page);
12702         kvm_release_page_clean(page);
12703         return r;
12704 }
12705
12706 static int check_vmentry_postreqs(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
12707                                   u32 *exit_qual)
12708 {
12709         bool ia32e;
12710
12711         *exit_qual = ENTRY_FAIL_DEFAULT;
12712
12713         if (!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0) ||
12714             !nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))
12715                 return 1;
12716
12717         if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
12718                 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
12719                 return 1;
12720         }
12721
12722         /*
12723          * If the load IA32_EFER VM-entry control is 1, the following checks
12724          * are performed on the field for the IA32_EFER MSR:
12725          * - Bits reserved in the IA32_EFER MSR must be 0.
12726          * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
12727          *   the IA-32e mode guest VM-exit control. It must also be identical
12728          *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
12729          *   CR0.PG) is 1.
12730          */
12731         if (to_vmx(vcpu)->nested.nested_run_pending &&
12732             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
12733                 ia32e = (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) != 0;
12734                 if (!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer) ||
12735                     ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA) ||
12736                     ((vmcs12->guest_cr0 & X86_CR0_PG) &&
12737                      ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))
12738                         return 1;
12739         }
12740
12741         /*
12742          * If the load IA32_EFER VM-exit control is 1, bits reserved in the
12743          * IA32_EFER MSR must be 0 in the field for that register. In addition,
12744          * the values of the LMA and LME bits in the field must each be that of
12745          * the host address-space size VM-exit control.
12746          */
12747         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
12748                 ia32e = (vmcs12->vm_exit_controls &
12749                          VM_EXIT_HOST_ADDR_SPACE_SIZE) != 0;
12750                 if (!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer) ||
12751                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA) ||
12752                     ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))
12753                         return 1;
12754         }
12755
12756         if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
12757                 (is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu) ||
12758                 (vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))
12759                         return 1;
12760
12761         return 0;
12762 }
12763
12764 /*
12765  * If exit_qual is NULL, this is being called from state restore (either RSM
12766  * or KVM_SET_NESTED_STATE).  Otherwise it's called from vmlaunch/vmresume.
12767  */
12768 static int enter_vmx_non_root_mode(struct kvm_vcpu *vcpu, u32 *exit_qual)
12769 {
12770         struct vcpu_vmx *vmx = to_vmx(vcpu);
12771         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
12772         bool from_vmentry = !!exit_qual;
12773         u32 dummy_exit_qual;
12774         bool evaluate_pending_interrupts;
12775         int r = 0;
12776
12777         evaluate_pending_interrupts = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL) &
12778                 (CPU_BASED_VIRTUAL_INTR_PENDING | CPU_BASED_VIRTUAL_NMI_PENDING);
12779         if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
12780                 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
12781
12782         enter_guest_mode(vcpu);
12783
12784         if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
12785                 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
12786         if (kvm_mpx_supported() &&
12787                 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
12788                 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
12789
12790         vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
12791         vmx_segment_cache_clear(vmx);
12792
12793         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
12794                 vcpu->arch.tsc_offset += vmcs12->tsc_offset;
12795
12796         r = EXIT_REASON_INVALID_STATE;
12797         if (prepare_vmcs02(vcpu, vmcs12, from_vmentry ? exit_qual : &dummy_exit_qual))
12798                 goto fail;
12799
12800         if (from_vmentry) {
12801                 nested_get_vmcs12_pages(vcpu);
12802
12803                 r = EXIT_REASON_MSR_LOAD_FAIL;
12804                 *exit_qual = nested_vmx_load_msr(vcpu,
12805                                                  vmcs12->vm_entry_msr_load_addr,
12806                                                  vmcs12->vm_entry_msr_load_count);
12807                 if (*exit_qual)
12808                         goto fail;
12809         } else {
12810                 /*
12811                  * The MMU is not initialized to point at the right entities yet and
12812                  * "get pages" would need to read data from the guest (i.e. we will
12813                  * need to perform gpa to hpa translation). Request a call
12814                  * to nested_get_vmcs12_pages before the next VM-entry.  The MSRs
12815                  * have already been set at vmentry time and should not be reset.
12816                  */
12817                 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
12818         }
12819
12820         /*
12821          * If L1 had a pending IRQ/NMI until it executed
12822          * VMLAUNCH/VMRESUME which wasn't delivered because it was
12823          * disallowed (e.g. interrupts disabled), L0 needs to
12824          * evaluate if this pending event should cause an exit from L2
12825          * to L1 or delivered directly to L2 (e.g. In case L1 don't
12826          * intercept EXTERNAL_INTERRUPT).
12827          *
12828          * Usually this would be handled by the processor noticing an
12829          * IRQ/NMI window request, or checking RVI during evaluation of
12830          * pending virtual interrupts.  However, this setting was done
12831          * on VMCS01 and now VMCS02 is active instead. Thus, we force L0
12832          * to perform pending event evaluation by requesting a KVM_REQ_EVENT.
12833          */
12834         if (unlikely(evaluate_pending_interrupts))
12835                 kvm_make_request(KVM_REQ_EVENT, vcpu);
12836
12837         /*
12838          * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
12839          * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
12840          * returned as far as L1 is concerned. It will only return (and set
12841          * the success flag) when L2 exits (see nested_vmx_vmexit()).
12842          */
12843         return 0;
12844
12845 fail:
12846         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
12847                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
12848         leave_guest_mode(vcpu);
12849         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
12850         return r;
12851 }
12852
12853 /*
12854  * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
12855  * for running an L2 nested guest.
12856  */
12857 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
12858 {
12859         struct vmcs12 *vmcs12;
12860         struct vcpu_vmx *vmx = to_vmx(vcpu);
12861         u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
12862         u32 exit_qual;
12863         int ret;
12864
12865         if (!nested_vmx_check_permission(vcpu))
12866                 return 1;
12867
12868         if (!nested_vmx_check_vmcs12(vcpu))
12869                 goto out;
12870
12871         vmcs12 = get_vmcs12(vcpu);
12872
12873         /*
12874          * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
12875          * that there *is* a valid VMCS pointer, RFLAGS.CF is set
12876          * rather than RFLAGS.ZF, and no error number is stored to the
12877          * VM-instruction error field.
12878          */
12879         if (vmcs12->hdr.shadow_vmcs) {
12880                 nested_vmx_failInvalid(vcpu);
12881                 goto out;
12882         }
12883
12884         if (enable_shadow_vmcs)
12885                 copy_shadow_to_vmcs12(vmx);
12886
12887         /*
12888          * The nested entry process starts with enforcing various prerequisites
12889          * on vmcs12 as required by the Intel SDM, and act appropriately when
12890          * they fail: As the SDM explains, some conditions should cause the
12891          * instruction to fail, while others will cause the instruction to seem
12892          * to succeed, but return an EXIT_REASON_INVALID_STATE.
12893          * To speed up the normal (success) code path, we should avoid checking
12894          * for misconfigurations which will anyway be caught by the processor
12895          * when using the merged vmcs02.
12896          */
12897         if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS) {
12898                 nested_vmx_failValid(vcpu,
12899                                      VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
12900                 goto out;
12901         }
12902
12903         if (vmcs12->launch_state == launch) {
12904                 nested_vmx_failValid(vcpu,
12905                         launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
12906                                : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
12907                 goto out;
12908         }
12909
12910         ret = check_vmentry_prereqs(vcpu, vmcs12);
12911         if (ret) {
12912                 nested_vmx_failValid(vcpu, ret);
12913                 goto out;
12914         }
12915
12916         /*
12917          * After this point, the trap flag no longer triggers a singlestep trap
12918          * on the vm entry instructions; don't call kvm_skip_emulated_instruction.
12919          * This is not 100% correct; for performance reasons, we delegate most
12920          * of the checks on host state to the processor.  If those fail,
12921          * the singlestep trap is missed.
12922          */
12923         skip_emulated_instruction(vcpu);
12924
12925         ret = check_vmentry_postreqs(vcpu, vmcs12, &exit_qual);
12926         if (ret) {
12927                 nested_vmx_entry_failure(vcpu, vmcs12,
12928                                          EXIT_REASON_INVALID_STATE, exit_qual);
12929                 return 1;
12930         }
12931
12932         /*
12933          * We're finally done with prerequisite checking, and can start with
12934          * the nested entry.
12935          */
12936
12937         vmx->nested.nested_run_pending = 1;
12938         ret = enter_vmx_non_root_mode(vcpu, &exit_qual);
12939         if (ret) {
12940                 nested_vmx_entry_failure(vcpu, vmcs12, ret, exit_qual);
12941                 vmx->nested.nested_run_pending = 0;
12942                 return 1;
12943         }
12944
12945         /* Hide L1D cache contents from the nested guest.  */
12946         vmx->vcpu.arch.l1tf_flush_l1d = true;
12947
12948         /*
12949          * Must happen outside of enter_vmx_non_root_mode() as it will
12950          * also be used as part of restoring nVMX state for
12951          * snapshot restore (migration).
12952          *
12953          * In this flow, it is assumed that vmcs12 cache was
12954          * trasferred as part of captured nVMX state and should
12955          * therefore not be read from guest memory (which may not
12956          * exist on destination host yet).
12957          */
12958         nested_cache_shadow_vmcs12(vcpu, vmcs12);
12959
12960         /*
12961          * If we're entering a halted L2 vcpu and the L2 vcpu won't be
12962          * awakened by event injection or by an NMI-window VM-exit or
12963          * by an interrupt-window VM-exit, halt the vcpu.
12964          */
12965         if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
12966             !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
12967             !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_NMI_PENDING) &&
12968             !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_VIRTUAL_INTR_PENDING) &&
12969               (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
12970                 vmx->nested.nested_run_pending = 0;
12971                 return kvm_vcpu_halt(vcpu);
12972         }
12973         return 1;
12974
12975 out:
12976         return kvm_skip_emulated_instruction(vcpu);
12977 }
12978
12979 /*
12980  * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
12981  * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
12982  * This function returns the new value we should put in vmcs12.guest_cr0.
12983  * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
12984  *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
12985  *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
12986  *     didn't trap the bit, because if L1 did, so would L0).
12987  *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
12988  *     been modified by L2, and L1 knows it. So just leave the old value of
12989  *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
12990  *     isn't relevant, because if L0 traps this bit it can set it to anything.
12991  *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
12992  *     changed these bits, and therefore they need to be updated, but L0
12993  *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
12994  *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
12995  */
12996 static inline unsigned long
12997 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
12998 {
12999         return
13000         /*1*/   (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
13001         /*2*/   (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
13002         /*3*/   (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
13003                         vcpu->arch.cr0_guest_owned_bits));
13004 }
13005
13006 static inline unsigned long
13007 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
13008 {
13009         return
13010         /*1*/   (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
13011         /*2*/   (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
13012         /*3*/   (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
13013                         vcpu->arch.cr4_guest_owned_bits));
13014 }
13015
13016 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
13017                                        struct vmcs12 *vmcs12)
13018 {
13019         u32 idt_vectoring;
13020         unsigned int nr;
13021
13022         if (vcpu->arch.exception.injected) {
13023                 nr = vcpu->arch.exception.nr;
13024                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
13025
13026                 if (kvm_exception_is_soft(nr)) {
13027                         vmcs12->vm_exit_instruction_len =
13028                                 vcpu->arch.event_exit_inst_len;
13029                         idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
13030                 } else
13031                         idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
13032
13033                 if (vcpu->arch.exception.has_error_code) {
13034                         idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
13035                         vmcs12->idt_vectoring_error_code =
13036                                 vcpu->arch.exception.error_code;
13037                 }
13038
13039                 vmcs12->idt_vectoring_info_field = idt_vectoring;
13040         } else if (vcpu->arch.nmi_injected) {
13041                 vmcs12->idt_vectoring_info_field =
13042                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
13043         } else if (vcpu->arch.interrupt.injected) {
13044                 nr = vcpu->arch.interrupt.nr;
13045                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
13046
13047                 if (vcpu->arch.interrupt.soft) {
13048                         idt_vectoring |= INTR_TYPE_SOFT_INTR;
13049                         vmcs12->vm_entry_instruction_len =
13050                                 vcpu->arch.event_exit_inst_len;
13051                 } else
13052                         idt_vectoring |= INTR_TYPE_EXT_INTR;
13053
13054                 vmcs12->idt_vectoring_info_field = idt_vectoring;
13055         }
13056 }
13057
13058 static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
13059 {
13060         struct vcpu_vmx *vmx = to_vmx(vcpu);
13061         unsigned long exit_qual;
13062         bool block_nested_events =
13063             vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
13064
13065         if (vcpu->arch.exception.pending &&
13066                 nested_vmx_check_exception(vcpu, &exit_qual)) {
13067                 if (block_nested_events)
13068                         return -EBUSY;
13069                 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
13070                 return 0;
13071         }
13072
13073         if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
13074             vmx->nested.preemption_timer_expired) {
13075                 if (block_nested_events)
13076                         return -EBUSY;
13077                 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
13078                 return 0;
13079         }
13080
13081         if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
13082                 if (block_nested_events)
13083                         return -EBUSY;
13084                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
13085                                   NMI_VECTOR | INTR_TYPE_NMI_INTR |
13086                                   INTR_INFO_VALID_MASK, 0);
13087                 /*
13088                  * The NMI-triggered VM exit counts as injection:
13089                  * clear this one and block further NMIs.
13090                  */
13091                 vcpu->arch.nmi_pending = 0;
13092                 vmx_set_nmi_mask(vcpu, true);
13093                 return 0;
13094         }
13095
13096         if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(vcpu)) {
13097                 if (block_nested_events)
13098                         return -EBUSY;
13099                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
13100                 return 0;
13101         }
13102
13103         vmx_complete_nested_posted_interrupt(vcpu);
13104         return 0;
13105 }
13106
13107 static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu)
13108 {
13109         to_vmx(vcpu)->req_immediate_exit = true;
13110 }
13111
13112 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
13113 {
13114         ktime_t remaining =
13115                 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
13116         u64 value;
13117
13118         if (ktime_to_ns(remaining) <= 0)
13119                 return 0;
13120
13121         value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
13122         do_div(value, 1000000);
13123         return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
13124 }
13125
13126 /*
13127  * Update the guest state fields of vmcs12 to reflect changes that
13128  * occurred while L2 was running. (The "IA-32e mode guest" bit of the
13129  * VM-entry controls is also updated, since this is really a guest
13130  * state bit.)
13131  */
13132 static void sync_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
13133 {
13134         vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
13135         vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
13136
13137         vmcs12->guest_rsp = kvm_register_read(vcpu, VCPU_REGS_RSP);
13138         vmcs12->guest_rip = kvm_register_read(vcpu, VCPU_REGS_RIP);
13139         vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
13140
13141         vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
13142         vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
13143         vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
13144         vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
13145         vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
13146         vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
13147         vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
13148         vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
13149         vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
13150         vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
13151         vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
13152         vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
13153         vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
13154         vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
13155         vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
13156         vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
13157         vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
13158         vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
13159         vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
13160         vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
13161         vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
13162         vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
13163         vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
13164         vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
13165         vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
13166         vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
13167         vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
13168         vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
13169         vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
13170         vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
13171         vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
13172         vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
13173         vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
13174         vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
13175         vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
13176         vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
13177
13178         vmcs12->guest_interruptibility_info =
13179                 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
13180         vmcs12->guest_pending_dbg_exceptions =
13181                 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
13182         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
13183                 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
13184         else
13185                 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
13186
13187         if (nested_cpu_has_preemption_timer(vmcs12)) {
13188                 if (vmcs12->vm_exit_controls &
13189                     VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
13190                         vmcs12->vmx_preemption_timer_value =
13191                                 vmx_get_preemption_timer_value(vcpu);
13192                 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
13193         }
13194
13195         /*
13196          * In some cases (usually, nested EPT), L2 is allowed to change its
13197          * own CR3 without exiting. If it has changed it, we must keep it.
13198          * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
13199          * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
13200          *
13201          * Additionally, restore L2's PDPTR to vmcs12.
13202          */
13203         if (enable_ept) {
13204                 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
13205                 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
13206                 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
13207                 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
13208                 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
13209         }
13210
13211         vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
13212
13213         if (nested_cpu_has_vid(vmcs12))
13214                 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
13215
13216         vmcs12->vm_entry_controls =
13217                 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
13218                 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
13219
13220         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) {
13221                 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
13222                 vmcs12->guest_ia32_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
13223         }
13224
13225         /* TODO: These cannot have changed unless we have MSR bitmaps and
13226          * the relevant bit asks not to trap the change */
13227         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_PAT)
13228                 vmcs12->guest_ia32_pat = vmcs_read64(GUEST_IA32_PAT);
13229         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
13230                 vmcs12->guest_ia32_efer = vcpu->arch.efer;
13231         vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
13232         vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
13233         vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
13234         if (kvm_mpx_supported())
13235                 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
13236 }
13237
13238 /*
13239  * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
13240  * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
13241  * and this function updates it to reflect the changes to the guest state while
13242  * L2 was running (and perhaps made some exits which were handled directly by L0
13243  * without going back to L1), and to reflect the exit reason.
13244  * Note that we do not have to copy here all VMCS fields, just those that
13245  * could have changed by the L2 guest or the exit - i.e., the guest-state and
13246  * exit-information fields only. Other fields are modified by L1 with VMWRITE,
13247  * which already writes to vmcs12 directly.
13248  */
13249 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
13250                            u32 exit_reason, u32 exit_intr_info,
13251                            unsigned long exit_qualification)
13252 {
13253         /* update guest state fields: */
13254         sync_vmcs12(vcpu, vmcs12);
13255
13256         /* update exit information fields: */
13257
13258         vmcs12->vm_exit_reason = exit_reason;
13259         vmcs12->exit_qualification = exit_qualification;
13260         vmcs12->vm_exit_intr_info = exit_intr_info;
13261
13262         vmcs12->idt_vectoring_info_field = 0;
13263         vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
13264         vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
13265
13266         if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
13267                 vmcs12->launch_state = 1;
13268
13269                 /* vm_entry_intr_info_field is cleared on exit. Emulate this
13270                  * instead of reading the real value. */
13271                 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
13272
13273                 /*
13274                  * Transfer the event that L0 or L1 may wanted to inject into
13275                  * L2 to IDT_VECTORING_INFO_FIELD.
13276                  */
13277                 vmcs12_save_pending_event(vcpu, vmcs12);
13278         }
13279 }
13280
13281 /*
13282  * A part of what we need to when the nested L2 guest exits and we want to
13283  * run its L1 parent, is to reset L1's guest state to the host state specified
13284  * in vmcs12.
13285  * This function is to be called not only on normal nested exit, but also on
13286  * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
13287  * Failures During or After Loading Guest State").
13288  * This function should be called when the active VMCS is L1's (vmcs01).
13289  */
13290 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
13291                                    struct vmcs12 *vmcs12)
13292 {
13293         struct kvm_segment seg;
13294         u32 entry_failure_code;
13295
13296         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
13297                 vcpu->arch.efer = vmcs12->host_ia32_efer;
13298         else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
13299                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
13300         else
13301                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
13302         vmx_set_efer(vcpu, vcpu->arch.efer);
13303
13304         kvm_register_write(vcpu, VCPU_REGS_RSP, vmcs12->host_rsp);
13305         kvm_register_write(vcpu, VCPU_REGS_RIP, vmcs12->host_rip);
13306         vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
13307         /*
13308          * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
13309          * actually changed, because vmx_set_cr0 refers to efer set above.
13310          *
13311          * CR0_GUEST_HOST_MASK is already set in the original vmcs01
13312          * (KVM doesn't change it);
13313          */
13314         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
13315         vmx_set_cr0(vcpu, vmcs12->host_cr0);
13316
13317         /* Same as above - no reason to call set_cr4_guest_host_mask().  */
13318         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
13319         vmx_set_cr4(vcpu, vmcs12->host_cr4);
13320
13321         nested_ept_uninit_mmu_context(vcpu);
13322
13323         /*
13324          * Only PDPTE load can fail as the value of cr3 was checked on entry and
13325          * couldn't have changed.
13326          */
13327         if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
13328                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
13329
13330         if (!enable_ept)
13331                 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
13332
13333         /*
13334          * If vmcs01 don't use VPID, CPU flushes TLB on every
13335          * VMEntry/VMExit. Thus, no need to flush TLB.
13336          *
13337          * If vmcs12 uses VPID, TLB entries populated by L2 are
13338          * tagged with vmx->nested.vpid02 while L1 entries are tagged
13339          * with vmx->vpid. Thus, no need to flush TLB.
13340          *
13341          * Therefore, flush TLB only in case vmcs01 uses VPID and
13342          * vmcs12 don't use VPID as in this case L1 & L2 TLB entries
13343          * are both tagged with vmx->vpid.
13344          */
13345         if (enable_vpid &&
13346             !(nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02)) {
13347                 vmx_flush_tlb(vcpu, true);
13348         }
13349
13350         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
13351         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
13352         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
13353         vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
13354         vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
13355         vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
13356         vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
13357
13358         /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
13359         if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
13360                 vmcs_write64(GUEST_BNDCFGS, 0);
13361
13362         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
13363                 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
13364                 vcpu->arch.pat = vmcs12->host_ia32_pat;
13365         }
13366         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
13367                 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
13368                         vmcs12->host_ia32_perf_global_ctrl);
13369
13370         /* Set L1 segment info according to Intel SDM
13371             27.5.2 Loading Host Segment and Descriptor-Table Registers */
13372         seg = (struct kvm_segment) {
13373                 .base = 0,
13374                 .limit = 0xFFFFFFFF,
13375                 .selector = vmcs12->host_cs_selector,
13376                 .type = 11,
13377                 .present = 1,
13378                 .s = 1,
13379                 .g = 1
13380         };
13381         if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
13382                 seg.l = 1;
13383         else
13384                 seg.db = 1;
13385         vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
13386         seg = (struct kvm_segment) {
13387                 .base = 0,
13388                 .limit = 0xFFFFFFFF,
13389                 .type = 3,
13390                 .present = 1,
13391                 .s = 1,
13392                 .db = 1,
13393                 .g = 1
13394         };
13395         seg.selector = vmcs12->host_ds_selector;
13396         vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
13397         seg.selector = vmcs12->host_es_selector;
13398         vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
13399         seg.selector = vmcs12->host_ss_selector;
13400         vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
13401         seg.selector = vmcs12->host_fs_selector;
13402         seg.base = vmcs12->host_fs_base;
13403         vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
13404         seg.selector = vmcs12->host_gs_selector;
13405         seg.base = vmcs12->host_gs_base;
13406         vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
13407         seg = (struct kvm_segment) {
13408                 .base = vmcs12->host_tr_base,
13409                 .limit = 0x67,
13410                 .selector = vmcs12->host_tr_selector,
13411                 .type = 11,
13412                 .present = 1
13413         };
13414         vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
13415
13416         kvm_set_dr(vcpu, 7, 0x400);
13417         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
13418
13419         if (cpu_has_vmx_msr_bitmap())
13420                 vmx_update_msr_bitmap(vcpu);
13421
13422         if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
13423                                 vmcs12->vm_exit_msr_load_count))
13424                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
13425 }
13426
13427 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
13428 {
13429         struct shared_msr_entry *efer_msr;
13430         unsigned int i;
13431
13432         if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
13433                 return vmcs_read64(GUEST_IA32_EFER);
13434
13435         if (cpu_has_load_ia32_efer)
13436                 return host_efer;
13437
13438         for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
13439                 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
13440                         return vmx->msr_autoload.guest.val[i].value;
13441         }
13442
13443         efer_msr = find_msr_entry(vmx, MSR_EFER);
13444         if (efer_msr)
13445                 return efer_msr->data;
13446
13447         return host_efer;
13448 }
13449
13450 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
13451 {
13452         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
13453         struct vcpu_vmx *vmx = to_vmx(vcpu);
13454         struct vmx_msr_entry g, h;
13455         struct msr_data msr;
13456         gpa_t gpa;
13457         u32 i, j;
13458
13459         vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
13460
13461         if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
13462                 /*
13463                  * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
13464                  * as vmcs01.GUEST_DR7 contains a userspace defined value
13465                  * and vcpu->arch.dr7 is not squirreled away before the
13466                  * nested VMENTER (not worth adding a variable in nested_vmx).
13467                  */
13468                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
13469                         kvm_set_dr(vcpu, 7, DR7_FIXED_1);
13470                 else
13471                         WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
13472         }
13473
13474         /*
13475          * Note that calling vmx_set_{efer,cr0,cr4} is important as they
13476          * handle a variety of side effects to KVM's software model.
13477          */
13478         vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
13479
13480         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
13481         vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
13482
13483         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
13484         vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
13485
13486         nested_ept_uninit_mmu_context(vcpu);
13487         vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
13488         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
13489
13490         /*
13491          * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
13492          * from vmcs01 (if necessary).  The PDPTRs are not loaded on
13493          * VMFail, like everything else we just need to ensure our
13494          * software model is up-to-date.
13495          */
13496         ept_save_pdptrs(vcpu);
13497
13498         kvm_mmu_reset_context(vcpu);
13499
13500         if (cpu_has_vmx_msr_bitmap())
13501                 vmx_update_msr_bitmap(vcpu);
13502
13503         /*
13504          * This nasty bit of open coding is a compromise between blindly
13505          * loading L1's MSRs using the exit load lists (incorrect emulation
13506          * of VMFail), leaving the nested VM's MSRs in the software model
13507          * (incorrect behavior) and snapshotting the modified MSRs (too
13508          * expensive since the lists are unbound by hardware).  For each
13509          * MSR that was (prematurely) loaded from the nested VMEntry load
13510          * list, reload it from the exit load list if it exists and differs
13511          * from the guest value.  The intent is to stuff host state as
13512          * silently as possible, not to fully process the exit load list.
13513          */
13514         msr.host_initiated = false;
13515         for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
13516                 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
13517                 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
13518                         pr_debug_ratelimited(
13519                                 "%s read MSR index failed (%u, 0x%08llx)\n",
13520                                 __func__, i, gpa);
13521                         goto vmabort;
13522                 }
13523
13524                 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
13525                         gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
13526                         if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
13527                                 pr_debug_ratelimited(
13528                                         "%s read MSR failed (%u, 0x%08llx)\n",
13529                                         __func__, j, gpa);
13530                                 goto vmabort;
13531                         }
13532                         if (h.index != g.index)
13533                                 continue;
13534                         if (h.value == g.value)
13535                                 break;
13536
13537                         if (nested_vmx_load_msr_check(vcpu, &h)) {
13538                                 pr_debug_ratelimited(
13539                                         "%s check failed (%u, 0x%x, 0x%x)\n",
13540                                         __func__, j, h.index, h.reserved);
13541                                 goto vmabort;
13542                         }
13543
13544                         msr.index = h.index;
13545                         msr.data = h.value;
13546                         if (kvm_set_msr(vcpu, &msr)) {
13547                                 pr_debug_ratelimited(
13548                                         "%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
13549                                         __func__, j, h.index, h.value);
13550                                 goto vmabort;
13551                         }
13552                 }
13553         }
13554
13555         return;
13556
13557 vmabort:
13558         nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
13559 }
13560
13561 /*
13562  * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
13563  * and modify vmcs12 to make it see what it would expect to see there if
13564  * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
13565  */
13566 static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
13567                               u32 exit_intr_info,
13568                               unsigned long exit_qualification)
13569 {
13570         struct vcpu_vmx *vmx = to_vmx(vcpu);
13571         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
13572
13573         /* trying to cancel vmlaunch/vmresume is a bug */
13574         WARN_ON_ONCE(vmx->nested.nested_run_pending);
13575
13576         /*
13577          * The only expected VM-instruction error is "VM entry with
13578          * invalid control field(s)." Anything else indicates a
13579          * problem with L0.
13580          */
13581         WARN_ON_ONCE(vmx->fail && (vmcs_read32(VM_INSTRUCTION_ERROR) !=
13582                                    VMXERR_ENTRY_INVALID_CONTROL_FIELD));
13583
13584         leave_guest_mode(vcpu);
13585
13586         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING)
13587                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
13588
13589         if (likely(!vmx->fail)) {
13590                 if (exit_reason == -1)
13591                         sync_vmcs12(vcpu, vmcs12);
13592                 else
13593                         prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
13594                                        exit_qualification);
13595
13596                 /*
13597                  * Must happen outside of sync_vmcs12() as it will
13598                  * also be used to capture vmcs12 cache as part of
13599                  * capturing nVMX state for snapshot (migration).
13600                  *
13601                  * Otherwise, this flush will dirty guest memory at a
13602                  * point it is already assumed by user-space to be
13603                  * immutable.
13604                  */
13605                 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
13606
13607                 if (nested_vmx_store_msr(vcpu, vmcs12->vm_exit_msr_store_addr,
13608                                          vmcs12->vm_exit_msr_store_count))
13609                         nested_vmx_abort(vcpu, VMX_ABORT_SAVE_GUEST_MSR_FAIL);
13610         }
13611
13612         /*
13613          * Drop events/exceptions that were queued for re-injection to L2
13614          * (picked up via vmx_complete_interrupts()), as well as exceptions
13615          * that were pending for L2.  Note, this must NOT be hoisted above
13616          * prepare_vmcs12(), events/exceptions queued for re-injection need to
13617          * be captured in vmcs12 (see vmcs12_save_pending_event()).
13618          */
13619         vcpu->arch.nmi_injected = false;
13620         kvm_clear_exception_queue(vcpu);
13621         kvm_clear_interrupt_queue(vcpu);
13622
13623         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
13624         vm_entry_controls_reset_shadow(vmx);
13625         vm_exit_controls_reset_shadow(vmx);
13626         vmx_segment_cache_clear(vmx);
13627
13628         /* Update any VMCS fields that might have changed while L2 ran */
13629         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
13630         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
13631         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
13632
13633         if (kvm_has_tsc_control)
13634                 decache_tsc_multiplier(vmx);
13635
13636         if (vmx->nested.change_vmcs01_virtual_apic_mode) {
13637                 vmx->nested.change_vmcs01_virtual_apic_mode = false;
13638                 vmx_set_virtual_apic_mode(vcpu);
13639         } else if (!nested_cpu_has_ept(vmcs12) &&
13640                    nested_cpu_has2(vmcs12,
13641                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
13642                 vmx_flush_tlb(vcpu, true);
13643         }
13644
13645         /* This is needed for same reason as it was needed in prepare_vmcs02 */
13646         vmx->host_rsp = 0;
13647
13648         /* Unpin physical memory we referred to in vmcs02 */
13649         if (vmx->nested.apic_access_page) {
13650                 kvm_release_page_dirty(vmx->nested.apic_access_page);
13651                 vmx->nested.apic_access_page = NULL;
13652         }
13653         if (vmx->nested.virtual_apic_page) {
13654                 kvm_release_page_dirty(vmx->nested.virtual_apic_page);
13655                 vmx->nested.virtual_apic_page = NULL;
13656         }
13657         if (vmx->nested.pi_desc_page) {
13658                 kunmap(vmx->nested.pi_desc_page);
13659                 kvm_release_page_dirty(vmx->nested.pi_desc_page);
13660                 vmx->nested.pi_desc_page = NULL;
13661                 vmx->nested.pi_desc = NULL;
13662         }
13663
13664         /*
13665          * We are now running in L2, mmu_notifier will force to reload the
13666          * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
13667          */
13668         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
13669
13670         if (enable_shadow_vmcs && exit_reason != -1)
13671                 vmx->nested.sync_shadow_vmcs = true;
13672
13673         /* in case we halted in L2 */
13674         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
13675
13676         if (likely(!vmx->fail)) {
13677                 if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
13678                     nested_exit_intr_ack_set(vcpu)) {
13679                         int irq = kvm_cpu_get_interrupt(vcpu);
13680                         WARN_ON(irq < 0);
13681                         vmcs12->vm_exit_intr_info = irq |
13682                                 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
13683                 }
13684
13685                 if (exit_reason != -1)
13686                         trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
13687                                                        vmcs12->exit_qualification,
13688                                                        vmcs12->idt_vectoring_info_field,
13689                                                        vmcs12->vm_exit_intr_info,
13690                                                        vmcs12->vm_exit_intr_error_code,
13691                                                        KVM_ISA_VMX);
13692
13693                 load_vmcs12_host_state(vcpu, vmcs12);
13694
13695                 return;
13696         }
13697
13698         /*
13699          * After an early L2 VM-entry failure, we're now back
13700          * in L1 which thinks it just finished a VMLAUNCH or
13701          * VMRESUME instruction, so we need to set the failure
13702          * flag and the VM-instruction error field of the VMCS
13703          * accordingly.
13704          */
13705         nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
13706
13707         /*
13708          * Restore L1's host state to KVM's software model.  We're here
13709          * because a consistency check was caught by hardware, which
13710          * means some amount of guest state has been propagated to KVM's
13711          * model and needs to be unwound to the host's state.
13712          */
13713         nested_vmx_restore_host_state(vcpu);
13714
13715         /*
13716          * The emulated instruction was already skipped in
13717          * nested_vmx_run, but the updated RIP was never
13718          * written back to the vmcs01.
13719          */
13720         skip_emulated_instruction(vcpu);
13721         vmx->fail = 0;
13722 }
13723
13724 /*
13725  * Forcibly leave nested mode in order to be able to reset the VCPU later on.
13726  */
13727 static void vmx_leave_nested(struct kvm_vcpu *vcpu)
13728 {
13729         if (is_guest_mode(vcpu)) {
13730                 to_vmx(vcpu)->nested.nested_run_pending = 0;
13731                 nested_vmx_vmexit(vcpu, -1, 0, 0);
13732         }
13733         free_nested(to_vmx(vcpu));
13734 }
13735
13736 /*
13737  * L1's failure to enter L2 is a subset of a normal exit, as explained in
13738  * 23.7 "VM-entry failures during or after loading guest state" (this also
13739  * lists the acceptable exit-reason and exit-qualification parameters).
13740  * It should only be called before L2 actually succeeded to run, and when
13741  * vmcs01 is current (it doesn't leave_guest_mode() or switch vmcss).
13742  */
13743 static void nested_vmx_entry_failure(struct kvm_vcpu *vcpu,
13744                         struct vmcs12 *vmcs12,
13745                         u32 reason, unsigned long qualification)
13746 {
13747         load_vmcs12_host_state(vcpu, vmcs12);
13748         vmcs12->vm_exit_reason = reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
13749         vmcs12->exit_qualification = qualification;
13750         nested_vmx_succeed(vcpu);
13751         if (enable_shadow_vmcs)
13752                 to_vmx(vcpu)->nested.sync_shadow_vmcs = true;
13753 }
13754
13755 static int vmx_check_intercept_io(struct kvm_vcpu *vcpu,
13756                                   struct x86_instruction_info *info)
13757 {
13758         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
13759         unsigned short port;
13760         bool intercept;
13761         int size;
13762
13763         if (info->intercept == x86_intercept_in ||
13764             info->intercept == x86_intercept_ins) {
13765                 port = info->src_val;
13766                 size = info->dst_bytes;
13767         } else {
13768                 port = info->dst_val;
13769                 size = info->src_bytes;
13770         }
13771
13772         /*
13773          * If the 'use IO bitmaps' VM-execution control is 0, IO instruction
13774          * VM-exits depend on the 'unconditional IO exiting' VM-execution
13775          * control.
13776          *
13777          * Otherwise, IO instruction VM-exits are controlled by the IO bitmaps.
13778          */
13779         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
13780                 intercept = nested_cpu_has(vmcs12,
13781                                            CPU_BASED_UNCOND_IO_EXITING);
13782         else
13783                 intercept = nested_vmx_check_io_bitmaps(vcpu, port, size);
13784
13785         /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED.  */
13786         return intercept ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE;
13787 }
13788
13789 static int vmx_check_intercept(struct kvm_vcpu *vcpu,
13790                                struct x86_instruction_info *info,
13791                                enum x86_intercept_stage stage)
13792 {
13793         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
13794         struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
13795
13796         switch (info->intercept) {
13797         /*
13798          * RDPID causes #UD if disabled through secondary execution controls.
13799          * Because it is marked as EmulateOnUD, we need to intercept it here.
13800          */
13801         case x86_intercept_rdtscp:
13802                 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDTSCP)) {
13803                         ctxt->exception.vector = UD_VECTOR;
13804                         ctxt->exception.error_code_valid = false;
13805                         return X86EMUL_PROPAGATE_FAULT;
13806                 }
13807                 break;
13808
13809         case x86_intercept_in:
13810         case x86_intercept_ins:
13811         case x86_intercept_out:
13812         case x86_intercept_outs:
13813                 return vmx_check_intercept_io(vcpu, info);
13814
13815         case x86_intercept_lgdt:
13816         case x86_intercept_lidt:
13817         case x86_intercept_lldt:
13818         case x86_intercept_ltr:
13819         case x86_intercept_sgdt:
13820         case x86_intercept_sidt:
13821         case x86_intercept_sldt:
13822         case x86_intercept_str:
13823                 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC))
13824                         return X86EMUL_CONTINUE;
13825
13826                 /* FIXME: produce nested vmexit and return X86EMUL_INTERCEPTED.  */
13827                 break;
13828
13829         /* TODO: check more intercepts... */
13830         default:
13831                 break;
13832         }
13833
13834         return X86EMUL_UNHANDLEABLE;
13835 }
13836
13837 #ifdef CONFIG_X86_64
13838 /* (a << shift) / divisor, return 1 if overflow otherwise 0 */
13839 static inline int u64_shl_div_u64(u64 a, unsigned int shift,
13840                                   u64 divisor, u64 *result)
13841 {
13842         u64 low = a << shift, high = a >> (64 - shift);
13843
13844         /* To avoid the overflow on divq */
13845         if (high >= divisor)
13846                 return 1;
13847
13848         /* Low hold the result, high hold rem which is discarded */
13849         asm("divq %2\n\t" : "=a" (low), "=d" (high) :
13850             "rm" (divisor), "0" (low), "1" (high));
13851         *result = low;
13852
13853         return 0;
13854 }
13855
13856 static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc)
13857 {
13858         struct vcpu_vmx *vmx;
13859         u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles;
13860
13861         if (kvm_mwait_in_guest(vcpu->kvm))
13862                 return -EOPNOTSUPP;
13863
13864         vmx = to_vmx(vcpu);
13865         tscl = rdtsc();
13866         guest_tscl = kvm_read_l1_tsc(vcpu, tscl);
13867         delta_tsc = max(guest_deadline_tsc, guest_tscl) - guest_tscl;
13868         lapic_timer_advance_cycles = nsec_to_cycles(vcpu, lapic_timer_advance_ns);
13869
13870         if (delta_tsc > lapic_timer_advance_cycles)
13871                 delta_tsc -= lapic_timer_advance_cycles;
13872         else
13873                 delta_tsc = 0;
13874
13875         /* Convert to host delta tsc if tsc scaling is enabled */
13876         if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio &&
13877                         u64_shl_div_u64(delta_tsc,
13878                                 kvm_tsc_scaling_ratio_frac_bits,
13879                                 vcpu->arch.tsc_scaling_ratio,
13880                                 &delta_tsc))
13881                 return -ERANGE;
13882
13883         /*
13884          * If the delta tsc can't fit in the 32 bit after the multi shift,
13885          * we can't use the preemption timer.
13886          * It's possible that it fits on later vmentries, but checking
13887          * on every vmentry is costly so we just use an hrtimer.
13888          */
13889         if (delta_tsc >> (cpu_preemption_timer_multi + 32))
13890                 return -ERANGE;
13891
13892         vmx->hv_deadline_tsc = tscl + delta_tsc;
13893         return delta_tsc == 0;
13894 }
13895
13896 static void vmx_cancel_hv_timer(struct kvm_vcpu *vcpu)
13897 {
13898         to_vmx(vcpu)->hv_deadline_tsc = -1;
13899 }
13900 #endif
13901
13902 static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu)
13903 {
13904         if (!kvm_pause_in_guest(vcpu->kvm))
13905                 shrink_ple_window(vcpu);
13906 }
13907
13908 static void vmx_slot_enable_log_dirty(struct kvm *kvm,
13909                                      struct kvm_memory_slot *slot)
13910 {
13911         kvm_mmu_slot_leaf_clear_dirty(kvm, slot);
13912         kvm_mmu_slot_largepage_remove_write_access(kvm, slot);
13913 }
13914
13915 static void vmx_slot_disable_log_dirty(struct kvm *kvm,
13916                                        struct kvm_memory_slot *slot)
13917 {
13918         kvm_mmu_slot_set_dirty(kvm, slot);
13919 }
13920
13921 static void vmx_flush_log_dirty(struct kvm *kvm)
13922 {
13923         kvm_flush_pml_buffers(kvm);
13924 }
13925
13926 static int vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa)
13927 {
13928         struct vmcs12 *vmcs12;
13929         struct vcpu_vmx *vmx = to_vmx(vcpu);
13930         struct page *page = NULL;
13931         u64 *pml_address;
13932
13933         if (is_guest_mode(vcpu)) {
13934                 WARN_ON_ONCE(vmx->nested.pml_full);
13935
13936                 /*
13937                  * Check if PML is enabled for the nested guest.
13938                  * Whether eptp bit 6 is set is already checked
13939                  * as part of A/D emulation.
13940                  */
13941                 vmcs12 = get_vmcs12(vcpu);
13942                 if (!nested_cpu_has_pml(vmcs12))
13943                         return 0;
13944
13945                 if (vmcs12->guest_pml_index >= PML_ENTITY_NUM) {
13946                         vmx->nested.pml_full = true;
13947                         return 1;
13948                 }
13949
13950                 gpa &= ~0xFFFull;
13951
13952                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->pml_address);
13953                 if (is_error_page(page))
13954                         return 0;
13955
13956                 pml_address = kmap(page);
13957                 pml_address[vmcs12->guest_pml_index--] = gpa;
13958                 kunmap(page);
13959                 kvm_release_page_clean(page);
13960         }
13961
13962         return 0;
13963 }
13964
13965 static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm,
13966                                            struct kvm_memory_slot *memslot,
13967                                            gfn_t offset, unsigned long mask)
13968 {
13969         kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask);
13970 }
13971
13972 static void __pi_post_block(struct kvm_vcpu *vcpu)
13973 {
13974         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
13975         struct pi_desc old, new;
13976         unsigned int dest;
13977
13978         do {
13979                 old.control = new.control = pi_desc->control;
13980                 WARN(old.nv != POSTED_INTR_WAKEUP_VECTOR,
13981                      "Wakeup handler not enabled while the VCPU is blocked\n");
13982
13983                 dest = cpu_physical_id(vcpu->cpu);
13984
13985                 if (x2apic_enabled())
13986                         new.ndst = dest;
13987                 else
13988                         new.ndst = (dest << 8) & 0xFF00;
13989
13990                 /* set 'NV' to 'notification vector' */
13991                 new.nv = POSTED_INTR_VECTOR;
13992         } while (cmpxchg64(&pi_desc->control, old.control,
13993                            new.control) != old.control);
13994
13995         if (!WARN_ON_ONCE(vcpu->pre_pcpu == -1)) {
13996                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
13997                 list_del(&vcpu->blocked_vcpu_list);
13998                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
13999                 vcpu->pre_pcpu = -1;
14000         }
14001 }
14002
14003 /*
14004  * This routine does the following things for vCPU which is going
14005  * to be blocked if VT-d PI is enabled.
14006  * - Store the vCPU to the wakeup list, so when interrupts happen
14007  *   we can find the right vCPU to wake up.
14008  * - Change the Posted-interrupt descriptor as below:
14009  *      'NDST' <-- vcpu->pre_pcpu
14010  *      'NV' <-- POSTED_INTR_WAKEUP_VECTOR
14011  * - If 'ON' is set during this process, which means at least one
14012  *   interrupt is posted for this vCPU, we cannot block it, in
14013  *   this case, return 1, otherwise, return 0.
14014  *
14015  */
14016 static int pi_pre_block(struct kvm_vcpu *vcpu)
14017 {
14018         unsigned int dest;
14019         struct pi_desc old, new;
14020         struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu);
14021
14022         if (!kvm_arch_has_assigned_device(vcpu->kvm) ||
14023                 !irq_remapping_cap(IRQ_POSTING_CAP)  ||
14024                 !kvm_vcpu_apicv_active(vcpu))
14025                 return 0;
14026
14027         WARN_ON(irqs_disabled());
14028         local_irq_disable();
14029         if (!WARN_ON_ONCE(vcpu->pre_pcpu != -1)) {
14030                 vcpu->pre_pcpu = vcpu->cpu;
14031                 spin_lock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
14032                 list_add_tail(&vcpu->blocked_vcpu_list,
14033                               &per_cpu(blocked_vcpu_on_cpu,
14034                                        vcpu->pre_pcpu));
14035                 spin_unlock(&per_cpu(blocked_vcpu_on_cpu_lock, vcpu->pre_pcpu));
14036         }
14037
14038         do {
14039                 old.control = new.control = pi_desc->control;
14040
14041                 WARN((pi_desc->sn == 1),
14042                      "Warning: SN field of posted-interrupts "
14043                      "is set before blocking\n");
14044
14045                 /*
14046                  * Since vCPU can be preempted during this process,
14047                  * vcpu->cpu could be different with pre_pcpu, we
14048                  * need to set pre_pcpu as the destination of wakeup
14049                  * notification event, then we can find the right vCPU
14050                  * to wakeup in wakeup handler if interrupts happen
14051                  * when the vCPU is in blocked state.
14052                  */
14053                 dest = cpu_physical_id(vcpu->pre_pcpu);
14054
14055                 if (x2apic_enabled())
14056                         new.ndst = dest;
14057                 else
14058                         new.ndst = (dest << 8) & 0xFF00;
14059
14060                 /* set 'NV' to 'wakeup vector' */
14061                 new.nv = POSTED_INTR_WAKEUP_VECTOR;
14062         } while (cmpxchg64(&pi_desc->control, old.control,
14063                            new.control) != old.control);
14064
14065         /* We should not block the vCPU if an interrupt is posted for it.  */
14066         if (pi_test_on(pi_desc) == 1)
14067                 __pi_post_block(vcpu);
14068
14069         local_irq_enable();
14070         return (vcpu->pre_pcpu == -1);
14071 }
14072
14073 static int vmx_pre_block(struct kvm_vcpu *vcpu)
14074 {
14075         if (pi_pre_block(vcpu))
14076                 return 1;
14077
14078         if (kvm_lapic_hv_timer_in_use(vcpu))
14079                 kvm_lapic_switch_to_sw_timer(vcpu);
14080
14081         return 0;
14082 }
14083
14084 static void pi_post_block(struct kvm_vcpu *vcpu)
14085 {
14086         if (vcpu->pre_pcpu == -1)
14087                 return;
14088
14089         WARN_ON(irqs_disabled());
14090         local_irq_disable();
14091         __pi_post_block(vcpu);
14092         local_irq_enable();
14093 }
14094
14095 static void vmx_post_block(struct kvm_vcpu *vcpu)
14096 {
14097         if (kvm_x86_ops->set_hv_timer)
14098                 kvm_lapic_switch_to_hv_timer(vcpu);
14099
14100         pi_post_block(vcpu);
14101 }
14102
14103 /*
14104  * vmx_update_pi_irte - set IRTE for Posted-Interrupts
14105  *
14106  * @kvm: kvm
14107  * @host_irq: host irq of the interrupt
14108  * @guest_irq: gsi of the interrupt
14109  * @set: set or unset PI
14110  * returns 0 on success, < 0 on failure
14111  */
14112 static int vmx_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
14113                               uint32_t guest_irq, bool set)
14114 {
14115         struct kvm_kernel_irq_routing_entry *e;
14116         struct kvm_irq_routing_table *irq_rt;
14117         struct kvm_lapic_irq irq;
14118         struct kvm_vcpu *vcpu;
14119         struct vcpu_data vcpu_info;
14120         int idx, ret = 0;
14121
14122         if (!kvm_arch_has_assigned_device(kvm) ||
14123                 !irq_remapping_cap(IRQ_POSTING_CAP) ||
14124                 !kvm_vcpu_apicv_active(kvm->vcpus[0]))
14125                 return 0;
14126
14127         idx = srcu_read_lock(&kvm->irq_srcu);
14128         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
14129         if (guest_irq >= irq_rt->nr_rt_entries ||
14130             hlist_empty(&irq_rt->map[guest_irq])) {
14131                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
14132                              guest_irq, irq_rt->nr_rt_entries);
14133                 goto out;
14134         }
14135
14136         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
14137                 if (e->type != KVM_IRQ_ROUTING_MSI)
14138                         continue;
14139                 /*
14140                  * VT-d PI cannot support posting multicast/broadcast
14141                  * interrupts to a vCPU, we still use interrupt remapping
14142                  * for these kind of interrupts.
14143                  *
14144                  * For lowest-priority interrupts, we only support
14145                  * those with single CPU as the destination, e.g. user
14146                  * configures the interrupts via /proc/irq or uses
14147                  * irqbalance to make the interrupts single-CPU.
14148                  *
14149                  * We will support full lowest-priority interrupt later.
14150                  */
14151
14152                 kvm_set_msi_irq(kvm, e, &irq);
14153                 if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu)) {
14154                         /*
14155                          * Make sure the IRTE is in remapped mode if
14156                          * we don't handle it in posted mode.
14157                          */
14158                         ret = irq_set_vcpu_affinity(host_irq, NULL);
14159                         if (ret < 0) {
14160                                 printk(KERN_INFO
14161                                    "failed to back to remapped mode, irq: %u\n",
14162                                    host_irq);
14163                                 goto out;
14164                         }
14165
14166                         continue;
14167                 }
14168
14169                 vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu));
14170                 vcpu_info.vector = irq.vector;
14171
14172                 trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi,
14173                                 vcpu_info.vector, vcpu_info.pi_desc_addr, set);
14174
14175                 if (set)
14176                         ret = irq_set_vcpu_affinity(host_irq, &vcpu_info);
14177                 else
14178                         ret = irq_set_vcpu_affinity(host_irq, NULL);
14179
14180                 if (ret < 0) {
14181                         printk(KERN_INFO "%s: failed to update PI IRTE\n",
14182                                         __func__);
14183                         goto out;
14184                 }
14185         }
14186
14187         ret = 0;
14188 out:
14189         srcu_read_unlock(&kvm->irq_srcu, idx);
14190         return ret;
14191 }
14192
14193 static void vmx_setup_mce(struct kvm_vcpu *vcpu)
14194 {
14195         if (vcpu->arch.mcg_cap & MCG_LMCE_P)
14196                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits |=
14197                         FEATURE_CONTROL_LMCE;
14198         else
14199                 to_vmx(vcpu)->msr_ia32_feature_control_valid_bits &=
14200                         ~FEATURE_CONTROL_LMCE;
14201 }
14202
14203 static int vmx_smi_allowed(struct kvm_vcpu *vcpu)
14204 {
14205         /* we need a nested vmexit to enter SMM, postpone if run is pending */
14206         if (to_vmx(vcpu)->nested.nested_run_pending)
14207                 return 0;
14208         return 1;
14209 }
14210
14211 static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
14212 {
14213         struct vcpu_vmx *vmx = to_vmx(vcpu);
14214
14215         vmx->nested.smm.guest_mode = is_guest_mode(vcpu);
14216         if (vmx->nested.smm.guest_mode)
14217                 nested_vmx_vmexit(vcpu, -1, 0, 0);
14218
14219         vmx->nested.smm.vmxon = vmx->nested.vmxon;
14220         vmx->nested.vmxon = false;
14221         vmx_clear_hlt(vcpu);
14222         return 0;
14223 }
14224
14225 static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, u64 smbase)
14226 {
14227         struct vcpu_vmx *vmx = to_vmx(vcpu);
14228         int ret;
14229
14230         if (vmx->nested.smm.vmxon) {
14231                 vmx->nested.vmxon = true;
14232                 vmx->nested.smm.vmxon = false;
14233         }
14234
14235         if (vmx->nested.smm.guest_mode) {
14236                 vcpu->arch.hflags &= ~HF_SMM_MASK;
14237                 ret = enter_vmx_non_root_mode(vcpu, NULL);
14238                 vcpu->arch.hflags |= HF_SMM_MASK;
14239                 if (ret)
14240                         return ret;
14241
14242                 vmx->nested.smm.guest_mode = false;
14243         }
14244         return 0;
14245 }
14246
14247 static int enable_smi_window(struct kvm_vcpu *vcpu)
14248 {
14249         return 0;
14250 }
14251
14252 static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
14253                                 struct kvm_nested_state __user *user_kvm_nested_state,
14254                                 u32 user_data_size)
14255 {
14256         struct vcpu_vmx *vmx;
14257         struct vmcs12 *vmcs12;
14258         struct kvm_nested_state kvm_state = {
14259                 .flags = 0,
14260                 .format = 0,
14261                 .size = sizeof(kvm_state),
14262                 .vmx.vmxon_pa = -1ull,
14263                 .vmx.vmcs_pa = -1ull,
14264         };
14265
14266         if (!vcpu)
14267                 return kvm_state.size + 2 * VMCS12_SIZE;
14268
14269         vmx = to_vmx(vcpu);
14270         vmcs12 = get_vmcs12(vcpu);
14271         if (nested_vmx_allowed(vcpu) &&
14272             (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
14273                 kvm_state.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
14274                 kvm_state.vmx.vmcs_pa = vmx->nested.current_vmptr;
14275
14276                 if (vmx->nested.current_vmptr != -1ull) {
14277                         kvm_state.size += VMCS12_SIZE;
14278
14279                         if (is_guest_mode(vcpu) &&
14280                             nested_cpu_has_shadow_vmcs(vmcs12) &&
14281                             vmcs12->vmcs_link_pointer != -1ull)
14282                                 kvm_state.size += VMCS12_SIZE;
14283                 }
14284
14285                 if (vmx->nested.smm.vmxon)
14286                         kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
14287
14288                 if (vmx->nested.smm.guest_mode)
14289                         kvm_state.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
14290
14291                 if (is_guest_mode(vcpu)) {
14292                         kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
14293
14294                         if (vmx->nested.nested_run_pending)
14295                                 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
14296                 }
14297         }
14298
14299         if (user_data_size < kvm_state.size)
14300                 goto out;
14301
14302         if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
14303                 return -EFAULT;
14304
14305         if (vmx->nested.current_vmptr == -1ull)
14306                 goto out;
14307
14308         /*
14309          * When running L2, the authoritative vmcs12 state is in the
14310          * vmcs02. When running L1, the authoritative vmcs12 state is
14311          * in the shadow vmcs linked to vmcs01, unless
14312          * sync_shadow_vmcs is set, in which case, the authoritative
14313          * vmcs12 state is in the vmcs12 already.
14314          */
14315         if (is_guest_mode(vcpu))
14316                 sync_vmcs12(vcpu, vmcs12);
14317         else if (enable_shadow_vmcs && !vmx->nested.sync_shadow_vmcs)
14318                 copy_shadow_to_vmcs12(vmx);
14319
14320         /*
14321          * Copy over the full allocated size of vmcs12 rather than just the size
14322          * of the struct.
14323          */
14324         if (copy_to_user(user_kvm_nested_state->data, vmcs12, VMCS12_SIZE))
14325                 return -EFAULT;
14326
14327         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
14328             vmcs12->vmcs_link_pointer != -1ull) {
14329                 if (copy_to_user(user_kvm_nested_state->data + VMCS12_SIZE,
14330                                  get_shadow_vmcs12(vcpu), VMCS12_SIZE))
14331                         return -EFAULT;
14332         }
14333
14334 out:
14335         return kvm_state.size;
14336 }
14337
14338 static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
14339                                 struct kvm_nested_state __user *user_kvm_nested_state,
14340                                 struct kvm_nested_state *kvm_state)
14341 {
14342         struct vcpu_vmx *vmx = to_vmx(vcpu);
14343         struct vmcs12 *vmcs12;
14344         u32 exit_qual;
14345         int ret;
14346
14347         if (kvm_state->format != 0)
14348                 return -EINVAL;
14349
14350         if (!nested_vmx_allowed(vcpu))
14351                 return kvm_state->vmx.vmxon_pa == -1ull ? 0 : -EINVAL;
14352
14353         if (kvm_state->vmx.vmxon_pa == -1ull) {
14354                 if (kvm_state->vmx.smm.flags)
14355                         return -EINVAL;
14356
14357                 if (kvm_state->vmx.vmcs_pa != -1ull)
14358                         return -EINVAL;
14359
14360                 vmx_leave_nested(vcpu);
14361                 return 0;
14362         }
14363
14364         if (!page_address_valid(vcpu, kvm_state->vmx.vmxon_pa))
14365                 return -EINVAL;
14366
14367         if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
14368             (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
14369                 return -EINVAL;
14370
14371         if (kvm_state->vmx.smm.flags &
14372             ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
14373                 return -EINVAL;
14374
14375         /*
14376          * SMM temporarily disables VMX, so we cannot be in guest mode,
14377          * nor can VMLAUNCH/VMRESUME be pending.  Outside SMM, SMM flags
14378          * must be zero.
14379          */
14380         if (is_smm(vcpu) ? kvm_state->flags : kvm_state->vmx.smm.flags)
14381                 return -EINVAL;
14382
14383         if ((kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
14384             !(kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
14385                 return -EINVAL;
14386
14387         vmx_leave_nested(vcpu);
14388         if (kvm_state->vmx.vmxon_pa == -1ull)
14389                 return 0;
14390
14391         vmx->nested.vmxon_ptr = kvm_state->vmx.vmxon_pa;
14392         ret = enter_vmx_operation(vcpu);
14393         if (ret)
14394                 return ret;
14395
14396         /* Empty 'VMXON' state is permitted */
14397         if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12))
14398                 return 0;
14399
14400         if (kvm_state->vmx.vmcs_pa == kvm_state->vmx.vmxon_pa ||
14401             !page_address_valid(vcpu, kvm_state->vmx.vmcs_pa))
14402                 return -EINVAL;
14403
14404         set_current_vmptr(vmx, kvm_state->vmx.vmcs_pa);
14405
14406         if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
14407                 vmx->nested.smm.vmxon = true;
14408                 vmx->nested.vmxon = false;
14409
14410                 if (kvm_state->vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
14411                         vmx->nested.smm.guest_mode = true;
14412         }
14413
14414         vmcs12 = get_vmcs12(vcpu);
14415         if (copy_from_user(vmcs12, user_kvm_nested_state->data, sizeof(*vmcs12)))
14416                 return -EFAULT;
14417
14418         if (vmcs12->hdr.revision_id != VMCS12_REVISION)
14419                 return -EINVAL;
14420
14421         if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
14422                 return 0;
14423
14424         vmx->nested.nested_run_pending =
14425                 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
14426
14427         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
14428             vmcs12->vmcs_link_pointer != -1ull) {
14429                 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
14430                 if (kvm_state->size < sizeof(*kvm_state) + 2 * sizeof(*vmcs12))
14431                         return -EINVAL;
14432
14433                 if (copy_from_user(shadow_vmcs12,
14434                                    user_kvm_nested_state->data + VMCS12_SIZE,
14435                                    sizeof(*vmcs12)))
14436                         return -EFAULT;
14437
14438                 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
14439                     !shadow_vmcs12->hdr.shadow_vmcs)
14440                         return -EINVAL;
14441         }
14442
14443         if (check_vmentry_prereqs(vcpu, vmcs12) ||
14444             check_vmentry_postreqs(vcpu, vmcs12, &exit_qual))
14445                 return -EINVAL;
14446
14447         vmx->nested.dirty_vmcs12 = true;
14448         ret = enter_vmx_non_root_mode(vcpu, NULL);
14449         if (ret)
14450                 return -EINVAL;
14451
14452         return 0;
14453 }
14454
14455 static struct kvm_x86_ops vmx_x86_ops __ro_after_init = {
14456         .cpu_has_kvm_support = cpu_has_kvm_support,
14457         .disabled_by_bios = vmx_disabled_by_bios,
14458         .hardware_setup = hardware_setup,
14459         .hardware_unsetup = hardware_unsetup,
14460         .check_processor_compatibility = vmx_check_processor_compat,
14461         .hardware_enable = hardware_enable,
14462         .hardware_disable = hardware_disable,
14463         .cpu_has_accelerated_tpr = report_flexpriority,
14464         .has_emulated_msr = vmx_has_emulated_msr,
14465
14466         .vm_init = vmx_vm_init,
14467         .vm_alloc = vmx_vm_alloc,
14468         .vm_free = vmx_vm_free,
14469
14470         .vcpu_create = vmx_create_vcpu,
14471         .vcpu_free = vmx_free_vcpu,
14472         .vcpu_reset = vmx_vcpu_reset,
14473
14474         .prepare_guest_switch = vmx_prepare_switch_to_guest,
14475         .vcpu_load = vmx_vcpu_load,
14476         .vcpu_put = vmx_vcpu_put,
14477
14478         .update_bp_intercept = update_exception_bitmap,
14479         .get_msr_feature = vmx_get_msr_feature,
14480         .get_msr = vmx_get_msr,
14481         .set_msr = vmx_set_msr,
14482         .get_segment_base = vmx_get_segment_base,
14483         .get_segment = vmx_get_segment,
14484         .set_segment = vmx_set_segment,
14485         .get_cpl = vmx_get_cpl,
14486         .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
14487         .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits,
14488         .decache_cr3 = vmx_decache_cr3,
14489         .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
14490         .set_cr0 = vmx_set_cr0,
14491         .set_cr3 = vmx_set_cr3,
14492         .set_cr4 = vmx_set_cr4,
14493         .set_efer = vmx_set_efer,
14494         .get_idt = vmx_get_idt,
14495         .set_idt = vmx_set_idt,
14496         .get_gdt = vmx_get_gdt,
14497         .set_gdt = vmx_set_gdt,
14498         .get_dr6 = vmx_get_dr6,
14499         .set_dr6 = vmx_set_dr6,
14500         .set_dr7 = vmx_set_dr7,
14501         .sync_dirty_debug_regs = vmx_sync_dirty_debug_regs,
14502         .cache_reg = vmx_cache_reg,
14503         .get_rflags = vmx_get_rflags,
14504         .set_rflags = vmx_set_rflags,
14505
14506         .tlb_flush = vmx_flush_tlb,
14507         .tlb_flush_gva = vmx_flush_tlb_gva,
14508
14509         .run = vmx_vcpu_run,
14510         .handle_exit = vmx_handle_exit,
14511         .skip_emulated_instruction = skip_emulated_instruction,
14512         .set_interrupt_shadow = vmx_set_interrupt_shadow,
14513         .get_interrupt_shadow = vmx_get_interrupt_shadow,
14514         .patch_hypercall = vmx_patch_hypercall,
14515         .set_irq = vmx_inject_irq,
14516         .set_nmi = vmx_inject_nmi,
14517         .queue_exception = vmx_queue_exception,
14518         .cancel_injection = vmx_cancel_injection,
14519         .interrupt_allowed = vmx_interrupt_allowed,
14520         .nmi_allowed = vmx_nmi_allowed,
14521         .get_nmi_mask = vmx_get_nmi_mask,
14522         .set_nmi_mask = vmx_set_nmi_mask,
14523         .enable_nmi_window = enable_nmi_window,
14524         .enable_irq_window = enable_irq_window,
14525         .update_cr8_intercept = update_cr8_intercept,
14526         .set_virtual_apic_mode = vmx_set_virtual_apic_mode,
14527         .set_apic_access_page_addr = vmx_set_apic_access_page_addr,
14528         .get_enable_apicv = vmx_get_enable_apicv,
14529         .refresh_apicv_exec_ctrl = vmx_refresh_apicv_exec_ctrl,
14530         .load_eoi_exitmap = vmx_load_eoi_exitmap,
14531         .apicv_post_state_restore = vmx_apicv_post_state_restore,
14532         .hwapic_irr_update = vmx_hwapic_irr_update,
14533         .hwapic_isr_update = vmx_hwapic_isr_update,
14534         .guest_apic_has_interrupt = vmx_guest_apic_has_interrupt,
14535         .sync_pir_to_irr = vmx_sync_pir_to_irr,
14536         .deliver_posted_interrupt = vmx_deliver_posted_interrupt,
14537         .dy_apicv_has_pending_interrupt = vmx_dy_apicv_has_pending_interrupt,
14538
14539         .set_tss_addr = vmx_set_tss_addr,
14540         .set_identity_map_addr = vmx_set_identity_map_addr,
14541         .get_tdp_level = get_ept_level,
14542         .get_mt_mask = vmx_get_mt_mask,
14543
14544         .get_exit_info = vmx_get_exit_info,
14545
14546         .get_lpage_level = vmx_get_lpage_level,
14547
14548         .cpuid_update = vmx_cpuid_update,
14549
14550         .rdtscp_supported = vmx_rdtscp_supported,
14551         .invpcid_supported = vmx_invpcid_supported,
14552
14553         .set_supported_cpuid = vmx_set_supported_cpuid,
14554
14555         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
14556
14557         .read_l1_tsc_offset = vmx_read_l1_tsc_offset,
14558         .write_l1_tsc_offset = vmx_write_l1_tsc_offset,
14559
14560         .set_tdp_cr3 = vmx_set_cr3,
14561
14562         .check_intercept = vmx_check_intercept,
14563         .handle_external_intr = vmx_handle_external_intr,
14564         .mpx_supported = vmx_mpx_supported,
14565         .xsaves_supported = vmx_xsaves_supported,
14566         .umip_emulated = vmx_umip_emulated,
14567
14568         .check_nested_events = vmx_check_nested_events,
14569         .request_immediate_exit = vmx_request_immediate_exit,
14570
14571         .sched_in = vmx_sched_in,
14572
14573         .slot_enable_log_dirty = vmx_slot_enable_log_dirty,
14574         .slot_disable_log_dirty = vmx_slot_disable_log_dirty,
14575         .flush_log_dirty = vmx_flush_log_dirty,
14576         .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked,
14577         .write_log_dirty = vmx_write_pml_buffer,
14578
14579         .pre_block = vmx_pre_block,
14580         .post_block = vmx_post_block,
14581
14582         .pmu_ops = &intel_pmu_ops,
14583
14584         .update_pi_irte = vmx_update_pi_irte,
14585
14586 #ifdef CONFIG_X86_64
14587         .set_hv_timer = vmx_set_hv_timer,
14588         .cancel_hv_timer = vmx_cancel_hv_timer,
14589 #endif
14590
14591         .setup_mce = vmx_setup_mce,
14592
14593         .get_nested_state = vmx_get_nested_state,
14594         .set_nested_state = vmx_set_nested_state,
14595         .get_vmcs12_pages = nested_get_vmcs12_pages,
14596
14597         .smi_allowed = vmx_smi_allowed,
14598         .pre_enter_smm = vmx_pre_enter_smm,
14599         .pre_leave_smm = vmx_pre_leave_smm,
14600         .enable_smi_window = enable_smi_window,
14601 };
14602
14603 static void vmx_cleanup_l1d_flush(void)
14604 {
14605         if (vmx_l1d_flush_pages) {
14606                 free_pages((unsigned long)vmx_l1d_flush_pages, L1D_CACHE_ORDER);
14607                 vmx_l1d_flush_pages = NULL;
14608         }
14609         /* Restore state so sysfs ignores VMX */
14610         l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
14611 }
14612
14613 static void vmx_exit(void)
14614 {
14615 #ifdef CONFIG_KEXEC_CORE
14616         RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL);
14617         synchronize_rcu();
14618 #endif
14619
14620         kvm_exit();
14621
14622 #if IS_ENABLED(CONFIG_HYPERV)
14623         if (static_branch_unlikely(&enable_evmcs)) {
14624                 int cpu;
14625                 struct hv_vp_assist_page *vp_ap;
14626                 /*
14627                  * Reset everything to support using non-enlightened VMCS
14628                  * access later (e.g. when we reload the module with
14629                  * enlightened_vmcs=0)
14630                  */
14631                 for_each_online_cpu(cpu) {
14632                         vp_ap = hv_get_vp_assist_page(cpu);
14633
14634                         if (!vp_ap)
14635                                 continue;
14636
14637                         vp_ap->current_nested_vmcs = 0;
14638                         vp_ap->enlighten_vmentry = 0;
14639                 }
14640
14641                 static_branch_disable(&enable_evmcs);
14642         }
14643 #endif
14644         vmx_cleanup_l1d_flush();
14645 }
14646 module_exit(vmx_exit);
14647
14648 static int __init vmx_init(void)
14649 {
14650         int r, cpu;
14651
14652 #if IS_ENABLED(CONFIG_HYPERV)
14653         /*
14654          * Enlightened VMCS usage should be recommended and the host needs
14655          * to support eVMCS v1 or above. We can also disable eVMCS support
14656          * with module parameter.
14657          */
14658         if (enlightened_vmcs &&
14659             ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED &&
14660             (ms_hyperv.nested_features & HV_X64_ENLIGHTENED_VMCS_VERSION) >=
14661             KVM_EVMCS_VERSION) {
14662                 int cpu;
14663
14664                 /* Check that we have assist pages on all online CPUs */
14665                 for_each_online_cpu(cpu) {
14666                         if (!hv_get_vp_assist_page(cpu)) {
14667                                 enlightened_vmcs = false;
14668                                 break;
14669                         }
14670                 }
14671
14672                 if (enlightened_vmcs) {
14673                         pr_info("KVM: vmx: using Hyper-V Enlightened VMCS\n");
14674                         static_branch_enable(&enable_evmcs);
14675                 }
14676         } else {
14677                 enlightened_vmcs = false;
14678         }
14679 #endif
14680
14681         r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx),
14682                      __alignof__(struct vcpu_vmx), THIS_MODULE);
14683         if (r)
14684                 return r;
14685
14686         /*
14687          * Must be called after kvm_init() so enable_ept is properly set
14688          * up. Hand the parameter mitigation value in which was stored in
14689          * the pre module init parser. If no parameter was given, it will
14690          * contain 'auto' which will be turned into the default 'cond'
14691          * mitigation mode.
14692          */
14693         if (boot_cpu_has(X86_BUG_L1TF)) {
14694                 r = vmx_setup_l1d_flush(vmentry_l1d_flush_param);
14695                 if (r) {
14696                         vmx_exit();
14697                         return r;
14698                 }
14699         }
14700
14701         vmx_setup_fb_clear_ctrl();
14702
14703         for_each_possible_cpu(cpu) {
14704                 INIT_LIST_HEAD(&per_cpu(loaded_vmcss_on_cpu, cpu));
14705
14706                 INIT_LIST_HEAD(&per_cpu(blocked_vcpu_on_cpu, cpu));
14707                 spin_lock_init(&per_cpu(blocked_vcpu_on_cpu_lock, cpu));
14708         }
14709
14710 #ifdef CONFIG_KEXEC_CORE
14711         rcu_assign_pointer(crash_vmclear_loaded_vmcss,
14712                            crash_vmclear_local_loaded_vmcss);
14713 #endif
14714         vmx_check_vmcs12_offsets();
14715
14716         return 0;
14717 }
14718 module_init(vmx_init);