1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 #include <asm/nospec-branch.h>
39 #define OpImplicit 1ull /* No generic decode */
40 #define OpReg 2ull /* Register */
41 #define OpMem 3ull /* Memory */
42 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
43 #define OpDI 5ull /* ES:DI/EDI/RDI */
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
46 #define OpDX 8ull /* DX register */
47 #define OpCL 9ull /* CL register (for shifts) */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
49 #define OpOne 11ull /* Implied 1 */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
53 #define OpImmU 15ull /* Immediate operand, zero extended */
54 #define OpSI 16ull /* SI/ESI/RSI */
55 #define OpImmFAddr 17ull /* Immediate far address */
56 #define OpMemFAddr 18ull /* Far address in memory */
57 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
58 #define OpES 20ull /* ES */
59 #define OpCS 21ull /* CS */
60 #define OpSS 22ull /* SS */
61 #define OpDS 23ull /* DS */
62 #define OpFS 24ull /* FS */
63 #define OpGS 25ull /* GS */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
67 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
70 #define OpBits 5 /* Width of operand field */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
75 * Note that we only emulate instructions that have at least one memory
76 * operand (excluding implicit stack references). We assume that stack
77 * references and instruction fetches will never occur in special memory
78 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
84 /* Destination operand type. */
86 #define ImplicitOps (OpImplicit << DstShift)
87 #define DstReg (OpReg << DstShift)
88 #define DstMem (OpMem << DstShift)
89 #define DstAcc (OpAcc << DstShift)
90 #define DstDI (OpDI << DstShift)
91 #define DstMem64 (OpMem64 << DstShift)
92 #define DstMem16 (OpMem16 << DstShift)
93 #define DstImmUByte (OpImmUByte << DstShift)
94 #define DstDX (OpDX << DstShift)
95 #define DstAccLo (OpAccLo << DstShift)
96 #define DstMask (OpMask << DstShift)
97 /* Source operand type. */
99 #define SrcNone (OpNone << SrcShift)
100 #define SrcReg (OpReg << SrcShift)
101 #define SrcMem (OpMem << SrcShift)
102 #define SrcMem16 (OpMem16 << SrcShift)
103 #define SrcMem32 (OpMem32 << SrcShift)
104 #define SrcImm (OpImm << SrcShift)
105 #define SrcImmByte (OpImmByte << SrcShift)
106 #define SrcOne (OpOne << SrcShift)
107 #define SrcImmUByte (OpImmUByte << SrcShift)
108 #define SrcImmU (OpImmU << SrcShift)
109 #define SrcSI (OpSI << SrcShift)
110 #define SrcXLat (OpXLat << SrcShift)
111 #define SrcImmFAddr (OpImmFAddr << SrcShift)
112 #define SrcMemFAddr (OpMemFAddr << SrcShift)
113 #define SrcAcc (OpAcc << SrcShift)
114 #define SrcImmU16 (OpImmU16 << SrcShift)
115 #define SrcImm64 (OpImm64 << SrcShift)
116 #define SrcDX (OpDX << SrcShift)
117 #define SrcMem8 (OpMem8 << SrcShift)
118 #define SrcAccHi (OpAccHi << SrcShift)
119 #define SrcMask (OpMask << SrcShift)
120 #define BitOp (1<<11)
121 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
122 #define String (1<<13) /* String instruction (rep capable) */
123 #define Stack (1<<14) /* Stack instruction (push/pop) */
124 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
125 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
126 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
127 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
128 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
129 #define Escape (5<<15) /* Escape to coprocessor instruction */
130 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
131 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
132 #define Sse (1<<18) /* SSE Vector instruction */
133 /* Generic ModRM decode. */
134 #define ModRM (1<<19)
135 /* Destination is only written; never read. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
139 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
140 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
142 #define Undefined (1<<25) /* No Such Instruction */
143 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
144 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
146 #define PageTable (1 << 29) /* instruction used to write page table */
147 #define NotImpl (1 << 30) /* instruction is not implemented */
148 /* Source 2 operand type */
149 #define Src2Shift (31)
150 #define Src2None (OpNone << Src2Shift)
151 #define Src2Mem (OpMem << Src2Shift)
152 #define Src2CL (OpCL << Src2Shift)
153 #define Src2ImmByte (OpImmByte << Src2Shift)
154 #define Src2One (OpOne << Src2Shift)
155 #define Src2Imm (OpImm << Src2Shift)
156 #define Src2ES (OpES << Src2Shift)
157 #define Src2CS (OpCS << Src2Shift)
158 #define Src2SS (OpSS << Src2Shift)
159 #define Src2DS (OpDS << Src2Shift)
160 #define Src2FS (OpFS << Src2Shift)
161 #define Src2GS (OpGS << Src2Shift)
162 #define Src2Mask (OpMask << Src2Shift)
163 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
164 #define AlignMask ((u64)7 << 41)
165 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
166 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
167 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
168 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
169 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
170 #define NoWrite ((u64)1 << 45) /* No writeback */
171 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
172 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
173 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
174 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
175 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
176 #define NearBranch ((u64)1 << 52) /* Near branches */
177 #define No16 ((u64)1 << 53) /* No 16 bit operand */
178 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
179 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
192 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
193 #define FASTOP_SIZE 8
196 * fastop functions have a special calling convention:
201 * flags: rflags (in/out)
202 * ex: rsi (in:fastop pointer, out:zero if exception)
204 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
205 * different operand sizes can be reached by calculation, rather than a jump
206 * table (which would be bigger than the code).
208 * fastop functions are declared as taking a never-defined fastop parameter,
209 * so they can't be called from C directly.
218 int (*execute)(struct x86_emulate_ctxt *ctxt);
219 const struct opcode *group;
220 const struct group_dual *gdual;
221 const struct gprefix *gprefix;
222 const struct escape *esc;
223 const struct instr_dual *idual;
224 const struct mode_dual *mdual;
225 void (*fastop)(struct fastop *fake);
227 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
231 struct opcode mod012[8];
232 struct opcode mod3[8];
236 struct opcode pfx_no;
237 struct opcode pfx_66;
238 struct opcode pfx_f2;
239 struct opcode pfx_f3;
244 struct opcode high[64];
248 struct opcode mod012;
253 struct opcode mode32;
254 struct opcode mode64;
257 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
259 enum x86_transfer_type {
261 X86_TRANSFER_CALL_JMP,
263 X86_TRANSFER_TASK_SWITCH,
266 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
268 if (!(ctxt->regs_valid & (1 << nr))) {
269 ctxt->regs_valid |= 1 << nr;
270 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
272 return ctxt->_regs[nr];
275 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
277 ctxt->regs_valid |= 1 << nr;
278 ctxt->regs_dirty |= 1 << nr;
279 return &ctxt->_regs[nr];
282 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
285 return reg_write(ctxt, nr);
288 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
292 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
293 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
296 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
298 ctxt->regs_dirty = 0;
299 ctxt->regs_valid = 0;
303 * These EFLAGS bits are restored from saved value during emulation, and
304 * any changes are written back to the saved value after emulation.
306 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
307 X86_EFLAGS_PF|X86_EFLAGS_CF)
315 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
317 #define FOP_FUNC(name) \
318 ".align " __stringify(FASTOP_SIZE) " \n\t" \
319 ".type " name ", @function \n\t" \
322 #define FOP_RET "ret \n\t"
324 #define FOP_START(op) \
325 extern void em_##op(struct fastop *fake); \
326 asm(".pushsection .text, \"ax\" \n\t" \
327 ".global em_" #op " \n\t" \
334 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
337 #define FOP1E(op, dst) \
338 FOP_FUNC(#op "_" #dst) \
339 "10: " #op " %" #dst " \n\t" FOP_RET
341 #define FOP1EEX(op, dst) \
342 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
344 #define FASTOP1(op) \
349 ON64(FOP1E(op##q, rax)) \
352 /* 1-operand, using src2 (for MUL/DIV r/m) */
353 #define FASTOP1SRC2(op, name) \
358 ON64(FOP1E(op, rcx)) \
361 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
362 #define FASTOP1SRC2EX(op, name) \
367 ON64(FOP1EEX(op, rcx)) \
370 #define FOP2E(op, dst, src) \
371 FOP_FUNC(#op "_" #dst "_" #src) \
372 #op " %" #src ", %" #dst " \n\t" FOP_RET
374 #define FASTOP2(op) \
376 FOP2E(op##b, al, dl) \
377 FOP2E(op##w, ax, dx) \
378 FOP2E(op##l, eax, edx) \
379 ON64(FOP2E(op##q, rax, rdx)) \
382 /* 2 operand, word only */
383 #define FASTOP2W(op) \
386 FOP2E(op##w, ax, dx) \
387 FOP2E(op##l, eax, edx) \
388 ON64(FOP2E(op##q, rax, rdx)) \
391 /* 2 operand, src is CL */
392 #define FASTOP2CL(op) \
394 FOP2E(op##b, al, cl) \
395 FOP2E(op##w, ax, cl) \
396 FOP2E(op##l, eax, cl) \
397 ON64(FOP2E(op##q, rax, cl)) \
400 /* 2 operand, src and dest are reversed */
401 #define FASTOP2R(op, name) \
403 FOP2E(op##b, dl, al) \
404 FOP2E(op##w, dx, ax) \
405 FOP2E(op##l, edx, eax) \
406 ON64(FOP2E(op##q, rdx, rax)) \
409 #define FOP3E(op, dst, src, src2) \
410 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
411 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
413 /* 3-operand, word-only, src2=cl */
414 #define FASTOP3WCL(op) \
417 FOP3E(op##w, ax, dx, cl) \
418 FOP3E(op##l, eax, edx, cl) \
419 ON64(FOP3E(op##q, rax, rdx, cl)) \
422 /* Special case for SETcc - 1 instruction per cc */
423 #define FOP_SETCC(op) \
425 ".type " #op ", @function \n\t" \
430 asm(".pushsection .fixup, \"ax\"\n"
431 ".global kvm_fastop_exception \n"
432 "kvm_fastop_exception: xor %esi, %esi; ret\n"
454 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
458 * XXX: inoutclob user must know where the argument is being expanded.
459 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
461 #define asm_safe(insn, inoutclob...) \
465 asm volatile("1:" insn "\n" \
467 ".pushsection .fixup, \"ax\"\n" \
468 "3: movl $1, %[_fault]\n" \
471 _ASM_EXTABLE(1b, 3b) \
472 : [_fault] "+qm"(_fault) inoutclob ); \
474 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
477 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
478 enum x86_intercept intercept,
479 enum x86_intercept_stage stage)
481 struct x86_instruction_info info = {
482 .intercept = intercept,
483 .rep_prefix = ctxt->rep_prefix,
484 .modrm_mod = ctxt->modrm_mod,
485 .modrm_reg = ctxt->modrm_reg,
486 .modrm_rm = ctxt->modrm_rm,
487 .src_val = ctxt->src.val64,
488 .dst_val = ctxt->dst.val64,
489 .src_bytes = ctxt->src.bytes,
490 .dst_bytes = ctxt->dst.bytes,
491 .ad_bytes = ctxt->ad_bytes,
492 .next_rip = ctxt->eip,
495 return ctxt->ops->intercept(ctxt, &info, stage);
498 static void assign_masked(ulong *dest, ulong src, ulong mask)
500 *dest = (*dest & ~mask) | (src & mask);
503 static void assign_register(unsigned long *reg, u64 val, int bytes)
505 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
508 *(u8 *)reg = (u8)val;
511 *(u16 *)reg = (u16)val;
515 break; /* 64b: zero-extend */
522 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
524 return (1UL << (ctxt->ad_bytes << 3)) - 1;
527 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
530 struct desc_struct ss;
532 if (ctxt->mode == X86EMUL_MODE_PROT64)
534 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
535 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
538 static int stack_size(struct x86_emulate_ctxt *ctxt)
540 return (__fls(stack_mask(ctxt)) + 1) >> 3;
543 /* Access/update address held in a register, based on addressing mode. */
544 static inline unsigned long
545 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
547 if (ctxt->ad_bytes == sizeof(unsigned long))
550 return reg & ad_mask(ctxt);
553 static inline unsigned long
554 register_address(struct x86_emulate_ctxt *ctxt, int reg)
556 return address_mask(ctxt, reg_read(ctxt, reg));
559 static void masked_increment(ulong *reg, ulong mask, int inc)
561 assign_masked(reg, *reg + inc, mask);
565 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
567 ulong *preg = reg_rmw(ctxt, reg);
569 assign_register(preg, *preg + inc, ctxt->ad_bytes);
572 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
574 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
577 static u32 desc_limit_scaled(struct desc_struct *desc)
579 u32 limit = get_desc_limit(desc);
581 return desc->g ? (limit << 12) | 0xfff : limit;
584 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
586 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
589 return ctxt->ops->get_cached_segment_base(ctxt, seg);
592 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
593 u32 error, bool valid)
596 ctxt->exception.vector = vec;
597 ctxt->exception.error_code = error;
598 ctxt->exception.error_code_valid = valid;
599 return X86EMUL_PROPAGATE_FAULT;
602 static int emulate_db(struct x86_emulate_ctxt *ctxt)
604 return emulate_exception(ctxt, DB_VECTOR, 0, false);
607 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
609 return emulate_exception(ctxt, GP_VECTOR, err, true);
612 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
614 return emulate_exception(ctxt, SS_VECTOR, err, true);
617 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
619 return emulate_exception(ctxt, UD_VECTOR, 0, false);
622 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
624 return emulate_exception(ctxt, TS_VECTOR, err, true);
627 static int emulate_de(struct x86_emulate_ctxt *ctxt)
629 return emulate_exception(ctxt, DE_VECTOR, 0, false);
632 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
634 return emulate_exception(ctxt, NM_VECTOR, 0, false);
637 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
640 struct desc_struct desc;
642 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
646 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
651 struct desc_struct desc;
653 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
654 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
658 * x86 defines three classes of vector instructions: explicitly
659 * aligned, explicitly unaligned, and the rest, which change behaviour
660 * depending on whether they're AVX encoded or not.
662 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
663 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
664 * 512 bytes of data must be aligned to a 16 byte boundary.
666 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
668 u64 alignment = ctxt->d & AlignMask;
670 if (likely(size < 16))
685 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
686 struct segmented_address addr,
687 unsigned *max_size, unsigned size,
688 bool write, bool fetch,
689 enum x86emul_mode mode, ulong *linear)
691 struct desc_struct desc;
698 la = seg_base(ctxt, addr.seg) + addr.ea;
701 case X86EMUL_MODE_PROT64:
703 va_bits = ctxt_virt_addr_bits(ctxt);
704 if (get_canonical(la, va_bits) != la)
707 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
708 if (size > *max_size)
712 *linear = la = (u32)la;
713 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
717 /* code segment in protected mode or read-only data segment */
718 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
719 || !(desc.type & 2)) && write)
721 /* unreadable code segment */
722 if (!fetch && (desc.type & 8) && !(desc.type & 2))
724 lim = desc_limit_scaled(&desc);
725 if (!(desc.type & 8) && (desc.type & 4)) {
726 /* expand-down segment */
729 lim = desc.d ? 0xffffffff : 0xffff;
733 if (lim == 0xffffffff)
736 *max_size = (u64)lim + 1 - addr.ea;
737 if (size > *max_size)
742 if (la & (insn_alignment(ctxt, size) - 1))
743 return emulate_gp(ctxt, 0);
744 return X86EMUL_CONTINUE;
746 if (addr.seg == VCPU_SREG_SS)
747 return emulate_ss(ctxt, 0);
749 return emulate_gp(ctxt, 0);
752 static int linearize(struct x86_emulate_ctxt *ctxt,
753 struct segmented_address addr,
754 unsigned size, bool write,
758 return __linearize(ctxt, addr, &max_size, size, write, false,
762 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst)
767 struct segmented_address addr = { .seg = VCPU_SREG_CS,
770 if (ctxt->op_bytes != sizeof(unsigned long))
771 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
772 rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear);
773 if (rc == X86EMUL_CONTINUE)
774 ctxt->_eip = addr.ea;
778 static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt)
781 struct desc_struct cs;
785 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
787 if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) {
788 /* Real mode. cpu must not have long mode active */
790 return X86EMUL_UNHANDLEABLE;
791 ctxt->mode = X86EMUL_MODE_REAL;
792 return X86EMUL_CONTINUE;
795 if (ctxt->eflags & X86_EFLAGS_VM) {
796 /* Protected/VM86 mode. cpu must not have long mode active */
798 return X86EMUL_UNHANDLEABLE;
799 ctxt->mode = X86EMUL_MODE_VM86;
800 return X86EMUL_CONTINUE;
803 if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS))
804 return X86EMUL_UNHANDLEABLE;
806 if (efer & EFER_LMA) {
808 /* Proper long mode */
809 ctxt->mode = X86EMUL_MODE_PROT64;
811 /* 32 bit compatibility mode*/
812 ctxt->mode = X86EMUL_MODE_PROT32;
814 ctxt->mode = X86EMUL_MODE_PROT16;
817 /* Legacy 32 bit / 16 bit mode */
818 ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
821 return X86EMUL_CONTINUE;
824 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
826 return assign_eip(ctxt, dst);
829 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst)
831 int rc = emulator_recalc_and_set_mode(ctxt);
833 if (rc != X86EMUL_CONTINUE)
836 return assign_eip(ctxt, dst);
839 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
841 return assign_eip_near(ctxt, ctxt->_eip + rel);
844 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
845 void *data, unsigned size)
847 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
850 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
851 ulong linear, void *data,
854 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
857 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
858 struct segmented_address addr,
865 rc = linearize(ctxt, addr, size, false, &linear);
866 if (rc != X86EMUL_CONTINUE)
868 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
871 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
872 struct segmented_address addr,
879 rc = linearize(ctxt, addr, size, true, &linear);
880 if (rc != X86EMUL_CONTINUE)
882 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
886 * Prefetch the remaining bytes of the instruction without crossing page
887 * boundary if they are not in fetch_cache yet.
889 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
892 unsigned size, max_size;
893 unsigned long linear;
894 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
895 struct segmented_address addr = { .seg = VCPU_SREG_CS,
896 .ea = ctxt->eip + cur_size };
899 * We do not know exactly how many bytes will be needed, and
900 * __linearize is expensive, so fetch as much as possible. We
901 * just have to avoid going beyond the 15 byte limit, the end
902 * of the segment, or the end of the page.
904 * __linearize is called with size 0 so that it does not do any
905 * boundary check itself. Instead, we use max_size to check
908 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
910 if (unlikely(rc != X86EMUL_CONTINUE))
913 size = min_t(unsigned, 15UL ^ cur_size, max_size);
914 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
917 * One instruction can only straddle two pages,
918 * and one has been loaded at the beginning of
919 * x86_decode_insn. So, if not enough bytes
920 * still, we must have hit the 15-byte boundary.
922 if (unlikely(size < op_size))
923 return emulate_gp(ctxt, 0);
925 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
926 size, &ctxt->exception);
927 if (unlikely(rc != X86EMUL_CONTINUE))
929 ctxt->fetch.end += size;
930 return X86EMUL_CONTINUE;
933 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
936 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
938 if (unlikely(done_size < size))
939 return __do_insn_fetch_bytes(ctxt, size - done_size);
941 return X86EMUL_CONTINUE;
944 /* Fetch next part of the instruction being emulated. */
945 #define insn_fetch(_type, _ctxt) \
948 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
949 if (rc != X86EMUL_CONTINUE) \
951 ctxt->_eip += sizeof(_type); \
952 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
953 ctxt->fetch.ptr += sizeof(_type); \
957 #define insn_fetch_arr(_arr, _size, _ctxt) \
959 rc = do_insn_fetch_bytes(_ctxt, _size); \
960 if (rc != X86EMUL_CONTINUE) \
962 ctxt->_eip += (_size); \
963 memcpy(_arr, ctxt->fetch.ptr, _size); \
964 ctxt->fetch.ptr += (_size); \
968 * Given the 'reg' portion of a ModRM byte, and a register block, return a
969 * pointer into the block that addresses the relevant register.
970 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
972 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
976 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
978 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
979 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
981 p = reg_rmw(ctxt, modrm_reg);
985 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
986 struct segmented_address addr,
987 u16 *size, unsigned long *address, int op_bytes)
994 rc = segmented_read_std(ctxt, addr, size, 2);
995 if (rc != X86EMUL_CONTINUE)
998 rc = segmented_read_std(ctxt, addr, address, op_bytes);
1012 FASTOP1SRC2(mul, mul_ex);
1013 FASTOP1SRC2(imul, imul_ex);
1014 FASTOP1SRC2EX(div, div_ex);
1015 FASTOP1SRC2EX(idiv, idiv_ex);
1044 FASTOP2R(cmp, cmp_r);
1046 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1048 /* If src is zero, do not writeback, but update flags */
1049 if (ctxt->src.val == 0)
1050 ctxt->dst.type = OP_NONE;
1051 return fastop(ctxt, em_bsf);
1054 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1056 /* If src is zero, do not writeback, but update flags */
1057 if (ctxt->src.val == 0)
1058 ctxt->dst.type = OP_NONE;
1059 return fastop(ctxt, em_bsr);
1062 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1065 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1067 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1068 asm("push %[flags]; popf; " CALL_NOSPEC
1069 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1073 static void fetch_register_operand(struct operand *op)
1075 switch (op->bytes) {
1077 op->val = *(u8 *)op->addr.reg;
1080 op->val = *(u16 *)op->addr.reg;
1083 op->val = *(u32 *)op->addr.reg;
1086 op->val = *(u64 *)op->addr.reg;
1091 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1094 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1095 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1096 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1097 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1098 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1099 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1100 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1101 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1102 #ifdef CONFIG_X86_64
1103 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1104 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1105 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1106 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1107 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1108 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1109 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1110 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1116 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1120 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1121 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1122 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1123 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1124 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1125 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1126 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1127 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1128 #ifdef CONFIG_X86_64
1129 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1130 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1131 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1132 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1133 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1134 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1135 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1136 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1142 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1145 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1146 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1147 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1148 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1149 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1150 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1151 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1152 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1157 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1160 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1161 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1162 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1163 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1164 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1165 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1166 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1167 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1172 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1174 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1175 return emulate_nm(ctxt);
1177 asm volatile("fninit");
1178 return X86EMUL_CONTINUE;
1181 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1185 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1186 return emulate_nm(ctxt);
1188 asm volatile("fnstcw %0": "+m"(fcw));
1190 ctxt->dst.val = fcw;
1192 return X86EMUL_CONTINUE;
1195 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1199 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1200 return emulate_nm(ctxt);
1202 asm volatile("fnstsw %0": "+m"(fsw));
1204 ctxt->dst.val = fsw;
1206 return X86EMUL_CONTINUE;
1209 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1212 unsigned reg = ctxt->modrm_reg;
1214 if (!(ctxt->d & ModRM))
1215 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1217 if (ctxt->d & Sse) {
1221 read_sse_reg(ctxt, &op->vec_val, reg);
1224 if (ctxt->d & Mmx) {
1233 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1234 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1236 fetch_register_operand(op);
1237 op->orig_val = op->val;
1240 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1242 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1243 ctxt->modrm_seg = VCPU_SREG_SS;
1246 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1250 int index_reg, base_reg, scale;
1251 int rc = X86EMUL_CONTINUE;
1254 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1255 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1256 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1258 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1259 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1260 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1261 ctxt->modrm_seg = VCPU_SREG_DS;
1263 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1265 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1266 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1268 if (ctxt->d & Sse) {
1271 op->addr.xmm = ctxt->modrm_rm;
1272 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1275 if (ctxt->d & Mmx) {
1278 op->addr.mm = ctxt->modrm_rm & 7;
1281 fetch_register_operand(op);
1287 if (ctxt->ad_bytes == 2) {
1288 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1289 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1290 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1291 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1293 /* 16-bit ModR/M decode. */
1294 switch (ctxt->modrm_mod) {
1296 if (ctxt->modrm_rm == 6)
1297 modrm_ea += insn_fetch(u16, ctxt);
1300 modrm_ea += insn_fetch(s8, ctxt);
1303 modrm_ea += insn_fetch(u16, ctxt);
1306 switch (ctxt->modrm_rm) {
1308 modrm_ea += bx + si;
1311 modrm_ea += bx + di;
1314 modrm_ea += bp + si;
1317 modrm_ea += bp + di;
1326 if (ctxt->modrm_mod != 0)
1333 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1334 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1335 ctxt->modrm_seg = VCPU_SREG_SS;
1336 modrm_ea = (u16)modrm_ea;
1338 /* 32/64-bit ModR/M decode. */
1339 if ((ctxt->modrm_rm & 7) == 4) {
1340 sib = insn_fetch(u8, ctxt);
1341 index_reg |= (sib >> 3) & 7;
1342 base_reg |= sib & 7;
1345 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1346 modrm_ea += insn_fetch(s32, ctxt);
1348 modrm_ea += reg_read(ctxt, base_reg);
1349 adjust_modrm_seg(ctxt, base_reg);
1350 /* Increment ESP on POP [ESP] */
1351 if ((ctxt->d & IncSP) &&
1352 base_reg == VCPU_REGS_RSP)
1353 modrm_ea += ctxt->op_bytes;
1356 modrm_ea += reg_read(ctxt, index_reg) << scale;
1357 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1358 modrm_ea += insn_fetch(s32, ctxt);
1359 if (ctxt->mode == X86EMUL_MODE_PROT64)
1360 ctxt->rip_relative = 1;
1362 base_reg = ctxt->modrm_rm;
1363 modrm_ea += reg_read(ctxt, base_reg);
1364 adjust_modrm_seg(ctxt, base_reg);
1366 switch (ctxt->modrm_mod) {
1368 modrm_ea += insn_fetch(s8, ctxt);
1371 modrm_ea += insn_fetch(s32, ctxt);
1375 op->addr.mem.ea = modrm_ea;
1376 if (ctxt->ad_bytes != 8)
1377 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1383 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1386 int rc = X86EMUL_CONTINUE;
1389 switch (ctxt->ad_bytes) {
1391 op->addr.mem.ea = insn_fetch(u16, ctxt);
1394 op->addr.mem.ea = insn_fetch(u32, ctxt);
1397 op->addr.mem.ea = insn_fetch(u64, ctxt);
1404 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1408 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1409 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1411 if (ctxt->src.bytes == 2)
1412 sv = (s16)ctxt->src.val & (s16)mask;
1413 else if (ctxt->src.bytes == 4)
1414 sv = (s32)ctxt->src.val & (s32)mask;
1416 sv = (s64)ctxt->src.val & (s64)mask;
1418 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1419 ctxt->dst.addr.mem.ea + (sv >> 3));
1422 /* only subword offset */
1423 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1426 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1427 unsigned long addr, void *dest, unsigned size)
1430 struct read_cache *mc = &ctxt->mem_read;
1432 if (mc->pos < mc->end)
1435 WARN_ON((mc->end + size) >= sizeof(mc->data));
1437 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1439 if (rc != X86EMUL_CONTINUE)
1445 memcpy(dest, mc->data + mc->pos, size);
1447 return X86EMUL_CONTINUE;
1450 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1451 struct segmented_address addr,
1458 rc = linearize(ctxt, addr, size, false, &linear);
1459 if (rc != X86EMUL_CONTINUE)
1461 return read_emulated(ctxt, linear, data, size);
1464 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1465 struct segmented_address addr,
1472 rc = linearize(ctxt, addr, size, true, &linear);
1473 if (rc != X86EMUL_CONTINUE)
1475 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1479 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1480 struct segmented_address addr,
1481 const void *orig_data, const void *data,
1487 rc = linearize(ctxt, addr, size, true, &linear);
1488 if (rc != X86EMUL_CONTINUE)
1490 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1491 size, &ctxt->exception);
1494 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1495 unsigned int size, unsigned short port,
1498 struct read_cache *rc = &ctxt->io_read;
1500 if (rc->pos == rc->end) { /* refill pio read ahead */
1501 unsigned int in_page, n;
1502 unsigned int count = ctxt->rep_prefix ?
1503 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1504 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1505 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1506 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1507 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1510 rc->pos = rc->end = 0;
1511 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1516 if (ctxt->rep_prefix && (ctxt->d & String) &&
1517 !(ctxt->eflags & X86_EFLAGS_DF)) {
1518 ctxt->dst.data = rc->data + rc->pos;
1519 ctxt->dst.type = OP_MEM_STR;
1520 ctxt->dst.count = (rc->end - rc->pos) / size;
1523 memcpy(dest, rc->data + rc->pos, size);
1529 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1530 u16 index, struct desc_struct *desc)
1535 ctxt->ops->get_idt(ctxt, &dt);
1537 if (dt.size < index * 8 + 7)
1538 return emulate_gp(ctxt, index << 3 | 0x2);
1540 addr = dt.address + index * 8;
1541 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1544 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1545 u16 selector, struct desc_ptr *dt)
1547 const struct x86_emulate_ops *ops = ctxt->ops;
1550 if (selector & 1 << 2) {
1551 struct desc_struct desc;
1554 memset (dt, 0, sizeof *dt);
1555 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1559 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1560 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1562 ops->get_gdt(ctxt, dt);
1565 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1566 u16 selector, ulong *desc_addr_p)
1569 u16 index = selector >> 3;
1572 get_descriptor_table_ptr(ctxt, selector, &dt);
1574 if (dt.size < index * 8 + 7)
1575 return emulate_gp(ctxt, selector & 0xfffc);
1577 addr = dt.address + index * 8;
1579 #ifdef CONFIG_X86_64
1580 if (addr >> 32 != 0) {
1583 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1584 if (!(efer & EFER_LMA))
1589 *desc_addr_p = addr;
1590 return X86EMUL_CONTINUE;
1593 /* allowed just for 8 bytes segments */
1594 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1595 u16 selector, struct desc_struct *desc,
1600 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1601 if (rc != X86EMUL_CONTINUE)
1604 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1607 /* allowed just for 8 bytes segments */
1608 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1609 u16 selector, struct desc_struct *desc)
1614 rc = get_descriptor_ptr(ctxt, selector, &addr);
1615 if (rc != X86EMUL_CONTINUE)
1618 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1621 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1622 u16 selector, int seg, u8 cpl,
1623 enum x86_transfer_type transfer,
1624 struct desc_struct *desc)
1626 struct desc_struct seg_desc, old_desc;
1628 unsigned err_vec = GP_VECTOR;
1630 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1636 memset(&seg_desc, 0, sizeof seg_desc);
1638 if (ctxt->mode == X86EMUL_MODE_REAL) {
1639 /* set real mode segment descriptor (keep limit etc. for
1641 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1642 set_desc_base(&seg_desc, selector << 4);
1644 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1645 /* VM86 needs a clean new segment descriptor */
1646 set_desc_base(&seg_desc, selector << 4);
1647 set_desc_limit(&seg_desc, 0xffff);
1657 /* TR should be in GDT only */
1658 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1661 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1662 if (null_selector) {
1663 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1666 if (seg == VCPU_SREG_SS) {
1667 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1671 * ctxt->ops->set_segment expects the CPL to be in
1672 * SS.DPL, so fake an expand-up 32-bit data segment.
1682 /* Skip all following checks */
1686 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1687 if (ret != X86EMUL_CONTINUE)
1690 err_code = selector & 0xfffc;
1691 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1694 /* can't load system descriptor into segment selector */
1695 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1696 if (transfer == X86_TRANSFER_CALL_JMP)
1697 return X86EMUL_UNHANDLEABLE;
1706 * segment is not a writable data segment or segment
1707 * selector's RPL != CPL or segment selector's RPL != CPL
1709 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1713 if (!(seg_desc.type & 8))
1716 if (seg_desc.type & 4) {
1722 if (rpl > cpl || dpl != cpl)
1725 /* in long-mode d/b must be clear if l is set */
1726 if (seg_desc.d && seg_desc.l) {
1729 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1730 if (efer & EFER_LMA)
1734 /* CS(RPL) <- CPL */
1735 selector = (selector & 0xfffc) | cpl;
1738 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1741 case VCPU_SREG_LDTR:
1742 if (seg_desc.s || seg_desc.type != 2)
1745 default: /* DS, ES, FS, or GS */
1747 * segment is not a data or readable code segment or
1748 * ((segment is a data or nonconforming code segment)
1749 * and (both RPL and CPL > DPL))
1751 if ((seg_desc.type & 0xa) == 0x8 ||
1752 (((seg_desc.type & 0xc) != 0xc) &&
1753 (rpl > dpl && cpl > dpl)))
1759 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1764 /* mark segment as accessed */
1765 if (!(seg_desc.type & 1)) {
1767 ret = write_segment_descriptor(ctxt, selector,
1769 if (ret != X86EMUL_CONTINUE)
1772 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1773 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1774 if (ret != X86EMUL_CONTINUE)
1776 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1777 ((u64)base3 << 32), ctxt))
1778 return emulate_gp(ctxt, err_code);
1781 if (seg == VCPU_SREG_TR) {
1782 old_desc = seg_desc;
1783 seg_desc.type |= 2; /* busy */
1784 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1785 sizeof(seg_desc), &ctxt->exception);
1786 if (ret != X86EMUL_CONTINUE)
1790 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1793 return X86EMUL_CONTINUE;
1795 return emulate_exception(ctxt, err_vec, err_code, true);
1798 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1799 u16 selector, int seg)
1801 u8 cpl = ctxt->ops->cpl(ctxt);
1804 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1805 * they can load it at CPL<3 (Intel's manual says only LSS can,
1808 * However, the Intel manual says that putting IST=1/DPL=3 in
1809 * an interrupt gate will result in SS=3 (the AMD manual instead
1810 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1811 * and only forbid it here.
1813 if (seg == VCPU_SREG_SS && selector == 3 &&
1814 ctxt->mode == X86EMUL_MODE_PROT64)
1815 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1817 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1818 X86_TRANSFER_NONE, NULL);
1821 static void write_register_operand(struct operand *op)
1823 return assign_register(op->addr.reg, op->val, op->bytes);
1826 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1830 write_register_operand(op);
1833 if (ctxt->lock_prefix)
1834 return segmented_cmpxchg(ctxt,
1840 return segmented_write(ctxt,
1846 return segmented_write(ctxt,
1849 op->bytes * op->count);
1852 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1855 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1863 return X86EMUL_CONTINUE;
1866 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1868 struct segmented_address addr;
1870 rsp_increment(ctxt, -bytes);
1871 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1872 addr.seg = VCPU_SREG_SS;
1874 return segmented_write(ctxt, addr, data, bytes);
1877 static int em_push(struct x86_emulate_ctxt *ctxt)
1879 /* Disable writeback. */
1880 ctxt->dst.type = OP_NONE;
1881 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1884 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1885 void *dest, int len)
1888 struct segmented_address addr;
1890 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1891 addr.seg = VCPU_SREG_SS;
1892 rc = segmented_read(ctxt, addr, dest, len);
1893 if (rc != X86EMUL_CONTINUE)
1896 rsp_increment(ctxt, len);
1900 static int em_pop(struct x86_emulate_ctxt *ctxt)
1902 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1905 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1906 void *dest, int len)
1909 unsigned long val, change_mask;
1910 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1911 int cpl = ctxt->ops->cpl(ctxt);
1913 rc = emulate_pop(ctxt, &val, len);
1914 if (rc != X86EMUL_CONTINUE)
1917 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1918 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1919 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1920 X86_EFLAGS_AC | X86_EFLAGS_ID;
1922 switch(ctxt->mode) {
1923 case X86EMUL_MODE_PROT64:
1924 case X86EMUL_MODE_PROT32:
1925 case X86EMUL_MODE_PROT16:
1927 change_mask |= X86_EFLAGS_IOPL;
1929 change_mask |= X86_EFLAGS_IF;
1931 case X86EMUL_MODE_VM86:
1933 return emulate_gp(ctxt, 0);
1934 change_mask |= X86_EFLAGS_IF;
1936 default: /* real mode */
1937 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1941 *(unsigned long *)dest =
1942 (ctxt->eflags & ~change_mask) | (val & change_mask);
1947 static int em_popf(struct x86_emulate_ctxt *ctxt)
1949 ctxt->dst.type = OP_REG;
1950 ctxt->dst.addr.reg = &ctxt->eflags;
1951 ctxt->dst.bytes = ctxt->op_bytes;
1952 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1955 static int em_enter(struct x86_emulate_ctxt *ctxt)
1958 unsigned frame_size = ctxt->src.val;
1959 unsigned nesting_level = ctxt->src2.val & 31;
1963 return X86EMUL_UNHANDLEABLE;
1965 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1966 rc = push(ctxt, &rbp, stack_size(ctxt));
1967 if (rc != X86EMUL_CONTINUE)
1969 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1971 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1972 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1974 return X86EMUL_CONTINUE;
1977 static int em_leave(struct x86_emulate_ctxt *ctxt)
1979 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1981 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1984 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1986 int seg = ctxt->src2.val;
1988 ctxt->src.val = get_segment_selector(ctxt, seg);
1989 if (ctxt->op_bytes == 4) {
1990 rsp_increment(ctxt, -2);
1994 return em_push(ctxt);
1997 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1999 int seg = ctxt->src2.val;
2000 unsigned long selector;
2003 rc = emulate_pop(ctxt, &selector, 2);
2004 if (rc != X86EMUL_CONTINUE)
2007 if (seg == VCPU_SREG_SS)
2008 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
2009 if (ctxt->op_bytes > 2)
2010 rsp_increment(ctxt, ctxt->op_bytes - 2);
2012 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
2016 static int em_pusha(struct x86_emulate_ctxt *ctxt)
2018 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2019 int rc = X86EMUL_CONTINUE;
2020 int reg = VCPU_REGS_RAX;
2022 while (reg <= VCPU_REGS_RDI) {
2023 (reg == VCPU_REGS_RSP) ?
2024 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2027 if (rc != X86EMUL_CONTINUE)
2036 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2038 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2039 return em_push(ctxt);
2042 static int em_popa(struct x86_emulate_ctxt *ctxt)
2044 int rc = X86EMUL_CONTINUE;
2045 int reg = VCPU_REGS_RDI;
2048 while (reg >= VCPU_REGS_RAX) {
2049 if (reg == VCPU_REGS_RSP) {
2050 rsp_increment(ctxt, ctxt->op_bytes);
2054 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2055 if (rc != X86EMUL_CONTINUE)
2057 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2063 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2065 const struct x86_emulate_ops *ops = ctxt->ops;
2072 /* TODO: Add limit checks */
2073 ctxt->src.val = ctxt->eflags;
2075 if (rc != X86EMUL_CONTINUE)
2078 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2080 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2082 if (rc != X86EMUL_CONTINUE)
2085 ctxt->src.val = ctxt->_eip;
2087 if (rc != X86EMUL_CONTINUE)
2090 ops->get_idt(ctxt, &dt);
2092 eip_addr = dt.address + (irq << 2);
2093 cs_addr = dt.address + (irq << 2) + 2;
2095 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2096 if (rc != X86EMUL_CONTINUE)
2099 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2100 if (rc != X86EMUL_CONTINUE)
2103 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2104 if (rc != X86EMUL_CONTINUE)
2112 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2116 invalidate_registers(ctxt);
2117 rc = __emulate_int_real(ctxt, irq);
2118 if (rc == X86EMUL_CONTINUE)
2119 writeback_registers(ctxt);
2123 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2125 switch(ctxt->mode) {
2126 case X86EMUL_MODE_REAL:
2127 return __emulate_int_real(ctxt, irq);
2128 case X86EMUL_MODE_VM86:
2129 case X86EMUL_MODE_PROT16:
2130 case X86EMUL_MODE_PROT32:
2131 case X86EMUL_MODE_PROT64:
2133 /* Protected mode interrupts unimplemented yet */
2134 return X86EMUL_UNHANDLEABLE;
2138 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2140 int rc = X86EMUL_CONTINUE;
2141 unsigned long temp_eip = 0;
2142 unsigned long temp_eflags = 0;
2143 unsigned long cs = 0;
2144 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2145 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2146 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2147 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2148 X86_EFLAGS_AC | X86_EFLAGS_ID |
2150 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2153 /* TODO: Add stack limit check */
2155 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2157 if (rc != X86EMUL_CONTINUE)
2160 if (temp_eip & ~0xffff)
2161 return emulate_gp(ctxt, 0);
2163 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2165 if (rc != X86EMUL_CONTINUE)
2168 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2170 if (rc != X86EMUL_CONTINUE)
2173 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2175 if (rc != X86EMUL_CONTINUE)
2178 ctxt->_eip = temp_eip;
2180 if (ctxt->op_bytes == 4)
2181 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2182 else if (ctxt->op_bytes == 2) {
2183 ctxt->eflags &= ~0xffff;
2184 ctxt->eflags |= temp_eflags;
2187 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2188 ctxt->eflags |= X86_EFLAGS_FIXED;
2189 ctxt->ops->set_nmi_mask(ctxt, false);
2194 static int em_iret(struct x86_emulate_ctxt *ctxt)
2196 switch(ctxt->mode) {
2197 case X86EMUL_MODE_REAL:
2198 return emulate_iret_real(ctxt);
2199 case X86EMUL_MODE_VM86:
2200 case X86EMUL_MODE_PROT16:
2201 case X86EMUL_MODE_PROT32:
2202 case X86EMUL_MODE_PROT64:
2204 /* iret from protected mode unimplemented yet */
2205 return X86EMUL_UNHANDLEABLE;
2209 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2213 struct desc_struct new_desc;
2214 u8 cpl = ctxt->ops->cpl(ctxt);
2216 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2218 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2219 X86_TRANSFER_CALL_JMP,
2221 if (rc != X86EMUL_CONTINUE)
2224 rc = assign_eip_far(ctxt, ctxt->src.val);
2225 /* Error handling is not implemented. */
2226 if (rc != X86EMUL_CONTINUE)
2227 return X86EMUL_UNHANDLEABLE;
2232 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2234 return assign_eip_near(ctxt, ctxt->src.val);
2237 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2242 old_eip = ctxt->_eip;
2243 rc = assign_eip_near(ctxt, ctxt->src.val);
2244 if (rc != X86EMUL_CONTINUE)
2246 ctxt->src.val = old_eip;
2251 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2253 u64 old = ctxt->dst.orig_val64;
2255 if (ctxt->dst.bytes == 16)
2256 return X86EMUL_UNHANDLEABLE;
2258 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2259 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2260 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2261 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2262 ctxt->eflags &= ~X86_EFLAGS_ZF;
2264 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2265 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2267 ctxt->eflags |= X86_EFLAGS_ZF;
2269 return X86EMUL_CONTINUE;
2272 static int em_ret(struct x86_emulate_ctxt *ctxt)
2277 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2278 if (rc != X86EMUL_CONTINUE)
2281 return assign_eip_near(ctxt, eip);
2284 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2287 unsigned long eip, cs;
2288 int cpl = ctxt->ops->cpl(ctxt);
2289 struct desc_struct new_desc;
2291 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2292 if (rc != X86EMUL_CONTINUE)
2294 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2295 if (rc != X86EMUL_CONTINUE)
2297 /* Outer-privilege level return is not implemented */
2298 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2299 return X86EMUL_UNHANDLEABLE;
2300 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2303 if (rc != X86EMUL_CONTINUE)
2305 rc = assign_eip_far(ctxt, eip);
2306 /* Error handling is not implemented. */
2307 if (rc != X86EMUL_CONTINUE)
2308 return X86EMUL_UNHANDLEABLE;
2313 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2317 rc = em_ret_far(ctxt);
2318 if (rc != X86EMUL_CONTINUE)
2320 rsp_increment(ctxt, ctxt->src.val);
2321 return X86EMUL_CONTINUE;
2324 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2326 /* Save real source value, then compare EAX against destination. */
2327 ctxt->dst.orig_val = ctxt->dst.val;
2328 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2329 ctxt->src.orig_val = ctxt->src.val;
2330 ctxt->src.val = ctxt->dst.orig_val;
2331 fastop(ctxt, em_cmp);
2333 if (ctxt->eflags & X86_EFLAGS_ZF) {
2334 /* Success: write back to memory; no update of EAX */
2335 ctxt->src.type = OP_NONE;
2336 ctxt->dst.val = ctxt->src.orig_val;
2338 /* Failure: write the value we saw to EAX. */
2339 ctxt->src.type = OP_REG;
2340 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2341 ctxt->src.val = ctxt->dst.orig_val;
2342 /* Create write-cycle to dest by writing the same value */
2343 ctxt->dst.val = ctxt->dst.orig_val;
2345 return X86EMUL_CONTINUE;
2348 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2350 int seg = ctxt->src2.val;
2354 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2356 rc = load_segment_descriptor(ctxt, sel, seg);
2357 if (rc != X86EMUL_CONTINUE)
2360 ctxt->dst.val = ctxt->src.val;
2364 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2366 #ifdef CONFIG_X86_64
2367 u32 eax, ebx, ecx, edx;
2371 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2372 return edx & bit(X86_FEATURE_LM);
2378 #define GET_SMSTATE(type, smbase, offset) \
2381 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2383 if (r != X86EMUL_CONTINUE) \
2384 return X86EMUL_UNHANDLEABLE; \
2388 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2390 desc->g = (flags >> 23) & 1;
2391 desc->d = (flags >> 22) & 1;
2392 desc->l = (flags >> 21) & 1;
2393 desc->avl = (flags >> 20) & 1;
2394 desc->p = (flags >> 15) & 1;
2395 desc->dpl = (flags >> 13) & 3;
2396 desc->s = (flags >> 12) & 1;
2397 desc->type = (flags >> 8) & 15;
2400 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2402 struct desc_struct desc;
2406 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2409 offset = 0x7f84 + n * 12;
2411 offset = 0x7f2c + (n - 3) * 12;
2413 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2414 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2415 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2416 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2417 return X86EMUL_CONTINUE;
2420 #ifdef CONFIG_X86_64
2421 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2423 struct desc_struct desc;
2428 offset = 0x7e00 + n * 16;
2430 selector = GET_SMSTATE(u16, smbase, offset);
2431 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2432 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2433 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2434 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2436 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2437 return X86EMUL_CONTINUE;
2441 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2442 u64 cr0, u64 cr3, u64 cr4)
2447 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2449 if (cr4 & X86_CR4_PCIDE) {
2454 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2456 return X86EMUL_UNHANDLEABLE;
2459 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2460 * Then enable protected mode. However, PCID cannot be enabled
2461 * if EFER.LMA=0, so set it separately.
2463 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2465 return X86EMUL_UNHANDLEABLE;
2467 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2469 return X86EMUL_UNHANDLEABLE;
2471 if (cr4 & X86_CR4_PCIDE) {
2472 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2474 return X86EMUL_UNHANDLEABLE;
2476 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2478 return X86EMUL_UNHANDLEABLE;
2483 return X86EMUL_CONTINUE;
2486 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2488 struct desc_struct desc;
2491 u32 val, cr0, cr3, cr4;
2494 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2495 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2496 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2497 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2499 for (i = 0; i < 8; i++)
2500 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2502 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2503 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2504 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2505 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2507 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2508 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2509 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2510 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2511 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2513 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2514 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2515 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2516 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2517 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2519 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2520 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2521 ctxt->ops->set_gdt(ctxt, &dt);
2523 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2524 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2525 ctxt->ops->set_idt(ctxt, &dt);
2527 for (i = 0; i < 6; i++) {
2528 int r = rsm_load_seg_32(ctxt, smbase, i);
2529 if (r != X86EMUL_CONTINUE)
2533 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2535 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2537 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2540 #ifdef CONFIG_X86_64
2541 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2543 struct desc_struct desc;
2545 u64 val, cr0, cr3, cr4;
2550 for (i = 0; i < 16; i++)
2551 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2553 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2554 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2556 val = GET_SMSTATE(u32, smbase, 0x7f68);
2557 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2558 val = GET_SMSTATE(u32, smbase, 0x7f60);
2559 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2561 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2562 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2563 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2564 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2565 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2566 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2568 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2569 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2570 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2571 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2572 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2573 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2575 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2576 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2577 ctxt->ops->set_idt(ctxt, &dt);
2579 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2580 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2581 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2582 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2583 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2584 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2586 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2587 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2588 ctxt->ops->set_gdt(ctxt, &dt);
2590 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2591 if (r != X86EMUL_CONTINUE)
2594 for (i = 0; i < 6; i++) {
2595 r = rsm_load_seg_64(ctxt, smbase, i);
2596 if (r != X86EMUL_CONTINUE)
2600 return X86EMUL_CONTINUE;
2604 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2606 unsigned long cr0, cr4, efer;
2610 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2611 return emulate_ud(ctxt);
2614 * Get back to real mode, to prepare a safe state in which to load
2615 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2616 * supports long mode.
2618 if (emulator_has_longmode(ctxt)) {
2619 struct desc_struct cs_desc;
2621 /* Zero CR4.PCIDE before CR0.PG. */
2622 cr4 = ctxt->ops->get_cr(ctxt, 4);
2623 if (cr4 & X86_CR4_PCIDE)
2624 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2626 /* A 32-bit code segment is required to clear EFER.LMA. */
2627 memset(&cs_desc, 0, sizeof(cs_desc));
2629 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2630 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2633 /* For the 64-bit case, this will clear EFER.LMA. */
2634 cr0 = ctxt->ops->get_cr(ctxt, 0);
2635 if (cr0 & X86_CR0_PE)
2636 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2638 if (emulator_has_longmode(ctxt)) {
2639 /* Clear CR4.PAE before clearing EFER.LME. */
2640 cr4 = ctxt->ops->get_cr(ctxt, 4);
2641 if (cr4 & X86_CR4_PAE)
2642 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2644 /* And finally go back to 32-bit mode. */
2646 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2649 smbase = ctxt->ops->get_smbase(ctxt);
2652 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2653 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2656 if (ctxt->ops->pre_leave_smm(ctxt, smbase))
2657 return X86EMUL_UNHANDLEABLE;
2659 #ifdef CONFIG_X86_64
2660 if (emulator_has_longmode(ctxt))
2661 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2664 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2666 if (ret != X86EMUL_CONTINUE) {
2667 /* FIXME: should triple fault */
2668 return X86EMUL_UNHANDLEABLE;
2671 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2672 ctxt->ops->set_nmi_mask(ctxt, false);
2674 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2675 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2676 return X86EMUL_CONTINUE;
2680 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2681 struct desc_struct *cs, struct desc_struct *ss)
2683 cs->l = 0; /* will be adjusted later */
2684 set_desc_base(cs, 0); /* flat segment */
2685 cs->g = 1; /* 4kb granularity */
2686 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2687 cs->type = 0x0b; /* Read, Execute, Accessed */
2689 cs->dpl = 0; /* will be adjusted later */
2694 set_desc_base(ss, 0); /* flat segment */
2695 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2696 ss->g = 1; /* 4kb granularity */
2698 ss->type = 0x03; /* Read/Write, Accessed */
2699 ss->d = 1; /* 32bit stack segment */
2706 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2708 u32 eax, ebx, ecx, edx;
2711 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2712 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2713 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2714 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2717 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2719 const struct x86_emulate_ops *ops = ctxt->ops;
2720 u32 eax, ebx, ecx, edx;
2723 * syscall should always be enabled in longmode - so only become
2724 * vendor specific (cpuid) if other modes are active...
2726 if (ctxt->mode == X86EMUL_MODE_PROT64)
2731 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2733 * Intel ("GenuineIntel")
2734 * remark: Intel CPUs only support "syscall" in 64bit
2735 * longmode. Also an 64bit guest with a
2736 * 32bit compat-app running will #UD !! While this
2737 * behaviour can be fixed (by emulating) into AMD
2738 * response - CPUs of AMD can't behave like Intel.
2740 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2741 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2742 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2745 /* AMD ("AuthenticAMD") */
2746 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2747 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2748 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2751 /* AMD ("AMDisbetter!") */
2752 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2753 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2754 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2757 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2761 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2763 const struct x86_emulate_ops *ops = ctxt->ops;
2764 struct desc_struct cs, ss;
2769 /* syscall is not available in real mode */
2770 if (ctxt->mode == X86EMUL_MODE_REAL ||
2771 ctxt->mode == X86EMUL_MODE_VM86)
2772 return emulate_ud(ctxt);
2774 if (!(em_syscall_is_enabled(ctxt)))
2775 return emulate_ud(ctxt);
2777 ops->get_msr(ctxt, MSR_EFER, &efer);
2778 setup_syscalls_segments(ctxt, &cs, &ss);
2780 if (!(efer & EFER_SCE))
2781 return emulate_ud(ctxt);
2783 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2785 cs_sel = (u16)(msr_data & 0xfffc);
2786 ss_sel = (u16)(msr_data + 8);
2788 if (efer & EFER_LMA) {
2792 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2793 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2795 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2796 if (efer & EFER_LMA) {
2797 #ifdef CONFIG_X86_64
2798 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2801 ctxt->mode == X86EMUL_MODE_PROT64 ?
2802 MSR_LSTAR : MSR_CSTAR, &msr_data);
2803 ctxt->_eip = msr_data;
2805 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2806 ctxt->eflags &= ~msr_data;
2807 ctxt->eflags |= X86_EFLAGS_FIXED;
2811 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2812 ctxt->_eip = (u32)msr_data;
2814 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2817 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2818 return X86EMUL_CONTINUE;
2821 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2823 const struct x86_emulate_ops *ops = ctxt->ops;
2824 struct desc_struct cs, ss;
2829 ops->get_msr(ctxt, MSR_EFER, &efer);
2830 /* inject #GP if in real mode */
2831 if (ctxt->mode == X86EMUL_MODE_REAL)
2832 return emulate_gp(ctxt, 0);
2835 * Not recognized on AMD in compat mode (but is recognized in legacy
2838 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2839 && !vendor_intel(ctxt))
2840 return emulate_ud(ctxt);
2842 /* sysenter/sysexit have not been tested in 64bit mode. */
2843 if (ctxt->mode == X86EMUL_MODE_PROT64)
2844 return X86EMUL_UNHANDLEABLE;
2846 setup_syscalls_segments(ctxt, &cs, &ss);
2848 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2849 if ((msr_data & 0xfffc) == 0x0)
2850 return emulate_gp(ctxt, 0);
2852 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2853 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2854 ss_sel = cs_sel + 8;
2855 if (efer & EFER_LMA) {
2860 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2861 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2863 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2864 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2866 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2867 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2870 return X86EMUL_CONTINUE;
2873 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2875 const struct x86_emulate_ops *ops = ctxt->ops;
2876 struct desc_struct cs, ss;
2877 u64 msr_data, rcx, rdx;
2879 u16 cs_sel = 0, ss_sel = 0;
2881 /* inject #GP if in real mode or Virtual 8086 mode */
2882 if (ctxt->mode == X86EMUL_MODE_REAL ||
2883 ctxt->mode == X86EMUL_MODE_VM86)
2884 return emulate_gp(ctxt, 0);
2886 setup_syscalls_segments(ctxt, &cs, &ss);
2888 if ((ctxt->rex_prefix & 0x8) != 0x0)
2889 usermode = X86EMUL_MODE_PROT64;
2891 usermode = X86EMUL_MODE_PROT32;
2893 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2894 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2898 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2900 case X86EMUL_MODE_PROT32:
2901 cs_sel = (u16)(msr_data + 16);
2902 if ((msr_data & 0xfffc) == 0x0)
2903 return emulate_gp(ctxt, 0);
2904 ss_sel = (u16)(msr_data + 24);
2908 case X86EMUL_MODE_PROT64:
2909 cs_sel = (u16)(msr_data + 32);
2910 if (msr_data == 0x0)
2911 return emulate_gp(ctxt, 0);
2912 ss_sel = cs_sel + 8;
2915 if (emul_is_noncanonical_address(rcx, ctxt) ||
2916 emul_is_noncanonical_address(rdx, ctxt))
2917 return emulate_gp(ctxt, 0);
2920 cs_sel |= SEGMENT_RPL_MASK;
2921 ss_sel |= SEGMENT_RPL_MASK;
2923 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2924 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2927 ctxt->mode = usermode;
2928 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2930 return X86EMUL_CONTINUE;
2933 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2936 if (ctxt->mode == X86EMUL_MODE_REAL)
2938 if (ctxt->mode == X86EMUL_MODE_VM86)
2940 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2941 return ctxt->ops->cpl(ctxt) > iopl;
2944 #define VMWARE_PORT_VMPORT (0x5658)
2945 #define VMWARE_PORT_VMRPC (0x5659)
2947 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2950 const struct x86_emulate_ops *ops = ctxt->ops;
2951 struct desc_struct tr_seg;
2954 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2955 unsigned mask = (1 << len) - 1;
2959 * VMware allows access to these ports even if denied
2960 * by TSS I/O permission bitmap. Mimic behavior.
2962 if (enable_vmware_backdoor &&
2963 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2966 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2969 if (desc_limit_scaled(&tr_seg) < 103)
2971 base = get_desc_base(&tr_seg);
2972 #ifdef CONFIG_X86_64
2973 base |= ((u64)base3) << 32;
2975 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2976 if (r != X86EMUL_CONTINUE)
2978 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2980 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2981 if (r != X86EMUL_CONTINUE)
2983 if ((perm >> bit_idx) & mask)
2988 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2994 if (emulator_bad_iopl(ctxt))
2995 if (!emulator_io_port_access_allowed(ctxt, port, len))
2998 ctxt->perm_ok = true;
3003 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
3006 * Intel CPUs mask the counter and pointers in quite strange
3007 * manner when ECX is zero due to REP-string optimizations.
3009 #ifdef CONFIG_X86_64
3010 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
3013 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
3016 case 0xa4: /* movsb */
3017 case 0xa5: /* movsd/w */
3018 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
3020 case 0xaa: /* stosb */
3021 case 0xab: /* stosd/w */
3022 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
3027 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
3028 struct tss_segment_16 *tss)
3030 tss->ip = ctxt->_eip;
3031 tss->flag = ctxt->eflags;
3032 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3033 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3034 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3035 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3036 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3037 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3038 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3039 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3041 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3042 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3043 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3044 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3045 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3048 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3049 struct tss_segment_16 *tss)
3054 ctxt->_eip = tss->ip;
3055 ctxt->eflags = tss->flag | 2;
3056 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3057 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3058 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3059 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3060 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3061 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3062 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3063 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3066 * SDM says that segment selectors are loaded before segment
3069 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3070 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3071 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3072 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3073 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3078 * Now load segment descriptors. If fault happens at this stage
3079 * it is handled in a context of new task
3081 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3082 X86_TRANSFER_TASK_SWITCH, NULL);
3083 if (ret != X86EMUL_CONTINUE)
3085 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3086 X86_TRANSFER_TASK_SWITCH, NULL);
3087 if (ret != X86EMUL_CONTINUE)
3089 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3090 X86_TRANSFER_TASK_SWITCH, NULL);
3091 if (ret != X86EMUL_CONTINUE)
3093 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3094 X86_TRANSFER_TASK_SWITCH, NULL);
3095 if (ret != X86EMUL_CONTINUE)
3097 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3098 X86_TRANSFER_TASK_SWITCH, NULL);
3099 if (ret != X86EMUL_CONTINUE)
3102 return X86EMUL_CONTINUE;
3105 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3106 u16 tss_selector, u16 old_tss_sel,
3107 ulong old_tss_base, struct desc_struct *new_desc)
3109 struct tss_segment_16 tss_seg;
3111 u32 new_tss_base = get_desc_base(new_desc);
3113 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3114 if (ret != X86EMUL_CONTINUE)
3117 save_state_to_tss16(ctxt, &tss_seg);
3119 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3120 if (ret != X86EMUL_CONTINUE)
3123 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3124 if (ret != X86EMUL_CONTINUE)
3127 if (old_tss_sel != 0xffff) {
3128 tss_seg.prev_task_link = old_tss_sel;
3130 ret = linear_write_system(ctxt, new_tss_base,
3131 &tss_seg.prev_task_link,
3132 sizeof tss_seg.prev_task_link);
3133 if (ret != X86EMUL_CONTINUE)
3137 return load_state_from_tss16(ctxt, &tss_seg);
3140 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3141 struct tss_segment_32 *tss)
3143 /* CR3 and ldt selector are not saved intentionally */
3144 tss->eip = ctxt->_eip;
3145 tss->eflags = ctxt->eflags;
3146 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3147 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3148 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3149 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3150 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3151 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3152 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3153 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3155 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3156 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3157 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3158 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3159 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3160 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3163 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3164 struct tss_segment_32 *tss)
3169 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3170 return emulate_gp(ctxt, 0);
3171 ctxt->_eip = tss->eip;
3172 ctxt->eflags = tss->eflags | 2;
3174 /* General purpose registers */
3175 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3176 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3177 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3178 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3179 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3180 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3181 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3182 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3185 * SDM says that segment selectors are loaded before segment
3186 * descriptors. This is important because CPL checks will
3189 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3190 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3191 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3192 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3193 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3194 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3195 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3198 * If we're switching between Protected Mode and VM86, we need to make
3199 * sure to update the mode before loading the segment descriptors so
3200 * that the selectors are interpreted correctly.
3202 if (ctxt->eflags & X86_EFLAGS_VM) {
3203 ctxt->mode = X86EMUL_MODE_VM86;
3206 ctxt->mode = X86EMUL_MODE_PROT32;
3211 * Now load segment descriptors. If fault happenes at this stage
3212 * it is handled in a context of new task
3214 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3215 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3216 if (ret != X86EMUL_CONTINUE)
3218 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3219 X86_TRANSFER_TASK_SWITCH, NULL);
3220 if (ret != X86EMUL_CONTINUE)
3222 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3223 X86_TRANSFER_TASK_SWITCH, NULL);
3224 if (ret != X86EMUL_CONTINUE)
3226 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3227 X86_TRANSFER_TASK_SWITCH, NULL);
3228 if (ret != X86EMUL_CONTINUE)
3230 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3231 X86_TRANSFER_TASK_SWITCH, NULL);
3232 if (ret != X86EMUL_CONTINUE)
3234 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3235 X86_TRANSFER_TASK_SWITCH, NULL);
3236 if (ret != X86EMUL_CONTINUE)
3238 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3239 X86_TRANSFER_TASK_SWITCH, NULL);
3244 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3245 u16 tss_selector, u16 old_tss_sel,
3246 ulong old_tss_base, struct desc_struct *new_desc)
3248 struct tss_segment_32 tss_seg;
3250 u32 new_tss_base = get_desc_base(new_desc);
3251 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3252 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3254 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3255 if (ret != X86EMUL_CONTINUE)
3258 save_state_to_tss32(ctxt, &tss_seg);
3260 /* Only GP registers and segment selectors are saved */
3261 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3262 ldt_sel_offset - eip_offset);
3263 if (ret != X86EMUL_CONTINUE)
3266 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3267 if (ret != X86EMUL_CONTINUE)
3270 if (old_tss_sel != 0xffff) {
3271 tss_seg.prev_task_link = old_tss_sel;
3273 ret = linear_write_system(ctxt, new_tss_base,
3274 &tss_seg.prev_task_link,
3275 sizeof tss_seg.prev_task_link);
3276 if (ret != X86EMUL_CONTINUE)
3280 return load_state_from_tss32(ctxt, &tss_seg);
3283 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3284 u16 tss_selector, int idt_index, int reason,
3285 bool has_error_code, u32 error_code)
3287 const struct x86_emulate_ops *ops = ctxt->ops;
3288 struct desc_struct curr_tss_desc, next_tss_desc;
3290 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3291 ulong old_tss_base =
3292 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3294 ulong desc_addr, dr7;
3296 /* FIXME: old_tss_base == ~0 ? */
3298 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3299 if (ret != X86EMUL_CONTINUE)
3301 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3302 if (ret != X86EMUL_CONTINUE)
3305 /* FIXME: check that next_tss_desc is tss */
3308 * Check privileges. The three cases are task switch caused by...
3310 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3311 * 2. Exception/IRQ/iret: No check is performed
3312 * 3. jmp/call to TSS/task-gate: No check is performed since the
3313 * hardware checks it before exiting.
3315 if (reason == TASK_SWITCH_GATE) {
3316 if (idt_index != -1) {
3317 /* Software interrupts */
3318 struct desc_struct task_gate_desc;
3321 ret = read_interrupt_descriptor(ctxt, idt_index,
3323 if (ret != X86EMUL_CONTINUE)
3326 dpl = task_gate_desc.dpl;
3327 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3328 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3332 desc_limit = desc_limit_scaled(&next_tss_desc);
3333 if (!next_tss_desc.p ||
3334 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3335 desc_limit < 0x2b)) {
3336 return emulate_ts(ctxt, tss_selector & 0xfffc);
3339 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3340 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3341 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3344 if (reason == TASK_SWITCH_IRET)
3345 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3347 /* set back link to prev task only if NT bit is set in eflags
3348 note that old_tss_sel is not used after this point */
3349 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3350 old_tss_sel = 0xffff;
3352 if (next_tss_desc.type & 8)
3353 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3354 old_tss_base, &next_tss_desc);
3356 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3357 old_tss_base, &next_tss_desc);
3358 if (ret != X86EMUL_CONTINUE)
3361 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3362 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3364 if (reason != TASK_SWITCH_IRET) {
3365 next_tss_desc.type |= (1 << 1); /* set busy flag */
3366 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3369 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3370 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3372 if (has_error_code) {
3373 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3374 ctxt->lock_prefix = 0;
3375 ctxt->src.val = (unsigned long) error_code;
3376 ret = em_push(ctxt);
3379 ops->get_dr(ctxt, 7, &dr7);
3380 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3385 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3386 u16 tss_selector, int idt_index, int reason,
3387 bool has_error_code, u32 error_code)
3391 invalidate_registers(ctxt);
3392 ctxt->_eip = ctxt->eip;
3393 ctxt->dst.type = OP_NONE;
3395 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3396 has_error_code, error_code);
3398 if (rc == X86EMUL_CONTINUE) {
3399 ctxt->eip = ctxt->_eip;
3400 writeback_registers(ctxt);
3403 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3406 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3409 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3411 register_address_increment(ctxt, reg, df * op->bytes);
3412 op->addr.mem.ea = register_address(ctxt, reg);
3415 static int em_das(struct x86_emulate_ctxt *ctxt)
3418 bool af, cf, old_cf;
3420 cf = ctxt->eflags & X86_EFLAGS_CF;
3426 af = ctxt->eflags & X86_EFLAGS_AF;
3427 if ((al & 0x0f) > 9 || af) {
3429 cf = old_cf | (al >= 250);
3434 if (old_al > 0x99 || old_cf) {
3440 /* Set PF, ZF, SF */
3441 ctxt->src.type = OP_IMM;
3443 ctxt->src.bytes = 1;
3444 fastop(ctxt, em_or);
3445 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3447 ctxt->eflags |= X86_EFLAGS_CF;
3449 ctxt->eflags |= X86_EFLAGS_AF;
3450 return X86EMUL_CONTINUE;
3453 static int em_aam(struct x86_emulate_ctxt *ctxt)
3457 if (ctxt->src.val == 0)
3458 return emulate_de(ctxt);
3460 al = ctxt->dst.val & 0xff;
3461 ah = al / ctxt->src.val;
3462 al %= ctxt->src.val;
3464 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3466 /* Set PF, ZF, SF */
3467 ctxt->src.type = OP_IMM;
3469 ctxt->src.bytes = 1;
3470 fastop(ctxt, em_or);
3472 return X86EMUL_CONTINUE;
3475 static int em_aad(struct x86_emulate_ctxt *ctxt)
3477 u8 al = ctxt->dst.val & 0xff;
3478 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3480 al = (al + (ah * ctxt->src.val)) & 0xff;
3482 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3484 /* Set PF, ZF, SF */
3485 ctxt->src.type = OP_IMM;
3487 ctxt->src.bytes = 1;
3488 fastop(ctxt, em_or);
3490 return X86EMUL_CONTINUE;
3493 static int em_call(struct x86_emulate_ctxt *ctxt)
3496 long rel = ctxt->src.val;
3498 ctxt->src.val = (unsigned long)ctxt->_eip;
3499 rc = jmp_rel(ctxt, rel);
3500 if (rc != X86EMUL_CONTINUE)
3502 return em_push(ctxt);
3505 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3510 struct desc_struct old_desc, new_desc;
3511 const struct x86_emulate_ops *ops = ctxt->ops;
3512 int cpl = ctxt->ops->cpl(ctxt);
3513 enum x86emul_mode prev_mode = ctxt->mode;
3515 old_eip = ctxt->_eip;
3516 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3518 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3519 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3520 X86_TRANSFER_CALL_JMP, &new_desc);
3521 if (rc != X86EMUL_CONTINUE)
3524 rc = assign_eip_far(ctxt, ctxt->src.val);
3525 if (rc != X86EMUL_CONTINUE)
3528 ctxt->src.val = old_cs;
3530 if (rc != X86EMUL_CONTINUE)
3533 ctxt->src.val = old_eip;
3535 /* If we failed, we tainted the memory, but the very least we should
3537 if (rc != X86EMUL_CONTINUE) {
3538 pr_warn_once("faulting far call emulation tainted memory\n");
3543 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3544 ctxt->mode = prev_mode;
3549 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3554 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3555 if (rc != X86EMUL_CONTINUE)
3557 rc = assign_eip_near(ctxt, eip);
3558 if (rc != X86EMUL_CONTINUE)
3560 rsp_increment(ctxt, ctxt->src.val);
3561 return X86EMUL_CONTINUE;
3564 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3566 /* Write back the register source. */
3567 ctxt->src.val = ctxt->dst.val;
3568 write_register_operand(&ctxt->src);
3570 /* Write back the memory destination with implicit LOCK prefix. */
3571 ctxt->dst.val = ctxt->src.orig_val;
3572 ctxt->lock_prefix = 1;
3573 return X86EMUL_CONTINUE;
3576 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3578 ctxt->dst.val = ctxt->src2.val;
3579 return fastop(ctxt, em_imul);
3582 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3584 ctxt->dst.type = OP_REG;
3585 ctxt->dst.bytes = ctxt->src.bytes;
3586 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3587 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3589 return X86EMUL_CONTINUE;
3592 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3596 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3597 return emulate_ud(ctxt);
3598 ctxt->dst.val = tsc_aux;
3599 return X86EMUL_CONTINUE;
3602 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3606 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3607 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3608 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3609 return X86EMUL_CONTINUE;
3612 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3616 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3617 return emulate_gp(ctxt, 0);
3618 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3619 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3620 return X86EMUL_CONTINUE;
3623 static int em_mov(struct x86_emulate_ctxt *ctxt)
3625 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3626 return X86EMUL_CONTINUE;
3629 #define FFL(x) bit(X86_FEATURE_##x)
3631 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3633 u32 ebx, ecx, edx, eax = 1;
3637 * Check MOVBE is set in the guest-visible CPUID leaf.
3639 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3640 if (!(ecx & FFL(MOVBE)))
3641 return emulate_ud(ctxt);
3643 switch (ctxt->op_bytes) {
3646 * From MOVBE definition: "...When the operand size is 16 bits,
3647 * the upper word of the destination register remains unchanged
3650 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3651 * rules so we have to do the operation almost per hand.
3653 tmp = (u16)ctxt->src.val;
3654 ctxt->dst.val &= ~0xffffUL;
3655 ctxt->dst.val |= (unsigned long)swab16(tmp);
3658 ctxt->dst.val = swab32((u32)ctxt->src.val);
3661 ctxt->dst.val = swab64(ctxt->src.val);
3666 return X86EMUL_CONTINUE;
3669 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3671 int cr_num = ctxt->modrm_reg;
3674 if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val))
3675 return emulate_gp(ctxt, 0);
3677 /* Disable writeback. */
3678 ctxt->dst.type = OP_NONE;
3682 * CR0 write might have updated CR0.PE and/or CR0.PG
3683 * which can affect the cpu's execution mode.
3685 r = emulator_recalc_and_set_mode(ctxt);
3686 if (r != X86EMUL_CONTINUE)
3690 return X86EMUL_CONTINUE;
3693 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3697 if (ctxt->mode == X86EMUL_MODE_PROT64)
3698 val = ctxt->src.val & ~0ULL;
3700 val = ctxt->src.val & ~0U;
3702 /* #UD condition is already handled. */
3703 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3704 return emulate_gp(ctxt, 0);
3706 /* Disable writeback. */
3707 ctxt->dst.type = OP_NONE;
3708 return X86EMUL_CONTINUE;
3711 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3715 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3716 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3717 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3718 return emulate_gp(ctxt, 0);
3720 return X86EMUL_CONTINUE;
3723 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3727 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3728 return emulate_gp(ctxt, 0);
3730 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3731 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3732 return X86EMUL_CONTINUE;
3735 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3737 if (segment > VCPU_SREG_GS &&
3738 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3739 ctxt->ops->cpl(ctxt) > 0)
3740 return emulate_gp(ctxt, 0);
3742 ctxt->dst.val = get_segment_selector(ctxt, segment);
3743 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3744 ctxt->dst.bytes = 2;
3745 return X86EMUL_CONTINUE;
3748 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3750 if (ctxt->modrm_reg > VCPU_SREG_GS)
3751 return emulate_ud(ctxt);
3753 return em_store_sreg(ctxt, ctxt->modrm_reg);
3756 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3758 u16 sel = ctxt->src.val;
3760 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3761 return emulate_ud(ctxt);
3763 if (ctxt->modrm_reg == VCPU_SREG_SS)
3764 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3766 /* Disable writeback. */
3767 ctxt->dst.type = OP_NONE;
3768 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3771 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3773 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3776 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3778 u16 sel = ctxt->src.val;
3780 /* Disable writeback. */
3781 ctxt->dst.type = OP_NONE;
3782 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3785 static int em_str(struct x86_emulate_ctxt *ctxt)
3787 return em_store_sreg(ctxt, VCPU_SREG_TR);
3790 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3792 u16 sel = ctxt->src.val;
3794 /* Disable writeback. */
3795 ctxt->dst.type = OP_NONE;
3796 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3799 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3804 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3805 if (rc == X86EMUL_CONTINUE)
3806 ctxt->ops->invlpg(ctxt, linear);
3807 /* Disable writeback. */
3808 ctxt->dst.type = OP_NONE;
3809 return X86EMUL_CONTINUE;
3812 static int em_clts(struct x86_emulate_ctxt *ctxt)
3816 cr0 = ctxt->ops->get_cr(ctxt, 0);
3818 ctxt->ops->set_cr(ctxt, 0, cr0);
3819 return X86EMUL_CONTINUE;
3822 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3824 int rc = ctxt->ops->fix_hypercall(ctxt);
3826 if (rc != X86EMUL_CONTINUE)
3829 /* Let the processor re-execute the fixed hypercall */
3830 ctxt->_eip = ctxt->eip;
3831 /* Disable writeback. */
3832 ctxt->dst.type = OP_NONE;
3833 return X86EMUL_CONTINUE;
3836 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3837 void (*get)(struct x86_emulate_ctxt *ctxt,
3838 struct desc_ptr *ptr))
3840 struct desc_ptr desc_ptr;
3842 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3843 ctxt->ops->cpl(ctxt) > 0)
3844 return emulate_gp(ctxt, 0);
3846 if (ctxt->mode == X86EMUL_MODE_PROT64)
3848 get(ctxt, &desc_ptr);
3849 if (ctxt->op_bytes == 2) {
3851 desc_ptr.address &= 0x00ffffff;
3853 /* Disable writeback. */
3854 ctxt->dst.type = OP_NONE;
3855 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3856 &desc_ptr, 2 + ctxt->op_bytes);
3859 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3861 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3864 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3866 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3869 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3871 struct desc_ptr desc_ptr;
3874 if (ctxt->mode == X86EMUL_MODE_PROT64)
3876 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3877 &desc_ptr.size, &desc_ptr.address,
3879 if (rc != X86EMUL_CONTINUE)
3881 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3882 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3883 return emulate_gp(ctxt, 0);
3885 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3887 ctxt->ops->set_idt(ctxt, &desc_ptr);
3888 /* Disable writeback. */
3889 ctxt->dst.type = OP_NONE;
3890 return X86EMUL_CONTINUE;
3893 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3895 return em_lgdt_lidt(ctxt, true);
3898 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3900 return em_lgdt_lidt(ctxt, false);
3903 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3905 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3906 ctxt->ops->cpl(ctxt) > 0)
3907 return emulate_gp(ctxt, 0);
3909 if (ctxt->dst.type == OP_MEM)
3910 ctxt->dst.bytes = 2;
3911 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3912 return X86EMUL_CONTINUE;
3915 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3917 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3918 | (ctxt->src.val & 0x0f));
3919 ctxt->dst.type = OP_NONE;
3920 return X86EMUL_CONTINUE;
3923 static int em_loop(struct x86_emulate_ctxt *ctxt)
3925 int rc = X86EMUL_CONTINUE;
3927 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3928 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3929 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3930 rc = jmp_rel(ctxt, ctxt->src.val);
3935 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3937 int rc = X86EMUL_CONTINUE;
3939 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3940 rc = jmp_rel(ctxt, ctxt->src.val);
3945 static int em_in(struct x86_emulate_ctxt *ctxt)
3947 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3949 return X86EMUL_IO_NEEDED;
3951 return X86EMUL_CONTINUE;
3954 static int em_out(struct x86_emulate_ctxt *ctxt)
3956 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3958 /* Disable writeback. */
3959 ctxt->dst.type = OP_NONE;
3960 return X86EMUL_CONTINUE;
3963 static int em_cli(struct x86_emulate_ctxt *ctxt)
3965 if (emulator_bad_iopl(ctxt))
3966 return emulate_gp(ctxt, 0);
3968 ctxt->eflags &= ~X86_EFLAGS_IF;
3969 return X86EMUL_CONTINUE;
3972 static int em_sti(struct x86_emulate_ctxt *ctxt)
3974 if (emulator_bad_iopl(ctxt))
3975 return emulate_gp(ctxt, 0);
3977 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3978 ctxt->eflags |= X86_EFLAGS_IF;
3979 return X86EMUL_CONTINUE;
3982 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3984 u32 eax, ebx, ecx, edx;
3987 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3988 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3989 ctxt->ops->cpl(ctxt)) {
3990 return emulate_gp(ctxt, 0);
3993 eax = reg_read(ctxt, VCPU_REGS_RAX);
3994 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3995 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3996 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3997 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3998 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3999 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
4000 return X86EMUL_CONTINUE;
4003 static int em_sahf(struct x86_emulate_ctxt *ctxt)
4007 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
4009 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
4011 ctxt->eflags &= ~0xffUL;
4012 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
4013 return X86EMUL_CONTINUE;
4016 static int em_lahf(struct x86_emulate_ctxt *ctxt)
4018 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
4019 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
4020 return X86EMUL_CONTINUE;
4023 static int em_bswap(struct x86_emulate_ctxt *ctxt)
4025 switch (ctxt->op_bytes) {
4026 #ifdef CONFIG_X86_64
4028 asm("bswap %0" : "+r"(ctxt->dst.val));
4032 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
4035 return X86EMUL_CONTINUE;
4038 static int em_clflush(struct x86_emulate_ctxt *ctxt)
4040 /* emulating clflush regardless of cpuid */
4041 return X86EMUL_CONTINUE;
4044 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4046 /* emulating clflushopt regardless of cpuid */
4047 return X86EMUL_CONTINUE;
4050 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4052 ctxt->dst.val = (s32) ctxt->src.val;
4053 return X86EMUL_CONTINUE;
4056 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4058 u32 eax = 1, ebx, ecx = 0, edx;
4060 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4061 if (!(edx & FFL(FXSR)))
4062 return emulate_ud(ctxt);
4064 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4065 return emulate_nm(ctxt);
4068 * Don't emulate a case that should never be hit, instead of working
4069 * around a lack of fxsave64/fxrstor64 on old compilers.
4071 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4072 return X86EMUL_UNHANDLEABLE;
4074 return X86EMUL_CONTINUE;
4078 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4079 * and restore MXCSR.
4081 static size_t __fxstate_size(int nregs)
4083 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4086 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4089 if (ctxt->mode == X86EMUL_MODE_PROT64)
4090 return __fxstate_size(16);
4092 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4093 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4097 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4100 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4101 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4103 * 3) 64-bit mode with REX.W prefix
4104 * - like (2), but XMM 8-15 are being saved and restored
4105 * 4) 64-bit mode without REX.W prefix
4106 * - like (3), but FIP and FDP are 64 bit
4108 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4109 * desired result. (4) is not emulated.
4111 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4112 * and FPU DS) should match.
4114 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4116 struct fxregs_state fx_state;
4119 rc = check_fxsr(ctxt);
4120 if (rc != X86EMUL_CONTINUE)
4123 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4125 if (rc != X86EMUL_CONTINUE)
4128 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4129 fxstate_size(ctxt));
4133 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4134 * in the host registers (via FXSAVE) instead, so they won't be modified.
4135 * (preemption has to stay disabled until FXRSTOR).
4137 * Use noinline to keep the stack for other functions called by callers small.
4139 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4140 const size_t used_size)
4142 struct fxregs_state fx_tmp;
4145 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4146 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4147 __fxstate_size(16) - used_size);
4152 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4154 struct fxregs_state fx_state;
4158 rc = check_fxsr(ctxt);
4159 if (rc != X86EMUL_CONTINUE)
4162 size = fxstate_size(ctxt);
4163 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4164 if (rc != X86EMUL_CONTINUE)
4167 if (size < __fxstate_size(16)) {
4168 rc = fxregs_fixup(&fx_state, size);
4169 if (rc != X86EMUL_CONTINUE)
4173 if (fx_state.mxcsr >> 16) {
4174 rc = emulate_gp(ctxt, 0);
4178 if (rc == X86EMUL_CONTINUE)
4179 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4185 static bool valid_cr(int nr)
4197 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4199 if (!valid_cr(ctxt->modrm_reg))
4200 return emulate_ud(ctxt);
4202 return X86EMUL_CONTINUE;
4205 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4207 u64 new_val = ctxt->src.val64;
4208 int cr = ctxt->modrm_reg;
4211 static u64 cr_reserved_bits[] = {
4212 0xffffffff00000000ULL,
4213 0, 0, 0, /* CR3 checked later */
4220 return emulate_ud(ctxt);
4222 if (new_val & cr_reserved_bits[cr])
4223 return emulate_gp(ctxt, 0);
4228 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4229 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4230 return emulate_gp(ctxt, 0);
4232 cr4 = ctxt->ops->get_cr(ctxt, 4);
4233 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4235 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4236 !(cr4 & X86_CR4_PAE))
4237 return emulate_gp(ctxt, 0);
4244 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4245 if (efer & EFER_LMA) {
4247 u32 eax, ebx, ecx, edx;
4251 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4253 maxphyaddr = eax & 0xff;
4256 rsvd = rsvd_bits(maxphyaddr, 63);
4257 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4258 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4262 return emulate_gp(ctxt, 0);
4267 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4269 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4270 return emulate_gp(ctxt, 0);
4276 return X86EMUL_CONTINUE;
4279 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4283 ctxt->ops->get_dr(ctxt, 7, &dr7);
4285 /* Check if DR7.Global_Enable is set */
4286 return dr7 & (1 << 13);
4289 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4291 int dr = ctxt->modrm_reg;
4295 return emulate_ud(ctxt);
4297 cr4 = ctxt->ops->get_cr(ctxt, 4);
4298 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4299 return emulate_ud(ctxt);
4301 if (check_dr7_gd(ctxt)) {
4304 ctxt->ops->get_dr(ctxt, 6, &dr6);
4306 dr6 |= DR6_BD | DR6_RTM;
4307 ctxt->ops->set_dr(ctxt, 6, dr6);
4308 return emulate_db(ctxt);
4311 return X86EMUL_CONTINUE;
4314 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4316 u64 new_val = ctxt->src.val64;
4317 int dr = ctxt->modrm_reg;
4319 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4320 return emulate_gp(ctxt, 0);
4322 return check_dr_read(ctxt);
4325 static int check_svme(struct x86_emulate_ctxt *ctxt)
4329 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4331 if (!(efer & EFER_SVME))
4332 return emulate_ud(ctxt);
4334 return X86EMUL_CONTINUE;
4337 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4339 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4341 /* Valid physical address? */
4342 if (rax & 0xffff000000000000ULL)
4343 return emulate_gp(ctxt, 0);
4345 return check_svme(ctxt);
4348 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4350 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4352 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4353 return emulate_ud(ctxt);
4355 return X86EMUL_CONTINUE;
4358 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4360 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4361 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4364 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4365 * in Ring3 when CR4.PCE=0.
4367 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4368 return X86EMUL_CONTINUE;
4370 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4371 ctxt->ops->check_pmc(ctxt, rcx))
4372 return emulate_gp(ctxt, 0);
4374 return X86EMUL_CONTINUE;
4377 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4379 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4380 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4381 return emulate_gp(ctxt, 0);
4383 return X86EMUL_CONTINUE;
4386 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4388 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4389 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4390 return emulate_gp(ctxt, 0);
4392 return X86EMUL_CONTINUE;
4395 #define D(_y) { .flags = (_y) }
4396 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4397 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4398 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4399 #define N D(NotImpl)
4400 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4401 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4402 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4403 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4404 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4405 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4406 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4407 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4408 #define II(_f, _e, _i) \
4409 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4410 #define IIP(_f, _e, _i, _p) \
4411 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4412 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4413 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4415 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4416 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4417 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4418 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4419 #define I2bvIP(_f, _e, _i, _p) \
4420 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4422 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4423 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4424 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4426 static const struct opcode group7_rm0[] = {
4428 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4432 static const struct opcode group7_rm1[] = {
4433 DI(SrcNone | Priv, monitor),
4434 DI(SrcNone | Priv, mwait),
4438 static const struct opcode group7_rm3[] = {
4439 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4440 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4441 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4442 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4443 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4444 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4445 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4446 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4449 static const struct opcode group7_rm7[] = {
4451 DIP(SrcNone, rdtscp, check_rdtsc),
4455 static const struct opcode group1[] = {
4457 F(Lock | PageTable, em_or),
4460 F(Lock | PageTable, em_and),
4466 static const struct opcode group1A[] = {
4467 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4470 static const struct opcode group2[] = {
4471 F(DstMem | ModRM, em_rol),
4472 F(DstMem | ModRM, em_ror),
4473 F(DstMem | ModRM, em_rcl),
4474 F(DstMem | ModRM, em_rcr),
4475 F(DstMem | ModRM, em_shl),
4476 F(DstMem | ModRM, em_shr),
4477 F(DstMem | ModRM, em_shl),
4478 F(DstMem | ModRM, em_sar),
4481 static const struct opcode group3[] = {
4482 F(DstMem | SrcImm | NoWrite, em_test),
4483 F(DstMem | SrcImm | NoWrite, em_test),
4484 F(DstMem | SrcNone | Lock, em_not),
4485 F(DstMem | SrcNone | Lock, em_neg),
4486 F(DstXacc | Src2Mem, em_mul_ex),
4487 F(DstXacc | Src2Mem, em_imul_ex),
4488 F(DstXacc | Src2Mem, em_div_ex),
4489 F(DstXacc | Src2Mem, em_idiv_ex),
4492 static const struct opcode group4[] = {
4493 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4494 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4498 static const struct opcode group5[] = {
4499 F(DstMem | SrcNone | Lock, em_inc),
4500 F(DstMem | SrcNone | Lock, em_dec),
4501 I(SrcMem | NearBranch, em_call_near_abs),
4502 I(SrcMemFAddr | ImplicitOps, em_call_far),
4503 I(SrcMem | NearBranch, em_jmp_abs),
4504 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4505 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4508 static const struct opcode group6[] = {
4509 II(Prot | DstMem, em_sldt, sldt),
4510 II(Prot | DstMem, em_str, str),
4511 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4512 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4516 static const struct group_dual group7 = { {
4517 II(Mov | DstMem, em_sgdt, sgdt),
4518 II(Mov | DstMem, em_sidt, sidt),
4519 II(SrcMem | Priv, em_lgdt, lgdt),
4520 II(SrcMem | Priv, em_lidt, lidt),
4521 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4522 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4523 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4527 N, EXT(0, group7_rm3),
4528 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4529 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4533 static const struct opcode group8[] = {
4535 F(DstMem | SrcImmByte | NoWrite, em_bt),
4536 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4537 F(DstMem | SrcImmByte | Lock, em_btr),
4538 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4542 * The "memory" destination is actually always a register, since we come
4543 * from the register case of group9.
4545 static const struct gprefix pfx_0f_c7_7 = {
4546 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4550 static const struct group_dual group9 = { {
4551 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4553 N, N, N, N, N, N, N,
4554 GP(0, &pfx_0f_c7_7),
4557 static const struct opcode group11[] = {
4558 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4562 static const struct gprefix pfx_0f_ae_7 = {
4563 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4566 static const struct group_dual group15 = { {
4567 I(ModRM | Aligned16, em_fxsave),
4568 I(ModRM | Aligned16, em_fxrstor),
4569 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4571 N, N, N, N, N, N, N, N,
4574 static const struct gprefix pfx_0f_6f_0f_7f = {
4575 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4578 static const struct instr_dual instr_dual_0f_2b = {
4582 static const struct gprefix pfx_0f_2b = {
4583 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4586 static const struct gprefix pfx_0f_10_0f_11 = {
4587 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4590 static const struct gprefix pfx_0f_28_0f_29 = {
4591 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4594 static const struct gprefix pfx_0f_e7 = {
4595 N, I(Sse, em_mov), N, N,
4598 static const struct escape escape_d9 = { {
4599 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4602 N, N, N, N, N, N, N, N,
4604 N, N, N, N, N, N, N, N,
4606 N, N, N, N, N, N, N, N,
4608 N, N, N, N, N, N, N, N,
4610 N, N, N, N, N, N, N, N,
4612 N, N, N, N, N, N, N, N,
4614 N, N, N, N, N, N, N, N,
4616 N, N, N, N, N, N, N, N,
4619 static const struct escape escape_db = { {
4620 N, N, N, N, N, N, N, N,
4623 N, N, N, N, N, N, N, N,
4625 N, N, N, N, N, N, N, N,
4627 N, N, N, N, N, N, N, N,
4629 N, N, N, N, N, N, N, N,
4631 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4633 N, N, N, N, N, N, N, N,
4635 N, N, N, N, N, N, N, N,
4637 N, N, N, N, N, N, N, N,
4640 static const struct escape escape_dd = { {
4641 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4644 N, N, N, N, N, N, N, N,
4646 N, N, N, N, N, N, N, N,
4648 N, N, N, N, N, N, N, N,
4650 N, N, N, N, N, N, N, N,
4652 N, N, N, N, N, N, N, N,
4654 N, N, N, N, N, N, N, N,
4656 N, N, N, N, N, N, N, N,
4658 N, N, N, N, N, N, N, N,
4661 static const struct instr_dual instr_dual_0f_c3 = {
4662 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4665 static const struct mode_dual mode_dual_63 = {
4666 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4669 static const struct opcode opcode_table[256] = {
4671 F6ALU(Lock, em_add),
4672 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4673 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4675 F6ALU(Lock | PageTable, em_or),
4676 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4679 F6ALU(Lock, em_adc),
4680 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4681 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4683 F6ALU(Lock, em_sbb),
4684 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4685 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4687 F6ALU(Lock | PageTable, em_and), N, N,
4689 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4691 F6ALU(Lock, em_xor), N, N,
4693 F6ALU(NoWrite, em_cmp), N, N,
4695 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4697 X8(I(SrcReg | Stack, em_push)),
4699 X8(I(DstReg | Stack, em_pop)),
4701 I(ImplicitOps | Stack | No64, em_pusha),
4702 I(ImplicitOps | Stack | No64, em_popa),
4703 N, MD(ModRM, &mode_dual_63),
4706 I(SrcImm | Mov | Stack, em_push),
4707 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4708 I(SrcImmByte | Mov | Stack, em_push),
4709 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4710 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4711 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4713 X16(D(SrcImmByte | NearBranch)),
4715 G(ByteOp | DstMem | SrcImm, group1),
4716 G(DstMem | SrcImm, group1),
4717 G(ByteOp | DstMem | SrcImm | No64, group1),
4718 G(DstMem | SrcImmByte, group1),
4719 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4720 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4722 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4723 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4724 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4725 D(ModRM | SrcMem | NoAccess | DstReg),
4726 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4729 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4731 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4732 I(SrcImmFAddr | No64, em_call_far), N,
4733 II(ImplicitOps | Stack, em_pushf, pushf),
4734 II(ImplicitOps | Stack, em_popf, popf),
4735 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4737 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4738 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4739 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4740 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4742 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4743 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4744 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4745 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4747 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4749 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4751 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4752 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4753 I(ImplicitOps | NearBranch, em_ret),
4754 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4755 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4756 G(ByteOp, group11), G(0, group11),
4758 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4759 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4760 I(ImplicitOps, em_ret_far),
4761 D(ImplicitOps), DI(SrcImmByte, intn),
4762 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4764 G(Src2One | ByteOp, group2), G(Src2One, group2),
4765 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4766 I(DstAcc | SrcImmUByte | No64, em_aam),
4767 I(DstAcc | SrcImmUByte | No64, em_aad),
4768 F(DstAcc | ByteOp | No64, em_salc),
4769 I(DstAcc | SrcXLat | ByteOp, em_mov),
4771 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4773 X3(I(SrcImmByte | NearBranch, em_loop)),
4774 I(SrcImmByte | NearBranch, em_jcxz),
4775 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4776 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4778 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4779 I(SrcImmFAddr | No64, em_jmp_far),
4780 D(SrcImmByte | ImplicitOps | NearBranch),
4781 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4782 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4784 N, DI(ImplicitOps, icebp), N, N,
4785 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4786 G(ByteOp, group3), G(0, group3),
4788 D(ImplicitOps), D(ImplicitOps),
4789 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4790 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4793 static const struct opcode twobyte_table[256] = {
4795 G(0, group6), GD(0, &group7), N, N,
4796 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4797 II(ImplicitOps | Priv, em_clts, clts), N,
4798 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4799 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4801 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4802 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4804 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4805 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4807 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4808 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4809 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4811 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4814 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4815 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4816 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4819 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4820 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4821 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4822 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4823 I(ImplicitOps | EmulateOnUD, em_sysenter),
4824 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4826 N, N, N, N, N, N, N, N,
4828 X16(D(DstReg | SrcMem | ModRM)),
4830 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4835 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4840 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4842 X16(D(SrcImm | NearBranch)),
4844 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4846 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4847 II(ImplicitOps, em_cpuid, cpuid),
4848 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4849 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4850 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4852 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4853 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4854 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4855 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4856 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4857 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4859 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4860 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4861 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4862 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4863 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4864 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4868 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4869 I(DstReg | SrcMem | ModRM, em_bsf_c),
4870 I(DstReg | SrcMem | ModRM, em_bsr_c),
4871 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4873 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4874 N, ID(0, &instr_dual_0f_c3),
4875 N, N, N, GD(0, &group9),
4877 X8(I(DstReg, em_bswap)),
4879 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4881 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4882 N, N, N, N, N, N, N, N,
4884 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4887 static const struct instr_dual instr_dual_0f_38_f0 = {
4888 I(DstReg | SrcMem | Mov, em_movbe), N
4891 static const struct instr_dual instr_dual_0f_38_f1 = {
4892 I(DstMem | SrcReg | Mov, em_movbe), N
4895 static const struct gprefix three_byte_0f_38_f0 = {
4896 ID(0, &instr_dual_0f_38_f0), N, N, N
4899 static const struct gprefix three_byte_0f_38_f1 = {
4900 ID(0, &instr_dual_0f_38_f1), N, N, N
4904 * Insns below are selected by the prefix which indexed by the third opcode
4907 static const struct opcode opcode_map_0f_38[256] = {
4909 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4911 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4913 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4914 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4935 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4939 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4945 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4946 unsigned size, bool sign_extension)
4948 int rc = X86EMUL_CONTINUE;
4952 op->addr.mem.ea = ctxt->_eip;
4953 /* NB. Immediates are sign-extended as necessary. */
4954 switch (op->bytes) {
4956 op->val = insn_fetch(s8, ctxt);
4959 op->val = insn_fetch(s16, ctxt);
4962 op->val = insn_fetch(s32, ctxt);
4965 op->val = insn_fetch(s64, ctxt);
4968 if (!sign_extension) {
4969 switch (op->bytes) {
4977 op->val &= 0xffffffff;
4985 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4988 int rc = X86EMUL_CONTINUE;
4992 decode_register_operand(ctxt, op);
4995 rc = decode_imm(ctxt, op, 1, false);
4998 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5002 if (ctxt->d & BitOp)
5003 fetch_bit_operand(ctxt);
5004 op->orig_val = op->val;
5007 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
5011 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5012 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5013 fetch_register_operand(op);
5014 op->orig_val = op->val;
5018 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
5019 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
5020 fetch_register_operand(op);
5021 op->orig_val = op->val;
5024 if (ctxt->d & ByteOp) {
5029 op->bytes = ctxt->op_bytes;
5030 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5031 fetch_register_operand(op);
5032 op->orig_val = op->val;
5036 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5038 register_address(ctxt, VCPU_REGS_RDI);
5039 op->addr.mem.seg = VCPU_SREG_ES;
5046 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5047 fetch_register_operand(op);
5052 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5055 rc = decode_imm(ctxt, op, 1, true);
5063 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5066 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5069 ctxt->memop.bytes = 1;
5070 if (ctxt->memop.type == OP_REG) {
5071 ctxt->memop.addr.reg = decode_register(ctxt,
5072 ctxt->modrm_rm, true);
5073 fetch_register_operand(&ctxt->memop);
5077 ctxt->memop.bytes = 2;
5080 ctxt->memop.bytes = 4;
5083 rc = decode_imm(ctxt, op, 2, false);
5086 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5090 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5092 register_address(ctxt, VCPU_REGS_RSI);
5093 op->addr.mem.seg = ctxt->seg_override;
5099 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5102 reg_read(ctxt, VCPU_REGS_RBX) +
5103 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5104 op->addr.mem.seg = ctxt->seg_override;
5109 op->addr.mem.ea = ctxt->_eip;
5110 op->bytes = ctxt->op_bytes + 2;
5111 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5114 ctxt->memop.bytes = ctxt->op_bytes + 2;
5118 op->val = VCPU_SREG_ES;
5122 op->val = VCPU_SREG_CS;
5126 op->val = VCPU_SREG_SS;
5130 op->val = VCPU_SREG_DS;
5134 op->val = VCPU_SREG_FS;
5138 op->val = VCPU_SREG_GS;
5141 /* Special instructions do their own operand decoding. */
5143 op->type = OP_NONE; /* Disable writeback. */
5151 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5153 int rc = X86EMUL_CONTINUE;
5154 int mode = ctxt->mode;
5155 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5156 bool op_prefix = false;
5157 bool has_seg_override = false;
5158 struct opcode opcode;
5160 struct desc_struct desc;
5162 ctxt->memop.type = OP_NONE;
5163 ctxt->memopp = NULL;
5164 ctxt->_eip = ctxt->eip;
5165 ctxt->fetch.ptr = ctxt->fetch.data;
5166 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5167 ctxt->opcode_len = 1;
5168 ctxt->intercept = x86_intercept_none;
5170 memcpy(ctxt->fetch.data, insn, insn_len);
5172 rc = __do_insn_fetch_bytes(ctxt, 1);
5173 if (rc != X86EMUL_CONTINUE)
5178 case X86EMUL_MODE_REAL:
5179 case X86EMUL_MODE_VM86:
5180 def_op_bytes = def_ad_bytes = 2;
5181 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5183 def_op_bytes = def_ad_bytes = 4;
5185 case X86EMUL_MODE_PROT16:
5186 def_op_bytes = def_ad_bytes = 2;
5188 case X86EMUL_MODE_PROT32:
5189 def_op_bytes = def_ad_bytes = 4;
5191 #ifdef CONFIG_X86_64
5192 case X86EMUL_MODE_PROT64:
5198 return EMULATION_FAILED;
5201 ctxt->op_bytes = def_op_bytes;
5202 ctxt->ad_bytes = def_ad_bytes;
5204 /* Legacy prefixes. */
5206 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5207 case 0x66: /* operand-size override */
5209 /* switch between 2/4 bytes */
5210 ctxt->op_bytes = def_op_bytes ^ 6;
5212 case 0x67: /* address-size override */
5213 if (mode == X86EMUL_MODE_PROT64)
5214 /* switch between 4/8 bytes */
5215 ctxt->ad_bytes = def_ad_bytes ^ 12;
5217 /* switch between 2/4 bytes */
5218 ctxt->ad_bytes = def_ad_bytes ^ 6;
5220 case 0x26: /* ES override */
5221 has_seg_override = true;
5222 ctxt->seg_override = VCPU_SREG_ES;
5224 case 0x2e: /* CS override */
5225 has_seg_override = true;
5226 ctxt->seg_override = VCPU_SREG_CS;
5228 case 0x36: /* SS override */
5229 has_seg_override = true;
5230 ctxt->seg_override = VCPU_SREG_SS;
5232 case 0x3e: /* DS override */
5233 has_seg_override = true;
5234 ctxt->seg_override = VCPU_SREG_DS;
5236 case 0x64: /* FS override */
5237 has_seg_override = true;
5238 ctxt->seg_override = VCPU_SREG_FS;
5240 case 0x65: /* GS override */
5241 has_seg_override = true;
5242 ctxt->seg_override = VCPU_SREG_GS;
5244 case 0x40 ... 0x4f: /* REX */
5245 if (mode != X86EMUL_MODE_PROT64)
5247 ctxt->rex_prefix = ctxt->b;
5249 case 0xf0: /* LOCK */
5250 ctxt->lock_prefix = 1;
5252 case 0xf2: /* REPNE/REPNZ */
5253 case 0xf3: /* REP/REPE/REPZ */
5254 ctxt->rep_prefix = ctxt->b;
5260 /* Any legacy prefix after a REX prefix nullifies its effect. */
5262 ctxt->rex_prefix = 0;
5268 if (ctxt->rex_prefix & 8)
5269 ctxt->op_bytes = 8; /* REX.W */
5271 /* Opcode byte(s). */
5272 opcode = opcode_table[ctxt->b];
5273 /* Two-byte opcode? */
5274 if (ctxt->b == 0x0f) {
5275 ctxt->opcode_len = 2;
5276 ctxt->b = insn_fetch(u8, ctxt);
5277 opcode = twobyte_table[ctxt->b];
5279 /* 0F_38 opcode map */
5280 if (ctxt->b == 0x38) {
5281 ctxt->opcode_len = 3;
5282 ctxt->b = insn_fetch(u8, ctxt);
5283 opcode = opcode_map_0f_38[ctxt->b];
5286 ctxt->d = opcode.flags;
5288 if (ctxt->d & ModRM)
5289 ctxt->modrm = insn_fetch(u8, ctxt);
5291 /* vex-prefix instructions are not implemented */
5292 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5293 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5297 while (ctxt->d & GroupMask) {
5298 switch (ctxt->d & GroupMask) {
5300 goffset = (ctxt->modrm >> 3) & 7;
5301 opcode = opcode.u.group[goffset];
5304 goffset = (ctxt->modrm >> 3) & 7;
5305 if ((ctxt->modrm >> 6) == 3)
5306 opcode = opcode.u.gdual->mod3[goffset];
5308 opcode = opcode.u.gdual->mod012[goffset];
5311 goffset = ctxt->modrm & 7;
5312 opcode = opcode.u.group[goffset];
5315 if (ctxt->rep_prefix && op_prefix)
5316 return EMULATION_FAILED;
5317 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5318 switch (simd_prefix) {
5319 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5320 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5321 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5322 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5326 if (ctxt->modrm > 0xbf) {
5327 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5328 u32 index = array_index_nospec(
5329 ctxt->modrm - 0xc0, size);
5331 opcode = opcode.u.esc->high[index];
5333 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5337 if ((ctxt->modrm >> 6) == 3)
5338 opcode = opcode.u.idual->mod3;
5340 opcode = opcode.u.idual->mod012;
5343 if (ctxt->mode == X86EMUL_MODE_PROT64)
5344 opcode = opcode.u.mdual->mode64;
5346 opcode = opcode.u.mdual->mode32;
5349 return EMULATION_FAILED;
5352 ctxt->d &= ~(u64)GroupMask;
5353 ctxt->d |= opcode.flags;
5358 return EMULATION_FAILED;
5360 ctxt->execute = opcode.u.execute;
5362 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5363 return EMULATION_FAILED;
5365 if (unlikely(ctxt->d &
5366 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5369 * These are copied unconditionally here, and checked unconditionally
5370 * in x86_emulate_insn.
5372 ctxt->check_perm = opcode.check_perm;
5373 ctxt->intercept = opcode.intercept;
5375 if (ctxt->d & NotImpl)
5376 return EMULATION_FAILED;
5378 if (mode == X86EMUL_MODE_PROT64) {
5379 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5381 else if (ctxt->d & NearBranch)
5385 if (ctxt->d & Op3264) {
5386 if (mode == X86EMUL_MODE_PROT64)
5392 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5396 ctxt->op_bytes = 16;
5397 else if (ctxt->d & Mmx)
5401 /* ModRM and SIB bytes. */
5402 if (ctxt->d & ModRM) {
5403 rc = decode_modrm(ctxt, &ctxt->memop);
5404 if (!has_seg_override) {
5405 has_seg_override = true;
5406 ctxt->seg_override = ctxt->modrm_seg;
5408 } else if (ctxt->d & MemAbs)
5409 rc = decode_abs(ctxt, &ctxt->memop);
5410 if (rc != X86EMUL_CONTINUE)
5413 if (!has_seg_override)
5414 ctxt->seg_override = VCPU_SREG_DS;
5416 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5419 * Decode and fetch the source operand: register, memory
5422 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5423 if (rc != X86EMUL_CONTINUE)
5427 * Decode and fetch the second source operand: register, memory
5430 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5431 if (rc != X86EMUL_CONTINUE)
5434 /* Decode and fetch the destination operand: register or memory. */
5435 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5437 if (ctxt->rip_relative && likely(ctxt->memopp))
5438 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5439 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5442 if (rc == X86EMUL_PROPAGATE_FAULT)
5443 ctxt->have_exception = true;
5444 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5447 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5449 return ctxt->d & PageTable;
5452 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5454 /* The second termination condition only applies for REPE
5455 * and REPNE. Test if the repeat string operation prefix is
5456 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5457 * corresponding termination condition according to:
5458 * - if REPE/REPZ and ZF = 0 then done
5459 * - if REPNE/REPNZ and ZF = 1 then done
5461 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5462 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5463 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5464 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5465 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5466 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5472 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5476 rc = asm_safe("fwait");
5478 if (unlikely(rc != X86EMUL_CONTINUE))
5479 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5481 return X86EMUL_CONTINUE;
5484 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5487 if (op->type == OP_MM)
5488 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5491 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5493 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5495 if (!(ctxt->d & ByteOp))
5496 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5498 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5499 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5500 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5501 : "c"(ctxt->src2.val));
5503 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5504 if (!fop) /* exception is returned in fop variable */
5505 return emulate_de(ctxt);
5506 return X86EMUL_CONTINUE;
5509 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5511 memset(&ctxt->rip_relative, 0,
5512 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5514 ctxt->io_read.pos = 0;
5515 ctxt->io_read.end = 0;
5516 ctxt->mem_read.end = 0;
5519 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5521 const struct x86_emulate_ops *ops = ctxt->ops;
5522 int rc = X86EMUL_CONTINUE;
5523 int saved_dst_type = ctxt->dst.type;
5524 unsigned emul_flags;
5526 ctxt->mem_read.pos = 0;
5528 /* LOCK prefix is allowed only with some instructions */
5529 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5530 rc = emulate_ud(ctxt);
5534 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5535 rc = emulate_ud(ctxt);
5539 emul_flags = ctxt->ops->get_hflags(ctxt);
5540 if (unlikely(ctxt->d &
5541 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5542 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5543 (ctxt->d & Undefined)) {
5544 rc = emulate_ud(ctxt);
5548 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5549 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5550 rc = emulate_ud(ctxt);
5554 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5555 rc = emulate_nm(ctxt);
5559 if (ctxt->d & Mmx) {
5560 rc = flush_pending_x87_faults(ctxt);
5561 if (rc != X86EMUL_CONTINUE)
5564 * Now that we know the fpu is exception safe, we can fetch
5567 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5568 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5569 if (!(ctxt->d & Mov))
5570 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5573 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5574 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5575 X86_ICPT_PRE_EXCEPT);
5576 if (rc != X86EMUL_CONTINUE)
5580 /* Instruction can only be executed in protected mode */
5581 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5582 rc = emulate_ud(ctxt);
5586 /* Privileged instruction can be executed only in CPL=0 */
5587 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5588 if (ctxt->d & PrivUD)
5589 rc = emulate_ud(ctxt);
5591 rc = emulate_gp(ctxt, 0);
5595 /* Do instruction specific permission checks */
5596 if (ctxt->d & CheckPerm) {
5597 rc = ctxt->check_perm(ctxt);
5598 if (rc != X86EMUL_CONTINUE)
5602 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5603 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5604 X86_ICPT_POST_EXCEPT);
5605 if (rc != X86EMUL_CONTINUE)
5609 if (ctxt->rep_prefix && (ctxt->d & String)) {
5610 /* All REP prefixes have the same first termination condition */
5611 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5612 string_registers_quirk(ctxt);
5613 ctxt->eip = ctxt->_eip;
5614 ctxt->eflags &= ~X86_EFLAGS_RF;
5620 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5621 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5622 ctxt->src.valptr, ctxt->src.bytes);
5623 if (rc != X86EMUL_CONTINUE)
5625 ctxt->src.orig_val64 = ctxt->src.val64;
5628 if (ctxt->src2.type == OP_MEM) {
5629 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5630 &ctxt->src2.val, ctxt->src2.bytes);
5631 if (rc != X86EMUL_CONTINUE)
5635 if ((ctxt->d & DstMask) == ImplicitOps)
5639 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5640 /* optimisation - avoid slow emulated read if Mov */
5641 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5642 &ctxt->dst.val, ctxt->dst.bytes);
5643 if (rc != X86EMUL_CONTINUE) {
5644 if (!(ctxt->d & NoWrite) &&
5645 rc == X86EMUL_PROPAGATE_FAULT &&
5646 ctxt->exception.vector == PF_VECTOR)
5647 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5651 /* Copy full 64-bit value for CMPXCHG8B. */
5652 ctxt->dst.orig_val64 = ctxt->dst.val64;
5656 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5657 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5658 X86_ICPT_POST_MEMACCESS);
5659 if (rc != X86EMUL_CONTINUE)
5663 if (ctxt->rep_prefix && (ctxt->d & String))
5664 ctxt->eflags |= X86_EFLAGS_RF;
5666 ctxt->eflags &= ~X86_EFLAGS_RF;
5668 if (ctxt->execute) {
5669 if (ctxt->d & Fastop) {
5670 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5671 rc = fastop(ctxt, fop);
5672 if (rc != X86EMUL_CONTINUE)
5676 rc = ctxt->execute(ctxt);
5677 if (rc != X86EMUL_CONTINUE)
5682 if (ctxt->opcode_len == 2)
5684 else if (ctxt->opcode_len == 3)
5685 goto threebyte_insn;
5688 case 0x70 ... 0x7f: /* jcc (short) */
5689 if (test_cc(ctxt->b, ctxt->eflags))
5690 rc = jmp_rel(ctxt, ctxt->src.val);
5692 case 0x8d: /* lea r16/r32, m */
5693 ctxt->dst.val = ctxt->src.addr.mem.ea;
5695 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5696 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5697 ctxt->dst.type = OP_NONE;
5701 case 0x98: /* cbw/cwde/cdqe */
5702 switch (ctxt->op_bytes) {
5703 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5704 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5705 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5708 case 0xcc: /* int3 */
5709 rc = emulate_int(ctxt, 3);
5711 case 0xcd: /* int n */
5712 rc = emulate_int(ctxt, ctxt->src.val);
5714 case 0xce: /* into */
5715 if (ctxt->eflags & X86_EFLAGS_OF)
5716 rc = emulate_int(ctxt, 4);
5718 case 0xe9: /* jmp rel */
5719 case 0xeb: /* jmp rel short */
5720 rc = jmp_rel(ctxt, ctxt->src.val);
5721 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5723 case 0xf4: /* hlt */
5724 ctxt->ops->halt(ctxt);
5726 case 0xf5: /* cmc */
5727 /* complement carry flag from eflags reg */
5728 ctxt->eflags ^= X86_EFLAGS_CF;
5730 case 0xf8: /* clc */
5731 ctxt->eflags &= ~X86_EFLAGS_CF;
5733 case 0xf9: /* stc */
5734 ctxt->eflags |= X86_EFLAGS_CF;
5736 case 0xfc: /* cld */
5737 ctxt->eflags &= ~X86_EFLAGS_DF;
5739 case 0xfd: /* std */
5740 ctxt->eflags |= X86_EFLAGS_DF;
5743 goto cannot_emulate;
5746 if (rc != X86EMUL_CONTINUE)
5750 if (ctxt->d & SrcWrite) {
5751 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5752 rc = writeback(ctxt, &ctxt->src);
5753 if (rc != X86EMUL_CONTINUE)
5756 if (!(ctxt->d & NoWrite)) {
5757 rc = writeback(ctxt, &ctxt->dst);
5758 if (rc != X86EMUL_CONTINUE)
5763 * restore dst type in case the decoding will be reused
5764 * (happens for string instruction )
5766 ctxt->dst.type = saved_dst_type;
5768 if ((ctxt->d & SrcMask) == SrcSI)
5769 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5771 if ((ctxt->d & DstMask) == DstDI)
5772 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5774 if (ctxt->rep_prefix && (ctxt->d & String)) {
5776 struct read_cache *r = &ctxt->io_read;
5777 if ((ctxt->d & SrcMask) == SrcSI)
5778 count = ctxt->src.count;
5780 count = ctxt->dst.count;
5781 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5783 if (!string_insn_completed(ctxt)) {
5785 * Re-enter guest when pio read ahead buffer is empty
5786 * or, if it is not used, after each 1024 iteration.
5788 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5789 (r->end == 0 || r->end != r->pos)) {
5791 * Reset read cache. Usually happens before
5792 * decode, but since instruction is restarted
5793 * we have to do it here.
5795 ctxt->mem_read.end = 0;
5796 writeback_registers(ctxt);
5797 return EMULATION_RESTART;
5799 goto done; /* skip rip writeback */
5801 ctxt->eflags &= ~X86_EFLAGS_RF;
5804 ctxt->eip = ctxt->_eip;
5807 if (rc == X86EMUL_PROPAGATE_FAULT) {
5808 WARN_ON(ctxt->exception.vector > 0x1f);
5809 ctxt->have_exception = true;
5811 if (rc == X86EMUL_INTERCEPTED)
5812 return EMULATION_INTERCEPTED;
5814 if (rc == X86EMUL_CONTINUE)
5815 writeback_registers(ctxt);
5817 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5821 case 0x09: /* wbinvd */
5822 (ctxt->ops->wbinvd)(ctxt);
5824 case 0x08: /* invd */
5825 case 0x0d: /* GrpP (prefetch) */
5826 case 0x18: /* Grp16 (prefetch/nop) */
5827 case 0x1f: /* nop */
5829 case 0x20: /* mov cr, reg */
5830 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5832 case 0x21: /* mov from dr to reg */
5833 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5835 case 0x40 ... 0x4f: /* cmov */
5836 if (test_cc(ctxt->b, ctxt->eflags))
5837 ctxt->dst.val = ctxt->src.val;
5838 else if (ctxt->op_bytes != 4)
5839 ctxt->dst.type = OP_NONE; /* no writeback */
5841 case 0x80 ... 0x8f: /* jnz rel, etc*/
5842 if (test_cc(ctxt->b, ctxt->eflags))
5843 rc = jmp_rel(ctxt, ctxt->src.val);
5845 case 0x90 ... 0x9f: /* setcc r/m8 */
5846 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5848 case 0xb6 ... 0xb7: /* movzx */
5849 ctxt->dst.bytes = ctxt->op_bytes;
5850 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5851 : (u16) ctxt->src.val;
5853 case 0xbe ... 0xbf: /* movsx */
5854 ctxt->dst.bytes = ctxt->op_bytes;
5855 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5856 (s16) ctxt->src.val;
5859 goto cannot_emulate;
5864 if (rc != X86EMUL_CONTINUE)
5870 return EMULATION_FAILED;
5873 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5875 invalidate_registers(ctxt);
5878 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5880 writeback_registers(ctxt);
5883 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5885 if (ctxt->rep_prefix && (ctxt->d & String))
5888 if (ctxt->d & TwoMemOp)