1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 #include <asm/nospec-branch.h>
39 #define OpImplicit 1ull /* No generic decode */
40 #define OpReg 2ull /* Register */
41 #define OpMem 3ull /* Memory */
42 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
43 #define OpDI 5ull /* ES:DI/EDI/RDI */
44 #define OpMem64 6ull /* Memory, 64-bit */
45 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
46 #define OpDX 8ull /* DX register */
47 #define OpCL 9ull /* CL register (for shifts) */
48 #define OpImmByte 10ull /* 8-bit sign extended immediate */
49 #define OpOne 11ull /* Implied 1 */
50 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
51 #define OpMem16 13ull /* Memory operand (16-bit). */
52 #define OpMem32 14ull /* Memory operand (32-bit). */
53 #define OpImmU 15ull /* Immediate operand, zero extended */
54 #define OpSI 16ull /* SI/ESI/RSI */
55 #define OpImmFAddr 17ull /* Immediate far address */
56 #define OpMemFAddr 18ull /* Far address in memory */
57 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
58 #define OpES 20ull /* ES */
59 #define OpCS 21ull /* CS */
60 #define OpSS 22ull /* SS */
61 #define OpDS 23ull /* DS */
62 #define OpFS 24ull /* FS */
63 #define OpGS 25ull /* GS */
64 #define OpMem8 26ull /* 8-bit zero extended memory operand */
65 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
66 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
67 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
68 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
70 #define OpBits 5 /* Width of operand field */
71 #define OpMask ((1ull << OpBits) - 1)
74 * Opcode effective-address decode tables.
75 * Note that we only emulate instructions that have at least one memory
76 * operand (excluding implicit stack references). We assume that stack
77 * references and instruction fetches will never occur in special memory
78 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
82 /* Operand sizes: 8-bit operands or specified/overridden size. */
83 #define ByteOp (1<<0) /* 8-bit operands. */
84 /* Destination operand type. */
86 #define ImplicitOps (OpImplicit << DstShift)
87 #define DstReg (OpReg << DstShift)
88 #define DstMem (OpMem << DstShift)
89 #define DstAcc (OpAcc << DstShift)
90 #define DstDI (OpDI << DstShift)
91 #define DstMem64 (OpMem64 << DstShift)
92 #define DstMem16 (OpMem16 << DstShift)
93 #define DstImmUByte (OpImmUByte << DstShift)
94 #define DstDX (OpDX << DstShift)
95 #define DstAccLo (OpAccLo << DstShift)
96 #define DstMask (OpMask << DstShift)
97 /* Source operand type. */
99 #define SrcNone (OpNone << SrcShift)
100 #define SrcReg (OpReg << SrcShift)
101 #define SrcMem (OpMem << SrcShift)
102 #define SrcMem16 (OpMem16 << SrcShift)
103 #define SrcMem32 (OpMem32 << SrcShift)
104 #define SrcImm (OpImm << SrcShift)
105 #define SrcImmByte (OpImmByte << SrcShift)
106 #define SrcOne (OpOne << SrcShift)
107 #define SrcImmUByte (OpImmUByte << SrcShift)
108 #define SrcImmU (OpImmU << SrcShift)
109 #define SrcSI (OpSI << SrcShift)
110 #define SrcXLat (OpXLat << SrcShift)
111 #define SrcImmFAddr (OpImmFAddr << SrcShift)
112 #define SrcMemFAddr (OpMemFAddr << SrcShift)
113 #define SrcAcc (OpAcc << SrcShift)
114 #define SrcImmU16 (OpImmU16 << SrcShift)
115 #define SrcImm64 (OpImm64 << SrcShift)
116 #define SrcDX (OpDX << SrcShift)
117 #define SrcMem8 (OpMem8 << SrcShift)
118 #define SrcAccHi (OpAccHi << SrcShift)
119 #define SrcMask (OpMask << SrcShift)
120 #define BitOp (1<<11)
121 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
122 #define String (1<<13) /* String instruction (rep capable) */
123 #define Stack (1<<14) /* Stack instruction (push/pop) */
124 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
125 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
126 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
127 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
128 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
129 #define Escape (5<<15) /* Escape to coprocessor instruction */
130 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
131 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
132 #define Sse (1<<18) /* SSE Vector instruction */
133 /* Generic ModRM decode. */
134 #define ModRM (1<<19)
135 /* Destination is only written; never read. */
138 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
139 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
140 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
141 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
142 #define Undefined (1<<25) /* No Such Instruction */
143 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
144 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
146 #define PageTable (1 << 29) /* instruction used to write page table */
147 #define NotImpl (1 << 30) /* instruction is not implemented */
148 /* Source 2 operand type */
149 #define Src2Shift (31)
150 #define Src2None (OpNone << Src2Shift)
151 #define Src2Mem (OpMem << Src2Shift)
152 #define Src2CL (OpCL << Src2Shift)
153 #define Src2ImmByte (OpImmByte << Src2Shift)
154 #define Src2One (OpOne << Src2Shift)
155 #define Src2Imm (OpImm << Src2Shift)
156 #define Src2ES (OpES << Src2Shift)
157 #define Src2CS (OpCS << Src2Shift)
158 #define Src2SS (OpSS << Src2Shift)
159 #define Src2DS (OpDS << Src2Shift)
160 #define Src2FS (OpFS << Src2Shift)
161 #define Src2GS (OpGS << Src2Shift)
162 #define Src2Mask (OpMask << Src2Shift)
163 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
164 #define AlignMask ((u64)7 << 41)
165 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
166 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
167 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
168 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
169 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
170 #define NoWrite ((u64)1 << 45) /* No writeback */
171 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
172 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
173 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
174 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
175 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
176 #define NearBranch ((u64)1 << 52) /* Near branches */
177 #define No16 ((u64)1 << 53) /* No 16 bit operand */
178 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
179 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
181 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
183 #define X2(x...) x, x
184 #define X3(x...) X2(x), x
185 #define X4(x...) X2(x), X2(x)
186 #define X5(x...) X4(x), x
187 #define X6(x...) X4(x), X2(x)
188 #define X7(x...) X4(x), X3(x)
189 #define X8(x...) X4(x), X4(x)
190 #define X16(x...) X8(x), X8(x)
192 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
193 #define FASTOP_SIZE 8
196 * fastop functions have a special calling convention:
201 * flags: rflags (in/out)
202 * ex: rsi (in:fastop pointer, out:zero if exception)
204 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
205 * different operand sizes can be reached by calculation, rather than a jump
206 * table (which would be bigger than the code).
208 * fastop functions are declared as taking a never-defined fastop parameter,
209 * so they can't be called from C directly.
218 int (*execute)(struct x86_emulate_ctxt *ctxt);
219 const struct opcode *group;
220 const struct group_dual *gdual;
221 const struct gprefix *gprefix;
222 const struct escape *esc;
223 const struct instr_dual *idual;
224 const struct mode_dual *mdual;
225 void (*fastop)(struct fastop *fake);
227 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
231 struct opcode mod012[8];
232 struct opcode mod3[8];
236 struct opcode pfx_no;
237 struct opcode pfx_66;
238 struct opcode pfx_f2;
239 struct opcode pfx_f3;
244 struct opcode high[64];
248 struct opcode mod012;
253 struct opcode mode32;
254 struct opcode mode64;
257 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
259 enum x86_transfer_type {
261 X86_TRANSFER_CALL_JMP,
263 X86_TRANSFER_TASK_SWITCH,
266 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
268 if (!(ctxt->regs_valid & (1 << nr))) {
269 ctxt->regs_valid |= 1 << nr;
270 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
272 return ctxt->_regs[nr];
275 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
277 ctxt->regs_valid |= 1 << nr;
278 ctxt->regs_dirty |= 1 << nr;
279 return &ctxt->_regs[nr];
282 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
285 return reg_write(ctxt, nr);
288 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
292 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
293 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
296 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
298 ctxt->regs_dirty = 0;
299 ctxt->regs_valid = 0;
303 * These EFLAGS bits are restored from saved value during emulation, and
304 * any changes are written back to the saved value after emulation.
306 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
307 X86_EFLAGS_PF|X86_EFLAGS_CF)
315 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
317 #define FOP_FUNC(name) \
318 ".align " __stringify(FASTOP_SIZE) " \n\t" \
319 ".type " name ", @function \n\t" \
322 #define FOP_RET "ret \n\t"
324 #define FOP_START(op) \
325 extern void em_##op(struct fastop *fake); \
326 asm(".pushsection .text, \"ax\" \n\t" \
327 ".global em_" #op " \n\t" \
334 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
337 #define FOP1E(op, dst) \
338 FOP_FUNC(#op "_" #dst) \
339 "10: " #op " %" #dst " \n\t" FOP_RET
341 #define FOP1EEX(op, dst) \
342 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
344 #define FASTOP1(op) \
349 ON64(FOP1E(op##q, rax)) \
352 /* 1-operand, using src2 (for MUL/DIV r/m) */
353 #define FASTOP1SRC2(op, name) \
358 ON64(FOP1E(op, rcx)) \
361 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
362 #define FASTOP1SRC2EX(op, name) \
367 ON64(FOP1EEX(op, rcx)) \
370 #define FOP2E(op, dst, src) \
371 FOP_FUNC(#op "_" #dst "_" #src) \
372 #op " %" #src ", %" #dst " \n\t" FOP_RET
374 #define FASTOP2(op) \
376 FOP2E(op##b, al, dl) \
377 FOP2E(op##w, ax, dx) \
378 FOP2E(op##l, eax, edx) \
379 ON64(FOP2E(op##q, rax, rdx)) \
382 /* 2 operand, word only */
383 #define FASTOP2W(op) \
386 FOP2E(op##w, ax, dx) \
387 FOP2E(op##l, eax, edx) \
388 ON64(FOP2E(op##q, rax, rdx)) \
391 /* 2 operand, src is CL */
392 #define FASTOP2CL(op) \
394 FOP2E(op##b, al, cl) \
395 FOP2E(op##w, ax, cl) \
396 FOP2E(op##l, eax, cl) \
397 ON64(FOP2E(op##q, rax, cl)) \
400 /* 2 operand, src and dest are reversed */
401 #define FASTOP2R(op, name) \
403 FOP2E(op##b, dl, al) \
404 FOP2E(op##w, dx, ax) \
405 FOP2E(op##l, edx, eax) \
406 ON64(FOP2E(op##q, rdx, rax)) \
409 #define FOP3E(op, dst, src, src2) \
410 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
411 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
413 /* 3-operand, word-only, src2=cl */
414 #define FASTOP3WCL(op) \
417 FOP3E(op##w, ax, dx, cl) \
418 FOP3E(op##l, eax, edx, cl) \
419 ON64(FOP3E(op##q, rax, rdx, cl)) \
422 /* Special case for SETcc - 1 instruction per cc */
423 #define FOP_SETCC(op) \
425 ".type " #op ", @function \n\t" \
430 asm(".pushsection .fixup, \"ax\"\n"
431 ".global kvm_fastop_exception \n"
432 "kvm_fastop_exception: xor %esi, %esi; ret\n"
454 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
458 * XXX: inoutclob user must know where the argument is being expanded.
459 * Relying on CONFIG_CC_HAS_ASM_GOTO would allow us to remove _fault.
461 #define asm_safe(insn, inoutclob...) \
465 asm volatile("1:" insn "\n" \
467 ".pushsection .fixup, \"ax\"\n" \
468 "3: movl $1, %[_fault]\n" \
471 _ASM_EXTABLE(1b, 3b) \
472 : [_fault] "+qm"(_fault) inoutclob ); \
474 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
477 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
478 enum x86_intercept intercept,
479 enum x86_intercept_stage stage)
481 struct x86_instruction_info info = {
482 .intercept = intercept,
483 .rep_prefix = ctxt->rep_prefix,
484 .modrm_mod = ctxt->modrm_mod,
485 .modrm_reg = ctxt->modrm_reg,
486 .modrm_rm = ctxt->modrm_rm,
487 .src_val = ctxt->src.val64,
488 .dst_val = ctxt->dst.val64,
489 .src_bytes = ctxt->src.bytes,
490 .dst_bytes = ctxt->dst.bytes,
491 .ad_bytes = ctxt->ad_bytes,
492 .next_rip = ctxt->eip,
495 return ctxt->ops->intercept(ctxt, &info, stage);
498 static void assign_masked(ulong *dest, ulong src, ulong mask)
500 *dest = (*dest & ~mask) | (src & mask);
503 static void assign_register(unsigned long *reg, u64 val, int bytes)
505 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
508 *(u8 *)reg = (u8)val;
511 *(u16 *)reg = (u16)val;
515 break; /* 64b: zero-extend */
522 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
524 return (1UL << (ctxt->ad_bytes << 3)) - 1;
527 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
530 struct desc_struct ss;
532 if (ctxt->mode == X86EMUL_MODE_PROT64)
534 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
535 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
538 static int stack_size(struct x86_emulate_ctxt *ctxt)
540 return (__fls(stack_mask(ctxt)) + 1) >> 3;
543 /* Access/update address held in a register, based on addressing mode. */
544 static inline unsigned long
545 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
547 if (ctxt->ad_bytes == sizeof(unsigned long))
550 return reg & ad_mask(ctxt);
553 static inline unsigned long
554 register_address(struct x86_emulate_ctxt *ctxt, int reg)
556 return address_mask(ctxt, reg_read(ctxt, reg));
559 static void masked_increment(ulong *reg, ulong mask, int inc)
561 assign_masked(reg, *reg + inc, mask);
565 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
567 ulong *preg = reg_rmw(ctxt, reg);
569 assign_register(preg, *preg + inc, ctxt->ad_bytes);
572 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
574 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
577 static u32 desc_limit_scaled(struct desc_struct *desc)
579 u32 limit = get_desc_limit(desc);
581 return desc->g ? (limit << 12) | 0xfff : limit;
584 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
586 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
589 return ctxt->ops->get_cached_segment_base(ctxt, seg);
592 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
593 u32 error, bool valid)
596 ctxt->exception.vector = vec;
597 ctxt->exception.error_code = error;
598 ctxt->exception.error_code_valid = valid;
599 return X86EMUL_PROPAGATE_FAULT;
602 static int emulate_db(struct x86_emulate_ctxt *ctxt)
604 return emulate_exception(ctxt, DB_VECTOR, 0, false);
607 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
609 return emulate_exception(ctxt, GP_VECTOR, err, true);
612 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
614 return emulate_exception(ctxt, SS_VECTOR, err, true);
617 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
619 return emulate_exception(ctxt, UD_VECTOR, 0, false);
622 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
624 return emulate_exception(ctxt, TS_VECTOR, err, true);
627 static int emulate_de(struct x86_emulate_ctxt *ctxt)
629 return emulate_exception(ctxt, DE_VECTOR, 0, false);
632 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
634 return emulate_exception(ctxt, NM_VECTOR, 0, false);
637 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
640 struct desc_struct desc;
642 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
646 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
651 struct desc_struct desc;
653 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
654 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
658 * x86 defines three classes of vector instructions: explicitly
659 * aligned, explicitly unaligned, and the rest, which change behaviour
660 * depending on whether they're AVX encoded or not.
662 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
663 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
664 * 512 bytes of data must be aligned to a 16 byte boundary.
666 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
668 u64 alignment = ctxt->d & AlignMask;
670 if (likely(size < 16))
685 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
686 struct segmented_address addr,
687 unsigned *max_size, unsigned size,
688 bool write, bool fetch,
689 enum x86emul_mode mode, ulong *linear)
691 struct desc_struct desc;
698 la = seg_base(ctxt, addr.seg) + addr.ea;
701 case X86EMUL_MODE_PROT64:
703 va_bits = ctxt_virt_addr_bits(ctxt);
704 if (get_canonical(la, va_bits) != la)
707 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
708 if (size > *max_size)
712 *linear = la = (u32)la;
713 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
717 /* code segment in protected mode or read-only data segment */
718 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
719 || !(desc.type & 2)) && write)
721 /* unreadable code segment */
722 if (!fetch && (desc.type & 8) && !(desc.type & 2))
724 lim = desc_limit_scaled(&desc);
725 if (!(desc.type & 8) && (desc.type & 4)) {
726 /* expand-down segment */
729 lim = desc.d ? 0xffffffff : 0xffff;
733 if (lim == 0xffffffff)
736 *max_size = (u64)lim + 1 - addr.ea;
737 if (size > *max_size)
742 if (la & (insn_alignment(ctxt, size) - 1))
743 return emulate_gp(ctxt, 0);
744 return X86EMUL_CONTINUE;
746 if (addr.seg == VCPU_SREG_SS)
747 return emulate_ss(ctxt, 0);
749 return emulate_gp(ctxt, 0);
752 static int linearize(struct x86_emulate_ctxt *ctxt,
753 struct segmented_address addr,
754 unsigned size, bool write,
758 return __linearize(ctxt, addr, &max_size, size, write, false,
762 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
763 enum x86emul_mode mode)
768 struct segmented_address addr = { .seg = VCPU_SREG_CS,
771 if (ctxt->op_bytes != sizeof(unsigned long))
772 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
773 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
774 if (rc == X86EMUL_CONTINUE)
775 ctxt->_eip = addr.ea;
779 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
781 return assign_eip(ctxt, dst, ctxt->mode);
784 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
785 const struct desc_struct *cs_desc)
787 enum x86emul_mode mode = ctxt->mode;
791 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
795 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
797 mode = X86EMUL_MODE_PROT64;
799 mode = X86EMUL_MODE_PROT32; /* temporary value */
802 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
803 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
804 rc = assign_eip(ctxt, dst, mode);
805 if (rc == X86EMUL_CONTINUE)
810 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
812 return assign_eip_near(ctxt, ctxt->_eip + rel);
815 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
816 void *data, unsigned size)
818 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
821 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
822 ulong linear, void *data,
825 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
828 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
829 struct segmented_address addr,
836 rc = linearize(ctxt, addr, size, false, &linear);
837 if (rc != X86EMUL_CONTINUE)
839 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
842 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
843 struct segmented_address addr,
850 rc = linearize(ctxt, addr, size, true, &linear);
851 if (rc != X86EMUL_CONTINUE)
853 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
857 * Prefetch the remaining bytes of the instruction without crossing page
858 * boundary if they are not in fetch_cache yet.
860 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
863 unsigned size, max_size;
864 unsigned long linear;
865 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
866 struct segmented_address addr = { .seg = VCPU_SREG_CS,
867 .ea = ctxt->eip + cur_size };
870 * We do not know exactly how many bytes will be needed, and
871 * __linearize is expensive, so fetch as much as possible. We
872 * just have to avoid going beyond the 15 byte limit, the end
873 * of the segment, or the end of the page.
875 * __linearize is called with size 0 so that it does not do any
876 * boundary check itself. Instead, we use max_size to check
879 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
881 if (unlikely(rc != X86EMUL_CONTINUE))
884 size = min_t(unsigned, 15UL ^ cur_size, max_size);
885 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
888 * One instruction can only straddle two pages,
889 * and one has been loaded at the beginning of
890 * x86_decode_insn. So, if not enough bytes
891 * still, we must have hit the 15-byte boundary.
893 if (unlikely(size < op_size))
894 return emulate_gp(ctxt, 0);
896 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
897 size, &ctxt->exception);
898 if (unlikely(rc != X86EMUL_CONTINUE))
900 ctxt->fetch.end += size;
901 return X86EMUL_CONTINUE;
904 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
907 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
909 if (unlikely(done_size < size))
910 return __do_insn_fetch_bytes(ctxt, size - done_size);
912 return X86EMUL_CONTINUE;
915 /* Fetch next part of the instruction being emulated. */
916 #define insn_fetch(_type, _ctxt) \
919 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
920 if (rc != X86EMUL_CONTINUE) \
922 ctxt->_eip += sizeof(_type); \
923 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
924 ctxt->fetch.ptr += sizeof(_type); \
928 #define insn_fetch_arr(_arr, _size, _ctxt) \
930 rc = do_insn_fetch_bytes(_ctxt, _size); \
931 if (rc != X86EMUL_CONTINUE) \
933 ctxt->_eip += (_size); \
934 memcpy(_arr, ctxt->fetch.ptr, _size); \
935 ctxt->fetch.ptr += (_size); \
939 * Given the 'reg' portion of a ModRM byte, and a register block, return a
940 * pointer into the block that addresses the relevant register.
941 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
943 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
947 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
949 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
950 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
952 p = reg_rmw(ctxt, modrm_reg);
956 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
957 struct segmented_address addr,
958 u16 *size, unsigned long *address, int op_bytes)
965 rc = segmented_read_std(ctxt, addr, size, 2);
966 if (rc != X86EMUL_CONTINUE)
969 rc = segmented_read_std(ctxt, addr, address, op_bytes);
983 FASTOP1SRC2(mul, mul_ex);
984 FASTOP1SRC2(imul, imul_ex);
985 FASTOP1SRC2EX(div, div_ex);
986 FASTOP1SRC2EX(idiv, idiv_ex);
1015 FASTOP2R(cmp, cmp_r);
1017 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1019 /* If src is zero, do not writeback, but update flags */
1020 if (ctxt->src.val == 0)
1021 ctxt->dst.type = OP_NONE;
1022 return fastop(ctxt, em_bsf);
1025 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1027 /* If src is zero, do not writeback, but update flags */
1028 if (ctxt->src.val == 0)
1029 ctxt->dst.type = OP_NONE;
1030 return fastop(ctxt, em_bsr);
1033 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1036 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1038 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1039 asm("push %[flags]; popf; " CALL_NOSPEC
1040 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1044 static void fetch_register_operand(struct operand *op)
1046 switch (op->bytes) {
1048 op->val = *(u8 *)op->addr.reg;
1051 op->val = *(u16 *)op->addr.reg;
1054 op->val = *(u32 *)op->addr.reg;
1057 op->val = *(u64 *)op->addr.reg;
1062 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1065 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1066 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1067 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1068 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1069 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1070 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1071 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1072 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1073 #ifdef CONFIG_X86_64
1074 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1075 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1076 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1077 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1078 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1079 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1080 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1081 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1087 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1091 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1092 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1093 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1094 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1095 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1096 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1097 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1098 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1099 #ifdef CONFIG_X86_64
1100 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1101 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1102 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1103 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1104 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1105 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1106 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1107 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1113 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1116 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1117 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1118 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1119 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1120 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1121 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1122 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1123 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1128 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1131 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1132 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1133 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1134 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1135 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1136 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1137 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1138 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1143 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1145 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1146 return emulate_nm(ctxt);
1148 asm volatile("fninit");
1149 return X86EMUL_CONTINUE;
1152 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1156 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1157 return emulate_nm(ctxt);
1159 asm volatile("fnstcw %0": "+m"(fcw));
1161 ctxt->dst.val = fcw;
1163 return X86EMUL_CONTINUE;
1166 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1170 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1171 return emulate_nm(ctxt);
1173 asm volatile("fnstsw %0": "+m"(fsw));
1175 ctxt->dst.val = fsw;
1177 return X86EMUL_CONTINUE;
1180 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1183 unsigned reg = ctxt->modrm_reg;
1185 if (!(ctxt->d & ModRM))
1186 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1188 if (ctxt->d & Sse) {
1192 read_sse_reg(ctxt, &op->vec_val, reg);
1195 if (ctxt->d & Mmx) {
1204 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1205 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1207 fetch_register_operand(op);
1208 op->orig_val = op->val;
1211 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1213 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1214 ctxt->modrm_seg = VCPU_SREG_SS;
1217 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1221 int index_reg, base_reg, scale;
1222 int rc = X86EMUL_CONTINUE;
1225 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1226 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1227 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1229 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1230 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1231 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1232 ctxt->modrm_seg = VCPU_SREG_DS;
1234 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1236 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1237 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1239 if (ctxt->d & Sse) {
1242 op->addr.xmm = ctxt->modrm_rm;
1243 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1246 if (ctxt->d & Mmx) {
1249 op->addr.mm = ctxt->modrm_rm & 7;
1252 fetch_register_operand(op);
1258 if (ctxt->ad_bytes == 2) {
1259 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1260 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1261 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1262 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1264 /* 16-bit ModR/M decode. */
1265 switch (ctxt->modrm_mod) {
1267 if (ctxt->modrm_rm == 6)
1268 modrm_ea += insn_fetch(u16, ctxt);
1271 modrm_ea += insn_fetch(s8, ctxt);
1274 modrm_ea += insn_fetch(u16, ctxt);
1277 switch (ctxt->modrm_rm) {
1279 modrm_ea += bx + si;
1282 modrm_ea += bx + di;
1285 modrm_ea += bp + si;
1288 modrm_ea += bp + di;
1297 if (ctxt->modrm_mod != 0)
1304 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1305 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1306 ctxt->modrm_seg = VCPU_SREG_SS;
1307 modrm_ea = (u16)modrm_ea;
1309 /* 32/64-bit ModR/M decode. */
1310 if ((ctxt->modrm_rm & 7) == 4) {
1311 sib = insn_fetch(u8, ctxt);
1312 index_reg |= (sib >> 3) & 7;
1313 base_reg |= sib & 7;
1316 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1317 modrm_ea += insn_fetch(s32, ctxt);
1319 modrm_ea += reg_read(ctxt, base_reg);
1320 adjust_modrm_seg(ctxt, base_reg);
1321 /* Increment ESP on POP [ESP] */
1322 if ((ctxt->d & IncSP) &&
1323 base_reg == VCPU_REGS_RSP)
1324 modrm_ea += ctxt->op_bytes;
1327 modrm_ea += reg_read(ctxt, index_reg) << scale;
1328 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1329 modrm_ea += insn_fetch(s32, ctxt);
1330 if (ctxt->mode == X86EMUL_MODE_PROT64)
1331 ctxt->rip_relative = 1;
1333 base_reg = ctxt->modrm_rm;
1334 modrm_ea += reg_read(ctxt, base_reg);
1335 adjust_modrm_seg(ctxt, base_reg);
1337 switch (ctxt->modrm_mod) {
1339 modrm_ea += insn_fetch(s8, ctxt);
1342 modrm_ea += insn_fetch(s32, ctxt);
1346 op->addr.mem.ea = modrm_ea;
1347 if (ctxt->ad_bytes != 8)
1348 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1354 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1357 int rc = X86EMUL_CONTINUE;
1360 switch (ctxt->ad_bytes) {
1362 op->addr.mem.ea = insn_fetch(u16, ctxt);
1365 op->addr.mem.ea = insn_fetch(u32, ctxt);
1368 op->addr.mem.ea = insn_fetch(u64, ctxt);
1375 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1379 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1380 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1382 if (ctxt->src.bytes == 2)
1383 sv = (s16)ctxt->src.val & (s16)mask;
1384 else if (ctxt->src.bytes == 4)
1385 sv = (s32)ctxt->src.val & (s32)mask;
1387 sv = (s64)ctxt->src.val & (s64)mask;
1389 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1390 ctxt->dst.addr.mem.ea + (sv >> 3));
1393 /* only subword offset */
1394 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1397 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1398 unsigned long addr, void *dest, unsigned size)
1401 struct read_cache *mc = &ctxt->mem_read;
1403 if (mc->pos < mc->end)
1406 WARN_ON((mc->end + size) >= sizeof(mc->data));
1408 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1410 if (rc != X86EMUL_CONTINUE)
1416 memcpy(dest, mc->data + mc->pos, size);
1418 return X86EMUL_CONTINUE;
1421 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1422 struct segmented_address addr,
1429 rc = linearize(ctxt, addr, size, false, &linear);
1430 if (rc != X86EMUL_CONTINUE)
1432 return read_emulated(ctxt, linear, data, size);
1435 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1436 struct segmented_address addr,
1443 rc = linearize(ctxt, addr, size, true, &linear);
1444 if (rc != X86EMUL_CONTINUE)
1446 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1450 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1451 struct segmented_address addr,
1452 const void *orig_data, const void *data,
1458 rc = linearize(ctxt, addr, size, true, &linear);
1459 if (rc != X86EMUL_CONTINUE)
1461 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1462 size, &ctxt->exception);
1465 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1466 unsigned int size, unsigned short port,
1469 struct read_cache *rc = &ctxt->io_read;
1471 if (rc->pos == rc->end) { /* refill pio read ahead */
1472 unsigned int in_page, n;
1473 unsigned int count = ctxt->rep_prefix ?
1474 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1475 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1476 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1477 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1478 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1481 rc->pos = rc->end = 0;
1482 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1487 if (ctxt->rep_prefix && (ctxt->d & String) &&
1488 !(ctxt->eflags & X86_EFLAGS_DF)) {
1489 ctxt->dst.data = rc->data + rc->pos;
1490 ctxt->dst.type = OP_MEM_STR;
1491 ctxt->dst.count = (rc->end - rc->pos) / size;
1494 memcpy(dest, rc->data + rc->pos, size);
1500 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1501 u16 index, struct desc_struct *desc)
1506 ctxt->ops->get_idt(ctxt, &dt);
1508 if (dt.size < index * 8 + 7)
1509 return emulate_gp(ctxt, index << 3 | 0x2);
1511 addr = dt.address + index * 8;
1512 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1515 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1516 u16 selector, struct desc_ptr *dt)
1518 const struct x86_emulate_ops *ops = ctxt->ops;
1521 if (selector & 1 << 2) {
1522 struct desc_struct desc;
1525 memset (dt, 0, sizeof *dt);
1526 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1530 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1531 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1533 ops->get_gdt(ctxt, dt);
1536 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1537 u16 selector, ulong *desc_addr_p)
1540 u16 index = selector >> 3;
1543 get_descriptor_table_ptr(ctxt, selector, &dt);
1545 if (dt.size < index * 8 + 7)
1546 return emulate_gp(ctxt, selector & 0xfffc);
1548 addr = dt.address + index * 8;
1550 #ifdef CONFIG_X86_64
1551 if (addr >> 32 != 0) {
1554 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1555 if (!(efer & EFER_LMA))
1560 *desc_addr_p = addr;
1561 return X86EMUL_CONTINUE;
1564 /* allowed just for 8 bytes segments */
1565 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1566 u16 selector, struct desc_struct *desc,
1571 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1572 if (rc != X86EMUL_CONTINUE)
1575 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1578 /* allowed just for 8 bytes segments */
1579 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1580 u16 selector, struct desc_struct *desc)
1585 rc = get_descriptor_ptr(ctxt, selector, &addr);
1586 if (rc != X86EMUL_CONTINUE)
1589 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1592 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1593 u16 selector, int seg, u8 cpl,
1594 enum x86_transfer_type transfer,
1595 struct desc_struct *desc)
1597 struct desc_struct seg_desc, old_desc;
1599 unsigned err_vec = GP_VECTOR;
1601 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1607 memset(&seg_desc, 0, sizeof seg_desc);
1609 if (ctxt->mode == X86EMUL_MODE_REAL) {
1610 /* set real mode segment descriptor (keep limit etc. for
1612 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1613 set_desc_base(&seg_desc, selector << 4);
1615 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1616 /* VM86 needs a clean new segment descriptor */
1617 set_desc_base(&seg_desc, selector << 4);
1618 set_desc_limit(&seg_desc, 0xffff);
1628 /* TR should be in GDT only */
1629 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1632 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1633 if (null_selector) {
1634 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1637 if (seg == VCPU_SREG_SS) {
1638 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1642 * ctxt->ops->set_segment expects the CPL to be in
1643 * SS.DPL, so fake an expand-up 32-bit data segment.
1653 /* Skip all following checks */
1657 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1658 if (ret != X86EMUL_CONTINUE)
1661 err_code = selector & 0xfffc;
1662 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1665 /* can't load system descriptor into segment selector */
1666 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1667 if (transfer == X86_TRANSFER_CALL_JMP)
1668 return X86EMUL_UNHANDLEABLE;
1677 * segment is not a writable data segment or segment
1678 * selector's RPL != CPL or segment selector's RPL != CPL
1680 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1684 if (!(seg_desc.type & 8))
1687 if (seg_desc.type & 4) {
1693 if (rpl > cpl || dpl != cpl)
1696 /* in long-mode d/b must be clear if l is set */
1697 if (seg_desc.d && seg_desc.l) {
1700 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1701 if (efer & EFER_LMA)
1705 /* CS(RPL) <- CPL */
1706 selector = (selector & 0xfffc) | cpl;
1709 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1712 case VCPU_SREG_LDTR:
1713 if (seg_desc.s || seg_desc.type != 2)
1716 default: /* DS, ES, FS, or GS */
1718 * segment is not a data or readable code segment or
1719 * ((segment is a data or nonconforming code segment)
1720 * and (both RPL and CPL > DPL))
1722 if ((seg_desc.type & 0xa) == 0x8 ||
1723 (((seg_desc.type & 0xc) != 0xc) &&
1724 (rpl > dpl && cpl > dpl)))
1730 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1735 /* mark segment as accessed */
1736 if (!(seg_desc.type & 1)) {
1738 ret = write_segment_descriptor(ctxt, selector,
1740 if (ret != X86EMUL_CONTINUE)
1743 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1744 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1745 if (ret != X86EMUL_CONTINUE)
1747 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1748 ((u64)base3 << 32), ctxt))
1749 return emulate_gp(ctxt, err_code);
1752 if (seg == VCPU_SREG_TR) {
1753 old_desc = seg_desc;
1754 seg_desc.type |= 2; /* busy */
1755 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1756 sizeof(seg_desc), &ctxt->exception);
1757 if (ret != X86EMUL_CONTINUE)
1761 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1764 return X86EMUL_CONTINUE;
1766 return emulate_exception(ctxt, err_vec, err_code, true);
1769 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1770 u16 selector, int seg)
1772 u8 cpl = ctxt->ops->cpl(ctxt);
1775 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1776 * they can load it at CPL<3 (Intel's manual says only LSS can,
1779 * However, the Intel manual says that putting IST=1/DPL=3 in
1780 * an interrupt gate will result in SS=3 (the AMD manual instead
1781 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1782 * and only forbid it here.
1784 if (seg == VCPU_SREG_SS && selector == 3 &&
1785 ctxt->mode == X86EMUL_MODE_PROT64)
1786 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1788 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1789 X86_TRANSFER_NONE, NULL);
1792 static void write_register_operand(struct operand *op)
1794 return assign_register(op->addr.reg, op->val, op->bytes);
1797 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1801 write_register_operand(op);
1804 if (ctxt->lock_prefix)
1805 return segmented_cmpxchg(ctxt,
1811 return segmented_write(ctxt,
1817 return segmented_write(ctxt,
1820 op->bytes * op->count);
1823 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1826 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1834 return X86EMUL_CONTINUE;
1837 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1839 struct segmented_address addr;
1841 rsp_increment(ctxt, -bytes);
1842 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1843 addr.seg = VCPU_SREG_SS;
1845 return segmented_write(ctxt, addr, data, bytes);
1848 static int em_push(struct x86_emulate_ctxt *ctxt)
1850 /* Disable writeback. */
1851 ctxt->dst.type = OP_NONE;
1852 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1855 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1856 void *dest, int len)
1859 struct segmented_address addr;
1861 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1862 addr.seg = VCPU_SREG_SS;
1863 rc = segmented_read(ctxt, addr, dest, len);
1864 if (rc != X86EMUL_CONTINUE)
1867 rsp_increment(ctxt, len);
1871 static int em_pop(struct x86_emulate_ctxt *ctxt)
1873 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1876 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1877 void *dest, int len)
1880 unsigned long val, change_mask;
1881 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1882 int cpl = ctxt->ops->cpl(ctxt);
1884 rc = emulate_pop(ctxt, &val, len);
1885 if (rc != X86EMUL_CONTINUE)
1888 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1889 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1890 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1891 X86_EFLAGS_AC | X86_EFLAGS_ID;
1893 switch(ctxt->mode) {
1894 case X86EMUL_MODE_PROT64:
1895 case X86EMUL_MODE_PROT32:
1896 case X86EMUL_MODE_PROT16:
1898 change_mask |= X86_EFLAGS_IOPL;
1900 change_mask |= X86_EFLAGS_IF;
1902 case X86EMUL_MODE_VM86:
1904 return emulate_gp(ctxt, 0);
1905 change_mask |= X86_EFLAGS_IF;
1907 default: /* real mode */
1908 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1912 *(unsigned long *)dest =
1913 (ctxt->eflags & ~change_mask) | (val & change_mask);
1918 static int em_popf(struct x86_emulate_ctxt *ctxt)
1920 ctxt->dst.type = OP_REG;
1921 ctxt->dst.addr.reg = &ctxt->eflags;
1922 ctxt->dst.bytes = ctxt->op_bytes;
1923 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1926 static int em_enter(struct x86_emulate_ctxt *ctxt)
1929 unsigned frame_size = ctxt->src.val;
1930 unsigned nesting_level = ctxt->src2.val & 31;
1934 return X86EMUL_UNHANDLEABLE;
1936 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1937 rc = push(ctxt, &rbp, stack_size(ctxt));
1938 if (rc != X86EMUL_CONTINUE)
1940 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1942 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1943 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1945 return X86EMUL_CONTINUE;
1948 static int em_leave(struct x86_emulate_ctxt *ctxt)
1950 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1952 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1955 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1957 int seg = ctxt->src2.val;
1959 ctxt->src.val = get_segment_selector(ctxt, seg);
1960 if (ctxt->op_bytes == 4) {
1961 rsp_increment(ctxt, -2);
1965 return em_push(ctxt);
1968 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1970 int seg = ctxt->src2.val;
1971 unsigned long selector;
1974 rc = emulate_pop(ctxt, &selector, 2);
1975 if (rc != X86EMUL_CONTINUE)
1978 if (seg == VCPU_SREG_SS)
1979 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1980 if (ctxt->op_bytes > 2)
1981 rsp_increment(ctxt, ctxt->op_bytes - 2);
1983 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1987 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1989 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1990 int rc = X86EMUL_CONTINUE;
1991 int reg = VCPU_REGS_RAX;
1993 while (reg <= VCPU_REGS_RDI) {
1994 (reg == VCPU_REGS_RSP) ?
1995 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1998 if (rc != X86EMUL_CONTINUE)
2007 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2009 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2010 return em_push(ctxt);
2013 static int em_popa(struct x86_emulate_ctxt *ctxt)
2015 int rc = X86EMUL_CONTINUE;
2016 int reg = VCPU_REGS_RDI;
2019 while (reg >= VCPU_REGS_RAX) {
2020 if (reg == VCPU_REGS_RSP) {
2021 rsp_increment(ctxt, ctxt->op_bytes);
2025 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2026 if (rc != X86EMUL_CONTINUE)
2028 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2034 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2036 const struct x86_emulate_ops *ops = ctxt->ops;
2043 /* TODO: Add limit checks */
2044 ctxt->src.val = ctxt->eflags;
2046 if (rc != X86EMUL_CONTINUE)
2049 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2051 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2053 if (rc != X86EMUL_CONTINUE)
2056 ctxt->src.val = ctxt->_eip;
2058 if (rc != X86EMUL_CONTINUE)
2061 ops->get_idt(ctxt, &dt);
2063 eip_addr = dt.address + (irq << 2);
2064 cs_addr = dt.address + (irq << 2) + 2;
2066 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2067 if (rc != X86EMUL_CONTINUE)
2070 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2071 if (rc != X86EMUL_CONTINUE)
2074 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2075 if (rc != X86EMUL_CONTINUE)
2083 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2087 invalidate_registers(ctxt);
2088 rc = __emulate_int_real(ctxt, irq);
2089 if (rc == X86EMUL_CONTINUE)
2090 writeback_registers(ctxt);
2094 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2096 switch(ctxt->mode) {
2097 case X86EMUL_MODE_REAL:
2098 return __emulate_int_real(ctxt, irq);
2099 case X86EMUL_MODE_VM86:
2100 case X86EMUL_MODE_PROT16:
2101 case X86EMUL_MODE_PROT32:
2102 case X86EMUL_MODE_PROT64:
2104 /* Protected mode interrupts unimplemented yet */
2105 return X86EMUL_UNHANDLEABLE;
2109 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2111 int rc = X86EMUL_CONTINUE;
2112 unsigned long temp_eip = 0;
2113 unsigned long temp_eflags = 0;
2114 unsigned long cs = 0;
2115 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2116 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2117 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2118 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2119 X86_EFLAGS_AC | X86_EFLAGS_ID |
2121 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2124 /* TODO: Add stack limit check */
2126 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2128 if (rc != X86EMUL_CONTINUE)
2131 if (temp_eip & ~0xffff)
2132 return emulate_gp(ctxt, 0);
2134 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2136 if (rc != X86EMUL_CONTINUE)
2139 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2141 if (rc != X86EMUL_CONTINUE)
2144 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2146 if (rc != X86EMUL_CONTINUE)
2149 ctxt->_eip = temp_eip;
2151 if (ctxt->op_bytes == 4)
2152 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2153 else if (ctxt->op_bytes == 2) {
2154 ctxt->eflags &= ~0xffff;
2155 ctxt->eflags |= temp_eflags;
2158 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2159 ctxt->eflags |= X86_EFLAGS_FIXED;
2160 ctxt->ops->set_nmi_mask(ctxt, false);
2165 static int em_iret(struct x86_emulate_ctxt *ctxt)
2167 switch(ctxt->mode) {
2168 case X86EMUL_MODE_REAL:
2169 return emulate_iret_real(ctxt);
2170 case X86EMUL_MODE_VM86:
2171 case X86EMUL_MODE_PROT16:
2172 case X86EMUL_MODE_PROT32:
2173 case X86EMUL_MODE_PROT64:
2175 /* iret from protected mode unimplemented yet */
2176 return X86EMUL_UNHANDLEABLE;
2180 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2184 struct desc_struct new_desc;
2185 u8 cpl = ctxt->ops->cpl(ctxt);
2187 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2189 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2190 X86_TRANSFER_CALL_JMP,
2192 if (rc != X86EMUL_CONTINUE)
2195 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2196 /* Error handling is not implemented. */
2197 if (rc != X86EMUL_CONTINUE)
2198 return X86EMUL_UNHANDLEABLE;
2203 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2205 return assign_eip_near(ctxt, ctxt->src.val);
2208 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2213 old_eip = ctxt->_eip;
2214 rc = assign_eip_near(ctxt, ctxt->src.val);
2215 if (rc != X86EMUL_CONTINUE)
2217 ctxt->src.val = old_eip;
2222 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2224 u64 old = ctxt->dst.orig_val64;
2226 if (ctxt->dst.bytes == 16)
2227 return X86EMUL_UNHANDLEABLE;
2229 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2230 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2231 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2232 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2233 ctxt->eflags &= ~X86_EFLAGS_ZF;
2235 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2236 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2238 ctxt->eflags |= X86_EFLAGS_ZF;
2240 return X86EMUL_CONTINUE;
2243 static int em_ret(struct x86_emulate_ctxt *ctxt)
2248 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2249 if (rc != X86EMUL_CONTINUE)
2252 return assign_eip_near(ctxt, eip);
2255 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2258 unsigned long eip, cs;
2259 int cpl = ctxt->ops->cpl(ctxt);
2260 struct desc_struct new_desc;
2262 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2263 if (rc != X86EMUL_CONTINUE)
2265 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2266 if (rc != X86EMUL_CONTINUE)
2268 /* Outer-privilege level return is not implemented */
2269 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2270 return X86EMUL_UNHANDLEABLE;
2271 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2274 if (rc != X86EMUL_CONTINUE)
2276 rc = assign_eip_far(ctxt, eip, &new_desc);
2277 /* Error handling is not implemented. */
2278 if (rc != X86EMUL_CONTINUE)
2279 return X86EMUL_UNHANDLEABLE;
2284 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2288 rc = em_ret_far(ctxt);
2289 if (rc != X86EMUL_CONTINUE)
2291 rsp_increment(ctxt, ctxt->src.val);
2292 return X86EMUL_CONTINUE;
2295 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2297 /* Save real source value, then compare EAX against destination. */
2298 ctxt->dst.orig_val = ctxt->dst.val;
2299 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2300 ctxt->src.orig_val = ctxt->src.val;
2301 ctxt->src.val = ctxt->dst.orig_val;
2302 fastop(ctxt, em_cmp);
2304 if (ctxt->eflags & X86_EFLAGS_ZF) {
2305 /* Success: write back to memory; no update of EAX */
2306 ctxt->src.type = OP_NONE;
2307 ctxt->dst.val = ctxt->src.orig_val;
2309 /* Failure: write the value we saw to EAX. */
2310 ctxt->src.type = OP_REG;
2311 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2312 ctxt->src.val = ctxt->dst.orig_val;
2313 /* Create write-cycle to dest by writing the same value */
2314 ctxt->dst.val = ctxt->dst.orig_val;
2316 return X86EMUL_CONTINUE;
2319 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2321 int seg = ctxt->src2.val;
2325 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2327 rc = load_segment_descriptor(ctxt, sel, seg);
2328 if (rc != X86EMUL_CONTINUE)
2331 ctxt->dst.val = ctxt->src.val;
2335 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2337 #ifdef CONFIG_X86_64
2338 u32 eax, ebx, ecx, edx;
2342 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2343 return edx & bit(X86_FEATURE_LM);
2349 #define GET_SMSTATE(type, smbase, offset) \
2352 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2354 if (r != X86EMUL_CONTINUE) \
2355 return X86EMUL_UNHANDLEABLE; \
2359 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2361 desc->g = (flags >> 23) & 1;
2362 desc->d = (flags >> 22) & 1;
2363 desc->l = (flags >> 21) & 1;
2364 desc->avl = (flags >> 20) & 1;
2365 desc->p = (flags >> 15) & 1;
2366 desc->dpl = (flags >> 13) & 3;
2367 desc->s = (flags >> 12) & 1;
2368 desc->type = (flags >> 8) & 15;
2371 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2373 struct desc_struct desc;
2377 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2380 offset = 0x7f84 + n * 12;
2382 offset = 0x7f2c + (n - 3) * 12;
2384 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2385 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2386 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2387 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2388 return X86EMUL_CONTINUE;
2391 #ifdef CONFIG_X86_64
2392 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2394 struct desc_struct desc;
2399 offset = 0x7e00 + n * 16;
2401 selector = GET_SMSTATE(u16, smbase, offset);
2402 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2403 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2404 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2405 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2407 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2408 return X86EMUL_CONTINUE;
2412 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2413 u64 cr0, u64 cr3, u64 cr4)
2418 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2420 if (cr4 & X86_CR4_PCIDE) {
2425 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2427 return X86EMUL_UNHANDLEABLE;
2430 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2431 * Then enable protected mode. However, PCID cannot be enabled
2432 * if EFER.LMA=0, so set it separately.
2434 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2436 return X86EMUL_UNHANDLEABLE;
2438 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2440 return X86EMUL_UNHANDLEABLE;
2442 if (cr4 & X86_CR4_PCIDE) {
2443 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2445 return X86EMUL_UNHANDLEABLE;
2447 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2449 return X86EMUL_UNHANDLEABLE;
2454 return X86EMUL_CONTINUE;
2457 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2459 struct desc_struct desc;
2462 u32 val, cr0, cr3, cr4;
2465 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2466 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2467 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2468 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2470 for (i = 0; i < 8; i++)
2471 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2473 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2474 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2475 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2476 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2478 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2479 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2480 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2481 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2482 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2484 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2485 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2486 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2487 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2488 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2490 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2491 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2492 ctxt->ops->set_gdt(ctxt, &dt);
2494 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2495 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2496 ctxt->ops->set_idt(ctxt, &dt);
2498 for (i = 0; i < 6; i++) {
2499 int r = rsm_load_seg_32(ctxt, smbase, i);
2500 if (r != X86EMUL_CONTINUE)
2504 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2506 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2508 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2511 #ifdef CONFIG_X86_64
2512 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2514 struct desc_struct desc;
2516 u64 val, cr0, cr3, cr4;
2521 for (i = 0; i < 16; i++)
2522 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2524 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2525 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2527 val = GET_SMSTATE(u32, smbase, 0x7f68);
2528 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2529 val = GET_SMSTATE(u32, smbase, 0x7f60);
2530 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2532 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2533 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2534 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2535 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2536 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2537 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2539 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2540 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2541 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2542 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2543 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2544 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2546 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2547 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2548 ctxt->ops->set_idt(ctxt, &dt);
2550 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2551 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2552 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2553 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2554 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2555 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2557 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2558 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2559 ctxt->ops->set_gdt(ctxt, &dt);
2561 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2562 if (r != X86EMUL_CONTINUE)
2565 for (i = 0; i < 6; i++) {
2566 r = rsm_load_seg_64(ctxt, smbase, i);
2567 if (r != X86EMUL_CONTINUE)
2571 return X86EMUL_CONTINUE;
2575 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2577 unsigned long cr0, cr4, efer;
2581 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2582 return emulate_ud(ctxt);
2585 * Get back to real mode, to prepare a safe state in which to load
2586 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2587 * supports long mode.
2589 if (emulator_has_longmode(ctxt)) {
2590 struct desc_struct cs_desc;
2592 /* Zero CR4.PCIDE before CR0.PG. */
2593 cr4 = ctxt->ops->get_cr(ctxt, 4);
2594 if (cr4 & X86_CR4_PCIDE)
2595 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2597 /* A 32-bit code segment is required to clear EFER.LMA. */
2598 memset(&cs_desc, 0, sizeof(cs_desc));
2600 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2601 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2604 /* For the 64-bit case, this will clear EFER.LMA. */
2605 cr0 = ctxt->ops->get_cr(ctxt, 0);
2606 if (cr0 & X86_CR0_PE)
2607 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2609 if (emulator_has_longmode(ctxt)) {
2610 /* Clear CR4.PAE before clearing EFER.LME. */
2611 cr4 = ctxt->ops->get_cr(ctxt, 4);
2612 if (cr4 & X86_CR4_PAE)
2613 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2615 /* And finally go back to 32-bit mode. */
2617 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2620 smbase = ctxt->ops->get_smbase(ctxt);
2623 * Give pre_leave_smm() a chance to make ISA-specific changes to the
2624 * vCPU state (e.g. enter guest mode) before loading state from the SMM
2627 if (ctxt->ops->pre_leave_smm(ctxt, smbase))
2628 return X86EMUL_UNHANDLEABLE;
2630 #ifdef CONFIG_X86_64
2631 if (emulator_has_longmode(ctxt))
2632 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2635 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2637 if (ret != X86EMUL_CONTINUE) {
2638 /* FIXME: should triple fault */
2639 return X86EMUL_UNHANDLEABLE;
2642 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2643 ctxt->ops->set_nmi_mask(ctxt, false);
2645 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2646 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2647 return X86EMUL_CONTINUE;
2651 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2652 struct desc_struct *cs, struct desc_struct *ss)
2654 cs->l = 0; /* will be adjusted later */
2655 set_desc_base(cs, 0); /* flat segment */
2656 cs->g = 1; /* 4kb granularity */
2657 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2658 cs->type = 0x0b; /* Read, Execute, Accessed */
2660 cs->dpl = 0; /* will be adjusted later */
2665 set_desc_base(ss, 0); /* flat segment */
2666 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2667 ss->g = 1; /* 4kb granularity */
2669 ss->type = 0x03; /* Read/Write, Accessed */
2670 ss->d = 1; /* 32bit stack segment */
2677 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2679 u32 eax, ebx, ecx, edx;
2682 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2683 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2684 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2685 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2688 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2690 const struct x86_emulate_ops *ops = ctxt->ops;
2691 u32 eax, ebx, ecx, edx;
2694 * syscall should always be enabled in longmode - so only become
2695 * vendor specific (cpuid) if other modes are active...
2697 if (ctxt->mode == X86EMUL_MODE_PROT64)
2702 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2704 * Intel ("GenuineIntel")
2705 * remark: Intel CPUs only support "syscall" in 64bit
2706 * longmode. Also an 64bit guest with a
2707 * 32bit compat-app running will #UD !! While this
2708 * behaviour can be fixed (by emulating) into AMD
2709 * response - CPUs of AMD can't behave like Intel.
2711 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2712 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2713 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2716 /* AMD ("AuthenticAMD") */
2717 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2718 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2719 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2722 /* AMD ("AMDisbetter!") */
2723 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2724 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2725 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2728 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2732 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2734 const struct x86_emulate_ops *ops = ctxt->ops;
2735 struct desc_struct cs, ss;
2740 /* syscall is not available in real mode */
2741 if (ctxt->mode == X86EMUL_MODE_REAL ||
2742 ctxt->mode == X86EMUL_MODE_VM86)
2743 return emulate_ud(ctxt);
2745 if (!(em_syscall_is_enabled(ctxt)))
2746 return emulate_ud(ctxt);
2748 ops->get_msr(ctxt, MSR_EFER, &efer);
2749 setup_syscalls_segments(ctxt, &cs, &ss);
2751 if (!(efer & EFER_SCE))
2752 return emulate_ud(ctxt);
2754 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2756 cs_sel = (u16)(msr_data & 0xfffc);
2757 ss_sel = (u16)(msr_data + 8);
2759 if (efer & EFER_LMA) {
2763 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2764 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2766 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2767 if (efer & EFER_LMA) {
2768 #ifdef CONFIG_X86_64
2769 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2772 ctxt->mode == X86EMUL_MODE_PROT64 ?
2773 MSR_LSTAR : MSR_CSTAR, &msr_data);
2774 ctxt->_eip = msr_data;
2776 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2777 ctxt->eflags &= ~msr_data;
2778 ctxt->eflags |= X86_EFLAGS_FIXED;
2782 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2783 ctxt->_eip = (u32)msr_data;
2785 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2788 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2789 return X86EMUL_CONTINUE;
2792 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2794 const struct x86_emulate_ops *ops = ctxt->ops;
2795 struct desc_struct cs, ss;
2800 ops->get_msr(ctxt, MSR_EFER, &efer);
2801 /* inject #GP if in real mode */
2802 if (ctxt->mode == X86EMUL_MODE_REAL)
2803 return emulate_gp(ctxt, 0);
2806 * Not recognized on AMD in compat mode (but is recognized in legacy
2809 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2810 && !vendor_intel(ctxt))
2811 return emulate_ud(ctxt);
2813 /* sysenter/sysexit have not been tested in 64bit mode. */
2814 if (ctxt->mode == X86EMUL_MODE_PROT64)
2815 return X86EMUL_UNHANDLEABLE;
2817 setup_syscalls_segments(ctxt, &cs, &ss);
2819 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2820 if ((msr_data & 0xfffc) == 0x0)
2821 return emulate_gp(ctxt, 0);
2823 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2824 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2825 ss_sel = cs_sel + 8;
2826 if (efer & EFER_LMA) {
2831 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2832 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2834 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2835 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2837 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2838 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2841 return X86EMUL_CONTINUE;
2844 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2846 const struct x86_emulate_ops *ops = ctxt->ops;
2847 struct desc_struct cs, ss;
2848 u64 msr_data, rcx, rdx;
2850 u16 cs_sel = 0, ss_sel = 0;
2852 /* inject #GP if in real mode or Virtual 8086 mode */
2853 if (ctxt->mode == X86EMUL_MODE_REAL ||
2854 ctxt->mode == X86EMUL_MODE_VM86)
2855 return emulate_gp(ctxt, 0);
2857 setup_syscalls_segments(ctxt, &cs, &ss);
2859 if ((ctxt->rex_prefix & 0x8) != 0x0)
2860 usermode = X86EMUL_MODE_PROT64;
2862 usermode = X86EMUL_MODE_PROT32;
2864 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2865 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2869 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2871 case X86EMUL_MODE_PROT32:
2872 cs_sel = (u16)(msr_data + 16);
2873 if ((msr_data & 0xfffc) == 0x0)
2874 return emulate_gp(ctxt, 0);
2875 ss_sel = (u16)(msr_data + 24);
2879 case X86EMUL_MODE_PROT64:
2880 cs_sel = (u16)(msr_data + 32);
2881 if (msr_data == 0x0)
2882 return emulate_gp(ctxt, 0);
2883 ss_sel = cs_sel + 8;
2886 if (emul_is_noncanonical_address(rcx, ctxt) ||
2887 emul_is_noncanonical_address(rdx, ctxt))
2888 return emulate_gp(ctxt, 0);
2891 cs_sel |= SEGMENT_RPL_MASK;
2892 ss_sel |= SEGMENT_RPL_MASK;
2894 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2895 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2898 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2900 return X86EMUL_CONTINUE;
2903 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2906 if (ctxt->mode == X86EMUL_MODE_REAL)
2908 if (ctxt->mode == X86EMUL_MODE_VM86)
2910 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2911 return ctxt->ops->cpl(ctxt) > iopl;
2914 #define VMWARE_PORT_VMPORT (0x5658)
2915 #define VMWARE_PORT_VMRPC (0x5659)
2917 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2920 const struct x86_emulate_ops *ops = ctxt->ops;
2921 struct desc_struct tr_seg;
2924 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2925 unsigned mask = (1 << len) - 1;
2929 * VMware allows access to these ports even if denied
2930 * by TSS I/O permission bitmap. Mimic behavior.
2932 if (enable_vmware_backdoor &&
2933 ((port == VMWARE_PORT_VMPORT) || (port == VMWARE_PORT_VMRPC)))
2936 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2939 if (desc_limit_scaled(&tr_seg) < 103)
2941 base = get_desc_base(&tr_seg);
2942 #ifdef CONFIG_X86_64
2943 base |= ((u64)base3) << 32;
2945 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2946 if (r != X86EMUL_CONTINUE)
2948 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2950 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2951 if (r != X86EMUL_CONTINUE)
2953 if ((perm >> bit_idx) & mask)
2958 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2964 if (emulator_bad_iopl(ctxt))
2965 if (!emulator_io_port_access_allowed(ctxt, port, len))
2968 ctxt->perm_ok = true;
2973 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2976 * Intel CPUs mask the counter and pointers in quite strange
2977 * manner when ECX is zero due to REP-string optimizations.
2979 #ifdef CONFIG_X86_64
2980 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2983 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2986 case 0xa4: /* movsb */
2987 case 0xa5: /* movsd/w */
2988 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2990 case 0xaa: /* stosb */
2991 case 0xab: /* stosd/w */
2992 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2997 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2998 struct tss_segment_16 *tss)
3000 tss->ip = ctxt->_eip;
3001 tss->flag = ctxt->eflags;
3002 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
3003 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
3004 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
3005 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
3006 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
3007 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
3008 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
3009 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
3011 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3012 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3013 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3014 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3015 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
3018 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
3019 struct tss_segment_16 *tss)
3024 ctxt->_eip = tss->ip;
3025 ctxt->eflags = tss->flag | 2;
3026 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3027 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3028 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3029 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3030 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3031 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3032 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3033 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3036 * SDM says that segment selectors are loaded before segment
3039 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3040 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3041 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3042 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3043 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3048 * Now load segment descriptors. If fault happens at this stage
3049 * it is handled in a context of new task
3051 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3052 X86_TRANSFER_TASK_SWITCH, NULL);
3053 if (ret != X86EMUL_CONTINUE)
3055 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3056 X86_TRANSFER_TASK_SWITCH, NULL);
3057 if (ret != X86EMUL_CONTINUE)
3059 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3060 X86_TRANSFER_TASK_SWITCH, NULL);
3061 if (ret != X86EMUL_CONTINUE)
3063 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3064 X86_TRANSFER_TASK_SWITCH, NULL);
3065 if (ret != X86EMUL_CONTINUE)
3067 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3068 X86_TRANSFER_TASK_SWITCH, NULL);
3069 if (ret != X86EMUL_CONTINUE)
3072 return X86EMUL_CONTINUE;
3075 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3076 u16 tss_selector, u16 old_tss_sel,
3077 ulong old_tss_base, struct desc_struct *new_desc)
3079 struct tss_segment_16 tss_seg;
3081 u32 new_tss_base = get_desc_base(new_desc);
3083 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3084 if (ret != X86EMUL_CONTINUE)
3087 save_state_to_tss16(ctxt, &tss_seg);
3089 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3090 if (ret != X86EMUL_CONTINUE)
3093 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3094 if (ret != X86EMUL_CONTINUE)
3097 if (old_tss_sel != 0xffff) {
3098 tss_seg.prev_task_link = old_tss_sel;
3100 ret = linear_write_system(ctxt, new_tss_base,
3101 &tss_seg.prev_task_link,
3102 sizeof tss_seg.prev_task_link);
3103 if (ret != X86EMUL_CONTINUE)
3107 return load_state_from_tss16(ctxt, &tss_seg);
3110 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3111 struct tss_segment_32 *tss)
3113 /* CR3 and ldt selector are not saved intentionally */
3114 tss->eip = ctxt->_eip;
3115 tss->eflags = ctxt->eflags;
3116 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3117 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3118 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3119 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3120 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3121 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3122 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3123 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3125 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3126 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3127 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3128 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3129 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3130 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3133 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3134 struct tss_segment_32 *tss)
3139 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3140 return emulate_gp(ctxt, 0);
3141 ctxt->_eip = tss->eip;
3142 ctxt->eflags = tss->eflags | 2;
3144 /* General purpose registers */
3145 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3146 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3147 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3148 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3149 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3150 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3151 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3152 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3155 * SDM says that segment selectors are loaded before segment
3156 * descriptors. This is important because CPL checks will
3159 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3160 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3161 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3162 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3163 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3164 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3165 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3168 * If we're switching between Protected Mode and VM86, we need to make
3169 * sure to update the mode before loading the segment descriptors so
3170 * that the selectors are interpreted correctly.
3172 if (ctxt->eflags & X86_EFLAGS_VM) {
3173 ctxt->mode = X86EMUL_MODE_VM86;
3176 ctxt->mode = X86EMUL_MODE_PROT32;
3181 * Now load segment descriptors. If fault happenes at this stage
3182 * it is handled in a context of new task
3184 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3185 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3186 if (ret != X86EMUL_CONTINUE)
3188 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3189 X86_TRANSFER_TASK_SWITCH, NULL);
3190 if (ret != X86EMUL_CONTINUE)
3192 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3193 X86_TRANSFER_TASK_SWITCH, NULL);
3194 if (ret != X86EMUL_CONTINUE)
3196 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3197 X86_TRANSFER_TASK_SWITCH, NULL);
3198 if (ret != X86EMUL_CONTINUE)
3200 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3201 X86_TRANSFER_TASK_SWITCH, NULL);
3202 if (ret != X86EMUL_CONTINUE)
3204 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3205 X86_TRANSFER_TASK_SWITCH, NULL);
3206 if (ret != X86EMUL_CONTINUE)
3208 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3209 X86_TRANSFER_TASK_SWITCH, NULL);
3214 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3215 u16 tss_selector, u16 old_tss_sel,
3216 ulong old_tss_base, struct desc_struct *new_desc)
3218 struct tss_segment_32 tss_seg;
3220 u32 new_tss_base = get_desc_base(new_desc);
3221 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3222 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3224 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3225 if (ret != X86EMUL_CONTINUE)
3228 save_state_to_tss32(ctxt, &tss_seg);
3230 /* Only GP registers and segment selectors are saved */
3231 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3232 ldt_sel_offset - eip_offset);
3233 if (ret != X86EMUL_CONTINUE)
3236 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3237 if (ret != X86EMUL_CONTINUE)
3240 if (old_tss_sel != 0xffff) {
3241 tss_seg.prev_task_link = old_tss_sel;
3243 ret = linear_write_system(ctxt, new_tss_base,
3244 &tss_seg.prev_task_link,
3245 sizeof tss_seg.prev_task_link);
3246 if (ret != X86EMUL_CONTINUE)
3250 return load_state_from_tss32(ctxt, &tss_seg);
3253 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3254 u16 tss_selector, int idt_index, int reason,
3255 bool has_error_code, u32 error_code)
3257 const struct x86_emulate_ops *ops = ctxt->ops;
3258 struct desc_struct curr_tss_desc, next_tss_desc;
3260 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3261 ulong old_tss_base =
3262 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3264 ulong desc_addr, dr7;
3266 /* FIXME: old_tss_base == ~0 ? */
3268 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3269 if (ret != X86EMUL_CONTINUE)
3271 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3272 if (ret != X86EMUL_CONTINUE)
3275 /* FIXME: check that next_tss_desc is tss */
3278 * Check privileges. The three cases are task switch caused by...
3280 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3281 * 2. Exception/IRQ/iret: No check is performed
3282 * 3. jmp/call to TSS/task-gate: No check is performed since the
3283 * hardware checks it before exiting.
3285 if (reason == TASK_SWITCH_GATE) {
3286 if (idt_index != -1) {
3287 /* Software interrupts */
3288 struct desc_struct task_gate_desc;
3291 ret = read_interrupt_descriptor(ctxt, idt_index,
3293 if (ret != X86EMUL_CONTINUE)
3296 dpl = task_gate_desc.dpl;
3297 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3298 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3302 desc_limit = desc_limit_scaled(&next_tss_desc);
3303 if (!next_tss_desc.p ||
3304 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3305 desc_limit < 0x2b)) {
3306 return emulate_ts(ctxt, tss_selector & 0xfffc);
3309 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3310 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3311 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3314 if (reason == TASK_SWITCH_IRET)
3315 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3317 /* set back link to prev task only if NT bit is set in eflags
3318 note that old_tss_sel is not used after this point */
3319 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3320 old_tss_sel = 0xffff;
3322 if (next_tss_desc.type & 8)
3323 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3324 old_tss_base, &next_tss_desc);
3326 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3327 old_tss_base, &next_tss_desc);
3328 if (ret != X86EMUL_CONTINUE)
3331 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3332 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3334 if (reason != TASK_SWITCH_IRET) {
3335 next_tss_desc.type |= (1 << 1); /* set busy flag */
3336 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3339 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3340 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3342 if (has_error_code) {
3343 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3344 ctxt->lock_prefix = 0;
3345 ctxt->src.val = (unsigned long) error_code;
3346 ret = em_push(ctxt);
3349 ops->get_dr(ctxt, 7, &dr7);
3350 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3355 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3356 u16 tss_selector, int idt_index, int reason,
3357 bool has_error_code, u32 error_code)
3361 invalidate_registers(ctxt);
3362 ctxt->_eip = ctxt->eip;
3363 ctxt->dst.type = OP_NONE;
3365 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3366 has_error_code, error_code);
3368 if (rc == X86EMUL_CONTINUE) {
3369 ctxt->eip = ctxt->_eip;
3370 writeback_registers(ctxt);
3373 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3376 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3379 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3381 register_address_increment(ctxt, reg, df * op->bytes);
3382 op->addr.mem.ea = register_address(ctxt, reg);
3385 static int em_das(struct x86_emulate_ctxt *ctxt)
3388 bool af, cf, old_cf;
3390 cf = ctxt->eflags & X86_EFLAGS_CF;
3396 af = ctxt->eflags & X86_EFLAGS_AF;
3397 if ((al & 0x0f) > 9 || af) {
3399 cf = old_cf | (al >= 250);
3404 if (old_al > 0x99 || old_cf) {
3410 /* Set PF, ZF, SF */
3411 ctxt->src.type = OP_IMM;
3413 ctxt->src.bytes = 1;
3414 fastop(ctxt, em_or);
3415 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3417 ctxt->eflags |= X86_EFLAGS_CF;
3419 ctxt->eflags |= X86_EFLAGS_AF;
3420 return X86EMUL_CONTINUE;
3423 static int em_aam(struct x86_emulate_ctxt *ctxt)
3427 if (ctxt->src.val == 0)
3428 return emulate_de(ctxt);
3430 al = ctxt->dst.val & 0xff;
3431 ah = al / ctxt->src.val;
3432 al %= ctxt->src.val;
3434 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3436 /* Set PF, ZF, SF */
3437 ctxt->src.type = OP_IMM;
3439 ctxt->src.bytes = 1;
3440 fastop(ctxt, em_or);
3442 return X86EMUL_CONTINUE;
3445 static int em_aad(struct x86_emulate_ctxt *ctxt)
3447 u8 al = ctxt->dst.val & 0xff;
3448 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3450 al = (al + (ah * ctxt->src.val)) & 0xff;
3452 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3454 /* Set PF, ZF, SF */
3455 ctxt->src.type = OP_IMM;
3457 ctxt->src.bytes = 1;
3458 fastop(ctxt, em_or);
3460 return X86EMUL_CONTINUE;
3463 static int em_call(struct x86_emulate_ctxt *ctxt)
3466 long rel = ctxt->src.val;
3468 ctxt->src.val = (unsigned long)ctxt->_eip;
3469 rc = jmp_rel(ctxt, rel);
3470 if (rc != X86EMUL_CONTINUE)
3472 return em_push(ctxt);
3475 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3480 struct desc_struct old_desc, new_desc;
3481 const struct x86_emulate_ops *ops = ctxt->ops;
3482 int cpl = ctxt->ops->cpl(ctxt);
3483 enum x86emul_mode prev_mode = ctxt->mode;
3485 old_eip = ctxt->_eip;
3486 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3488 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3489 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3490 X86_TRANSFER_CALL_JMP, &new_desc);
3491 if (rc != X86EMUL_CONTINUE)
3494 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3495 if (rc != X86EMUL_CONTINUE)
3498 ctxt->src.val = old_cs;
3500 if (rc != X86EMUL_CONTINUE)
3503 ctxt->src.val = old_eip;
3505 /* If we failed, we tainted the memory, but the very least we should
3507 if (rc != X86EMUL_CONTINUE) {
3508 pr_warn_once("faulting far call emulation tainted memory\n");
3513 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3514 ctxt->mode = prev_mode;
3519 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3524 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3525 if (rc != X86EMUL_CONTINUE)
3527 rc = assign_eip_near(ctxt, eip);
3528 if (rc != X86EMUL_CONTINUE)
3530 rsp_increment(ctxt, ctxt->src.val);
3531 return X86EMUL_CONTINUE;
3534 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3536 /* Write back the register source. */
3537 ctxt->src.val = ctxt->dst.val;
3538 write_register_operand(&ctxt->src);
3540 /* Write back the memory destination with implicit LOCK prefix. */
3541 ctxt->dst.val = ctxt->src.orig_val;
3542 ctxt->lock_prefix = 1;
3543 return X86EMUL_CONTINUE;
3546 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3548 ctxt->dst.val = ctxt->src2.val;
3549 return fastop(ctxt, em_imul);
3552 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3554 ctxt->dst.type = OP_REG;
3555 ctxt->dst.bytes = ctxt->src.bytes;
3556 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3557 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3559 return X86EMUL_CONTINUE;
3562 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3566 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3567 return emulate_ud(ctxt);
3568 ctxt->dst.val = tsc_aux;
3569 return X86EMUL_CONTINUE;
3572 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3576 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3577 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3578 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3579 return X86EMUL_CONTINUE;
3582 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3586 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3587 return emulate_gp(ctxt, 0);
3588 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3589 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3590 return X86EMUL_CONTINUE;
3593 static int em_mov(struct x86_emulate_ctxt *ctxt)
3595 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3596 return X86EMUL_CONTINUE;
3599 #define FFL(x) bit(X86_FEATURE_##x)
3601 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3603 u32 ebx, ecx, edx, eax = 1;
3607 * Check MOVBE is set in the guest-visible CPUID leaf.
3609 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3610 if (!(ecx & FFL(MOVBE)))
3611 return emulate_ud(ctxt);
3613 switch (ctxt->op_bytes) {
3616 * From MOVBE definition: "...When the operand size is 16 bits,
3617 * the upper word of the destination register remains unchanged
3620 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3621 * rules so we have to do the operation almost per hand.
3623 tmp = (u16)ctxt->src.val;
3624 ctxt->dst.val &= ~0xffffUL;
3625 ctxt->dst.val |= (unsigned long)swab16(tmp);
3628 ctxt->dst.val = swab32((u32)ctxt->src.val);
3631 ctxt->dst.val = swab64(ctxt->src.val);
3636 return X86EMUL_CONTINUE;
3639 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3641 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3642 return emulate_gp(ctxt, 0);
3644 /* Disable writeback. */
3645 ctxt->dst.type = OP_NONE;
3646 return X86EMUL_CONTINUE;
3649 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3653 if (ctxt->mode == X86EMUL_MODE_PROT64)
3654 val = ctxt->src.val & ~0ULL;
3656 val = ctxt->src.val & ~0U;
3658 /* #UD condition is already handled. */
3659 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3660 return emulate_gp(ctxt, 0);
3662 /* Disable writeback. */
3663 ctxt->dst.type = OP_NONE;
3664 return X86EMUL_CONTINUE;
3667 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3671 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3672 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3673 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3674 return emulate_gp(ctxt, 0);
3676 return X86EMUL_CONTINUE;
3679 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3683 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3684 return emulate_gp(ctxt, 0);
3686 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3687 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3688 return X86EMUL_CONTINUE;
3691 static int em_store_sreg(struct x86_emulate_ctxt *ctxt, int segment)
3693 if (segment > VCPU_SREG_GS &&
3694 (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3695 ctxt->ops->cpl(ctxt) > 0)
3696 return emulate_gp(ctxt, 0);
3698 ctxt->dst.val = get_segment_selector(ctxt, segment);
3699 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3700 ctxt->dst.bytes = 2;
3701 return X86EMUL_CONTINUE;
3704 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3706 if (ctxt->modrm_reg > VCPU_SREG_GS)
3707 return emulate_ud(ctxt);
3709 return em_store_sreg(ctxt, ctxt->modrm_reg);
3712 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3714 u16 sel = ctxt->src.val;
3716 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3717 return emulate_ud(ctxt);
3719 if (ctxt->modrm_reg == VCPU_SREG_SS)
3720 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3722 /* Disable writeback. */
3723 ctxt->dst.type = OP_NONE;
3724 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3727 static int em_sldt(struct x86_emulate_ctxt *ctxt)
3729 return em_store_sreg(ctxt, VCPU_SREG_LDTR);
3732 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3734 u16 sel = ctxt->src.val;
3736 /* Disable writeback. */
3737 ctxt->dst.type = OP_NONE;
3738 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3741 static int em_str(struct x86_emulate_ctxt *ctxt)
3743 return em_store_sreg(ctxt, VCPU_SREG_TR);
3746 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3748 u16 sel = ctxt->src.val;
3750 /* Disable writeback. */
3751 ctxt->dst.type = OP_NONE;
3752 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3755 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3760 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3761 if (rc == X86EMUL_CONTINUE)
3762 ctxt->ops->invlpg(ctxt, linear);
3763 /* Disable writeback. */
3764 ctxt->dst.type = OP_NONE;
3765 return X86EMUL_CONTINUE;
3768 static int em_clts(struct x86_emulate_ctxt *ctxt)
3772 cr0 = ctxt->ops->get_cr(ctxt, 0);
3774 ctxt->ops->set_cr(ctxt, 0, cr0);
3775 return X86EMUL_CONTINUE;
3778 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3780 int rc = ctxt->ops->fix_hypercall(ctxt);
3782 if (rc != X86EMUL_CONTINUE)
3785 /* Let the processor re-execute the fixed hypercall */
3786 ctxt->_eip = ctxt->eip;
3787 /* Disable writeback. */
3788 ctxt->dst.type = OP_NONE;
3789 return X86EMUL_CONTINUE;
3792 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3793 void (*get)(struct x86_emulate_ctxt *ctxt,
3794 struct desc_ptr *ptr))
3796 struct desc_ptr desc_ptr;
3798 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3799 ctxt->ops->cpl(ctxt) > 0)
3800 return emulate_gp(ctxt, 0);
3802 if (ctxt->mode == X86EMUL_MODE_PROT64)
3804 get(ctxt, &desc_ptr);
3805 if (ctxt->op_bytes == 2) {
3807 desc_ptr.address &= 0x00ffffff;
3809 /* Disable writeback. */
3810 ctxt->dst.type = OP_NONE;
3811 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3812 &desc_ptr, 2 + ctxt->op_bytes);
3815 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3817 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3820 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3822 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3825 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3827 struct desc_ptr desc_ptr;
3830 if (ctxt->mode == X86EMUL_MODE_PROT64)
3832 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3833 &desc_ptr.size, &desc_ptr.address,
3835 if (rc != X86EMUL_CONTINUE)
3837 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3838 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3839 return emulate_gp(ctxt, 0);
3841 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3843 ctxt->ops->set_idt(ctxt, &desc_ptr);
3844 /* Disable writeback. */
3845 ctxt->dst.type = OP_NONE;
3846 return X86EMUL_CONTINUE;
3849 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3851 return em_lgdt_lidt(ctxt, true);
3854 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3856 return em_lgdt_lidt(ctxt, false);
3859 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3861 if ((ctxt->ops->get_cr(ctxt, 4) & X86_CR4_UMIP) &&
3862 ctxt->ops->cpl(ctxt) > 0)
3863 return emulate_gp(ctxt, 0);
3865 if (ctxt->dst.type == OP_MEM)
3866 ctxt->dst.bytes = 2;
3867 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3868 return X86EMUL_CONTINUE;
3871 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3873 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3874 | (ctxt->src.val & 0x0f));
3875 ctxt->dst.type = OP_NONE;
3876 return X86EMUL_CONTINUE;
3879 static int em_loop(struct x86_emulate_ctxt *ctxt)
3881 int rc = X86EMUL_CONTINUE;
3883 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3884 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3885 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3886 rc = jmp_rel(ctxt, ctxt->src.val);
3891 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3893 int rc = X86EMUL_CONTINUE;
3895 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3896 rc = jmp_rel(ctxt, ctxt->src.val);
3901 static int em_in(struct x86_emulate_ctxt *ctxt)
3903 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3905 return X86EMUL_IO_NEEDED;
3907 return X86EMUL_CONTINUE;
3910 static int em_out(struct x86_emulate_ctxt *ctxt)
3912 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3914 /* Disable writeback. */
3915 ctxt->dst.type = OP_NONE;
3916 return X86EMUL_CONTINUE;
3919 static int em_cli(struct x86_emulate_ctxt *ctxt)
3921 if (emulator_bad_iopl(ctxt))
3922 return emulate_gp(ctxt, 0);
3924 ctxt->eflags &= ~X86_EFLAGS_IF;
3925 return X86EMUL_CONTINUE;
3928 static int em_sti(struct x86_emulate_ctxt *ctxt)
3930 if (emulator_bad_iopl(ctxt))
3931 return emulate_gp(ctxt, 0);
3933 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3934 ctxt->eflags |= X86_EFLAGS_IF;
3935 return X86EMUL_CONTINUE;
3938 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3940 u32 eax, ebx, ecx, edx;
3943 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3944 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3945 ctxt->ops->cpl(ctxt)) {
3946 return emulate_gp(ctxt, 0);
3949 eax = reg_read(ctxt, VCPU_REGS_RAX);
3950 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3951 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3952 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3953 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3954 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3955 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3956 return X86EMUL_CONTINUE;
3959 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3963 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3965 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3967 ctxt->eflags &= ~0xffUL;
3968 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3969 return X86EMUL_CONTINUE;
3972 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3974 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3975 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3976 return X86EMUL_CONTINUE;
3979 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3981 switch (ctxt->op_bytes) {
3982 #ifdef CONFIG_X86_64
3984 asm("bswap %0" : "+r"(ctxt->dst.val));
3988 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3991 return X86EMUL_CONTINUE;
3994 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3996 /* emulating clflush regardless of cpuid */
3997 return X86EMUL_CONTINUE;
4000 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
4002 /* emulating clflushopt regardless of cpuid */
4003 return X86EMUL_CONTINUE;
4006 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
4008 ctxt->dst.val = (s32) ctxt->src.val;
4009 return X86EMUL_CONTINUE;
4012 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
4014 u32 eax = 1, ebx, ecx = 0, edx;
4016 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
4017 if (!(edx & FFL(FXSR)))
4018 return emulate_ud(ctxt);
4020 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
4021 return emulate_nm(ctxt);
4024 * Don't emulate a case that should never be hit, instead of working
4025 * around a lack of fxsave64/fxrstor64 on old compilers.
4027 if (ctxt->mode >= X86EMUL_MODE_PROT64)
4028 return X86EMUL_UNHANDLEABLE;
4030 return X86EMUL_CONTINUE;
4034 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
4035 * and restore MXCSR.
4037 static size_t __fxstate_size(int nregs)
4039 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
4042 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
4045 if (ctxt->mode == X86EMUL_MODE_PROT64)
4046 return __fxstate_size(16);
4048 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4049 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4053 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4056 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4057 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4059 * 3) 64-bit mode with REX.W prefix
4060 * - like (2), but XMM 8-15 are being saved and restored
4061 * 4) 64-bit mode without REX.W prefix
4062 * - like (3), but FIP and FDP are 64 bit
4064 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4065 * desired result. (4) is not emulated.
4067 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4068 * and FPU DS) should match.
4070 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4072 struct fxregs_state fx_state;
4075 rc = check_fxsr(ctxt);
4076 if (rc != X86EMUL_CONTINUE)
4079 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4081 if (rc != X86EMUL_CONTINUE)
4084 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4085 fxstate_size(ctxt));
4089 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4090 * in the host registers (via FXSAVE) instead, so they won't be modified.
4091 * (preemption has to stay disabled until FXRSTOR).
4093 * Use noinline to keep the stack for other functions called by callers small.
4095 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4096 const size_t used_size)
4098 struct fxregs_state fx_tmp;
4101 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4102 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4103 __fxstate_size(16) - used_size);
4108 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4110 struct fxregs_state fx_state;
4114 rc = check_fxsr(ctxt);
4115 if (rc != X86EMUL_CONTINUE)
4118 size = fxstate_size(ctxt);
4119 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4120 if (rc != X86EMUL_CONTINUE)
4123 if (size < __fxstate_size(16)) {
4124 rc = fxregs_fixup(&fx_state, size);
4125 if (rc != X86EMUL_CONTINUE)
4129 if (fx_state.mxcsr >> 16) {
4130 rc = emulate_gp(ctxt, 0);
4134 if (rc == X86EMUL_CONTINUE)
4135 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4141 static bool valid_cr(int nr)
4153 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4155 if (!valid_cr(ctxt->modrm_reg))
4156 return emulate_ud(ctxt);
4158 return X86EMUL_CONTINUE;
4161 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4163 u64 new_val = ctxt->src.val64;
4164 int cr = ctxt->modrm_reg;
4167 static u64 cr_reserved_bits[] = {
4168 0xffffffff00000000ULL,
4169 0, 0, 0, /* CR3 checked later */
4176 return emulate_ud(ctxt);
4178 if (new_val & cr_reserved_bits[cr])
4179 return emulate_gp(ctxt, 0);
4184 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4185 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4186 return emulate_gp(ctxt, 0);
4188 cr4 = ctxt->ops->get_cr(ctxt, 4);
4189 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4191 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4192 !(cr4 & X86_CR4_PAE))
4193 return emulate_gp(ctxt, 0);
4200 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4201 if (efer & EFER_LMA) {
4203 u32 eax, ebx, ecx, edx;
4207 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4209 maxphyaddr = eax & 0xff;
4212 rsvd = rsvd_bits(maxphyaddr, 63);
4213 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4214 rsvd &= ~X86_CR3_PCID_NOFLUSH;
4218 return emulate_gp(ctxt, 0);
4223 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4225 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4226 return emulate_gp(ctxt, 0);
4232 return X86EMUL_CONTINUE;
4235 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4239 ctxt->ops->get_dr(ctxt, 7, &dr7);
4241 /* Check if DR7.Global_Enable is set */
4242 return dr7 & (1 << 13);
4245 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4247 int dr = ctxt->modrm_reg;
4251 return emulate_ud(ctxt);
4253 cr4 = ctxt->ops->get_cr(ctxt, 4);
4254 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4255 return emulate_ud(ctxt);
4257 if (check_dr7_gd(ctxt)) {
4260 ctxt->ops->get_dr(ctxt, 6, &dr6);
4262 dr6 |= DR6_BD | DR6_RTM;
4263 ctxt->ops->set_dr(ctxt, 6, dr6);
4264 return emulate_db(ctxt);
4267 return X86EMUL_CONTINUE;
4270 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4272 u64 new_val = ctxt->src.val64;
4273 int dr = ctxt->modrm_reg;
4275 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4276 return emulate_gp(ctxt, 0);
4278 return check_dr_read(ctxt);
4281 static int check_svme(struct x86_emulate_ctxt *ctxt)
4285 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4287 if (!(efer & EFER_SVME))
4288 return emulate_ud(ctxt);
4290 return X86EMUL_CONTINUE;
4293 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4295 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4297 /* Valid physical address? */
4298 if (rax & 0xffff000000000000ULL)
4299 return emulate_gp(ctxt, 0);
4301 return check_svme(ctxt);
4304 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4306 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4308 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4309 return emulate_ud(ctxt);
4311 return X86EMUL_CONTINUE;
4314 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4316 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4317 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4320 * VMware allows access to these Pseduo-PMCs even when read via RDPMC
4321 * in Ring3 when CR4.PCE=0.
4323 if (enable_vmware_backdoor && is_vmware_backdoor_pmc(rcx))
4324 return X86EMUL_CONTINUE;
4326 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4327 ctxt->ops->check_pmc(ctxt, rcx))
4328 return emulate_gp(ctxt, 0);
4330 return X86EMUL_CONTINUE;
4333 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4335 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4336 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4337 return emulate_gp(ctxt, 0);
4339 return X86EMUL_CONTINUE;
4342 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4344 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4345 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4346 return emulate_gp(ctxt, 0);
4348 return X86EMUL_CONTINUE;
4351 #define D(_y) { .flags = (_y) }
4352 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4353 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4354 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4355 #define N D(NotImpl)
4356 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4357 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4358 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4359 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4360 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4361 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4362 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4363 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4364 #define II(_f, _e, _i) \
4365 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4366 #define IIP(_f, _e, _i, _p) \
4367 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4368 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4369 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4371 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4372 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4373 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4374 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4375 #define I2bvIP(_f, _e, _i, _p) \
4376 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4378 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4379 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4380 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4382 static const struct opcode group7_rm0[] = {
4384 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4388 static const struct opcode group7_rm1[] = {
4389 DI(SrcNone | Priv, monitor),
4390 DI(SrcNone | Priv, mwait),
4394 static const struct opcode group7_rm3[] = {
4395 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4396 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4397 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4398 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4399 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4400 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4401 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4402 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4405 static const struct opcode group7_rm7[] = {
4407 DIP(SrcNone, rdtscp, check_rdtsc),
4411 static const struct opcode group1[] = {
4413 F(Lock | PageTable, em_or),
4416 F(Lock | PageTable, em_and),
4422 static const struct opcode group1A[] = {
4423 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4426 static const struct opcode group2[] = {
4427 F(DstMem | ModRM, em_rol),
4428 F(DstMem | ModRM, em_ror),
4429 F(DstMem | ModRM, em_rcl),
4430 F(DstMem | ModRM, em_rcr),
4431 F(DstMem | ModRM, em_shl),
4432 F(DstMem | ModRM, em_shr),
4433 F(DstMem | ModRM, em_shl),
4434 F(DstMem | ModRM, em_sar),
4437 static const struct opcode group3[] = {
4438 F(DstMem | SrcImm | NoWrite, em_test),
4439 F(DstMem | SrcImm | NoWrite, em_test),
4440 F(DstMem | SrcNone | Lock, em_not),
4441 F(DstMem | SrcNone | Lock, em_neg),
4442 F(DstXacc | Src2Mem, em_mul_ex),
4443 F(DstXacc | Src2Mem, em_imul_ex),
4444 F(DstXacc | Src2Mem, em_div_ex),
4445 F(DstXacc | Src2Mem, em_idiv_ex),
4448 static const struct opcode group4[] = {
4449 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4450 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4454 static const struct opcode group5[] = {
4455 F(DstMem | SrcNone | Lock, em_inc),
4456 F(DstMem | SrcNone | Lock, em_dec),
4457 I(SrcMem | NearBranch, em_call_near_abs),
4458 I(SrcMemFAddr | ImplicitOps, em_call_far),
4459 I(SrcMem | NearBranch, em_jmp_abs),
4460 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4461 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4464 static const struct opcode group6[] = {
4465 II(Prot | DstMem, em_sldt, sldt),
4466 II(Prot | DstMem, em_str, str),
4467 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4468 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4472 static const struct group_dual group7 = { {
4473 II(Mov | DstMem, em_sgdt, sgdt),
4474 II(Mov | DstMem, em_sidt, sidt),
4475 II(SrcMem | Priv, em_lgdt, lgdt),
4476 II(SrcMem | Priv, em_lidt, lidt),
4477 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4478 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4479 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4483 N, EXT(0, group7_rm3),
4484 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4485 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4489 static const struct opcode group8[] = {
4491 F(DstMem | SrcImmByte | NoWrite, em_bt),
4492 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4493 F(DstMem | SrcImmByte | Lock, em_btr),
4494 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4498 * The "memory" destination is actually always a register, since we come
4499 * from the register case of group9.
4501 static const struct gprefix pfx_0f_c7_7 = {
4502 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4506 static const struct group_dual group9 = { {
4507 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4509 N, N, N, N, N, N, N,
4510 GP(0, &pfx_0f_c7_7),
4513 static const struct opcode group11[] = {
4514 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4518 static const struct gprefix pfx_0f_ae_7 = {
4519 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4522 static const struct group_dual group15 = { {
4523 I(ModRM | Aligned16, em_fxsave),
4524 I(ModRM | Aligned16, em_fxrstor),
4525 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4527 N, N, N, N, N, N, N, N,
4530 static const struct gprefix pfx_0f_6f_0f_7f = {
4531 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4534 static const struct instr_dual instr_dual_0f_2b = {
4538 static const struct gprefix pfx_0f_2b = {
4539 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4542 static const struct gprefix pfx_0f_10_0f_11 = {
4543 I(Unaligned, em_mov), I(Unaligned, em_mov), N, N,
4546 static const struct gprefix pfx_0f_28_0f_29 = {
4547 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4550 static const struct gprefix pfx_0f_e7 = {
4551 N, I(Sse, em_mov), N, N,
4554 static const struct escape escape_d9 = { {
4555 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4558 N, N, N, N, N, N, N, N,
4560 N, N, N, N, N, N, N, N,
4562 N, N, N, N, N, N, N, N,
4564 N, N, N, N, N, N, N, N,
4566 N, N, N, N, N, N, N, N,
4568 N, N, N, N, N, N, N, N,
4570 N, N, N, N, N, N, N, N,
4572 N, N, N, N, N, N, N, N,
4575 static const struct escape escape_db = { {
4576 N, N, N, N, N, N, N, N,
4579 N, N, N, N, N, N, N, N,
4581 N, N, N, N, N, N, N, N,
4583 N, N, N, N, N, N, N, N,
4585 N, N, N, N, N, N, N, N,
4587 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4589 N, N, N, N, N, N, N, N,
4591 N, N, N, N, N, N, N, N,
4593 N, N, N, N, N, N, N, N,
4596 static const struct escape escape_dd = { {
4597 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4600 N, N, N, N, N, N, N, N,
4602 N, N, N, N, N, N, N, N,
4604 N, N, N, N, N, N, N, N,
4606 N, N, N, N, N, N, N, N,
4608 N, N, N, N, N, N, N, N,
4610 N, N, N, N, N, N, N, N,
4612 N, N, N, N, N, N, N, N,
4614 N, N, N, N, N, N, N, N,
4617 static const struct instr_dual instr_dual_0f_c3 = {
4618 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4621 static const struct mode_dual mode_dual_63 = {
4622 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4625 static const struct opcode opcode_table[256] = {
4627 F6ALU(Lock, em_add),
4628 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4629 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4631 F6ALU(Lock | PageTable, em_or),
4632 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4635 F6ALU(Lock, em_adc),
4636 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4637 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4639 F6ALU(Lock, em_sbb),
4640 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4641 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4643 F6ALU(Lock | PageTable, em_and), N, N,
4645 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4647 F6ALU(Lock, em_xor), N, N,
4649 F6ALU(NoWrite, em_cmp), N, N,
4651 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4653 X8(I(SrcReg | Stack, em_push)),
4655 X8(I(DstReg | Stack, em_pop)),
4657 I(ImplicitOps | Stack | No64, em_pusha),
4658 I(ImplicitOps | Stack | No64, em_popa),
4659 N, MD(ModRM, &mode_dual_63),
4662 I(SrcImm | Mov | Stack, em_push),
4663 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4664 I(SrcImmByte | Mov | Stack, em_push),
4665 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4666 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4667 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4669 X16(D(SrcImmByte | NearBranch)),
4671 G(ByteOp | DstMem | SrcImm, group1),
4672 G(DstMem | SrcImm, group1),
4673 G(ByteOp | DstMem | SrcImm | No64, group1),
4674 G(DstMem | SrcImmByte, group1),
4675 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4676 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4678 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4679 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4680 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4681 D(ModRM | SrcMem | NoAccess | DstReg),
4682 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4685 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4687 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4688 I(SrcImmFAddr | No64, em_call_far), N,
4689 II(ImplicitOps | Stack, em_pushf, pushf),
4690 II(ImplicitOps | Stack, em_popf, popf),
4691 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4693 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4694 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4695 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4696 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4698 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4699 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4700 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4701 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4703 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4705 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4707 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4708 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4709 I(ImplicitOps | NearBranch, em_ret),
4710 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4711 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4712 G(ByteOp, group11), G(0, group11),
4714 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4715 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4716 I(ImplicitOps, em_ret_far),
4717 D(ImplicitOps), DI(SrcImmByte, intn),
4718 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4720 G(Src2One | ByteOp, group2), G(Src2One, group2),
4721 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4722 I(DstAcc | SrcImmUByte | No64, em_aam),
4723 I(DstAcc | SrcImmUByte | No64, em_aad),
4724 F(DstAcc | ByteOp | No64, em_salc),
4725 I(DstAcc | SrcXLat | ByteOp, em_mov),
4727 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4729 X3(I(SrcImmByte | NearBranch, em_loop)),
4730 I(SrcImmByte | NearBranch, em_jcxz),
4731 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4732 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4734 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4735 I(SrcImmFAddr | No64, em_jmp_far),
4736 D(SrcImmByte | ImplicitOps | NearBranch),
4737 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4738 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4740 N, DI(ImplicitOps, icebp), N, N,
4741 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4742 G(ByteOp, group3), G(0, group3),
4744 D(ImplicitOps), D(ImplicitOps),
4745 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4746 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4749 static const struct opcode twobyte_table[256] = {
4751 G(0, group6), GD(0, &group7), N, N,
4752 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4753 II(ImplicitOps | Priv, em_clts, clts), N,
4754 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4755 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4757 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11),
4758 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11),
4760 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4761 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4763 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4764 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4765 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4767 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4770 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4771 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4772 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4775 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4776 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4777 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4778 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4779 I(ImplicitOps | EmulateOnUD, em_sysenter),
4780 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4782 N, N, N, N, N, N, N, N,
4784 X16(D(DstReg | SrcMem | ModRM)),
4786 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4791 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4796 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4798 X16(D(SrcImm | NearBranch)),
4800 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4802 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4803 II(ImplicitOps, em_cpuid, cpuid),
4804 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4805 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4806 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4808 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4809 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4810 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4811 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4812 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4813 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4815 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4816 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4817 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4818 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4819 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4820 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4824 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4825 I(DstReg | SrcMem | ModRM, em_bsf_c),
4826 I(DstReg | SrcMem | ModRM, em_bsr_c),
4827 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4829 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4830 N, ID(0, &instr_dual_0f_c3),
4831 N, N, N, GD(0, &group9),
4833 X8(I(DstReg, em_bswap)),
4835 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4837 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4838 N, N, N, N, N, N, N, N,
4840 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4843 static const struct instr_dual instr_dual_0f_38_f0 = {
4844 I(DstReg | SrcMem | Mov, em_movbe), N
4847 static const struct instr_dual instr_dual_0f_38_f1 = {
4848 I(DstMem | SrcReg | Mov, em_movbe), N
4851 static const struct gprefix three_byte_0f_38_f0 = {
4852 ID(0, &instr_dual_0f_38_f0), N, N, N
4855 static const struct gprefix three_byte_0f_38_f1 = {
4856 ID(0, &instr_dual_0f_38_f1), N, N, N
4860 * Insns below are selected by the prefix which indexed by the third opcode
4863 static const struct opcode opcode_map_0f_38[256] = {
4865 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4867 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4869 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4870 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4891 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4895 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4901 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4902 unsigned size, bool sign_extension)
4904 int rc = X86EMUL_CONTINUE;
4908 op->addr.mem.ea = ctxt->_eip;
4909 /* NB. Immediates are sign-extended as necessary. */
4910 switch (op->bytes) {
4912 op->val = insn_fetch(s8, ctxt);
4915 op->val = insn_fetch(s16, ctxt);
4918 op->val = insn_fetch(s32, ctxt);
4921 op->val = insn_fetch(s64, ctxt);
4924 if (!sign_extension) {
4925 switch (op->bytes) {
4933 op->val &= 0xffffffff;
4941 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4944 int rc = X86EMUL_CONTINUE;
4948 decode_register_operand(ctxt, op);
4951 rc = decode_imm(ctxt, op, 1, false);
4954 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4958 if (ctxt->d & BitOp)
4959 fetch_bit_operand(ctxt);
4960 op->orig_val = op->val;
4963 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4967 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4968 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4969 fetch_register_operand(op);
4970 op->orig_val = op->val;
4974 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4975 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4976 fetch_register_operand(op);
4977 op->orig_val = op->val;
4980 if (ctxt->d & ByteOp) {
4985 op->bytes = ctxt->op_bytes;
4986 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4987 fetch_register_operand(op);
4988 op->orig_val = op->val;
4992 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4994 register_address(ctxt, VCPU_REGS_RDI);
4995 op->addr.mem.seg = VCPU_SREG_ES;
5002 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
5003 fetch_register_operand(op);
5008 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
5011 rc = decode_imm(ctxt, op, 1, true);
5019 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
5022 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
5025 ctxt->memop.bytes = 1;
5026 if (ctxt->memop.type == OP_REG) {
5027 ctxt->memop.addr.reg = decode_register(ctxt,
5028 ctxt->modrm_rm, true);
5029 fetch_register_operand(&ctxt->memop);
5033 ctxt->memop.bytes = 2;
5036 ctxt->memop.bytes = 4;
5039 rc = decode_imm(ctxt, op, 2, false);
5042 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
5046 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5048 register_address(ctxt, VCPU_REGS_RSI);
5049 op->addr.mem.seg = ctxt->seg_override;
5055 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5058 reg_read(ctxt, VCPU_REGS_RBX) +
5059 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5060 op->addr.mem.seg = ctxt->seg_override;
5065 op->addr.mem.ea = ctxt->_eip;
5066 op->bytes = ctxt->op_bytes + 2;
5067 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5070 ctxt->memop.bytes = ctxt->op_bytes + 2;
5074 op->val = VCPU_SREG_ES;
5078 op->val = VCPU_SREG_CS;
5082 op->val = VCPU_SREG_SS;
5086 op->val = VCPU_SREG_DS;
5090 op->val = VCPU_SREG_FS;
5094 op->val = VCPU_SREG_GS;
5097 /* Special instructions do their own operand decoding. */
5099 op->type = OP_NONE; /* Disable writeback. */
5107 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5109 int rc = X86EMUL_CONTINUE;
5110 int mode = ctxt->mode;
5111 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5112 bool op_prefix = false;
5113 bool has_seg_override = false;
5114 struct opcode opcode;
5116 struct desc_struct desc;
5118 ctxt->memop.type = OP_NONE;
5119 ctxt->memopp = NULL;
5120 ctxt->_eip = ctxt->eip;
5121 ctxt->fetch.ptr = ctxt->fetch.data;
5122 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5123 ctxt->opcode_len = 1;
5124 ctxt->intercept = x86_intercept_none;
5126 memcpy(ctxt->fetch.data, insn, insn_len);
5128 rc = __do_insn_fetch_bytes(ctxt, 1);
5129 if (rc != X86EMUL_CONTINUE)
5134 case X86EMUL_MODE_REAL:
5135 case X86EMUL_MODE_VM86:
5136 def_op_bytes = def_ad_bytes = 2;
5137 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5139 def_op_bytes = def_ad_bytes = 4;
5141 case X86EMUL_MODE_PROT16:
5142 def_op_bytes = def_ad_bytes = 2;
5144 case X86EMUL_MODE_PROT32:
5145 def_op_bytes = def_ad_bytes = 4;
5147 #ifdef CONFIG_X86_64
5148 case X86EMUL_MODE_PROT64:
5154 return EMULATION_FAILED;
5157 ctxt->op_bytes = def_op_bytes;
5158 ctxt->ad_bytes = def_ad_bytes;
5160 /* Legacy prefixes. */
5162 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5163 case 0x66: /* operand-size override */
5165 /* switch between 2/4 bytes */
5166 ctxt->op_bytes = def_op_bytes ^ 6;
5168 case 0x67: /* address-size override */
5169 if (mode == X86EMUL_MODE_PROT64)
5170 /* switch between 4/8 bytes */
5171 ctxt->ad_bytes = def_ad_bytes ^ 12;
5173 /* switch between 2/4 bytes */
5174 ctxt->ad_bytes = def_ad_bytes ^ 6;
5176 case 0x26: /* ES override */
5177 has_seg_override = true;
5178 ctxt->seg_override = VCPU_SREG_ES;
5180 case 0x2e: /* CS override */
5181 has_seg_override = true;
5182 ctxt->seg_override = VCPU_SREG_CS;
5184 case 0x36: /* SS override */
5185 has_seg_override = true;
5186 ctxt->seg_override = VCPU_SREG_SS;
5188 case 0x3e: /* DS override */
5189 has_seg_override = true;
5190 ctxt->seg_override = VCPU_SREG_DS;
5192 case 0x64: /* FS override */
5193 has_seg_override = true;
5194 ctxt->seg_override = VCPU_SREG_FS;
5196 case 0x65: /* GS override */
5197 has_seg_override = true;
5198 ctxt->seg_override = VCPU_SREG_GS;
5200 case 0x40 ... 0x4f: /* REX */
5201 if (mode != X86EMUL_MODE_PROT64)
5203 ctxt->rex_prefix = ctxt->b;
5205 case 0xf0: /* LOCK */
5206 ctxt->lock_prefix = 1;
5208 case 0xf2: /* REPNE/REPNZ */
5209 case 0xf3: /* REP/REPE/REPZ */
5210 ctxt->rep_prefix = ctxt->b;
5216 /* Any legacy prefix after a REX prefix nullifies its effect. */
5218 ctxt->rex_prefix = 0;
5224 if (ctxt->rex_prefix & 8)
5225 ctxt->op_bytes = 8; /* REX.W */
5227 /* Opcode byte(s). */
5228 opcode = opcode_table[ctxt->b];
5229 /* Two-byte opcode? */
5230 if (ctxt->b == 0x0f) {
5231 ctxt->opcode_len = 2;
5232 ctxt->b = insn_fetch(u8, ctxt);
5233 opcode = twobyte_table[ctxt->b];
5235 /* 0F_38 opcode map */
5236 if (ctxt->b == 0x38) {
5237 ctxt->opcode_len = 3;
5238 ctxt->b = insn_fetch(u8, ctxt);
5239 opcode = opcode_map_0f_38[ctxt->b];
5242 ctxt->d = opcode.flags;
5244 if (ctxt->d & ModRM)
5245 ctxt->modrm = insn_fetch(u8, ctxt);
5247 /* vex-prefix instructions are not implemented */
5248 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5249 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5253 while (ctxt->d & GroupMask) {
5254 switch (ctxt->d & GroupMask) {
5256 goffset = (ctxt->modrm >> 3) & 7;
5257 opcode = opcode.u.group[goffset];
5260 goffset = (ctxt->modrm >> 3) & 7;
5261 if ((ctxt->modrm >> 6) == 3)
5262 opcode = opcode.u.gdual->mod3[goffset];
5264 opcode = opcode.u.gdual->mod012[goffset];
5267 goffset = ctxt->modrm & 7;
5268 opcode = opcode.u.group[goffset];
5271 if (ctxt->rep_prefix && op_prefix)
5272 return EMULATION_FAILED;
5273 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5274 switch (simd_prefix) {
5275 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5276 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5277 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5278 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5282 if (ctxt->modrm > 0xbf) {
5283 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5284 u32 index = array_index_nospec(
5285 ctxt->modrm - 0xc0, size);
5287 opcode = opcode.u.esc->high[index];
5289 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5293 if ((ctxt->modrm >> 6) == 3)
5294 opcode = opcode.u.idual->mod3;
5296 opcode = opcode.u.idual->mod012;
5299 if (ctxt->mode == X86EMUL_MODE_PROT64)
5300 opcode = opcode.u.mdual->mode64;
5302 opcode = opcode.u.mdual->mode32;
5305 return EMULATION_FAILED;
5308 ctxt->d &= ~(u64)GroupMask;
5309 ctxt->d |= opcode.flags;
5314 return EMULATION_FAILED;
5316 ctxt->execute = opcode.u.execute;
5318 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5319 return EMULATION_FAILED;
5321 if (unlikely(ctxt->d &
5322 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5325 * These are copied unconditionally here, and checked unconditionally
5326 * in x86_emulate_insn.
5328 ctxt->check_perm = opcode.check_perm;
5329 ctxt->intercept = opcode.intercept;
5331 if (ctxt->d & NotImpl)
5332 return EMULATION_FAILED;
5334 if (mode == X86EMUL_MODE_PROT64) {
5335 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5337 else if (ctxt->d & NearBranch)
5341 if (ctxt->d & Op3264) {
5342 if (mode == X86EMUL_MODE_PROT64)
5348 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5352 ctxt->op_bytes = 16;
5353 else if (ctxt->d & Mmx)
5357 /* ModRM and SIB bytes. */
5358 if (ctxt->d & ModRM) {
5359 rc = decode_modrm(ctxt, &ctxt->memop);
5360 if (!has_seg_override) {
5361 has_seg_override = true;
5362 ctxt->seg_override = ctxt->modrm_seg;
5364 } else if (ctxt->d & MemAbs)
5365 rc = decode_abs(ctxt, &ctxt->memop);
5366 if (rc != X86EMUL_CONTINUE)
5369 if (!has_seg_override)
5370 ctxt->seg_override = VCPU_SREG_DS;
5372 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5375 * Decode and fetch the source operand: register, memory
5378 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5379 if (rc != X86EMUL_CONTINUE)
5383 * Decode and fetch the second source operand: register, memory
5386 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5387 if (rc != X86EMUL_CONTINUE)
5390 /* Decode and fetch the destination operand: register or memory. */
5391 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5393 if (ctxt->rip_relative && likely(ctxt->memopp))
5394 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5395 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5398 if (rc == X86EMUL_PROPAGATE_FAULT)
5399 ctxt->have_exception = true;
5400 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5403 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5405 return ctxt->d & PageTable;
5408 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5410 /* The second termination condition only applies for REPE
5411 * and REPNE. Test if the repeat string operation prefix is
5412 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5413 * corresponding termination condition according to:
5414 * - if REPE/REPZ and ZF = 0 then done
5415 * - if REPNE/REPNZ and ZF = 1 then done
5417 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5418 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5419 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5420 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5421 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5422 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5428 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5432 rc = asm_safe("fwait");
5434 if (unlikely(rc != X86EMUL_CONTINUE))
5435 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5437 return X86EMUL_CONTINUE;
5440 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5443 if (op->type == OP_MM)
5444 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5447 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5449 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5451 if (!(ctxt->d & ByteOp))
5452 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5454 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5455 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5456 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5457 : "c"(ctxt->src2.val));
5459 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5460 if (!fop) /* exception is returned in fop variable */
5461 return emulate_de(ctxt);
5462 return X86EMUL_CONTINUE;
5465 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5467 memset(&ctxt->rip_relative, 0,
5468 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5470 ctxt->io_read.pos = 0;
5471 ctxt->io_read.end = 0;
5472 ctxt->mem_read.end = 0;
5475 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5477 const struct x86_emulate_ops *ops = ctxt->ops;
5478 int rc = X86EMUL_CONTINUE;
5479 int saved_dst_type = ctxt->dst.type;
5480 unsigned emul_flags;
5482 ctxt->mem_read.pos = 0;
5484 /* LOCK prefix is allowed only with some instructions */
5485 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5486 rc = emulate_ud(ctxt);
5490 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5491 rc = emulate_ud(ctxt);
5495 emul_flags = ctxt->ops->get_hflags(ctxt);
5496 if (unlikely(ctxt->d &
5497 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5498 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5499 (ctxt->d & Undefined)) {
5500 rc = emulate_ud(ctxt);
5504 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5505 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5506 rc = emulate_ud(ctxt);
5510 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5511 rc = emulate_nm(ctxt);
5515 if (ctxt->d & Mmx) {
5516 rc = flush_pending_x87_faults(ctxt);
5517 if (rc != X86EMUL_CONTINUE)
5520 * Now that we know the fpu is exception safe, we can fetch
5523 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5524 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5525 if (!(ctxt->d & Mov))
5526 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5529 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5530 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5531 X86_ICPT_PRE_EXCEPT);
5532 if (rc != X86EMUL_CONTINUE)
5536 /* Instruction can only be executed in protected mode */
5537 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5538 rc = emulate_ud(ctxt);
5542 /* Privileged instruction can be executed only in CPL=0 */
5543 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5544 if (ctxt->d & PrivUD)
5545 rc = emulate_ud(ctxt);
5547 rc = emulate_gp(ctxt, 0);
5551 /* Do instruction specific permission checks */
5552 if (ctxt->d & CheckPerm) {
5553 rc = ctxt->check_perm(ctxt);
5554 if (rc != X86EMUL_CONTINUE)
5558 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5559 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5560 X86_ICPT_POST_EXCEPT);
5561 if (rc != X86EMUL_CONTINUE)
5565 if (ctxt->rep_prefix && (ctxt->d & String)) {
5566 /* All REP prefixes have the same first termination condition */
5567 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5568 string_registers_quirk(ctxt);
5569 ctxt->eip = ctxt->_eip;
5570 ctxt->eflags &= ~X86_EFLAGS_RF;
5576 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5577 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5578 ctxt->src.valptr, ctxt->src.bytes);
5579 if (rc != X86EMUL_CONTINUE)
5581 ctxt->src.orig_val64 = ctxt->src.val64;
5584 if (ctxt->src2.type == OP_MEM) {
5585 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5586 &ctxt->src2.val, ctxt->src2.bytes);
5587 if (rc != X86EMUL_CONTINUE)
5591 if ((ctxt->d & DstMask) == ImplicitOps)
5595 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5596 /* optimisation - avoid slow emulated read if Mov */
5597 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5598 &ctxt->dst.val, ctxt->dst.bytes);
5599 if (rc != X86EMUL_CONTINUE) {
5600 if (!(ctxt->d & NoWrite) &&
5601 rc == X86EMUL_PROPAGATE_FAULT &&
5602 ctxt->exception.vector == PF_VECTOR)
5603 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5607 /* Copy full 64-bit value for CMPXCHG8B. */
5608 ctxt->dst.orig_val64 = ctxt->dst.val64;
5612 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5613 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5614 X86_ICPT_POST_MEMACCESS);
5615 if (rc != X86EMUL_CONTINUE)
5619 if (ctxt->rep_prefix && (ctxt->d & String))
5620 ctxt->eflags |= X86_EFLAGS_RF;
5622 ctxt->eflags &= ~X86_EFLAGS_RF;
5624 if (ctxt->execute) {
5625 if (ctxt->d & Fastop) {
5626 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5627 rc = fastop(ctxt, fop);
5628 if (rc != X86EMUL_CONTINUE)
5632 rc = ctxt->execute(ctxt);
5633 if (rc != X86EMUL_CONTINUE)
5638 if (ctxt->opcode_len == 2)
5640 else if (ctxt->opcode_len == 3)
5641 goto threebyte_insn;
5644 case 0x70 ... 0x7f: /* jcc (short) */
5645 if (test_cc(ctxt->b, ctxt->eflags))
5646 rc = jmp_rel(ctxt, ctxt->src.val);
5648 case 0x8d: /* lea r16/r32, m */
5649 ctxt->dst.val = ctxt->src.addr.mem.ea;
5651 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5652 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5653 ctxt->dst.type = OP_NONE;
5657 case 0x98: /* cbw/cwde/cdqe */
5658 switch (ctxt->op_bytes) {
5659 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5660 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5661 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5664 case 0xcc: /* int3 */
5665 rc = emulate_int(ctxt, 3);
5667 case 0xcd: /* int n */
5668 rc = emulate_int(ctxt, ctxt->src.val);
5670 case 0xce: /* into */
5671 if (ctxt->eflags & X86_EFLAGS_OF)
5672 rc = emulate_int(ctxt, 4);
5674 case 0xe9: /* jmp rel */
5675 case 0xeb: /* jmp rel short */
5676 rc = jmp_rel(ctxt, ctxt->src.val);
5677 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5679 case 0xf4: /* hlt */
5680 ctxt->ops->halt(ctxt);
5682 case 0xf5: /* cmc */
5683 /* complement carry flag from eflags reg */
5684 ctxt->eflags ^= X86_EFLAGS_CF;
5686 case 0xf8: /* clc */
5687 ctxt->eflags &= ~X86_EFLAGS_CF;
5689 case 0xf9: /* stc */
5690 ctxt->eflags |= X86_EFLAGS_CF;
5692 case 0xfc: /* cld */
5693 ctxt->eflags &= ~X86_EFLAGS_DF;
5695 case 0xfd: /* std */
5696 ctxt->eflags |= X86_EFLAGS_DF;
5699 goto cannot_emulate;
5702 if (rc != X86EMUL_CONTINUE)
5706 if (ctxt->d & SrcWrite) {
5707 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5708 rc = writeback(ctxt, &ctxt->src);
5709 if (rc != X86EMUL_CONTINUE)
5712 if (!(ctxt->d & NoWrite)) {
5713 rc = writeback(ctxt, &ctxt->dst);
5714 if (rc != X86EMUL_CONTINUE)
5719 * restore dst type in case the decoding will be reused
5720 * (happens for string instruction )
5722 ctxt->dst.type = saved_dst_type;
5724 if ((ctxt->d & SrcMask) == SrcSI)
5725 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5727 if ((ctxt->d & DstMask) == DstDI)
5728 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5730 if (ctxt->rep_prefix && (ctxt->d & String)) {
5732 struct read_cache *r = &ctxt->io_read;
5733 if ((ctxt->d & SrcMask) == SrcSI)
5734 count = ctxt->src.count;
5736 count = ctxt->dst.count;
5737 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5739 if (!string_insn_completed(ctxt)) {
5741 * Re-enter guest when pio read ahead buffer is empty
5742 * or, if it is not used, after each 1024 iteration.
5744 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5745 (r->end == 0 || r->end != r->pos)) {
5747 * Reset read cache. Usually happens before
5748 * decode, but since instruction is restarted
5749 * we have to do it here.
5751 ctxt->mem_read.end = 0;
5752 writeback_registers(ctxt);
5753 return EMULATION_RESTART;
5755 goto done; /* skip rip writeback */
5757 ctxt->eflags &= ~X86_EFLAGS_RF;
5760 ctxt->eip = ctxt->_eip;
5763 if (rc == X86EMUL_PROPAGATE_FAULT) {
5764 WARN_ON(ctxt->exception.vector > 0x1f);
5765 ctxt->have_exception = true;
5767 if (rc == X86EMUL_INTERCEPTED)
5768 return EMULATION_INTERCEPTED;
5770 if (rc == X86EMUL_CONTINUE)
5771 writeback_registers(ctxt);
5773 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5777 case 0x09: /* wbinvd */
5778 (ctxt->ops->wbinvd)(ctxt);
5780 case 0x08: /* invd */
5781 case 0x0d: /* GrpP (prefetch) */
5782 case 0x18: /* Grp16 (prefetch/nop) */
5783 case 0x1f: /* nop */
5785 case 0x20: /* mov cr, reg */
5786 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5788 case 0x21: /* mov from dr to reg */
5789 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5791 case 0x40 ... 0x4f: /* cmov */
5792 if (test_cc(ctxt->b, ctxt->eflags))
5793 ctxt->dst.val = ctxt->src.val;
5794 else if (ctxt->op_bytes != 4)
5795 ctxt->dst.type = OP_NONE; /* no writeback */
5797 case 0x80 ... 0x8f: /* jnz rel, etc*/
5798 if (test_cc(ctxt->b, ctxt->eflags))
5799 rc = jmp_rel(ctxt, ctxt->src.val);
5801 case 0x90 ... 0x9f: /* setcc r/m8 */
5802 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5804 case 0xb6 ... 0xb7: /* movzx */
5805 ctxt->dst.bytes = ctxt->op_bytes;
5806 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5807 : (u16) ctxt->src.val;
5809 case 0xbe ... 0xbf: /* movsx */
5810 ctxt->dst.bytes = ctxt->op_bytes;
5811 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5812 (s16) ctxt->src.val;
5815 goto cannot_emulate;
5820 if (rc != X86EMUL_CONTINUE)
5826 return EMULATION_FAILED;
5829 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5831 invalidate_registers(ctxt);
5834 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5836 writeback_registers(ctxt);
5839 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5841 if (ctxt->rep_prefix && (ctxt->d & String))
5844 if (ctxt->d & TwoMemOp)