1 /******************************************************************************
4 * Generic x86 (32-bit and 64-bit) instruction decoder and emulator.
6 * Copyright (c) 2005 Keir Fraser
8 * Linux coding style, mod r/m decoder, segment base fixes, real-mode
9 * privileged instructions:
11 * Copyright (C) 2006 Qumranet
12 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
14 * Avi Kivity <avi@qumranet.com>
15 * Yaniv Kamay <yaniv@qumranet.com>
17 * This work is licensed under the terms of the GNU GPL, version 2. See
18 * the COPYING file in the top-level directory.
20 * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4
23 #include <linux/kvm_host.h>
24 #include "kvm_cache_regs.h"
25 #include <asm/kvm_emulate.h>
26 #include <linux/stringify.h>
27 #include <asm/debugreg.h>
28 #include <asm/nospec-branch.h>
38 #define OpImplicit 1ull /* No generic decode */
39 #define OpReg 2ull /* Register */
40 #define OpMem 3ull /* Memory */
41 #define OpAcc 4ull /* Accumulator: AL/AX/EAX/RAX */
42 #define OpDI 5ull /* ES:DI/EDI/RDI */
43 #define OpMem64 6ull /* Memory, 64-bit */
44 #define OpImmUByte 7ull /* Zero-extended 8-bit immediate */
45 #define OpDX 8ull /* DX register */
46 #define OpCL 9ull /* CL register (for shifts) */
47 #define OpImmByte 10ull /* 8-bit sign extended immediate */
48 #define OpOne 11ull /* Implied 1 */
49 #define OpImm 12ull /* Sign extended up to 32-bit immediate */
50 #define OpMem16 13ull /* Memory operand (16-bit). */
51 #define OpMem32 14ull /* Memory operand (32-bit). */
52 #define OpImmU 15ull /* Immediate operand, zero extended */
53 #define OpSI 16ull /* SI/ESI/RSI */
54 #define OpImmFAddr 17ull /* Immediate far address */
55 #define OpMemFAddr 18ull /* Far address in memory */
56 #define OpImmU16 19ull /* Immediate operand, 16 bits, zero extended */
57 #define OpES 20ull /* ES */
58 #define OpCS 21ull /* CS */
59 #define OpSS 22ull /* SS */
60 #define OpDS 23ull /* DS */
61 #define OpFS 24ull /* FS */
62 #define OpGS 25ull /* GS */
63 #define OpMem8 26ull /* 8-bit zero extended memory operand */
64 #define OpImm64 27ull /* Sign extended 16/32/64-bit immediate */
65 #define OpXLat 28ull /* memory at BX/EBX/RBX + zero-extended AL */
66 #define OpAccLo 29ull /* Low part of extended acc (AX/AX/EAX/RAX) */
67 #define OpAccHi 30ull /* High part of extended acc (-/DX/EDX/RDX) */
69 #define OpBits 5 /* Width of operand field */
70 #define OpMask ((1ull << OpBits) - 1)
73 * Opcode effective-address decode tables.
74 * Note that we only emulate instructions that have at least one memory
75 * operand (excluding implicit stack references). We assume that stack
76 * references and instruction fetches will never occur in special memory
77 * areas that require emulation. So, for example, 'mov <imm>,<reg>' need
81 /* Operand sizes: 8-bit operands or specified/overridden size. */
82 #define ByteOp (1<<0) /* 8-bit operands. */
83 /* Destination operand type. */
85 #define ImplicitOps (OpImplicit << DstShift)
86 #define DstReg (OpReg << DstShift)
87 #define DstMem (OpMem << DstShift)
88 #define DstAcc (OpAcc << DstShift)
89 #define DstDI (OpDI << DstShift)
90 #define DstMem64 (OpMem64 << DstShift)
91 #define DstMem16 (OpMem16 << DstShift)
92 #define DstImmUByte (OpImmUByte << DstShift)
93 #define DstDX (OpDX << DstShift)
94 #define DstAccLo (OpAccLo << DstShift)
95 #define DstMask (OpMask << DstShift)
96 /* Source operand type. */
98 #define SrcNone (OpNone << SrcShift)
99 #define SrcReg (OpReg << SrcShift)
100 #define SrcMem (OpMem << SrcShift)
101 #define SrcMem16 (OpMem16 << SrcShift)
102 #define SrcMem32 (OpMem32 << SrcShift)
103 #define SrcImm (OpImm << SrcShift)
104 #define SrcImmByte (OpImmByte << SrcShift)
105 #define SrcOne (OpOne << SrcShift)
106 #define SrcImmUByte (OpImmUByte << SrcShift)
107 #define SrcImmU (OpImmU << SrcShift)
108 #define SrcSI (OpSI << SrcShift)
109 #define SrcXLat (OpXLat << SrcShift)
110 #define SrcImmFAddr (OpImmFAddr << SrcShift)
111 #define SrcMemFAddr (OpMemFAddr << SrcShift)
112 #define SrcAcc (OpAcc << SrcShift)
113 #define SrcImmU16 (OpImmU16 << SrcShift)
114 #define SrcImm64 (OpImm64 << SrcShift)
115 #define SrcDX (OpDX << SrcShift)
116 #define SrcMem8 (OpMem8 << SrcShift)
117 #define SrcAccHi (OpAccHi << SrcShift)
118 #define SrcMask (OpMask << SrcShift)
119 #define BitOp (1<<11)
120 #define MemAbs (1<<12) /* Memory operand is absolute displacement */
121 #define String (1<<13) /* String instruction (rep capable) */
122 #define Stack (1<<14) /* Stack instruction (push/pop) */
123 #define GroupMask (7<<15) /* Opcode uses one of the group mechanisms */
124 #define Group (1<<15) /* Bits 3:5 of modrm byte extend opcode */
125 #define GroupDual (2<<15) /* Alternate decoding of mod == 3 */
126 #define Prefix (3<<15) /* Instruction varies with 66/f2/f3 prefix */
127 #define RMExt (4<<15) /* Opcode extension in ModRM r/m if mod == 3 */
128 #define Escape (5<<15) /* Escape to coprocessor instruction */
129 #define InstrDual (6<<15) /* Alternate instruction decoding of mod == 3 */
130 #define ModeDual (7<<15) /* Different instruction for 32/64 bit */
131 #define Sse (1<<18) /* SSE Vector instruction */
132 /* Generic ModRM decode. */
133 #define ModRM (1<<19)
134 /* Destination is only written; never read. */
137 #define Prot (1<<21) /* instruction generates #UD if not in prot-mode */
138 #define EmulateOnUD (1<<22) /* Emulate if unsupported by the host */
139 #define NoAccess (1<<23) /* Don't access memory (lea/invlpg/verr etc) */
140 #define Op3264 (1<<24) /* Operand is 64b in long mode, 32b otherwise */
141 #define Undefined (1<<25) /* No Such Instruction */
142 #define Lock (1<<26) /* lock prefix is allowed for the instruction */
143 #define Priv (1<<27) /* instruction generates #GP if current CPL != 0 */
145 #define PageTable (1 << 29) /* instruction used to write page table */
146 #define NotImpl (1 << 30) /* instruction is not implemented */
147 /* Source 2 operand type */
148 #define Src2Shift (31)
149 #define Src2None (OpNone << Src2Shift)
150 #define Src2Mem (OpMem << Src2Shift)
151 #define Src2CL (OpCL << Src2Shift)
152 #define Src2ImmByte (OpImmByte << Src2Shift)
153 #define Src2One (OpOne << Src2Shift)
154 #define Src2Imm (OpImm << Src2Shift)
155 #define Src2ES (OpES << Src2Shift)
156 #define Src2CS (OpCS << Src2Shift)
157 #define Src2SS (OpSS << Src2Shift)
158 #define Src2DS (OpDS << Src2Shift)
159 #define Src2FS (OpFS << Src2Shift)
160 #define Src2GS (OpGS << Src2Shift)
161 #define Src2Mask (OpMask << Src2Shift)
162 #define Mmx ((u64)1 << 40) /* MMX Vector instruction */
163 #define AlignMask ((u64)7 << 41)
164 #define Aligned ((u64)1 << 41) /* Explicitly aligned (e.g. MOVDQA) */
165 #define Unaligned ((u64)2 << 41) /* Explicitly unaligned (e.g. MOVDQU) */
166 #define Avx ((u64)3 << 41) /* Advanced Vector Extensions */
167 #define Aligned16 ((u64)4 << 41) /* Aligned to 16 byte boundary (e.g. FXSAVE) */
168 #define Fastop ((u64)1 << 44) /* Use opcode::u.fastop */
169 #define NoWrite ((u64)1 << 45) /* No writeback */
170 #define SrcWrite ((u64)1 << 46) /* Write back src operand */
171 #define NoMod ((u64)1 << 47) /* Mod field is ignored */
172 #define Intercept ((u64)1 << 48) /* Has valid intercept field */
173 #define CheckPerm ((u64)1 << 49) /* Has valid check_perm field */
174 #define PrivUD ((u64)1 << 51) /* #UD instead of #GP on CPL > 0 */
175 #define NearBranch ((u64)1 << 52) /* Near branches */
176 #define No16 ((u64)1 << 53) /* No 16 bit operand */
177 #define IncSP ((u64)1 << 54) /* SP is incremented before ModRM calc */
178 #define TwoMemOp ((u64)1 << 55) /* Instruction has two memory operand */
180 #define DstXacc (DstAccLo | SrcAccHi | SrcWrite)
182 #define X2(x...) x, x
183 #define X3(x...) X2(x), x
184 #define X4(x...) X2(x), X2(x)
185 #define X5(x...) X4(x), x
186 #define X6(x...) X4(x), X2(x)
187 #define X7(x...) X4(x), X3(x)
188 #define X8(x...) X4(x), X4(x)
189 #define X16(x...) X8(x), X8(x)
191 #define NR_FASTOP (ilog2(sizeof(ulong)) + 1)
192 #define FASTOP_SIZE 8
195 * fastop functions have a special calling convention:
200 * flags: rflags (in/out)
201 * ex: rsi (in:fastop pointer, out:zero if exception)
203 * Moreover, they are all exactly FASTOP_SIZE bytes long, so functions for
204 * different operand sizes can be reached by calculation, rather than a jump
205 * table (which would be bigger than the code).
207 * fastop functions are declared as taking a never-defined fastop parameter,
208 * so they can't be called from C directly.
217 int (*execute)(struct x86_emulate_ctxt *ctxt);
218 const struct opcode *group;
219 const struct group_dual *gdual;
220 const struct gprefix *gprefix;
221 const struct escape *esc;
222 const struct instr_dual *idual;
223 const struct mode_dual *mdual;
224 void (*fastop)(struct fastop *fake);
226 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
230 struct opcode mod012[8];
231 struct opcode mod3[8];
235 struct opcode pfx_no;
236 struct opcode pfx_66;
237 struct opcode pfx_f2;
238 struct opcode pfx_f3;
243 struct opcode high[64];
247 struct opcode mod012;
252 struct opcode mode32;
253 struct opcode mode64;
256 #define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
258 enum x86_transfer_type {
260 X86_TRANSFER_CALL_JMP,
262 X86_TRANSFER_TASK_SWITCH,
265 static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
267 if (!(ctxt->regs_valid & (1 << nr))) {
268 ctxt->regs_valid |= 1 << nr;
269 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
271 return ctxt->_regs[nr];
274 static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
276 ctxt->regs_valid |= 1 << nr;
277 ctxt->regs_dirty |= 1 << nr;
278 return &ctxt->_regs[nr];
281 static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
284 return reg_write(ctxt, nr);
287 static void writeback_registers(struct x86_emulate_ctxt *ctxt)
291 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
292 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
295 static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
297 ctxt->regs_dirty = 0;
298 ctxt->regs_valid = 0;
302 * These EFLAGS bits are restored from saved value during emulation, and
303 * any changes are written back to the saved value after emulation.
305 #define EFLAGS_MASK (X86_EFLAGS_OF|X86_EFLAGS_SF|X86_EFLAGS_ZF|X86_EFLAGS_AF|\
306 X86_EFLAGS_PF|X86_EFLAGS_CF)
314 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *));
316 #define FOP_FUNC(name) \
317 ".align " __stringify(FASTOP_SIZE) " \n\t" \
318 ".type " name ", @function \n\t" \
321 #define FOP_RET "ret \n\t"
323 #define FOP_START(op) \
324 extern void em_##op(struct fastop *fake); \
325 asm(".pushsection .text, \"ax\" \n\t" \
326 ".global em_" #op " \n\t" \
333 FOP_FUNC(__stringify(__UNIQUE_ID(nop))) \
336 #define FOP1E(op, dst) \
337 FOP_FUNC(#op "_" #dst) \
338 "10: " #op " %" #dst " \n\t" FOP_RET
340 #define FOP1EEX(op, dst) \
341 FOP1E(op, dst) _ASM_EXTABLE(10b, kvm_fastop_exception)
343 #define FASTOP1(op) \
348 ON64(FOP1E(op##q, rax)) \
351 /* 1-operand, using src2 (for MUL/DIV r/m) */
352 #define FASTOP1SRC2(op, name) \
357 ON64(FOP1E(op, rcx)) \
360 /* 1-operand, using src2 (for MUL/DIV r/m), with exceptions */
361 #define FASTOP1SRC2EX(op, name) \
366 ON64(FOP1EEX(op, rcx)) \
369 #define FOP2E(op, dst, src) \
370 FOP_FUNC(#op "_" #dst "_" #src) \
371 #op " %" #src ", %" #dst " \n\t" FOP_RET
373 #define FASTOP2(op) \
375 FOP2E(op##b, al, dl) \
376 FOP2E(op##w, ax, dx) \
377 FOP2E(op##l, eax, edx) \
378 ON64(FOP2E(op##q, rax, rdx)) \
381 /* 2 operand, word only */
382 #define FASTOP2W(op) \
385 FOP2E(op##w, ax, dx) \
386 FOP2E(op##l, eax, edx) \
387 ON64(FOP2E(op##q, rax, rdx)) \
390 /* 2 operand, src is CL */
391 #define FASTOP2CL(op) \
393 FOP2E(op##b, al, cl) \
394 FOP2E(op##w, ax, cl) \
395 FOP2E(op##l, eax, cl) \
396 ON64(FOP2E(op##q, rax, cl)) \
399 /* 2 operand, src and dest are reversed */
400 #define FASTOP2R(op, name) \
402 FOP2E(op##b, dl, al) \
403 FOP2E(op##w, dx, ax) \
404 FOP2E(op##l, edx, eax) \
405 ON64(FOP2E(op##q, rdx, rax)) \
408 #define FOP3E(op, dst, src, src2) \
409 FOP_FUNC(#op "_" #dst "_" #src "_" #src2) \
410 #op " %" #src2 ", %" #src ", %" #dst " \n\t" FOP_RET
412 /* 3-operand, word-only, src2=cl */
413 #define FASTOP3WCL(op) \
416 FOP3E(op##w, ax, dx, cl) \
417 FOP3E(op##l, eax, edx, cl) \
418 ON64(FOP3E(op##q, rax, rdx, cl)) \
421 /* Special case for SETcc - 1 instruction per cc */
422 #define FOP_SETCC(op) \
424 ".type " #op ", @function \n\t" \
429 asm(".pushsection .fixup, \"ax\"\n"
430 ".global kvm_fastop_exception \n"
431 "kvm_fastop_exception: xor %esi, %esi; ret\n"
453 FOP_START(salc) "pushf; sbb %al, %al; popf \n\t" FOP_RET
457 * XXX: inoutclob user must know where the argument is being expanded.
458 * Relying on CC_HAVE_ASM_GOTO would allow us to remove _fault.
460 #define asm_safe(insn, inoutclob...) \
464 asm volatile("1:" insn "\n" \
466 ".pushsection .fixup, \"ax\"\n" \
467 "3: movl $1, %[_fault]\n" \
470 _ASM_EXTABLE(1b, 3b) \
471 : [_fault] "+qm"(_fault) inoutclob ); \
473 _fault ? X86EMUL_UNHANDLEABLE : X86EMUL_CONTINUE; \
476 static int emulator_check_intercept(struct x86_emulate_ctxt *ctxt,
477 enum x86_intercept intercept,
478 enum x86_intercept_stage stage)
480 struct x86_instruction_info info = {
481 .intercept = intercept,
482 .rep_prefix = ctxt->rep_prefix,
483 .modrm_mod = ctxt->modrm_mod,
484 .modrm_reg = ctxt->modrm_reg,
485 .modrm_rm = ctxt->modrm_rm,
486 .src_val = ctxt->src.val64,
487 .dst_val = ctxt->dst.val64,
488 .src_bytes = ctxt->src.bytes,
489 .dst_bytes = ctxt->dst.bytes,
490 .ad_bytes = ctxt->ad_bytes,
491 .next_rip = ctxt->eip,
494 return ctxt->ops->intercept(ctxt, &info, stage);
497 static void assign_masked(ulong *dest, ulong src, ulong mask)
499 *dest = (*dest & ~mask) | (src & mask);
502 static void assign_register(unsigned long *reg, u64 val, int bytes)
504 /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */
507 *(u8 *)reg = (u8)val;
510 *(u16 *)reg = (u16)val;
514 break; /* 64b: zero-extend */
521 static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt)
523 return (1UL << (ctxt->ad_bytes << 3)) - 1;
526 static ulong stack_mask(struct x86_emulate_ctxt *ctxt)
529 struct desc_struct ss;
531 if (ctxt->mode == X86EMUL_MODE_PROT64)
533 ctxt->ops->get_segment(ctxt, &sel, &ss, NULL, VCPU_SREG_SS);
534 return ~0U >> ((ss.d ^ 1) * 16); /* d=0: 0xffff; d=1: 0xffffffff */
537 static int stack_size(struct x86_emulate_ctxt *ctxt)
539 return (__fls(stack_mask(ctxt)) + 1) >> 3;
542 /* Access/update address held in a register, based on addressing mode. */
543 static inline unsigned long
544 address_mask(struct x86_emulate_ctxt *ctxt, unsigned long reg)
546 if (ctxt->ad_bytes == sizeof(unsigned long))
549 return reg & ad_mask(ctxt);
552 static inline unsigned long
553 register_address(struct x86_emulate_ctxt *ctxt, int reg)
555 return address_mask(ctxt, reg_read(ctxt, reg));
558 static void masked_increment(ulong *reg, ulong mask, int inc)
560 assign_masked(reg, *reg + inc, mask);
564 register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc)
566 ulong *preg = reg_rmw(ctxt, reg);
568 assign_register(preg, *preg + inc, ctxt->ad_bytes);
571 static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
573 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
576 static u32 desc_limit_scaled(struct desc_struct *desc)
578 u32 limit = get_desc_limit(desc);
580 return desc->g ? (limit << 12) | 0xfff : limit;
583 static unsigned long seg_base(struct x86_emulate_ctxt *ctxt, int seg)
585 if (ctxt->mode == X86EMUL_MODE_PROT64 && seg < VCPU_SREG_FS)
588 return ctxt->ops->get_cached_segment_base(ctxt, seg);
591 static int emulate_exception(struct x86_emulate_ctxt *ctxt, int vec,
592 u32 error, bool valid)
595 ctxt->exception.vector = vec;
596 ctxt->exception.error_code = error;
597 ctxt->exception.error_code_valid = valid;
598 return X86EMUL_PROPAGATE_FAULT;
601 static int emulate_db(struct x86_emulate_ctxt *ctxt)
603 return emulate_exception(ctxt, DB_VECTOR, 0, false);
606 static int emulate_gp(struct x86_emulate_ctxt *ctxt, int err)
608 return emulate_exception(ctxt, GP_VECTOR, err, true);
611 static int emulate_ss(struct x86_emulate_ctxt *ctxt, int err)
613 return emulate_exception(ctxt, SS_VECTOR, err, true);
616 static int emulate_ud(struct x86_emulate_ctxt *ctxt)
618 return emulate_exception(ctxt, UD_VECTOR, 0, false);
621 static int emulate_ts(struct x86_emulate_ctxt *ctxt, int err)
623 return emulate_exception(ctxt, TS_VECTOR, err, true);
626 static int emulate_de(struct x86_emulate_ctxt *ctxt)
628 return emulate_exception(ctxt, DE_VECTOR, 0, false);
631 static int emulate_nm(struct x86_emulate_ctxt *ctxt)
633 return emulate_exception(ctxt, NM_VECTOR, 0, false);
636 static u16 get_segment_selector(struct x86_emulate_ctxt *ctxt, unsigned seg)
639 struct desc_struct desc;
641 ctxt->ops->get_segment(ctxt, &selector, &desc, NULL, seg);
645 static void set_segment_selector(struct x86_emulate_ctxt *ctxt, u16 selector,
650 struct desc_struct desc;
652 ctxt->ops->get_segment(ctxt, &dummy, &desc, &base3, seg);
653 ctxt->ops->set_segment(ctxt, selector, &desc, base3, seg);
657 * x86 defines three classes of vector instructions: explicitly
658 * aligned, explicitly unaligned, and the rest, which change behaviour
659 * depending on whether they're AVX encoded or not.
661 * Also included is CMPXCHG16B which is not a vector instruction, yet it is
662 * subject to the same check. FXSAVE and FXRSTOR are checked here too as their
663 * 512 bytes of data must be aligned to a 16 byte boundary.
665 static unsigned insn_alignment(struct x86_emulate_ctxt *ctxt, unsigned size)
667 u64 alignment = ctxt->d & AlignMask;
669 if (likely(size < 16))
684 static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt,
685 struct segmented_address addr,
686 unsigned *max_size, unsigned size,
687 bool write, bool fetch,
688 enum x86emul_mode mode, ulong *linear)
690 struct desc_struct desc;
697 la = seg_base(ctxt, addr.seg) + addr.ea;
700 case X86EMUL_MODE_PROT64:
702 va_bits = ctxt_virt_addr_bits(ctxt);
703 if (get_canonical(la, va_bits) != la)
706 *max_size = min_t(u64, ~0u, (1ull << va_bits) - la);
707 if (size > *max_size)
711 *linear = la = (u32)la;
712 usable = ctxt->ops->get_segment(ctxt, &sel, &desc, NULL,
716 /* code segment in protected mode or read-only data segment */
717 if ((((ctxt->mode != X86EMUL_MODE_REAL) && (desc.type & 8))
718 || !(desc.type & 2)) && write)
720 /* unreadable code segment */
721 if (!fetch && (desc.type & 8) && !(desc.type & 2))
723 lim = desc_limit_scaled(&desc);
724 if (!(desc.type & 8) && (desc.type & 4)) {
725 /* expand-down segment */
728 lim = desc.d ? 0xffffffff : 0xffff;
732 if (lim == 0xffffffff)
735 *max_size = (u64)lim + 1 - addr.ea;
736 if (size > *max_size)
741 if (la & (insn_alignment(ctxt, size) - 1))
742 return emulate_gp(ctxt, 0);
743 return X86EMUL_CONTINUE;
745 if (addr.seg == VCPU_SREG_SS)
746 return emulate_ss(ctxt, 0);
748 return emulate_gp(ctxt, 0);
751 static int linearize(struct x86_emulate_ctxt *ctxt,
752 struct segmented_address addr,
753 unsigned size, bool write,
757 return __linearize(ctxt, addr, &max_size, size, write, false,
761 static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst,
762 enum x86emul_mode mode)
767 struct segmented_address addr = { .seg = VCPU_SREG_CS,
770 if (ctxt->op_bytes != sizeof(unsigned long))
771 addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1);
772 rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear);
773 if (rc == X86EMUL_CONTINUE)
774 ctxt->_eip = addr.ea;
778 static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst)
780 return assign_eip(ctxt, dst, ctxt->mode);
783 static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst,
784 const struct desc_struct *cs_desc)
786 enum x86emul_mode mode = ctxt->mode;
790 if (ctxt->mode >= X86EMUL_MODE_PROT16) {
794 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
796 mode = X86EMUL_MODE_PROT64;
798 mode = X86EMUL_MODE_PROT32; /* temporary value */
801 if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32)
802 mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
803 rc = assign_eip(ctxt, dst, mode);
804 if (rc == X86EMUL_CONTINUE)
809 static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
811 return assign_eip_near(ctxt, ctxt->_eip + rel);
814 static int linear_read_system(struct x86_emulate_ctxt *ctxt, ulong linear,
815 void *data, unsigned size)
817 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, true);
820 static int linear_write_system(struct x86_emulate_ctxt *ctxt,
821 ulong linear, void *data,
824 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, true);
827 static int segmented_read_std(struct x86_emulate_ctxt *ctxt,
828 struct segmented_address addr,
835 rc = linearize(ctxt, addr, size, false, &linear);
836 if (rc != X86EMUL_CONTINUE)
838 return ctxt->ops->read_std(ctxt, linear, data, size, &ctxt->exception, false);
841 static int segmented_write_std(struct x86_emulate_ctxt *ctxt,
842 struct segmented_address addr,
849 rc = linearize(ctxt, addr, size, true, &linear);
850 if (rc != X86EMUL_CONTINUE)
852 return ctxt->ops->write_std(ctxt, linear, data, size, &ctxt->exception, false);
856 * Prefetch the remaining bytes of the instruction without crossing page
857 * boundary if they are not in fetch_cache yet.
859 static int __do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt, int op_size)
862 unsigned size, max_size;
863 unsigned long linear;
864 int cur_size = ctxt->fetch.end - ctxt->fetch.data;
865 struct segmented_address addr = { .seg = VCPU_SREG_CS,
866 .ea = ctxt->eip + cur_size };
869 * We do not know exactly how many bytes will be needed, and
870 * __linearize is expensive, so fetch as much as possible. We
871 * just have to avoid going beyond the 15 byte limit, the end
872 * of the segment, or the end of the page.
874 * __linearize is called with size 0 so that it does not do any
875 * boundary check itself. Instead, we use max_size to check
878 rc = __linearize(ctxt, addr, &max_size, 0, false, true, ctxt->mode,
880 if (unlikely(rc != X86EMUL_CONTINUE))
883 size = min_t(unsigned, 15UL ^ cur_size, max_size);
884 size = min_t(unsigned, size, PAGE_SIZE - offset_in_page(linear));
887 * One instruction can only straddle two pages,
888 * and one has been loaded at the beginning of
889 * x86_decode_insn. So, if not enough bytes
890 * still, we must have hit the 15-byte boundary.
892 if (unlikely(size < op_size))
893 return emulate_gp(ctxt, 0);
895 rc = ctxt->ops->fetch(ctxt, linear, ctxt->fetch.end,
896 size, &ctxt->exception);
897 if (unlikely(rc != X86EMUL_CONTINUE))
899 ctxt->fetch.end += size;
900 return X86EMUL_CONTINUE;
903 static __always_inline int do_insn_fetch_bytes(struct x86_emulate_ctxt *ctxt,
906 unsigned done_size = ctxt->fetch.end - ctxt->fetch.ptr;
908 if (unlikely(done_size < size))
909 return __do_insn_fetch_bytes(ctxt, size - done_size);
911 return X86EMUL_CONTINUE;
914 /* Fetch next part of the instruction being emulated. */
915 #define insn_fetch(_type, _ctxt) \
918 rc = do_insn_fetch_bytes(_ctxt, sizeof(_type)); \
919 if (rc != X86EMUL_CONTINUE) \
921 ctxt->_eip += sizeof(_type); \
922 memcpy(&_x, ctxt->fetch.ptr, sizeof(_type)); \
923 ctxt->fetch.ptr += sizeof(_type); \
927 #define insn_fetch_arr(_arr, _size, _ctxt) \
929 rc = do_insn_fetch_bytes(_ctxt, _size); \
930 if (rc != X86EMUL_CONTINUE) \
932 ctxt->_eip += (_size); \
933 memcpy(_arr, ctxt->fetch.ptr, _size); \
934 ctxt->fetch.ptr += (_size); \
938 * Given the 'reg' portion of a ModRM byte, and a register block, return a
939 * pointer into the block that addresses the relevant register.
940 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
942 static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
946 int highbyte_regs = (ctxt->rex_prefix == 0) && byteop;
948 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
949 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
951 p = reg_rmw(ctxt, modrm_reg);
955 static int read_descriptor(struct x86_emulate_ctxt *ctxt,
956 struct segmented_address addr,
957 u16 *size, unsigned long *address, int op_bytes)
964 rc = segmented_read_std(ctxt, addr, size, 2);
965 if (rc != X86EMUL_CONTINUE)
968 rc = segmented_read_std(ctxt, addr, address, op_bytes);
982 FASTOP1SRC2(mul, mul_ex);
983 FASTOP1SRC2(imul, imul_ex);
984 FASTOP1SRC2EX(div, div_ex);
985 FASTOP1SRC2EX(idiv, idiv_ex);
1014 FASTOP2R(cmp, cmp_r);
1016 static int em_bsf_c(struct x86_emulate_ctxt *ctxt)
1018 /* If src is zero, do not writeback, but update flags */
1019 if (ctxt->src.val == 0)
1020 ctxt->dst.type = OP_NONE;
1021 return fastop(ctxt, em_bsf);
1024 static int em_bsr_c(struct x86_emulate_ctxt *ctxt)
1026 /* If src is zero, do not writeback, but update flags */
1027 if (ctxt->src.val == 0)
1028 ctxt->dst.type = OP_NONE;
1029 return fastop(ctxt, em_bsr);
1032 static __always_inline u8 test_cc(unsigned int condition, unsigned long flags)
1035 void (*fop)(void) = (void *)em_setcc + 4 * (condition & 0xf);
1037 flags = (flags & EFLAGS_MASK) | X86_EFLAGS_IF;
1038 asm("push %[flags]; popf; " CALL_NOSPEC
1039 : "=a"(rc) : [thunk_target]"r"(fop), [flags]"r"(flags));
1043 static void fetch_register_operand(struct operand *op)
1045 switch (op->bytes) {
1047 op->val = *(u8 *)op->addr.reg;
1050 op->val = *(u16 *)op->addr.reg;
1053 op->val = *(u32 *)op->addr.reg;
1056 op->val = *(u64 *)op->addr.reg;
1061 static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
1063 ctxt->ops->get_fpu(ctxt);
1065 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
1066 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
1067 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
1068 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
1069 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
1070 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
1071 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
1072 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
1073 #ifdef CONFIG_X86_64
1074 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
1075 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
1076 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
1077 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
1078 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
1079 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
1080 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
1081 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
1085 ctxt->ops->put_fpu(ctxt);
1088 static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
1091 ctxt->ops->get_fpu(ctxt);
1093 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
1094 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
1095 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
1096 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
1097 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
1098 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
1099 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
1100 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
1101 #ifdef CONFIG_X86_64
1102 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
1103 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
1104 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
1105 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
1106 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
1107 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
1108 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
1109 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
1113 ctxt->ops->put_fpu(ctxt);
1116 static void read_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1118 ctxt->ops->get_fpu(ctxt);
1120 case 0: asm("movq %%mm0, %0" : "=m"(*data)); break;
1121 case 1: asm("movq %%mm1, %0" : "=m"(*data)); break;
1122 case 2: asm("movq %%mm2, %0" : "=m"(*data)); break;
1123 case 3: asm("movq %%mm3, %0" : "=m"(*data)); break;
1124 case 4: asm("movq %%mm4, %0" : "=m"(*data)); break;
1125 case 5: asm("movq %%mm5, %0" : "=m"(*data)); break;
1126 case 6: asm("movq %%mm6, %0" : "=m"(*data)); break;
1127 case 7: asm("movq %%mm7, %0" : "=m"(*data)); break;
1130 ctxt->ops->put_fpu(ctxt);
1133 static void write_mmx_reg(struct x86_emulate_ctxt *ctxt, u64 *data, int reg)
1135 ctxt->ops->get_fpu(ctxt);
1137 case 0: asm("movq %0, %%mm0" : : "m"(*data)); break;
1138 case 1: asm("movq %0, %%mm1" : : "m"(*data)); break;
1139 case 2: asm("movq %0, %%mm2" : : "m"(*data)); break;
1140 case 3: asm("movq %0, %%mm3" : : "m"(*data)); break;
1141 case 4: asm("movq %0, %%mm4" : : "m"(*data)); break;
1142 case 5: asm("movq %0, %%mm5" : : "m"(*data)); break;
1143 case 6: asm("movq %0, %%mm6" : : "m"(*data)); break;
1144 case 7: asm("movq %0, %%mm7" : : "m"(*data)); break;
1147 ctxt->ops->put_fpu(ctxt);
1150 static int em_fninit(struct x86_emulate_ctxt *ctxt)
1152 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1153 return emulate_nm(ctxt);
1155 ctxt->ops->get_fpu(ctxt);
1156 asm volatile("fninit");
1157 ctxt->ops->put_fpu(ctxt);
1158 return X86EMUL_CONTINUE;
1161 static int em_fnstcw(struct x86_emulate_ctxt *ctxt)
1165 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1166 return emulate_nm(ctxt);
1168 ctxt->ops->get_fpu(ctxt);
1169 asm volatile("fnstcw %0": "+m"(fcw));
1170 ctxt->ops->put_fpu(ctxt);
1172 ctxt->dst.val = fcw;
1174 return X86EMUL_CONTINUE;
1177 static int em_fnstsw(struct x86_emulate_ctxt *ctxt)
1181 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
1182 return emulate_nm(ctxt);
1184 ctxt->ops->get_fpu(ctxt);
1185 asm volatile("fnstsw %0": "+m"(fsw));
1186 ctxt->ops->put_fpu(ctxt);
1188 ctxt->dst.val = fsw;
1190 return X86EMUL_CONTINUE;
1193 static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
1196 unsigned reg = ctxt->modrm_reg;
1198 if (!(ctxt->d & ModRM))
1199 reg = (ctxt->b & 7) | ((ctxt->rex_prefix & 1) << 3);
1201 if (ctxt->d & Sse) {
1205 read_sse_reg(ctxt, &op->vec_val, reg);
1208 if (ctxt->d & Mmx) {
1217 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1218 op->addr.reg = decode_register(ctxt, reg, ctxt->d & ByteOp);
1220 fetch_register_operand(op);
1221 op->orig_val = op->val;
1224 static void adjust_modrm_seg(struct x86_emulate_ctxt *ctxt, int base_reg)
1226 if (base_reg == VCPU_REGS_RSP || base_reg == VCPU_REGS_RBP)
1227 ctxt->modrm_seg = VCPU_SREG_SS;
1230 static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1234 int index_reg, base_reg, scale;
1235 int rc = X86EMUL_CONTINUE;
1238 ctxt->modrm_reg = ((ctxt->rex_prefix << 1) & 8); /* REX.R */
1239 index_reg = (ctxt->rex_prefix << 2) & 8; /* REX.X */
1240 base_reg = (ctxt->rex_prefix << 3) & 8; /* REX.B */
1242 ctxt->modrm_mod = (ctxt->modrm & 0xc0) >> 6;
1243 ctxt->modrm_reg |= (ctxt->modrm & 0x38) >> 3;
1244 ctxt->modrm_rm = base_reg | (ctxt->modrm & 0x07);
1245 ctxt->modrm_seg = VCPU_SREG_DS;
1247 if (ctxt->modrm_mod == 3 || (ctxt->d & NoMod)) {
1249 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1250 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm,
1252 if (ctxt->d & Sse) {
1255 op->addr.xmm = ctxt->modrm_rm;
1256 read_sse_reg(ctxt, &op->vec_val, ctxt->modrm_rm);
1259 if (ctxt->d & Mmx) {
1262 op->addr.mm = ctxt->modrm_rm & 7;
1265 fetch_register_operand(op);
1271 if (ctxt->ad_bytes == 2) {
1272 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1273 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1274 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1275 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1277 /* 16-bit ModR/M decode. */
1278 switch (ctxt->modrm_mod) {
1280 if (ctxt->modrm_rm == 6)
1281 modrm_ea += insn_fetch(u16, ctxt);
1284 modrm_ea += insn_fetch(s8, ctxt);
1287 modrm_ea += insn_fetch(u16, ctxt);
1290 switch (ctxt->modrm_rm) {
1292 modrm_ea += bx + si;
1295 modrm_ea += bx + di;
1298 modrm_ea += bp + si;
1301 modrm_ea += bp + di;
1310 if (ctxt->modrm_mod != 0)
1317 if (ctxt->modrm_rm == 2 || ctxt->modrm_rm == 3 ||
1318 (ctxt->modrm_rm == 6 && ctxt->modrm_mod != 0))
1319 ctxt->modrm_seg = VCPU_SREG_SS;
1320 modrm_ea = (u16)modrm_ea;
1322 /* 32/64-bit ModR/M decode. */
1323 if ((ctxt->modrm_rm & 7) == 4) {
1324 sib = insn_fetch(u8, ctxt);
1325 index_reg |= (sib >> 3) & 7;
1326 base_reg |= sib & 7;
1329 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1330 modrm_ea += insn_fetch(s32, ctxt);
1332 modrm_ea += reg_read(ctxt, base_reg);
1333 adjust_modrm_seg(ctxt, base_reg);
1334 /* Increment ESP on POP [ESP] */
1335 if ((ctxt->d & IncSP) &&
1336 base_reg == VCPU_REGS_RSP)
1337 modrm_ea += ctxt->op_bytes;
1340 modrm_ea += reg_read(ctxt, index_reg) << scale;
1341 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1342 modrm_ea += insn_fetch(s32, ctxt);
1343 if (ctxt->mode == X86EMUL_MODE_PROT64)
1344 ctxt->rip_relative = 1;
1346 base_reg = ctxt->modrm_rm;
1347 modrm_ea += reg_read(ctxt, base_reg);
1348 adjust_modrm_seg(ctxt, base_reg);
1350 switch (ctxt->modrm_mod) {
1352 modrm_ea += insn_fetch(s8, ctxt);
1355 modrm_ea += insn_fetch(s32, ctxt);
1359 op->addr.mem.ea = modrm_ea;
1360 if (ctxt->ad_bytes != 8)
1361 ctxt->memop.addr.mem.ea = (u32)ctxt->memop.addr.mem.ea;
1367 static int decode_abs(struct x86_emulate_ctxt *ctxt,
1370 int rc = X86EMUL_CONTINUE;
1373 switch (ctxt->ad_bytes) {
1375 op->addr.mem.ea = insn_fetch(u16, ctxt);
1378 op->addr.mem.ea = insn_fetch(u32, ctxt);
1381 op->addr.mem.ea = insn_fetch(u64, ctxt);
1388 static void fetch_bit_operand(struct x86_emulate_ctxt *ctxt)
1392 if (ctxt->dst.type == OP_MEM && ctxt->src.type == OP_REG) {
1393 mask = ~((long)ctxt->dst.bytes * 8 - 1);
1395 if (ctxt->src.bytes == 2)
1396 sv = (s16)ctxt->src.val & (s16)mask;
1397 else if (ctxt->src.bytes == 4)
1398 sv = (s32)ctxt->src.val & (s32)mask;
1400 sv = (s64)ctxt->src.val & (s64)mask;
1402 ctxt->dst.addr.mem.ea = address_mask(ctxt,
1403 ctxt->dst.addr.mem.ea + (sv >> 3));
1406 /* only subword offset */
1407 ctxt->src.val &= (ctxt->dst.bytes << 3) - 1;
1410 static int read_emulated(struct x86_emulate_ctxt *ctxt,
1411 unsigned long addr, void *dest, unsigned size)
1414 struct read_cache *mc = &ctxt->mem_read;
1416 if (mc->pos < mc->end)
1419 WARN_ON((mc->end + size) >= sizeof(mc->data));
1421 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1423 if (rc != X86EMUL_CONTINUE)
1429 memcpy(dest, mc->data + mc->pos, size);
1431 return X86EMUL_CONTINUE;
1434 static int segmented_read(struct x86_emulate_ctxt *ctxt,
1435 struct segmented_address addr,
1442 rc = linearize(ctxt, addr, size, false, &linear);
1443 if (rc != X86EMUL_CONTINUE)
1445 return read_emulated(ctxt, linear, data, size);
1448 static int segmented_write(struct x86_emulate_ctxt *ctxt,
1449 struct segmented_address addr,
1456 rc = linearize(ctxt, addr, size, true, &linear);
1457 if (rc != X86EMUL_CONTINUE)
1459 return ctxt->ops->write_emulated(ctxt, linear, data, size,
1463 static int segmented_cmpxchg(struct x86_emulate_ctxt *ctxt,
1464 struct segmented_address addr,
1465 const void *orig_data, const void *data,
1471 rc = linearize(ctxt, addr, size, true, &linear);
1472 if (rc != X86EMUL_CONTINUE)
1474 return ctxt->ops->cmpxchg_emulated(ctxt, linear, orig_data, data,
1475 size, &ctxt->exception);
1478 static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1479 unsigned int size, unsigned short port,
1482 struct read_cache *rc = &ctxt->io_read;
1484 if (rc->pos == rc->end) { /* refill pio read ahead */
1485 unsigned int in_page, n;
1486 unsigned int count = ctxt->rep_prefix ?
1487 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1488 in_page = (ctxt->eflags & X86_EFLAGS_DF) ?
1489 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1490 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1491 n = min3(in_page, (unsigned int)sizeof(rc->data) / size, count);
1494 rc->pos = rc->end = 0;
1495 if (!ctxt->ops->pio_in_emulated(ctxt, size, port, rc->data, n))
1500 if (ctxt->rep_prefix && (ctxt->d & String) &&
1501 !(ctxt->eflags & X86_EFLAGS_DF)) {
1502 ctxt->dst.data = rc->data + rc->pos;
1503 ctxt->dst.type = OP_MEM_STR;
1504 ctxt->dst.count = (rc->end - rc->pos) / size;
1507 memcpy(dest, rc->data + rc->pos, size);
1513 static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1514 u16 index, struct desc_struct *desc)
1519 ctxt->ops->get_idt(ctxt, &dt);
1521 if (dt.size < index * 8 + 7)
1522 return emulate_gp(ctxt, index << 3 | 0x2);
1524 addr = dt.address + index * 8;
1525 return linear_read_system(ctxt, addr, desc, sizeof *desc);
1528 static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1529 u16 selector, struct desc_ptr *dt)
1531 const struct x86_emulate_ops *ops = ctxt->ops;
1534 if (selector & 1 << 2) {
1535 struct desc_struct desc;
1538 memset (dt, 0, sizeof *dt);
1539 if (!ops->get_segment(ctxt, &sel, &desc, &base3,
1543 dt->size = desc_limit_scaled(&desc); /* what if limit > 65535? */
1544 dt->address = get_desc_base(&desc) | ((u64)base3 << 32);
1546 ops->get_gdt(ctxt, dt);
1549 static int get_descriptor_ptr(struct x86_emulate_ctxt *ctxt,
1550 u16 selector, ulong *desc_addr_p)
1553 u16 index = selector >> 3;
1556 get_descriptor_table_ptr(ctxt, selector, &dt);
1558 if (dt.size < index * 8 + 7)
1559 return emulate_gp(ctxt, selector & 0xfffc);
1561 addr = dt.address + index * 8;
1563 #ifdef CONFIG_X86_64
1564 if (addr >> 32 != 0) {
1567 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1568 if (!(efer & EFER_LMA))
1573 *desc_addr_p = addr;
1574 return X86EMUL_CONTINUE;
1577 /* allowed just for 8 bytes segments */
1578 static int read_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1579 u16 selector, struct desc_struct *desc,
1584 rc = get_descriptor_ptr(ctxt, selector, desc_addr_p);
1585 if (rc != X86EMUL_CONTINUE)
1588 return linear_read_system(ctxt, *desc_addr_p, desc, sizeof(*desc));
1591 /* allowed just for 8 bytes segments */
1592 static int write_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1593 u16 selector, struct desc_struct *desc)
1598 rc = get_descriptor_ptr(ctxt, selector, &addr);
1599 if (rc != X86EMUL_CONTINUE)
1602 return linear_write_system(ctxt, addr, desc, sizeof *desc);
1605 static int __load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1606 u16 selector, int seg, u8 cpl,
1607 enum x86_transfer_type transfer,
1608 struct desc_struct *desc)
1610 struct desc_struct seg_desc, old_desc;
1612 unsigned err_vec = GP_VECTOR;
1614 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1620 memset(&seg_desc, 0, sizeof seg_desc);
1622 if (ctxt->mode == X86EMUL_MODE_REAL) {
1623 /* set real mode segment descriptor (keep limit etc. for
1625 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1626 set_desc_base(&seg_desc, selector << 4);
1628 } else if (seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) {
1629 /* VM86 needs a clean new segment descriptor */
1630 set_desc_base(&seg_desc, selector << 4);
1631 set_desc_limit(&seg_desc, 0xffff);
1641 /* TR should be in GDT only */
1642 if (seg == VCPU_SREG_TR && (selector & (1 << 2)))
1645 /* NULL selector is not valid for TR, CS and (except for long mode) SS */
1646 if (null_selector) {
1647 if (seg == VCPU_SREG_CS || seg == VCPU_SREG_TR)
1650 if (seg == VCPU_SREG_SS) {
1651 if (ctxt->mode != X86EMUL_MODE_PROT64 || rpl != cpl)
1655 * ctxt->ops->set_segment expects the CPL to be in
1656 * SS.DPL, so fake an expand-up 32-bit data segment.
1666 /* Skip all following checks */
1670 ret = read_segment_descriptor(ctxt, selector, &seg_desc, &desc_addr);
1671 if (ret != X86EMUL_CONTINUE)
1674 err_code = selector & 0xfffc;
1675 err_vec = (transfer == X86_TRANSFER_TASK_SWITCH) ? TS_VECTOR :
1678 /* can't load system descriptor into segment selector */
1679 if (seg <= VCPU_SREG_GS && !seg_desc.s) {
1680 if (transfer == X86_TRANSFER_CALL_JMP)
1681 return X86EMUL_UNHANDLEABLE;
1686 err_vec = (seg == VCPU_SREG_SS) ? SS_VECTOR : NP_VECTOR;
1695 * segment is not a writable data segment or segment
1696 * selector's RPL != CPL or segment selector's RPL != CPL
1698 if (rpl != cpl || (seg_desc.type & 0xa) != 0x2 || dpl != cpl)
1702 if (!(seg_desc.type & 8))
1705 if (seg_desc.type & 4) {
1711 if (rpl > cpl || dpl != cpl)
1714 /* in long-mode d/b must be clear if l is set */
1715 if (seg_desc.d && seg_desc.l) {
1718 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
1719 if (efer & EFER_LMA)
1723 /* CS(RPL) <- CPL */
1724 selector = (selector & 0xfffc) | cpl;
1727 if (seg_desc.s || (seg_desc.type != 1 && seg_desc.type != 9))
1729 old_desc = seg_desc;
1730 seg_desc.type |= 2; /* busy */
1731 ret = ctxt->ops->cmpxchg_emulated(ctxt, desc_addr, &old_desc, &seg_desc,
1732 sizeof(seg_desc), &ctxt->exception);
1733 if (ret != X86EMUL_CONTINUE)
1736 case VCPU_SREG_LDTR:
1737 if (seg_desc.s || seg_desc.type != 2)
1740 default: /* DS, ES, FS, or GS */
1742 * segment is not a data or readable code segment or
1743 * ((segment is a data or nonconforming code segment)
1744 * and (both RPL and CPL > DPL))
1746 if ((seg_desc.type & 0xa) == 0x8 ||
1747 (((seg_desc.type & 0xc) != 0xc) &&
1748 (rpl > dpl && cpl > dpl)))
1754 /* mark segment as accessed */
1755 if (!(seg_desc.type & 1)) {
1757 ret = write_segment_descriptor(ctxt, selector,
1759 if (ret != X86EMUL_CONTINUE)
1762 } else if (ctxt->mode == X86EMUL_MODE_PROT64) {
1763 ret = linear_read_system(ctxt, desc_addr+8, &base3, sizeof(base3));
1764 if (ret != X86EMUL_CONTINUE)
1766 if (emul_is_noncanonical_address(get_desc_base(&seg_desc) |
1767 ((u64)base3 << 32), ctxt))
1768 return emulate_gp(ctxt, 0);
1771 ctxt->ops->set_segment(ctxt, selector, &seg_desc, base3, seg);
1774 return X86EMUL_CONTINUE;
1776 return emulate_exception(ctxt, err_vec, err_code, true);
1779 static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1780 u16 selector, int seg)
1782 u8 cpl = ctxt->ops->cpl(ctxt);
1785 * None of MOV, POP and LSS can load a NULL selector in CPL=3, but
1786 * they can load it at CPL<3 (Intel's manual says only LSS can,
1789 * However, the Intel manual says that putting IST=1/DPL=3 in
1790 * an interrupt gate will result in SS=3 (the AMD manual instead
1791 * says it doesn't), so allow SS=3 in __load_segment_descriptor
1792 * and only forbid it here.
1794 if (seg == VCPU_SREG_SS && selector == 3 &&
1795 ctxt->mode == X86EMUL_MODE_PROT64)
1796 return emulate_exception(ctxt, GP_VECTOR, 0, true);
1798 return __load_segment_descriptor(ctxt, selector, seg, cpl,
1799 X86_TRANSFER_NONE, NULL);
1802 static void write_register_operand(struct operand *op)
1804 return assign_register(op->addr.reg, op->val, op->bytes);
1807 static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op)
1811 write_register_operand(op);
1814 if (ctxt->lock_prefix)
1815 return segmented_cmpxchg(ctxt,
1821 return segmented_write(ctxt,
1827 return segmented_write(ctxt,
1830 op->bytes * op->count);
1833 write_sse_reg(ctxt, &op->vec_val, op->addr.xmm);
1836 write_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
1844 return X86EMUL_CONTINUE;
1847 static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1849 struct segmented_address addr;
1851 rsp_increment(ctxt, -bytes);
1852 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1853 addr.seg = VCPU_SREG_SS;
1855 return segmented_write(ctxt, addr, data, bytes);
1858 static int em_push(struct x86_emulate_ctxt *ctxt)
1860 /* Disable writeback. */
1861 ctxt->dst.type = OP_NONE;
1862 return push(ctxt, &ctxt->src.val, ctxt->op_bytes);
1865 static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1866 void *dest, int len)
1869 struct segmented_address addr;
1871 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1872 addr.seg = VCPU_SREG_SS;
1873 rc = segmented_read(ctxt, addr, dest, len);
1874 if (rc != X86EMUL_CONTINUE)
1877 rsp_increment(ctxt, len);
1881 static int em_pop(struct x86_emulate_ctxt *ctxt)
1883 return emulate_pop(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1886 static int emulate_popf(struct x86_emulate_ctxt *ctxt,
1887 void *dest, int len)
1890 unsigned long val, change_mask;
1891 int iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
1892 int cpl = ctxt->ops->cpl(ctxt);
1894 rc = emulate_pop(ctxt, &val, len);
1895 if (rc != X86EMUL_CONTINUE)
1898 change_mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
1899 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF |
1900 X86_EFLAGS_TF | X86_EFLAGS_DF | X86_EFLAGS_NT |
1901 X86_EFLAGS_AC | X86_EFLAGS_ID;
1903 switch(ctxt->mode) {
1904 case X86EMUL_MODE_PROT64:
1905 case X86EMUL_MODE_PROT32:
1906 case X86EMUL_MODE_PROT16:
1908 change_mask |= X86_EFLAGS_IOPL;
1910 change_mask |= X86_EFLAGS_IF;
1912 case X86EMUL_MODE_VM86:
1914 return emulate_gp(ctxt, 0);
1915 change_mask |= X86_EFLAGS_IF;
1917 default: /* real mode */
1918 change_mask |= (X86_EFLAGS_IOPL | X86_EFLAGS_IF);
1922 *(unsigned long *)dest =
1923 (ctxt->eflags & ~change_mask) | (val & change_mask);
1928 static int em_popf(struct x86_emulate_ctxt *ctxt)
1930 ctxt->dst.type = OP_REG;
1931 ctxt->dst.addr.reg = &ctxt->eflags;
1932 ctxt->dst.bytes = ctxt->op_bytes;
1933 return emulate_popf(ctxt, &ctxt->dst.val, ctxt->op_bytes);
1936 static int em_enter(struct x86_emulate_ctxt *ctxt)
1939 unsigned frame_size = ctxt->src.val;
1940 unsigned nesting_level = ctxt->src2.val & 31;
1944 return X86EMUL_UNHANDLEABLE;
1946 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1947 rc = push(ctxt, &rbp, stack_size(ctxt));
1948 if (rc != X86EMUL_CONTINUE)
1950 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1952 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1953 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1955 return X86EMUL_CONTINUE;
1958 static int em_leave(struct x86_emulate_ctxt *ctxt)
1960 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1962 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1965 static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
1967 int seg = ctxt->src2.val;
1969 ctxt->src.val = get_segment_selector(ctxt, seg);
1970 if (ctxt->op_bytes == 4) {
1971 rsp_increment(ctxt, -2);
1975 return em_push(ctxt);
1978 static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1980 int seg = ctxt->src2.val;
1981 unsigned long selector;
1984 rc = emulate_pop(ctxt, &selector, 2);
1985 if (rc != X86EMUL_CONTINUE)
1988 if (ctxt->modrm_reg == VCPU_SREG_SS)
1989 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
1990 if (ctxt->op_bytes > 2)
1991 rsp_increment(ctxt, ctxt->op_bytes - 2);
1993 rc = load_segment_descriptor(ctxt, (u16)selector, seg);
1997 static int em_pusha(struct x86_emulate_ctxt *ctxt)
1999 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
2000 int rc = X86EMUL_CONTINUE;
2001 int reg = VCPU_REGS_RAX;
2003 while (reg <= VCPU_REGS_RDI) {
2004 (reg == VCPU_REGS_RSP) ?
2005 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
2008 if (rc != X86EMUL_CONTINUE)
2017 static int em_pushf(struct x86_emulate_ctxt *ctxt)
2019 ctxt->src.val = (unsigned long)ctxt->eflags & ~X86_EFLAGS_VM;
2020 return em_push(ctxt);
2023 static int em_popa(struct x86_emulate_ctxt *ctxt)
2025 int rc = X86EMUL_CONTINUE;
2026 int reg = VCPU_REGS_RDI;
2029 while (reg >= VCPU_REGS_RAX) {
2030 if (reg == VCPU_REGS_RSP) {
2031 rsp_increment(ctxt, ctxt->op_bytes);
2035 rc = emulate_pop(ctxt, &val, ctxt->op_bytes);
2036 if (rc != X86EMUL_CONTINUE)
2038 assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes);
2044 static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2046 const struct x86_emulate_ops *ops = ctxt->ops;
2053 /* TODO: Add limit checks */
2054 ctxt->src.val = ctxt->eflags;
2056 if (rc != X86EMUL_CONTINUE)
2059 ctxt->eflags &= ~(X86_EFLAGS_IF | X86_EFLAGS_TF | X86_EFLAGS_AC);
2061 ctxt->src.val = get_segment_selector(ctxt, VCPU_SREG_CS);
2063 if (rc != X86EMUL_CONTINUE)
2066 ctxt->src.val = ctxt->_eip;
2068 if (rc != X86EMUL_CONTINUE)
2071 ops->get_idt(ctxt, &dt);
2073 eip_addr = dt.address + (irq << 2);
2074 cs_addr = dt.address + (irq << 2) + 2;
2076 rc = linear_read_system(ctxt, cs_addr, &cs, 2);
2077 if (rc != X86EMUL_CONTINUE)
2080 rc = linear_read_system(ctxt, eip_addr, &eip, 2);
2081 if (rc != X86EMUL_CONTINUE)
2084 rc = load_segment_descriptor(ctxt, cs, VCPU_SREG_CS);
2085 if (rc != X86EMUL_CONTINUE)
2093 int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
2097 invalidate_registers(ctxt);
2098 rc = __emulate_int_real(ctxt, irq);
2099 if (rc == X86EMUL_CONTINUE)
2100 writeback_registers(ctxt);
2104 static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
2106 switch(ctxt->mode) {
2107 case X86EMUL_MODE_REAL:
2108 return __emulate_int_real(ctxt, irq);
2109 case X86EMUL_MODE_VM86:
2110 case X86EMUL_MODE_PROT16:
2111 case X86EMUL_MODE_PROT32:
2112 case X86EMUL_MODE_PROT64:
2114 /* Protected mode interrupts unimplemented yet */
2115 return X86EMUL_UNHANDLEABLE;
2119 static int emulate_iret_real(struct x86_emulate_ctxt *ctxt)
2121 int rc = X86EMUL_CONTINUE;
2122 unsigned long temp_eip = 0;
2123 unsigned long temp_eflags = 0;
2124 unsigned long cs = 0;
2125 unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
2126 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF |
2127 X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF |
2128 X86_EFLAGS_IOPL | X86_EFLAGS_NT | X86_EFLAGS_RF |
2129 X86_EFLAGS_AC | X86_EFLAGS_ID |
2131 unsigned long vm86_mask = X86_EFLAGS_VM | X86_EFLAGS_VIF |
2134 /* TODO: Add stack limit check */
2136 rc = emulate_pop(ctxt, &temp_eip, ctxt->op_bytes);
2138 if (rc != X86EMUL_CONTINUE)
2141 if (temp_eip & ~0xffff)
2142 return emulate_gp(ctxt, 0);
2144 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2146 if (rc != X86EMUL_CONTINUE)
2149 rc = emulate_pop(ctxt, &temp_eflags, ctxt->op_bytes);
2151 if (rc != X86EMUL_CONTINUE)
2154 rc = load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS);
2156 if (rc != X86EMUL_CONTINUE)
2159 ctxt->_eip = temp_eip;
2161 if (ctxt->op_bytes == 4)
2162 ctxt->eflags = ((temp_eflags & mask) | (ctxt->eflags & vm86_mask));
2163 else if (ctxt->op_bytes == 2) {
2164 ctxt->eflags &= ~0xffff;
2165 ctxt->eflags |= temp_eflags;
2168 ctxt->eflags &= ~EFLG_RESERVED_ZEROS_MASK; /* Clear reserved zeros */
2169 ctxt->eflags |= X86_EFLAGS_FIXED;
2170 ctxt->ops->set_nmi_mask(ctxt, false);
2175 static int em_iret(struct x86_emulate_ctxt *ctxt)
2177 switch(ctxt->mode) {
2178 case X86EMUL_MODE_REAL:
2179 return emulate_iret_real(ctxt);
2180 case X86EMUL_MODE_VM86:
2181 case X86EMUL_MODE_PROT16:
2182 case X86EMUL_MODE_PROT32:
2183 case X86EMUL_MODE_PROT64:
2185 /* iret from protected mode unimplemented yet */
2186 return X86EMUL_UNHANDLEABLE;
2190 static int em_jmp_far(struct x86_emulate_ctxt *ctxt)
2194 struct desc_struct new_desc;
2195 u8 cpl = ctxt->ops->cpl(ctxt);
2197 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2199 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
2200 X86_TRANSFER_CALL_JMP,
2202 if (rc != X86EMUL_CONTINUE)
2205 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
2206 /* Error handling is not implemented. */
2207 if (rc != X86EMUL_CONTINUE)
2208 return X86EMUL_UNHANDLEABLE;
2213 static int em_jmp_abs(struct x86_emulate_ctxt *ctxt)
2215 return assign_eip_near(ctxt, ctxt->src.val);
2218 static int em_call_near_abs(struct x86_emulate_ctxt *ctxt)
2223 old_eip = ctxt->_eip;
2224 rc = assign_eip_near(ctxt, ctxt->src.val);
2225 if (rc != X86EMUL_CONTINUE)
2227 ctxt->src.val = old_eip;
2232 static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
2234 u64 old = ctxt->dst.orig_val64;
2236 if (ctxt->dst.bytes == 16)
2237 return X86EMUL_UNHANDLEABLE;
2239 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
2240 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
2241 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
2242 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
2243 ctxt->eflags &= ~X86_EFLAGS_ZF;
2245 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
2246 (u32) reg_read(ctxt, VCPU_REGS_RBX);
2248 ctxt->eflags |= X86_EFLAGS_ZF;
2250 return X86EMUL_CONTINUE;
2253 static int em_ret(struct x86_emulate_ctxt *ctxt)
2258 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2259 if (rc != X86EMUL_CONTINUE)
2262 return assign_eip_near(ctxt, eip);
2265 static int em_ret_far(struct x86_emulate_ctxt *ctxt)
2268 unsigned long eip, cs;
2269 int cpl = ctxt->ops->cpl(ctxt);
2270 struct desc_struct new_desc;
2272 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
2273 if (rc != X86EMUL_CONTINUE)
2275 rc = emulate_pop(ctxt, &cs, ctxt->op_bytes);
2276 if (rc != X86EMUL_CONTINUE)
2278 /* Outer-privilege level return is not implemented */
2279 if (ctxt->mode >= X86EMUL_MODE_PROT16 && (cs & 3) > cpl)
2280 return X86EMUL_UNHANDLEABLE;
2281 rc = __load_segment_descriptor(ctxt, (u16)cs, VCPU_SREG_CS, cpl,
2284 if (rc != X86EMUL_CONTINUE)
2286 rc = assign_eip_far(ctxt, eip, &new_desc);
2287 /* Error handling is not implemented. */
2288 if (rc != X86EMUL_CONTINUE)
2289 return X86EMUL_UNHANDLEABLE;
2294 static int em_ret_far_imm(struct x86_emulate_ctxt *ctxt)
2298 rc = em_ret_far(ctxt);
2299 if (rc != X86EMUL_CONTINUE)
2301 rsp_increment(ctxt, ctxt->src.val);
2302 return X86EMUL_CONTINUE;
2305 static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2307 /* Save real source value, then compare EAX against destination. */
2308 ctxt->dst.orig_val = ctxt->dst.val;
2309 ctxt->dst.val = reg_read(ctxt, VCPU_REGS_RAX);
2310 ctxt->src.orig_val = ctxt->src.val;
2311 ctxt->src.val = ctxt->dst.orig_val;
2312 fastop(ctxt, em_cmp);
2314 if (ctxt->eflags & X86_EFLAGS_ZF) {
2315 /* Success: write back to memory; no update of EAX */
2316 ctxt->src.type = OP_NONE;
2317 ctxt->dst.val = ctxt->src.orig_val;
2319 /* Failure: write the value we saw to EAX. */
2320 ctxt->src.type = OP_REG;
2321 ctxt->src.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2322 ctxt->src.val = ctxt->dst.orig_val;
2323 /* Create write-cycle to dest by writing the same value */
2324 ctxt->dst.val = ctxt->dst.orig_val;
2326 return X86EMUL_CONTINUE;
2329 static int em_lseg(struct x86_emulate_ctxt *ctxt)
2331 int seg = ctxt->src2.val;
2335 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
2337 rc = load_segment_descriptor(ctxt, sel, seg);
2338 if (rc != X86EMUL_CONTINUE)
2341 ctxt->dst.val = ctxt->src.val;
2345 static int emulator_has_longmode(struct x86_emulate_ctxt *ctxt)
2347 u32 eax, ebx, ecx, edx;
2351 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2352 return edx & bit(X86_FEATURE_LM);
2355 #define GET_SMSTATE(type, smbase, offset) \
2358 int r = ctxt->ops->read_phys(ctxt, smbase + offset, &__val, \
2360 if (r != X86EMUL_CONTINUE) \
2361 return X86EMUL_UNHANDLEABLE; \
2365 static void rsm_set_desc_flags(struct desc_struct *desc, u32 flags)
2367 desc->g = (flags >> 23) & 1;
2368 desc->d = (flags >> 22) & 1;
2369 desc->l = (flags >> 21) & 1;
2370 desc->avl = (flags >> 20) & 1;
2371 desc->p = (flags >> 15) & 1;
2372 desc->dpl = (flags >> 13) & 3;
2373 desc->s = (flags >> 12) & 1;
2374 desc->type = (flags >> 8) & 15;
2377 static int rsm_load_seg_32(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2379 struct desc_struct desc;
2383 selector = GET_SMSTATE(u32, smbase, 0x7fa8 + n * 4);
2386 offset = 0x7f84 + n * 12;
2388 offset = 0x7f2c + (n - 3) * 12;
2390 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2391 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2392 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, offset));
2393 ctxt->ops->set_segment(ctxt, selector, &desc, 0, n);
2394 return X86EMUL_CONTINUE;
2397 static int rsm_load_seg_64(struct x86_emulate_ctxt *ctxt, u64 smbase, int n)
2399 struct desc_struct desc;
2404 offset = 0x7e00 + n * 16;
2406 selector = GET_SMSTATE(u16, smbase, offset);
2407 rsm_set_desc_flags(&desc, GET_SMSTATE(u16, smbase, offset + 2) << 8);
2408 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, offset + 4));
2409 set_desc_base(&desc, GET_SMSTATE(u32, smbase, offset + 8));
2410 base3 = GET_SMSTATE(u32, smbase, offset + 12);
2412 ctxt->ops->set_segment(ctxt, selector, &desc, base3, n);
2413 return X86EMUL_CONTINUE;
2416 static int rsm_enter_protected_mode(struct x86_emulate_ctxt *ctxt,
2417 u64 cr0, u64 cr3, u64 cr4)
2422 /* In order to later set CR4.PCIDE, CR3[11:0] must be zero. */
2424 if (cr4 & X86_CR4_PCIDE) {
2429 bad = ctxt->ops->set_cr(ctxt, 3, cr3);
2431 return X86EMUL_UNHANDLEABLE;
2434 * First enable PAE, long mode needs it before CR0.PG = 1 is set.
2435 * Then enable protected mode. However, PCID cannot be enabled
2436 * if EFER.LMA=0, so set it separately.
2438 bad = ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2440 return X86EMUL_UNHANDLEABLE;
2442 bad = ctxt->ops->set_cr(ctxt, 0, cr0);
2444 return X86EMUL_UNHANDLEABLE;
2446 if (cr4 & X86_CR4_PCIDE) {
2447 bad = ctxt->ops->set_cr(ctxt, 4, cr4);
2449 return X86EMUL_UNHANDLEABLE;
2451 bad = ctxt->ops->set_cr(ctxt, 3, cr3 | pcid);
2453 return X86EMUL_UNHANDLEABLE;
2458 return X86EMUL_CONTINUE;
2461 static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, u64 smbase)
2463 struct desc_struct desc;
2466 u32 val, cr0, cr3, cr4;
2469 cr0 = GET_SMSTATE(u32, smbase, 0x7ffc);
2470 cr3 = GET_SMSTATE(u32, smbase, 0x7ff8);
2471 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7ff4) | X86_EFLAGS_FIXED;
2472 ctxt->_eip = GET_SMSTATE(u32, smbase, 0x7ff0);
2474 for (i = 0; i < 8; i++)
2475 *reg_write(ctxt, i) = GET_SMSTATE(u32, smbase, 0x7fd0 + i * 4);
2477 val = GET_SMSTATE(u32, smbase, 0x7fcc);
2478 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2479 val = GET_SMSTATE(u32, smbase, 0x7fc8);
2480 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2482 selector = GET_SMSTATE(u32, smbase, 0x7fc4);
2483 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f64));
2484 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f60));
2485 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f5c));
2486 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_TR);
2488 selector = GET_SMSTATE(u32, smbase, 0x7fc0);
2489 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7f80));
2490 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7f7c));
2491 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7f78));
2492 ctxt->ops->set_segment(ctxt, selector, &desc, 0, VCPU_SREG_LDTR);
2494 dt.address = GET_SMSTATE(u32, smbase, 0x7f74);
2495 dt.size = GET_SMSTATE(u32, smbase, 0x7f70);
2496 ctxt->ops->set_gdt(ctxt, &dt);
2498 dt.address = GET_SMSTATE(u32, smbase, 0x7f58);
2499 dt.size = GET_SMSTATE(u32, smbase, 0x7f54);
2500 ctxt->ops->set_idt(ctxt, &dt);
2502 for (i = 0; i < 6; i++) {
2503 int r = rsm_load_seg_32(ctxt, smbase, i);
2504 if (r != X86EMUL_CONTINUE)
2508 cr4 = GET_SMSTATE(u32, smbase, 0x7f14);
2510 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7ef8));
2512 return rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2515 static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u64 smbase)
2517 struct desc_struct desc;
2519 u64 val, cr0, cr3, cr4;
2524 for (i = 0; i < 16; i++)
2525 *reg_write(ctxt, i) = GET_SMSTATE(u64, smbase, 0x7ff8 - i * 8);
2527 ctxt->_eip = GET_SMSTATE(u64, smbase, 0x7f78);
2528 ctxt->eflags = GET_SMSTATE(u32, smbase, 0x7f70) | X86_EFLAGS_FIXED;
2530 val = GET_SMSTATE(u32, smbase, 0x7f68);
2531 ctxt->ops->set_dr(ctxt, 6, (val & DR6_VOLATILE) | DR6_FIXED_1);
2532 val = GET_SMSTATE(u32, smbase, 0x7f60);
2533 ctxt->ops->set_dr(ctxt, 7, (val & DR7_VOLATILE) | DR7_FIXED_1);
2535 cr0 = GET_SMSTATE(u64, smbase, 0x7f58);
2536 cr3 = GET_SMSTATE(u64, smbase, 0x7f50);
2537 cr4 = GET_SMSTATE(u64, smbase, 0x7f48);
2538 ctxt->ops->set_smbase(ctxt, GET_SMSTATE(u32, smbase, 0x7f00));
2539 val = GET_SMSTATE(u64, smbase, 0x7ed0);
2540 ctxt->ops->set_msr(ctxt, MSR_EFER, val & ~EFER_LMA);
2542 selector = GET_SMSTATE(u32, smbase, 0x7e90);
2543 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e92) << 8);
2544 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e94));
2545 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e98));
2546 base3 = GET_SMSTATE(u32, smbase, 0x7e9c);
2547 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_TR);
2549 dt.size = GET_SMSTATE(u32, smbase, 0x7e84);
2550 dt.address = GET_SMSTATE(u64, smbase, 0x7e88);
2551 ctxt->ops->set_idt(ctxt, &dt);
2553 selector = GET_SMSTATE(u32, smbase, 0x7e70);
2554 rsm_set_desc_flags(&desc, GET_SMSTATE(u32, smbase, 0x7e72) << 8);
2555 set_desc_limit(&desc, GET_SMSTATE(u32, smbase, 0x7e74));
2556 set_desc_base(&desc, GET_SMSTATE(u32, smbase, 0x7e78));
2557 base3 = GET_SMSTATE(u32, smbase, 0x7e7c);
2558 ctxt->ops->set_segment(ctxt, selector, &desc, base3, VCPU_SREG_LDTR);
2560 dt.size = GET_SMSTATE(u32, smbase, 0x7e64);
2561 dt.address = GET_SMSTATE(u64, smbase, 0x7e68);
2562 ctxt->ops->set_gdt(ctxt, &dt);
2564 r = rsm_enter_protected_mode(ctxt, cr0, cr3, cr4);
2565 if (r != X86EMUL_CONTINUE)
2568 for (i = 0; i < 6; i++) {
2569 r = rsm_load_seg_64(ctxt, smbase, i);
2570 if (r != X86EMUL_CONTINUE)
2574 return X86EMUL_CONTINUE;
2577 static int em_rsm(struct x86_emulate_ctxt *ctxt)
2579 unsigned long cr0, cr4, efer;
2583 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_MASK) == 0)
2584 return emulate_ud(ctxt);
2587 * Get back to real mode, to prepare a safe state in which to load
2588 * CR0/CR3/CR4/EFER. It's all a bit more complicated if the vCPU
2589 * supports long mode.
2591 if (emulator_has_longmode(ctxt)) {
2592 struct desc_struct cs_desc;
2594 /* Zero CR4.PCIDE before CR0.PG. */
2595 cr4 = ctxt->ops->get_cr(ctxt, 4);
2596 if (cr4 & X86_CR4_PCIDE)
2597 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PCIDE);
2599 /* A 32-bit code segment is required to clear EFER.LMA. */
2600 memset(&cs_desc, 0, sizeof(cs_desc));
2602 cs_desc.s = cs_desc.g = cs_desc.p = 1;
2603 ctxt->ops->set_segment(ctxt, 0, &cs_desc, 0, VCPU_SREG_CS);
2606 /* For the 64-bit case, this will clear EFER.LMA. */
2607 cr0 = ctxt->ops->get_cr(ctxt, 0);
2608 if (cr0 & X86_CR0_PE)
2609 ctxt->ops->set_cr(ctxt, 0, cr0 & ~(X86_CR0_PG | X86_CR0_PE));
2611 if (emulator_has_longmode(ctxt)) {
2612 /* Clear CR4.PAE before clearing EFER.LME. */
2613 cr4 = ctxt->ops->get_cr(ctxt, 4);
2614 if (cr4 & X86_CR4_PAE)
2615 ctxt->ops->set_cr(ctxt, 4, cr4 & ~X86_CR4_PAE);
2617 /* And finally go back to 32-bit mode. */
2619 ctxt->ops->set_msr(ctxt, MSR_EFER, efer);
2622 smbase = ctxt->ops->get_smbase(ctxt);
2623 if (emulator_has_longmode(ctxt))
2624 ret = rsm_load_state_64(ctxt, smbase + 0x8000);
2626 ret = rsm_load_state_32(ctxt, smbase + 0x8000);
2628 if (ret != X86EMUL_CONTINUE) {
2629 /* FIXME: should triple fault */
2630 return X86EMUL_UNHANDLEABLE;
2633 if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0)
2634 ctxt->ops->set_nmi_mask(ctxt, false);
2636 ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) &
2637 ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK));
2638 return X86EMUL_CONTINUE;
2642 setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2643 struct desc_struct *cs, struct desc_struct *ss)
2645 cs->l = 0; /* will be adjusted later */
2646 set_desc_base(cs, 0); /* flat segment */
2647 cs->g = 1; /* 4kb granularity */
2648 set_desc_limit(cs, 0xfffff); /* 4GB limit */
2649 cs->type = 0x0b; /* Read, Execute, Accessed */
2651 cs->dpl = 0; /* will be adjusted later */
2656 set_desc_base(ss, 0); /* flat segment */
2657 set_desc_limit(ss, 0xfffff); /* 4GB limit */
2658 ss->g = 1; /* 4kb granularity */
2660 ss->type = 0x03; /* Read/Write, Accessed */
2661 ss->d = 1; /* 32bit stack segment */
2668 static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2670 u32 eax, ebx, ecx, edx;
2673 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2674 return ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx
2675 && ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx
2676 && edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx;
2679 static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2681 const struct x86_emulate_ops *ops = ctxt->ops;
2682 u32 eax, ebx, ecx, edx;
2685 * syscall should always be enabled in longmode - so only become
2686 * vendor specific (cpuid) if other modes are active...
2688 if (ctxt->mode == X86EMUL_MODE_PROT64)
2693 ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
2695 * Intel ("GenuineIntel")
2696 * remark: Intel CPUs only support "syscall" in 64bit
2697 * longmode. Also an 64bit guest with a
2698 * 32bit compat-app running will #UD !! While this
2699 * behaviour can be fixed (by emulating) into AMD
2700 * response - CPUs of AMD can't behave like Intel.
2702 if (ebx == X86EMUL_CPUID_VENDOR_GenuineIntel_ebx &&
2703 ecx == X86EMUL_CPUID_VENDOR_GenuineIntel_ecx &&
2704 edx == X86EMUL_CPUID_VENDOR_GenuineIntel_edx)
2707 /* AMD ("AuthenticAMD") */
2708 if (ebx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx &&
2709 ecx == X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx &&
2710 edx == X86EMUL_CPUID_VENDOR_AuthenticAMD_edx)
2713 /* AMD ("AMDisbetter!") */
2714 if (ebx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ebx &&
2715 ecx == X86EMUL_CPUID_VENDOR_AMDisbetterI_ecx &&
2716 edx == X86EMUL_CPUID_VENDOR_AMDisbetterI_edx)
2719 /* default: (not Intel, not AMD), apply Intel's stricter rules... */
2723 static int em_syscall(struct x86_emulate_ctxt *ctxt)
2725 const struct x86_emulate_ops *ops = ctxt->ops;
2726 struct desc_struct cs, ss;
2731 /* syscall is not available in real mode */
2732 if (ctxt->mode == X86EMUL_MODE_REAL ||
2733 ctxt->mode == X86EMUL_MODE_VM86)
2734 return emulate_ud(ctxt);
2736 if (!(em_syscall_is_enabled(ctxt)))
2737 return emulate_ud(ctxt);
2739 ops->get_msr(ctxt, MSR_EFER, &efer);
2740 setup_syscalls_segments(ctxt, &cs, &ss);
2742 if (!(efer & EFER_SCE))
2743 return emulate_ud(ctxt);
2745 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2747 cs_sel = (u16)(msr_data & 0xfffc);
2748 ss_sel = (u16)(msr_data + 8);
2750 if (efer & EFER_LMA) {
2754 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2755 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2757 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2758 if (efer & EFER_LMA) {
2759 #ifdef CONFIG_X86_64
2760 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags;
2763 ctxt->mode == X86EMUL_MODE_PROT64 ?
2764 MSR_LSTAR : MSR_CSTAR, &msr_data);
2765 ctxt->_eip = msr_data;
2767 ops->get_msr(ctxt, MSR_SYSCALL_MASK, &msr_data);
2768 ctxt->eflags &= ~msr_data;
2769 ctxt->eflags |= X86_EFLAGS_FIXED;
2773 ops->get_msr(ctxt, MSR_STAR, &msr_data);
2774 ctxt->_eip = (u32)msr_data;
2776 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2779 ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0;
2780 return X86EMUL_CONTINUE;
2783 static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2785 const struct x86_emulate_ops *ops = ctxt->ops;
2786 struct desc_struct cs, ss;
2791 ops->get_msr(ctxt, MSR_EFER, &efer);
2792 /* inject #GP if in real mode */
2793 if (ctxt->mode == X86EMUL_MODE_REAL)
2794 return emulate_gp(ctxt, 0);
2797 * Not recognized on AMD in compat mode (but is recognized in legacy
2800 if ((ctxt->mode != X86EMUL_MODE_PROT64) && (efer & EFER_LMA)
2801 && !vendor_intel(ctxt))
2802 return emulate_ud(ctxt);
2804 /* sysenter/sysexit have not been tested in 64bit mode. */
2805 if (ctxt->mode == X86EMUL_MODE_PROT64)
2806 return X86EMUL_UNHANDLEABLE;
2808 setup_syscalls_segments(ctxt, &cs, &ss);
2810 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2811 if ((msr_data & 0xfffc) == 0x0)
2812 return emulate_gp(ctxt, 0);
2814 ctxt->eflags &= ~(X86_EFLAGS_VM | X86_EFLAGS_IF);
2815 cs_sel = (u16)msr_data & ~SEGMENT_RPL_MASK;
2816 ss_sel = cs_sel + 8;
2817 if (efer & EFER_LMA) {
2822 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2823 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2825 ops->get_msr(ctxt, MSR_IA32_SYSENTER_EIP, &msr_data);
2826 ctxt->_eip = (efer & EFER_LMA) ? msr_data : (u32)msr_data;
2828 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2829 *reg_write(ctxt, VCPU_REGS_RSP) = (efer & EFER_LMA) ? msr_data :
2832 return X86EMUL_CONTINUE;
2835 static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2837 const struct x86_emulate_ops *ops = ctxt->ops;
2838 struct desc_struct cs, ss;
2839 u64 msr_data, rcx, rdx;
2841 u16 cs_sel = 0, ss_sel = 0;
2843 /* inject #GP if in real mode or Virtual 8086 mode */
2844 if (ctxt->mode == X86EMUL_MODE_REAL ||
2845 ctxt->mode == X86EMUL_MODE_VM86)
2846 return emulate_gp(ctxt, 0);
2848 setup_syscalls_segments(ctxt, &cs, &ss);
2850 if ((ctxt->rex_prefix & 0x8) != 0x0)
2851 usermode = X86EMUL_MODE_PROT64;
2853 usermode = X86EMUL_MODE_PROT32;
2855 rcx = reg_read(ctxt, VCPU_REGS_RCX);
2856 rdx = reg_read(ctxt, VCPU_REGS_RDX);
2860 ops->get_msr(ctxt, MSR_IA32_SYSENTER_CS, &msr_data);
2862 case X86EMUL_MODE_PROT32:
2863 cs_sel = (u16)(msr_data + 16);
2864 if ((msr_data & 0xfffc) == 0x0)
2865 return emulate_gp(ctxt, 0);
2866 ss_sel = (u16)(msr_data + 24);
2870 case X86EMUL_MODE_PROT64:
2871 cs_sel = (u16)(msr_data + 32);
2872 if (msr_data == 0x0)
2873 return emulate_gp(ctxt, 0);
2874 ss_sel = cs_sel + 8;
2877 if (emul_is_noncanonical_address(rcx, ctxt) ||
2878 emul_is_noncanonical_address(rdx, ctxt))
2879 return emulate_gp(ctxt, 0);
2882 cs_sel |= SEGMENT_RPL_MASK;
2883 ss_sel |= SEGMENT_RPL_MASK;
2885 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2886 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2889 *reg_write(ctxt, VCPU_REGS_RSP) = rcx;
2891 return X86EMUL_CONTINUE;
2894 static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2897 if (ctxt->mode == X86EMUL_MODE_REAL)
2899 if (ctxt->mode == X86EMUL_MODE_VM86)
2901 iopl = (ctxt->eflags & X86_EFLAGS_IOPL) >> X86_EFLAGS_IOPL_BIT;
2902 return ctxt->ops->cpl(ctxt) > iopl;
2905 static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2908 const struct x86_emulate_ops *ops = ctxt->ops;
2909 struct desc_struct tr_seg;
2912 u16 tr, io_bitmap_ptr, perm, bit_idx = port & 0x7;
2913 unsigned mask = (1 << len) - 1;
2916 ops->get_segment(ctxt, &tr, &tr_seg, &base3, VCPU_SREG_TR);
2919 if (desc_limit_scaled(&tr_seg) < 103)
2921 base = get_desc_base(&tr_seg);
2922 #ifdef CONFIG_X86_64
2923 base |= ((u64)base3) << 32;
2925 r = ops->read_std(ctxt, base + 102, &io_bitmap_ptr, 2, NULL, true);
2926 if (r != X86EMUL_CONTINUE)
2928 if (io_bitmap_ptr + port/8 > desc_limit_scaled(&tr_seg))
2930 r = ops->read_std(ctxt, base + io_bitmap_ptr + port/8, &perm, 2, NULL, true);
2931 if (r != X86EMUL_CONTINUE)
2933 if ((perm >> bit_idx) & mask)
2938 static bool emulator_io_permited(struct x86_emulate_ctxt *ctxt,
2944 if (emulator_bad_iopl(ctxt))
2945 if (!emulator_io_port_access_allowed(ctxt, port, len))
2948 ctxt->perm_ok = true;
2953 static void string_registers_quirk(struct x86_emulate_ctxt *ctxt)
2956 * Intel CPUs mask the counter and pointers in quite strange
2957 * manner when ECX is zero due to REP-string optimizations.
2959 #ifdef CONFIG_X86_64
2960 if (ctxt->ad_bytes != 4 || !vendor_intel(ctxt))
2963 *reg_write(ctxt, VCPU_REGS_RCX) = 0;
2966 case 0xa4: /* movsb */
2967 case 0xa5: /* movsd/w */
2968 *reg_rmw(ctxt, VCPU_REGS_RSI) &= (u32)-1;
2970 case 0xaa: /* stosb */
2971 case 0xab: /* stosd/w */
2972 *reg_rmw(ctxt, VCPU_REGS_RDI) &= (u32)-1;
2977 static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2978 struct tss_segment_16 *tss)
2980 tss->ip = ctxt->_eip;
2981 tss->flag = ctxt->eflags;
2982 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2983 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2984 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2985 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2986 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2987 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2988 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2989 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2991 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2992 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
2993 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
2994 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
2995 tss->ldt = get_segment_selector(ctxt, VCPU_SREG_LDTR);
2998 static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2999 struct tss_segment_16 *tss)
3004 ctxt->_eip = tss->ip;
3005 ctxt->eflags = tss->flag | 2;
3006 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
3007 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
3008 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
3009 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
3010 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
3011 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
3012 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
3013 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
3016 * SDM says that segment selectors are loaded before segment
3019 set_segment_selector(ctxt, tss->ldt, VCPU_SREG_LDTR);
3020 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3021 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3022 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3023 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3028 * Now load segment descriptors. If fault happens at this stage
3029 * it is handled in a context of new task
3031 ret = __load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR, cpl,
3032 X86_TRANSFER_TASK_SWITCH, NULL);
3033 if (ret != X86EMUL_CONTINUE)
3035 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3036 X86_TRANSFER_TASK_SWITCH, NULL);
3037 if (ret != X86EMUL_CONTINUE)
3039 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3040 X86_TRANSFER_TASK_SWITCH, NULL);
3041 if (ret != X86EMUL_CONTINUE)
3043 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3044 X86_TRANSFER_TASK_SWITCH, NULL);
3045 if (ret != X86EMUL_CONTINUE)
3047 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3048 X86_TRANSFER_TASK_SWITCH, NULL);
3049 if (ret != X86EMUL_CONTINUE)
3052 return X86EMUL_CONTINUE;
3055 static int task_switch_16(struct x86_emulate_ctxt *ctxt,
3056 u16 tss_selector, u16 old_tss_sel,
3057 ulong old_tss_base, struct desc_struct *new_desc)
3059 struct tss_segment_16 tss_seg;
3061 u32 new_tss_base = get_desc_base(new_desc);
3063 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3064 if (ret != X86EMUL_CONTINUE)
3067 save_state_to_tss16(ctxt, &tss_seg);
3069 ret = linear_write_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3070 if (ret != X86EMUL_CONTINUE)
3073 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3074 if (ret != X86EMUL_CONTINUE)
3077 if (old_tss_sel != 0xffff) {
3078 tss_seg.prev_task_link = old_tss_sel;
3080 ret = linear_write_system(ctxt, new_tss_base,
3081 &tss_seg.prev_task_link,
3082 sizeof tss_seg.prev_task_link);
3083 if (ret != X86EMUL_CONTINUE)
3087 return load_state_from_tss16(ctxt, &tss_seg);
3090 static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
3091 struct tss_segment_32 *tss)
3093 /* CR3 and ldt selector are not saved intentionally */
3094 tss->eip = ctxt->_eip;
3095 tss->eflags = ctxt->eflags;
3096 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
3097 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
3098 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
3099 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
3100 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
3101 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
3102 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
3103 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
3105 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
3106 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
3107 tss->ss = get_segment_selector(ctxt, VCPU_SREG_SS);
3108 tss->ds = get_segment_selector(ctxt, VCPU_SREG_DS);
3109 tss->fs = get_segment_selector(ctxt, VCPU_SREG_FS);
3110 tss->gs = get_segment_selector(ctxt, VCPU_SREG_GS);
3113 static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
3114 struct tss_segment_32 *tss)
3119 if (ctxt->ops->set_cr(ctxt, 3, tss->cr3))
3120 return emulate_gp(ctxt, 0);
3121 ctxt->_eip = tss->eip;
3122 ctxt->eflags = tss->eflags | 2;
3124 /* General purpose registers */
3125 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
3126 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
3127 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
3128 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
3129 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
3130 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
3131 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
3132 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
3135 * SDM says that segment selectors are loaded before segment
3136 * descriptors. This is important because CPL checks will
3139 set_segment_selector(ctxt, tss->ldt_selector, VCPU_SREG_LDTR);
3140 set_segment_selector(ctxt, tss->es, VCPU_SREG_ES);
3141 set_segment_selector(ctxt, tss->cs, VCPU_SREG_CS);
3142 set_segment_selector(ctxt, tss->ss, VCPU_SREG_SS);
3143 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
3144 set_segment_selector(ctxt, tss->fs, VCPU_SREG_FS);
3145 set_segment_selector(ctxt, tss->gs, VCPU_SREG_GS);
3148 * If we're switching between Protected Mode and VM86, we need to make
3149 * sure to update the mode before loading the segment descriptors so
3150 * that the selectors are interpreted correctly.
3152 if (ctxt->eflags & X86_EFLAGS_VM) {
3153 ctxt->mode = X86EMUL_MODE_VM86;
3156 ctxt->mode = X86EMUL_MODE_PROT32;
3161 * Now load segment descriptors. If fault happenes at this stage
3162 * it is handled in a context of new task
3164 ret = __load_segment_descriptor(ctxt, tss->ldt_selector, VCPU_SREG_LDTR,
3165 cpl, X86_TRANSFER_TASK_SWITCH, NULL);
3166 if (ret != X86EMUL_CONTINUE)
3168 ret = __load_segment_descriptor(ctxt, tss->es, VCPU_SREG_ES, cpl,
3169 X86_TRANSFER_TASK_SWITCH, NULL);
3170 if (ret != X86EMUL_CONTINUE)
3172 ret = __load_segment_descriptor(ctxt, tss->cs, VCPU_SREG_CS, cpl,
3173 X86_TRANSFER_TASK_SWITCH, NULL);
3174 if (ret != X86EMUL_CONTINUE)
3176 ret = __load_segment_descriptor(ctxt, tss->ss, VCPU_SREG_SS, cpl,
3177 X86_TRANSFER_TASK_SWITCH, NULL);
3178 if (ret != X86EMUL_CONTINUE)
3180 ret = __load_segment_descriptor(ctxt, tss->ds, VCPU_SREG_DS, cpl,
3181 X86_TRANSFER_TASK_SWITCH, NULL);
3182 if (ret != X86EMUL_CONTINUE)
3184 ret = __load_segment_descriptor(ctxt, tss->fs, VCPU_SREG_FS, cpl,
3185 X86_TRANSFER_TASK_SWITCH, NULL);
3186 if (ret != X86EMUL_CONTINUE)
3188 ret = __load_segment_descriptor(ctxt, tss->gs, VCPU_SREG_GS, cpl,
3189 X86_TRANSFER_TASK_SWITCH, NULL);
3194 static int task_switch_32(struct x86_emulate_ctxt *ctxt,
3195 u16 tss_selector, u16 old_tss_sel,
3196 ulong old_tss_base, struct desc_struct *new_desc)
3198 struct tss_segment_32 tss_seg;
3200 u32 new_tss_base = get_desc_base(new_desc);
3201 u32 eip_offset = offsetof(struct tss_segment_32, eip);
3202 u32 ldt_sel_offset = offsetof(struct tss_segment_32, ldt_selector);
3204 ret = linear_read_system(ctxt, old_tss_base, &tss_seg, sizeof tss_seg);
3205 if (ret != X86EMUL_CONTINUE)
3208 save_state_to_tss32(ctxt, &tss_seg);
3210 /* Only GP registers and segment selectors are saved */
3211 ret = linear_write_system(ctxt, old_tss_base + eip_offset, &tss_seg.eip,
3212 ldt_sel_offset - eip_offset);
3213 if (ret != X86EMUL_CONTINUE)
3216 ret = linear_read_system(ctxt, new_tss_base, &tss_seg, sizeof tss_seg);
3217 if (ret != X86EMUL_CONTINUE)
3220 if (old_tss_sel != 0xffff) {
3221 tss_seg.prev_task_link = old_tss_sel;
3223 ret = linear_write_system(ctxt, new_tss_base,
3224 &tss_seg.prev_task_link,
3225 sizeof tss_seg.prev_task_link);
3226 if (ret != X86EMUL_CONTINUE)
3230 return load_state_from_tss32(ctxt, &tss_seg);
3233 static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
3234 u16 tss_selector, int idt_index, int reason,
3235 bool has_error_code, u32 error_code)
3237 const struct x86_emulate_ops *ops = ctxt->ops;
3238 struct desc_struct curr_tss_desc, next_tss_desc;
3240 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
3241 ulong old_tss_base =
3242 ops->get_cached_segment_base(ctxt, VCPU_SREG_TR);
3244 ulong desc_addr, dr7;
3246 /* FIXME: old_tss_base == ~0 ? */
3248 ret = read_segment_descriptor(ctxt, tss_selector, &next_tss_desc, &desc_addr);
3249 if (ret != X86EMUL_CONTINUE)
3251 ret = read_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc, &desc_addr);
3252 if (ret != X86EMUL_CONTINUE)
3255 /* FIXME: check that next_tss_desc is tss */
3258 * Check privileges. The three cases are task switch caused by...
3260 * 1. jmp/call/int to task gate: Check against DPL of the task gate
3261 * 2. Exception/IRQ/iret: No check is performed
3262 * 3. jmp/call to TSS/task-gate: No check is performed since the
3263 * hardware checks it before exiting.
3265 if (reason == TASK_SWITCH_GATE) {
3266 if (idt_index != -1) {
3267 /* Software interrupts */
3268 struct desc_struct task_gate_desc;
3271 ret = read_interrupt_descriptor(ctxt, idt_index,
3273 if (ret != X86EMUL_CONTINUE)
3276 dpl = task_gate_desc.dpl;
3277 if ((tss_selector & 3) > dpl || ops->cpl(ctxt) > dpl)
3278 return emulate_gp(ctxt, (idt_index << 3) | 0x2);
3282 desc_limit = desc_limit_scaled(&next_tss_desc);
3283 if (!next_tss_desc.p ||
3284 ((desc_limit < 0x67 && (next_tss_desc.type & 8)) ||
3285 desc_limit < 0x2b)) {
3286 return emulate_ts(ctxt, tss_selector & 0xfffc);
3289 if (reason == TASK_SWITCH_IRET || reason == TASK_SWITCH_JMP) {
3290 curr_tss_desc.type &= ~(1 << 1); /* clear busy flag */
3291 write_segment_descriptor(ctxt, old_tss_sel, &curr_tss_desc);
3294 if (reason == TASK_SWITCH_IRET)
3295 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
3297 /* set back link to prev task only if NT bit is set in eflags
3298 note that old_tss_sel is not used after this point */
3299 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
3300 old_tss_sel = 0xffff;
3302 if (next_tss_desc.type & 8)
3303 ret = task_switch_32(ctxt, tss_selector, old_tss_sel,
3304 old_tss_base, &next_tss_desc);
3306 ret = task_switch_16(ctxt, tss_selector, old_tss_sel,
3307 old_tss_base, &next_tss_desc);
3308 if (ret != X86EMUL_CONTINUE)
3311 if (reason == TASK_SWITCH_CALL || reason == TASK_SWITCH_GATE)
3312 ctxt->eflags = ctxt->eflags | X86_EFLAGS_NT;
3314 if (reason != TASK_SWITCH_IRET) {
3315 next_tss_desc.type |= (1 << 1); /* set busy flag */
3316 write_segment_descriptor(ctxt, tss_selector, &next_tss_desc);
3319 ops->set_cr(ctxt, 0, ops->get_cr(ctxt, 0) | X86_CR0_TS);
3320 ops->set_segment(ctxt, tss_selector, &next_tss_desc, 0, VCPU_SREG_TR);
3322 if (has_error_code) {
3323 ctxt->op_bytes = ctxt->ad_bytes = (next_tss_desc.type & 8) ? 4 : 2;
3324 ctxt->lock_prefix = 0;
3325 ctxt->src.val = (unsigned long) error_code;
3326 ret = em_push(ctxt);
3329 ops->get_dr(ctxt, 7, &dr7);
3330 ops->set_dr(ctxt, 7, dr7 & ~(DR_LOCAL_ENABLE_MASK | DR_LOCAL_SLOWDOWN));
3335 int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
3336 u16 tss_selector, int idt_index, int reason,
3337 bool has_error_code, u32 error_code)
3341 invalidate_registers(ctxt);
3342 ctxt->_eip = ctxt->eip;
3343 ctxt->dst.type = OP_NONE;
3345 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
3346 has_error_code, error_code);
3348 if (rc == X86EMUL_CONTINUE) {
3349 ctxt->eip = ctxt->_eip;
3350 writeback_registers(ctxt);
3353 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
3356 static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
3359 int df = (ctxt->eflags & X86_EFLAGS_DF) ? -op->count : op->count;
3361 register_address_increment(ctxt, reg, df * op->bytes);
3362 op->addr.mem.ea = register_address(ctxt, reg);
3365 static int em_das(struct x86_emulate_ctxt *ctxt)
3368 bool af, cf, old_cf;
3370 cf = ctxt->eflags & X86_EFLAGS_CF;
3376 af = ctxt->eflags & X86_EFLAGS_AF;
3377 if ((al & 0x0f) > 9 || af) {
3379 cf = old_cf | (al >= 250);
3384 if (old_al > 0x99 || old_cf) {
3390 /* Set PF, ZF, SF */
3391 ctxt->src.type = OP_IMM;
3393 ctxt->src.bytes = 1;
3394 fastop(ctxt, em_or);
3395 ctxt->eflags &= ~(X86_EFLAGS_AF | X86_EFLAGS_CF);
3397 ctxt->eflags |= X86_EFLAGS_CF;
3399 ctxt->eflags |= X86_EFLAGS_AF;
3400 return X86EMUL_CONTINUE;
3403 static int em_aam(struct x86_emulate_ctxt *ctxt)
3407 if (ctxt->src.val == 0)
3408 return emulate_de(ctxt);
3410 al = ctxt->dst.val & 0xff;
3411 ah = al / ctxt->src.val;
3412 al %= ctxt->src.val;
3414 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al | (ah << 8);
3416 /* Set PF, ZF, SF */
3417 ctxt->src.type = OP_IMM;
3419 ctxt->src.bytes = 1;
3420 fastop(ctxt, em_or);
3422 return X86EMUL_CONTINUE;
3425 static int em_aad(struct x86_emulate_ctxt *ctxt)
3427 u8 al = ctxt->dst.val & 0xff;
3428 u8 ah = (ctxt->dst.val >> 8) & 0xff;
3430 al = (al + (ah * ctxt->src.val)) & 0xff;
3432 ctxt->dst.val = (ctxt->dst.val & 0xffff0000) | al;
3434 /* Set PF, ZF, SF */
3435 ctxt->src.type = OP_IMM;
3437 ctxt->src.bytes = 1;
3438 fastop(ctxt, em_or);
3440 return X86EMUL_CONTINUE;
3443 static int em_call(struct x86_emulate_ctxt *ctxt)
3446 long rel = ctxt->src.val;
3448 ctxt->src.val = (unsigned long)ctxt->_eip;
3449 rc = jmp_rel(ctxt, rel);
3450 if (rc != X86EMUL_CONTINUE)
3452 return em_push(ctxt);
3455 static int em_call_far(struct x86_emulate_ctxt *ctxt)
3460 struct desc_struct old_desc, new_desc;
3461 const struct x86_emulate_ops *ops = ctxt->ops;
3462 int cpl = ctxt->ops->cpl(ctxt);
3463 enum x86emul_mode prev_mode = ctxt->mode;
3465 old_eip = ctxt->_eip;
3466 ops->get_segment(ctxt, &old_cs, &old_desc, NULL, VCPU_SREG_CS);
3468 memcpy(&sel, ctxt->src.valptr + ctxt->op_bytes, 2);
3469 rc = __load_segment_descriptor(ctxt, sel, VCPU_SREG_CS, cpl,
3470 X86_TRANSFER_CALL_JMP, &new_desc);
3471 if (rc != X86EMUL_CONTINUE)
3474 rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc);
3475 if (rc != X86EMUL_CONTINUE)
3478 ctxt->src.val = old_cs;
3480 if (rc != X86EMUL_CONTINUE)
3483 ctxt->src.val = old_eip;
3485 /* If we failed, we tainted the memory, but the very least we should
3487 if (rc != X86EMUL_CONTINUE) {
3488 pr_warn_once("faulting far call emulation tainted memory\n");
3493 ops->set_segment(ctxt, old_cs, &old_desc, 0, VCPU_SREG_CS);
3494 ctxt->mode = prev_mode;
3499 static int em_ret_near_imm(struct x86_emulate_ctxt *ctxt)
3504 rc = emulate_pop(ctxt, &eip, ctxt->op_bytes);
3505 if (rc != X86EMUL_CONTINUE)
3507 rc = assign_eip_near(ctxt, eip);
3508 if (rc != X86EMUL_CONTINUE)
3510 rsp_increment(ctxt, ctxt->src.val);
3511 return X86EMUL_CONTINUE;
3514 static int em_xchg(struct x86_emulate_ctxt *ctxt)
3516 /* Write back the register source. */
3517 ctxt->src.val = ctxt->dst.val;
3518 write_register_operand(&ctxt->src);
3520 /* Write back the memory destination with implicit LOCK prefix. */
3521 ctxt->dst.val = ctxt->src.orig_val;
3522 ctxt->lock_prefix = 1;
3523 return X86EMUL_CONTINUE;
3526 static int em_imul_3op(struct x86_emulate_ctxt *ctxt)
3528 ctxt->dst.val = ctxt->src2.val;
3529 return fastop(ctxt, em_imul);
3532 static int em_cwd(struct x86_emulate_ctxt *ctxt)
3534 ctxt->dst.type = OP_REG;
3535 ctxt->dst.bytes = ctxt->src.bytes;
3536 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3537 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
3539 return X86EMUL_CONTINUE;
3542 static int em_rdpid(struct x86_emulate_ctxt *ctxt)
3546 if (ctxt->ops->get_msr(ctxt, MSR_TSC_AUX, &tsc_aux))
3547 return emulate_ud(ctxt);
3548 ctxt->dst.val = tsc_aux;
3549 return X86EMUL_CONTINUE;
3552 static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
3556 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
3557 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
3558 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
3559 return X86EMUL_CONTINUE;
3562 static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
3566 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
3567 return emulate_gp(ctxt, 0);
3568 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
3569 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
3570 return X86EMUL_CONTINUE;
3573 static int em_mov(struct x86_emulate_ctxt *ctxt)
3575 memcpy(ctxt->dst.valptr, ctxt->src.valptr, sizeof(ctxt->src.valptr));
3576 return X86EMUL_CONTINUE;
3579 #define FFL(x) bit(X86_FEATURE_##x)
3581 static int em_movbe(struct x86_emulate_ctxt *ctxt)
3583 u32 ebx, ecx, edx, eax = 1;
3587 * Check MOVBE is set in the guest-visible CPUID leaf.
3589 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3590 if (!(ecx & FFL(MOVBE)))
3591 return emulate_ud(ctxt);
3593 switch (ctxt->op_bytes) {
3596 * From MOVBE definition: "...When the operand size is 16 bits,
3597 * the upper word of the destination register remains unchanged
3600 * Both casting ->valptr and ->val to u16 breaks strict aliasing
3601 * rules so we have to do the operation almost per hand.
3603 tmp = (u16)ctxt->src.val;
3604 ctxt->dst.val &= ~0xffffUL;
3605 ctxt->dst.val |= (unsigned long)swab16(tmp);
3608 ctxt->dst.val = swab32((u32)ctxt->src.val);
3611 ctxt->dst.val = swab64(ctxt->src.val);
3616 return X86EMUL_CONTINUE;
3619 static int em_cr_write(struct x86_emulate_ctxt *ctxt)
3621 if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val))
3622 return emulate_gp(ctxt, 0);
3624 /* Disable writeback. */
3625 ctxt->dst.type = OP_NONE;
3626 return X86EMUL_CONTINUE;
3629 static int em_dr_write(struct x86_emulate_ctxt *ctxt)
3633 if (ctxt->mode == X86EMUL_MODE_PROT64)
3634 val = ctxt->src.val & ~0ULL;
3636 val = ctxt->src.val & ~0U;
3638 /* #UD condition is already handled. */
3639 if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0)
3640 return emulate_gp(ctxt, 0);
3642 /* Disable writeback. */
3643 ctxt->dst.type = OP_NONE;
3644 return X86EMUL_CONTINUE;
3647 static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
3651 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
3652 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
3653 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
3654 return emulate_gp(ctxt, 0);
3656 return X86EMUL_CONTINUE;
3659 static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3663 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3664 return emulate_gp(ctxt, 0);
3666 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3667 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3668 return X86EMUL_CONTINUE;
3671 static int em_mov_rm_sreg(struct x86_emulate_ctxt *ctxt)
3673 if (ctxt->modrm_reg > VCPU_SREG_GS)
3674 return emulate_ud(ctxt);
3676 ctxt->dst.val = get_segment_selector(ctxt, ctxt->modrm_reg);
3677 if (ctxt->dst.bytes == 4 && ctxt->dst.type == OP_MEM)
3678 ctxt->dst.bytes = 2;
3679 return X86EMUL_CONTINUE;
3682 static int em_mov_sreg_rm(struct x86_emulate_ctxt *ctxt)
3684 u16 sel = ctxt->src.val;
3686 if (ctxt->modrm_reg == VCPU_SREG_CS || ctxt->modrm_reg > VCPU_SREG_GS)
3687 return emulate_ud(ctxt);
3689 if (ctxt->modrm_reg == VCPU_SREG_SS)
3690 ctxt->interruptibility = KVM_X86_SHADOW_INT_MOV_SS;
3692 /* Disable writeback. */
3693 ctxt->dst.type = OP_NONE;
3694 return load_segment_descriptor(ctxt, sel, ctxt->modrm_reg);
3697 static int em_lldt(struct x86_emulate_ctxt *ctxt)
3699 u16 sel = ctxt->src.val;
3701 /* Disable writeback. */
3702 ctxt->dst.type = OP_NONE;
3703 return load_segment_descriptor(ctxt, sel, VCPU_SREG_LDTR);
3706 static int em_ltr(struct x86_emulate_ctxt *ctxt)
3708 u16 sel = ctxt->src.val;
3710 /* Disable writeback. */
3711 ctxt->dst.type = OP_NONE;
3712 return load_segment_descriptor(ctxt, sel, VCPU_SREG_TR);
3715 static int em_invlpg(struct x86_emulate_ctxt *ctxt)
3720 rc = linearize(ctxt, ctxt->src.addr.mem, 1, false, &linear);
3721 if (rc == X86EMUL_CONTINUE)
3722 ctxt->ops->invlpg(ctxt, linear);
3723 /* Disable writeback. */
3724 ctxt->dst.type = OP_NONE;
3725 return X86EMUL_CONTINUE;
3728 static int em_clts(struct x86_emulate_ctxt *ctxt)
3732 cr0 = ctxt->ops->get_cr(ctxt, 0);
3734 ctxt->ops->set_cr(ctxt, 0, cr0);
3735 return X86EMUL_CONTINUE;
3738 static int em_hypercall(struct x86_emulate_ctxt *ctxt)
3740 int rc = ctxt->ops->fix_hypercall(ctxt);
3742 if (rc != X86EMUL_CONTINUE)
3745 /* Let the processor re-execute the fixed hypercall */
3746 ctxt->_eip = ctxt->eip;
3747 /* Disable writeback. */
3748 ctxt->dst.type = OP_NONE;
3749 return X86EMUL_CONTINUE;
3752 static int emulate_store_desc_ptr(struct x86_emulate_ctxt *ctxt,
3753 void (*get)(struct x86_emulate_ctxt *ctxt,
3754 struct desc_ptr *ptr))
3756 struct desc_ptr desc_ptr;
3758 if (ctxt->mode == X86EMUL_MODE_PROT64)
3760 get(ctxt, &desc_ptr);
3761 if (ctxt->op_bytes == 2) {
3763 desc_ptr.address &= 0x00ffffff;
3765 /* Disable writeback. */
3766 ctxt->dst.type = OP_NONE;
3767 return segmented_write_std(ctxt, ctxt->dst.addr.mem,
3768 &desc_ptr, 2 + ctxt->op_bytes);
3771 static int em_sgdt(struct x86_emulate_ctxt *ctxt)
3773 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_gdt);
3776 static int em_sidt(struct x86_emulate_ctxt *ctxt)
3778 return emulate_store_desc_ptr(ctxt, ctxt->ops->get_idt);
3781 static int em_lgdt_lidt(struct x86_emulate_ctxt *ctxt, bool lgdt)
3783 struct desc_ptr desc_ptr;
3786 if (ctxt->mode == X86EMUL_MODE_PROT64)
3788 rc = read_descriptor(ctxt, ctxt->src.addr.mem,
3789 &desc_ptr.size, &desc_ptr.address,
3791 if (rc != X86EMUL_CONTINUE)
3793 if (ctxt->mode == X86EMUL_MODE_PROT64 &&
3794 emul_is_noncanonical_address(desc_ptr.address, ctxt))
3795 return emulate_gp(ctxt, 0);
3797 ctxt->ops->set_gdt(ctxt, &desc_ptr);
3799 ctxt->ops->set_idt(ctxt, &desc_ptr);
3800 /* Disable writeback. */
3801 ctxt->dst.type = OP_NONE;
3802 return X86EMUL_CONTINUE;
3805 static int em_lgdt(struct x86_emulate_ctxt *ctxt)
3807 return em_lgdt_lidt(ctxt, true);
3810 static int em_lidt(struct x86_emulate_ctxt *ctxt)
3812 return em_lgdt_lidt(ctxt, false);
3815 static int em_smsw(struct x86_emulate_ctxt *ctxt)
3817 if (ctxt->dst.type == OP_MEM)
3818 ctxt->dst.bytes = 2;
3819 ctxt->dst.val = ctxt->ops->get_cr(ctxt, 0);
3820 return X86EMUL_CONTINUE;
3823 static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3825 ctxt->ops->set_cr(ctxt, 0, (ctxt->ops->get_cr(ctxt, 0) & ~0x0eul)
3826 | (ctxt->src.val & 0x0f));
3827 ctxt->dst.type = OP_NONE;
3828 return X86EMUL_CONTINUE;
3831 static int em_loop(struct x86_emulate_ctxt *ctxt)
3833 int rc = X86EMUL_CONTINUE;
3835 register_address_increment(ctxt, VCPU_REGS_RCX, -1);
3836 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3837 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3838 rc = jmp_rel(ctxt, ctxt->src.val);
3843 static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3845 int rc = X86EMUL_CONTINUE;
3847 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3848 rc = jmp_rel(ctxt, ctxt->src.val);
3853 static int em_in(struct x86_emulate_ctxt *ctxt)
3855 if (!pio_in_emulated(ctxt, ctxt->dst.bytes, ctxt->src.val,
3857 return X86EMUL_IO_NEEDED;
3859 return X86EMUL_CONTINUE;
3862 static int em_out(struct x86_emulate_ctxt *ctxt)
3864 ctxt->ops->pio_out_emulated(ctxt, ctxt->src.bytes, ctxt->dst.val,
3866 /* Disable writeback. */
3867 ctxt->dst.type = OP_NONE;
3868 return X86EMUL_CONTINUE;
3871 static int em_cli(struct x86_emulate_ctxt *ctxt)
3873 if (emulator_bad_iopl(ctxt))
3874 return emulate_gp(ctxt, 0);
3876 ctxt->eflags &= ~X86_EFLAGS_IF;
3877 return X86EMUL_CONTINUE;
3880 static int em_sti(struct x86_emulate_ctxt *ctxt)
3882 if (emulator_bad_iopl(ctxt))
3883 return emulate_gp(ctxt, 0);
3885 ctxt->interruptibility = KVM_X86_SHADOW_INT_STI;
3886 ctxt->eflags |= X86_EFLAGS_IF;
3887 return X86EMUL_CONTINUE;
3890 static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3892 u32 eax, ebx, ecx, edx;
3895 ctxt->ops->get_msr(ctxt, MSR_MISC_FEATURES_ENABLES, &msr);
3896 if (msr & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT &&
3897 ctxt->ops->cpl(ctxt)) {
3898 return emulate_gp(ctxt, 0);
3901 eax = reg_read(ctxt, VCPU_REGS_RAX);
3902 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3903 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, true);
3904 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3905 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3906 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3907 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3908 return X86EMUL_CONTINUE;
3911 static int em_sahf(struct x86_emulate_ctxt *ctxt)
3915 flags = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
3917 flags &= *reg_rmw(ctxt, VCPU_REGS_RAX) >> 8;
3919 ctxt->eflags &= ~0xffUL;
3920 ctxt->eflags |= flags | X86_EFLAGS_FIXED;
3921 return X86EMUL_CONTINUE;
3924 static int em_lahf(struct x86_emulate_ctxt *ctxt)
3926 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3927 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3928 return X86EMUL_CONTINUE;
3931 static int em_bswap(struct x86_emulate_ctxt *ctxt)
3933 switch (ctxt->op_bytes) {
3934 #ifdef CONFIG_X86_64
3936 asm("bswap %0" : "+r"(ctxt->dst.val));
3940 asm("bswap %0" : "+r"(*(u32 *)&ctxt->dst.val));
3943 return X86EMUL_CONTINUE;
3946 static int em_clflush(struct x86_emulate_ctxt *ctxt)
3948 /* emulating clflush regardless of cpuid */
3949 return X86EMUL_CONTINUE;
3952 static int em_clflushopt(struct x86_emulate_ctxt *ctxt)
3954 /* emulating clflushopt regardless of cpuid */
3955 return X86EMUL_CONTINUE;
3958 static int em_movsxd(struct x86_emulate_ctxt *ctxt)
3960 ctxt->dst.val = (s32) ctxt->src.val;
3961 return X86EMUL_CONTINUE;
3964 static int check_fxsr(struct x86_emulate_ctxt *ctxt)
3966 u32 eax = 1, ebx, ecx = 0, edx;
3968 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx, false);
3969 if (!(edx & FFL(FXSR)))
3970 return emulate_ud(ctxt);
3972 if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM))
3973 return emulate_nm(ctxt);
3976 * Don't emulate a case that should never be hit, instead of working
3977 * around a lack of fxsave64/fxrstor64 on old compilers.
3979 if (ctxt->mode >= X86EMUL_MODE_PROT64)
3980 return X86EMUL_UNHANDLEABLE;
3982 return X86EMUL_CONTINUE;
3986 * Hardware doesn't save and restore XMM 0-7 without CR4.OSFXSR, but does save
3987 * and restore MXCSR.
3989 static size_t __fxstate_size(int nregs)
3991 return offsetof(struct fxregs_state, xmm_space[0]) + nregs * 16;
3994 static inline size_t fxstate_size(struct x86_emulate_ctxt *ctxt)
3997 if (ctxt->mode == X86EMUL_MODE_PROT64)
3998 return __fxstate_size(16);
4000 cr4_osfxsr = ctxt->ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR;
4001 return __fxstate_size(cr4_osfxsr ? 8 : 0);
4005 * FXSAVE and FXRSTOR have 4 different formats depending on execution mode,
4008 * - like (1), but FIP and FDP (foo) are only 16 bit. At least Intel CPUs
4009 * preserve whole 32 bit values, though, so (1) and (2) are the same wrt.
4011 * 3) 64-bit mode with REX.W prefix
4012 * - like (2), but XMM 8-15 are being saved and restored
4013 * 4) 64-bit mode without REX.W prefix
4014 * - like (3), but FIP and FDP are 64 bit
4016 * Emulation uses (3) for (1) and (2) and preserves XMM 8-15 to reach the
4017 * desired result. (4) is not emulated.
4019 * Note: Guest and host CPUID.(EAX=07H,ECX=0H):EBX[bit 13] (deprecate FPU CS
4020 * and FPU DS) should match.
4022 static int em_fxsave(struct x86_emulate_ctxt *ctxt)
4024 struct fxregs_state fx_state;
4027 rc = check_fxsr(ctxt);
4028 if (rc != X86EMUL_CONTINUE)
4031 ctxt->ops->get_fpu(ctxt);
4033 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state));
4035 ctxt->ops->put_fpu(ctxt);
4037 if (rc != X86EMUL_CONTINUE)
4040 return segmented_write_std(ctxt, ctxt->memop.addr.mem, &fx_state,
4041 fxstate_size(ctxt));
4045 * FXRSTOR might restore XMM registers not provided by the guest. Fill
4046 * in the host registers (via FXSAVE) instead, so they won't be modified.
4047 * (preemption has to stay disabled until FXRSTOR).
4049 * Use noinline to keep the stack for other functions called by callers small.
4051 static noinline int fxregs_fixup(struct fxregs_state *fx_state,
4052 const size_t used_size)
4054 struct fxregs_state fx_tmp;
4057 rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_tmp));
4058 memcpy((void *)fx_state + used_size, (void *)&fx_tmp + used_size,
4059 __fxstate_size(16) - used_size);
4064 static int em_fxrstor(struct x86_emulate_ctxt *ctxt)
4066 struct fxregs_state fx_state;
4070 rc = check_fxsr(ctxt);
4071 if (rc != X86EMUL_CONTINUE)
4074 size = fxstate_size(ctxt);
4075 rc = segmented_read_std(ctxt, ctxt->memop.addr.mem, &fx_state, size);
4076 if (rc != X86EMUL_CONTINUE)
4079 ctxt->ops->get_fpu(ctxt);
4081 if (size < __fxstate_size(16)) {
4082 rc = fxregs_fixup(&fx_state, size);
4083 if (rc != X86EMUL_CONTINUE)
4087 if (fx_state.mxcsr >> 16) {
4088 rc = emulate_gp(ctxt, 0);
4092 if (rc == X86EMUL_CONTINUE)
4093 rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state));
4096 ctxt->ops->put_fpu(ctxt);
4101 static bool valid_cr(int nr)
4113 static int check_cr_read(struct x86_emulate_ctxt *ctxt)
4115 if (!valid_cr(ctxt->modrm_reg))
4116 return emulate_ud(ctxt);
4118 return X86EMUL_CONTINUE;
4121 static int check_cr_write(struct x86_emulate_ctxt *ctxt)
4123 u64 new_val = ctxt->src.val64;
4124 int cr = ctxt->modrm_reg;
4127 static u64 cr_reserved_bits[] = {
4128 0xffffffff00000000ULL,
4129 0, 0, 0, /* CR3 checked later */
4136 return emulate_ud(ctxt);
4138 if (new_val & cr_reserved_bits[cr])
4139 return emulate_gp(ctxt, 0);
4144 if (((new_val & X86_CR0_PG) && !(new_val & X86_CR0_PE)) ||
4145 ((new_val & X86_CR0_NW) && !(new_val & X86_CR0_CD)))
4146 return emulate_gp(ctxt, 0);
4148 cr4 = ctxt->ops->get_cr(ctxt, 4);
4149 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4151 if ((new_val & X86_CR0_PG) && (efer & EFER_LME) &&
4152 !(cr4 & X86_CR4_PAE))
4153 return emulate_gp(ctxt, 0);
4160 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4161 if (efer & EFER_LMA) {
4163 u32 eax, ebx, ecx, edx;
4167 if (ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx,
4169 maxphyaddr = eax & 0xff;
4172 rsvd = rsvd_bits(maxphyaddr, 63);
4173 if (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_PCIDE)
4174 rsvd &= ~CR3_PCID_INVD;
4178 return emulate_gp(ctxt, 0);
4183 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4185 if ((efer & EFER_LMA) && !(new_val & X86_CR4_PAE))
4186 return emulate_gp(ctxt, 0);
4192 return X86EMUL_CONTINUE;
4195 static int check_dr7_gd(struct x86_emulate_ctxt *ctxt)
4199 ctxt->ops->get_dr(ctxt, 7, &dr7);
4201 /* Check if DR7.Global_Enable is set */
4202 return dr7 & (1 << 13);
4205 static int check_dr_read(struct x86_emulate_ctxt *ctxt)
4207 int dr = ctxt->modrm_reg;
4211 return emulate_ud(ctxt);
4213 cr4 = ctxt->ops->get_cr(ctxt, 4);
4214 if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5))
4215 return emulate_ud(ctxt);
4217 if (check_dr7_gd(ctxt)) {
4220 ctxt->ops->get_dr(ctxt, 6, &dr6);
4222 dr6 |= DR6_BD | DR6_RTM;
4223 ctxt->ops->set_dr(ctxt, 6, dr6);
4224 return emulate_db(ctxt);
4227 return X86EMUL_CONTINUE;
4230 static int check_dr_write(struct x86_emulate_ctxt *ctxt)
4232 u64 new_val = ctxt->src.val64;
4233 int dr = ctxt->modrm_reg;
4235 if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL))
4236 return emulate_gp(ctxt, 0);
4238 return check_dr_read(ctxt);
4241 static int check_svme(struct x86_emulate_ctxt *ctxt)
4245 ctxt->ops->get_msr(ctxt, MSR_EFER, &efer);
4247 if (!(efer & EFER_SVME))
4248 return emulate_ud(ctxt);
4250 return X86EMUL_CONTINUE;
4253 static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
4255 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
4257 /* Valid physical address? */
4258 if (rax & 0xffff000000000000ULL)
4259 return emulate_gp(ctxt, 0);
4261 return check_svme(ctxt);
4264 static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
4266 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4268 if (cr4 & X86_CR4_TSD && ctxt->ops->cpl(ctxt))
4269 return emulate_ud(ctxt);
4271 return X86EMUL_CONTINUE;
4274 static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
4276 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
4277 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
4279 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
4280 ctxt->ops->check_pmc(ctxt, rcx))
4281 return emulate_gp(ctxt, 0);
4283 return X86EMUL_CONTINUE;
4286 static int check_perm_in(struct x86_emulate_ctxt *ctxt)
4288 ctxt->dst.bytes = min(ctxt->dst.bytes, 4u);
4289 if (!emulator_io_permited(ctxt, ctxt->src.val, ctxt->dst.bytes))
4290 return emulate_gp(ctxt, 0);
4292 return X86EMUL_CONTINUE;
4295 static int check_perm_out(struct x86_emulate_ctxt *ctxt)
4297 ctxt->src.bytes = min(ctxt->src.bytes, 4u);
4298 if (!emulator_io_permited(ctxt, ctxt->dst.val, ctxt->src.bytes))
4299 return emulate_gp(ctxt, 0);
4301 return X86EMUL_CONTINUE;
4304 #define D(_y) { .flags = (_y) }
4305 #define DI(_y, _i) { .flags = (_y)|Intercept, .intercept = x86_intercept_##_i }
4306 #define DIP(_y, _i, _p) { .flags = (_y)|Intercept|CheckPerm, \
4307 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4308 #define N D(NotImpl)
4309 #define EXT(_f, _e) { .flags = ((_f) | RMExt), .u.group = (_e) }
4310 #define G(_f, _g) { .flags = ((_f) | Group | ModRM), .u.group = (_g) }
4311 #define GD(_f, _g) { .flags = ((_f) | GroupDual | ModRM), .u.gdual = (_g) }
4312 #define ID(_f, _i) { .flags = ((_f) | InstrDual | ModRM), .u.idual = (_i) }
4313 #define MD(_f, _m) { .flags = ((_f) | ModeDual), .u.mdual = (_m) }
4314 #define E(_f, _e) { .flags = ((_f) | Escape | ModRM), .u.esc = (_e) }
4315 #define I(_f, _e) { .flags = (_f), .u.execute = (_e) }
4316 #define F(_f, _e) { .flags = (_f) | Fastop, .u.fastop = (_e) }
4317 #define II(_f, _e, _i) \
4318 { .flags = (_f)|Intercept, .u.execute = (_e), .intercept = x86_intercept_##_i }
4319 #define IIP(_f, _e, _i, _p) \
4320 { .flags = (_f)|Intercept|CheckPerm, .u.execute = (_e), \
4321 .intercept = x86_intercept_##_i, .check_perm = (_p) }
4322 #define GP(_f, _g) { .flags = ((_f) | Prefix), .u.gprefix = (_g) }
4324 #define D2bv(_f) D((_f) | ByteOp), D(_f)
4325 #define D2bvIP(_f, _i, _p) DIP((_f) | ByteOp, _i, _p), DIP(_f, _i, _p)
4326 #define I2bv(_f, _e) I((_f) | ByteOp, _e), I(_f, _e)
4327 #define F2bv(_f, _e) F((_f) | ByteOp, _e), F(_f, _e)
4328 #define I2bvIP(_f, _e, _i, _p) \
4329 IIP((_f) | ByteOp, _e, _i, _p), IIP(_f, _e, _i, _p)
4331 #define F6ALU(_f, _e) F2bv((_f) | DstMem | SrcReg | ModRM, _e), \
4332 F2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
4333 F2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
4335 static const struct opcode group7_rm0[] = {
4337 I(SrcNone | Priv | EmulateOnUD, em_hypercall),
4341 static const struct opcode group7_rm1[] = {
4342 DI(SrcNone | Priv, monitor),
4343 DI(SrcNone | Priv, mwait),
4347 static const struct opcode group7_rm3[] = {
4348 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
4349 II(SrcNone | Prot | EmulateOnUD, em_hypercall, vmmcall),
4350 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
4351 DIP(SrcNone | Prot | Priv, vmsave, check_svme_pa),
4352 DIP(SrcNone | Prot | Priv, stgi, check_svme),
4353 DIP(SrcNone | Prot | Priv, clgi, check_svme),
4354 DIP(SrcNone | Prot | Priv, skinit, check_svme),
4355 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
4358 static const struct opcode group7_rm7[] = {
4360 DIP(SrcNone, rdtscp, check_rdtsc),
4364 static const struct opcode group1[] = {
4366 F(Lock | PageTable, em_or),
4369 F(Lock | PageTable, em_and),
4375 static const struct opcode group1A[] = {
4376 I(DstMem | SrcNone | Mov | Stack | IncSP | TwoMemOp, em_pop), N, N, N, N, N, N, N,
4379 static const struct opcode group2[] = {
4380 F(DstMem | ModRM, em_rol),
4381 F(DstMem | ModRM, em_ror),
4382 F(DstMem | ModRM, em_rcl),
4383 F(DstMem | ModRM, em_rcr),
4384 F(DstMem | ModRM, em_shl),
4385 F(DstMem | ModRM, em_shr),
4386 F(DstMem | ModRM, em_shl),
4387 F(DstMem | ModRM, em_sar),
4390 static const struct opcode group3[] = {
4391 F(DstMem | SrcImm | NoWrite, em_test),
4392 F(DstMem | SrcImm | NoWrite, em_test),
4393 F(DstMem | SrcNone | Lock, em_not),
4394 F(DstMem | SrcNone | Lock, em_neg),
4395 F(DstXacc | Src2Mem, em_mul_ex),
4396 F(DstXacc | Src2Mem, em_imul_ex),
4397 F(DstXacc | Src2Mem, em_div_ex),
4398 F(DstXacc | Src2Mem, em_idiv_ex),
4401 static const struct opcode group4[] = {
4402 F(ByteOp | DstMem | SrcNone | Lock, em_inc),
4403 F(ByteOp | DstMem | SrcNone | Lock, em_dec),
4407 static const struct opcode group5[] = {
4408 F(DstMem | SrcNone | Lock, em_inc),
4409 F(DstMem | SrcNone | Lock, em_dec),
4410 I(SrcMem | NearBranch, em_call_near_abs),
4411 I(SrcMemFAddr | ImplicitOps, em_call_far),
4412 I(SrcMem | NearBranch, em_jmp_abs),
4413 I(SrcMemFAddr | ImplicitOps, em_jmp_far),
4414 I(SrcMem | Stack | TwoMemOp, em_push), D(Undefined),
4417 static const struct opcode group6[] = {
4418 DI(Prot | DstMem, sldt),
4419 DI(Prot | DstMem, str),
4420 II(Prot | Priv | SrcMem16, em_lldt, lldt),
4421 II(Prot | Priv | SrcMem16, em_ltr, ltr),
4425 static const struct group_dual group7 = { {
4426 II(Mov | DstMem, em_sgdt, sgdt),
4427 II(Mov | DstMem, em_sidt, sidt),
4428 II(SrcMem | Priv, em_lgdt, lgdt),
4429 II(SrcMem | Priv, em_lidt, lidt),
4430 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4431 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4432 II(SrcMem | ByteOp | Priv | NoAccess, em_invlpg, invlpg),
4436 N, EXT(0, group7_rm3),
4437 II(SrcNone | DstMem | Mov, em_smsw, smsw), N,
4438 II(SrcMem16 | Mov | Priv, em_lmsw, lmsw),
4442 static const struct opcode group8[] = {
4444 F(DstMem | SrcImmByte | NoWrite, em_bt),
4445 F(DstMem | SrcImmByte | Lock | PageTable, em_bts),
4446 F(DstMem | SrcImmByte | Lock, em_btr),
4447 F(DstMem | SrcImmByte | Lock | PageTable, em_btc),
4451 * The "memory" destination is actually always a register, since we come
4452 * from the register case of group9.
4454 static const struct gprefix pfx_0f_c7_7 = {
4455 N, N, N, II(DstMem | ModRM | Op3264 | EmulateOnUD, em_rdpid, rdtscp),
4459 static const struct group_dual group9 = { {
4460 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
4462 N, N, N, N, N, N, N,
4463 GP(0, &pfx_0f_c7_7),
4466 static const struct opcode group11[] = {
4467 I(DstMem | SrcImm | Mov | PageTable, em_mov),
4471 static const struct gprefix pfx_0f_ae_7 = {
4472 I(SrcMem | ByteOp, em_clflush), I(SrcMem | ByteOp, em_clflushopt), N, N,
4475 static const struct group_dual group15 = { {
4476 I(ModRM | Aligned16, em_fxsave),
4477 I(ModRM | Aligned16, em_fxrstor),
4478 N, N, N, N, N, GP(0, &pfx_0f_ae_7),
4480 N, N, N, N, N, N, N, N,
4483 static const struct gprefix pfx_0f_6f_0f_7f = {
4484 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
4487 static const struct instr_dual instr_dual_0f_2b = {
4491 static const struct gprefix pfx_0f_2b = {
4492 ID(0, &instr_dual_0f_2b), ID(0, &instr_dual_0f_2b), N, N,
4495 static const struct gprefix pfx_0f_28_0f_29 = {
4496 I(Aligned, em_mov), I(Aligned, em_mov), N, N,
4499 static const struct gprefix pfx_0f_e7 = {
4500 N, I(Sse, em_mov), N, N,
4503 static const struct escape escape_d9 = { {
4504 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstcw),
4507 N, N, N, N, N, N, N, N,
4509 N, N, N, N, N, N, N, N,
4511 N, N, N, N, N, N, N, N,
4513 N, N, N, N, N, N, N, N,
4515 N, N, N, N, N, N, N, N,
4517 N, N, N, N, N, N, N, N,
4519 N, N, N, N, N, N, N, N,
4521 N, N, N, N, N, N, N, N,
4524 static const struct escape escape_db = { {
4525 N, N, N, N, N, N, N, N,
4528 N, N, N, N, N, N, N, N,
4530 N, N, N, N, N, N, N, N,
4532 N, N, N, N, N, N, N, N,
4534 N, N, N, N, N, N, N, N,
4536 N, N, N, I(ImplicitOps, em_fninit), N, N, N, N,
4538 N, N, N, N, N, N, N, N,
4540 N, N, N, N, N, N, N, N,
4542 N, N, N, N, N, N, N, N,
4545 static const struct escape escape_dd = { {
4546 N, N, N, N, N, N, N, I(DstMem16 | Mov, em_fnstsw),
4549 N, N, N, N, N, N, N, N,
4551 N, N, N, N, N, N, N, N,
4553 N, N, N, N, N, N, N, N,
4555 N, N, N, N, N, N, N, N,
4557 N, N, N, N, N, N, N, N,
4559 N, N, N, N, N, N, N, N,
4561 N, N, N, N, N, N, N, N,
4563 N, N, N, N, N, N, N, N,
4566 static const struct instr_dual instr_dual_0f_c3 = {
4567 I(DstMem | SrcReg | ModRM | No16 | Mov, em_mov), N
4570 static const struct mode_dual mode_dual_63 = {
4571 N, I(DstReg | SrcMem32 | ModRM | Mov, em_movsxd)
4574 static const struct opcode opcode_table[256] = {
4576 F6ALU(Lock, em_add),
4577 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
4578 I(ImplicitOps | Stack | No64 | Src2ES, em_pop_sreg),
4580 F6ALU(Lock | PageTable, em_or),
4581 I(ImplicitOps | Stack | No64 | Src2CS, em_push_sreg),
4584 F6ALU(Lock, em_adc),
4585 I(ImplicitOps | Stack | No64 | Src2SS, em_push_sreg),
4586 I(ImplicitOps | Stack | No64 | Src2SS, em_pop_sreg),
4588 F6ALU(Lock, em_sbb),
4589 I(ImplicitOps | Stack | No64 | Src2DS, em_push_sreg),
4590 I(ImplicitOps | Stack | No64 | Src2DS, em_pop_sreg),
4592 F6ALU(Lock | PageTable, em_and), N, N,
4594 F6ALU(Lock, em_sub), N, I(ByteOp | DstAcc | No64, em_das),
4596 F6ALU(Lock, em_xor), N, N,
4598 F6ALU(NoWrite, em_cmp), N, N,
4600 X8(F(DstReg, em_inc)), X8(F(DstReg, em_dec)),
4602 X8(I(SrcReg | Stack, em_push)),
4604 X8(I(DstReg | Stack, em_pop)),
4606 I(ImplicitOps | Stack | No64, em_pusha),
4607 I(ImplicitOps | Stack | No64, em_popa),
4608 N, MD(ModRM, &mode_dual_63),
4611 I(SrcImm | Mov | Stack, em_push),
4612 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
4613 I(SrcImmByte | Mov | Stack, em_push),
4614 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
4615 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
4616 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
4618 X16(D(SrcImmByte | NearBranch)),
4620 G(ByteOp | DstMem | SrcImm, group1),
4621 G(DstMem | SrcImm, group1),
4622 G(ByteOp | DstMem | SrcImm | No64, group1),
4623 G(DstMem | SrcImmByte, group1),
4624 F2bv(DstMem | SrcReg | ModRM | NoWrite, em_test),
4625 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable, em_xchg),
4627 I2bv(DstMem | SrcReg | ModRM | Mov | PageTable, em_mov),
4628 I2bv(DstReg | SrcMem | ModRM | Mov, em_mov),
4629 I(DstMem | SrcNone | ModRM | Mov | PageTable, em_mov_rm_sreg),
4630 D(ModRM | SrcMem | NoAccess | DstReg),
4631 I(ImplicitOps | SrcMem16 | ModRM, em_mov_sreg_rm),
4634 DI(SrcAcc | DstReg, pause), X7(D(SrcAcc | DstReg)),
4636 D(DstAcc | SrcNone), I(ImplicitOps | SrcAcc, em_cwd),
4637 I(SrcImmFAddr | No64, em_call_far), N,
4638 II(ImplicitOps | Stack, em_pushf, pushf),
4639 II(ImplicitOps | Stack, em_popf, popf),
4640 I(ImplicitOps, em_sahf), I(ImplicitOps, em_lahf),
4642 I2bv(DstAcc | SrcMem | Mov | MemAbs, em_mov),
4643 I2bv(DstMem | SrcAcc | Mov | MemAbs | PageTable, em_mov),
4644 I2bv(SrcSI | DstDI | Mov | String | TwoMemOp, em_mov),
4645 F2bv(SrcSI | DstDI | String | NoWrite | TwoMemOp, em_cmp_r),
4647 F2bv(DstAcc | SrcImm | NoWrite, em_test),
4648 I2bv(SrcAcc | DstDI | Mov | String, em_mov),
4649 I2bv(SrcSI | DstAcc | Mov | String, em_mov),
4650 F2bv(SrcAcc | DstDI | String | NoWrite, em_cmp_r),
4652 X8(I(ByteOp | DstReg | SrcImm | Mov, em_mov)),
4654 X8(I(DstReg | SrcImm64 | Mov, em_mov)),
4656 G(ByteOp | Src2ImmByte, group2), G(Src2ImmByte, group2),
4657 I(ImplicitOps | NearBranch | SrcImmU16, em_ret_near_imm),
4658 I(ImplicitOps | NearBranch, em_ret),
4659 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2ES, em_lseg),
4660 I(DstReg | SrcMemFAddr | ModRM | No64 | Src2DS, em_lseg),
4661 G(ByteOp, group11), G(0, group11),
4663 I(Stack | SrcImmU16 | Src2ImmByte, em_enter), I(Stack, em_leave),
4664 I(ImplicitOps | SrcImmU16, em_ret_far_imm),
4665 I(ImplicitOps, em_ret_far),
4666 D(ImplicitOps), DI(SrcImmByte, intn),
4667 D(ImplicitOps | No64), II(ImplicitOps, em_iret, iret),
4669 G(Src2One | ByteOp, group2), G(Src2One, group2),
4670 G(Src2CL | ByteOp, group2), G(Src2CL, group2),
4671 I(DstAcc | SrcImmUByte | No64, em_aam),
4672 I(DstAcc | SrcImmUByte | No64, em_aad),
4673 F(DstAcc | ByteOp | No64, em_salc),
4674 I(DstAcc | SrcXLat | ByteOp, em_mov),
4676 N, E(0, &escape_d9), N, E(0, &escape_db), N, E(0, &escape_dd), N, N,
4678 X3(I(SrcImmByte | NearBranch, em_loop)),
4679 I(SrcImmByte | NearBranch, em_jcxz),
4680 I2bvIP(SrcImmUByte | DstAcc, em_in, in, check_perm_in),
4681 I2bvIP(SrcAcc | DstImmUByte, em_out, out, check_perm_out),
4683 I(SrcImm | NearBranch, em_call), D(SrcImm | ImplicitOps | NearBranch),
4684 I(SrcImmFAddr | No64, em_jmp_far),
4685 D(SrcImmByte | ImplicitOps | NearBranch),
4686 I2bvIP(SrcDX | DstAcc, em_in, in, check_perm_in),
4687 I2bvIP(SrcAcc | DstDX, em_out, out, check_perm_out),
4689 N, DI(ImplicitOps, icebp), N, N,
4690 DI(ImplicitOps | Priv, hlt), D(ImplicitOps),
4691 G(ByteOp, group3), G(0, group3),
4693 D(ImplicitOps), D(ImplicitOps),
4694 I(ImplicitOps, em_cli), I(ImplicitOps, em_sti),
4695 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
4698 static const struct opcode twobyte_table[256] = {
4700 G(0, group6), GD(0, &group7), N, N,
4701 N, I(ImplicitOps | EmulateOnUD, em_syscall),
4702 II(ImplicitOps | Priv, em_clts, clts), N,
4703 DI(ImplicitOps | Priv, invd), DI(ImplicitOps | Priv, wbinvd), N, N,
4704 N, D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N,
4706 N, N, N, N, N, N, N, N,
4707 D(ImplicitOps | ModRM | SrcMem | NoAccess),
4708 N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess),
4710 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read),
4711 DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read),
4712 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write,
4714 IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write,
4717 GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_28_0f_29),
4718 GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_28_0f_29),
4719 N, GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_2b),
4722 II(ImplicitOps | Priv, em_wrmsr, wrmsr),
4723 IIP(ImplicitOps, em_rdtsc, rdtsc, check_rdtsc),
4724 II(ImplicitOps | Priv, em_rdmsr, rdmsr),
4725 IIP(ImplicitOps, em_rdpmc, rdpmc, check_rdpmc),
4726 I(ImplicitOps | EmulateOnUD, em_sysenter),
4727 I(ImplicitOps | Priv | EmulateOnUD, em_sysexit),
4729 N, N, N, N, N, N, N, N,
4731 X16(D(DstReg | SrcMem | ModRM)),
4733 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4738 N, N, N, GP(SrcMem | DstReg | ModRM | Mov, &pfx_0f_6f_0f_7f),
4743 N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_6f_0f_7f),
4745 X16(D(SrcImm | NearBranch)),
4747 X16(D(ByteOp | DstMem | SrcNone | ModRM| Mov)),
4749 I(Stack | Src2FS, em_push_sreg), I(Stack | Src2FS, em_pop_sreg),
4750 II(ImplicitOps, em_cpuid, cpuid),
4751 F(DstMem | SrcReg | ModRM | BitOp | NoWrite, em_bt),
4752 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shld),
4753 F(DstMem | SrcReg | Src2CL | ModRM, em_shld), N, N,
4755 I(Stack | Src2GS, em_push_sreg), I(Stack | Src2GS, em_pop_sreg),
4756 II(EmulateOnUD | ImplicitOps, em_rsm, rsm),
4757 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_bts),
4758 F(DstMem | SrcReg | Src2ImmByte | ModRM, em_shrd),
4759 F(DstMem | SrcReg | Src2CL | ModRM, em_shrd),
4760 GD(0, &group15), F(DstReg | SrcMem | ModRM, em_imul),
4762 I2bv(DstMem | SrcReg | ModRM | Lock | PageTable | SrcWrite, em_cmpxchg),
4763 I(DstReg | SrcMemFAddr | ModRM | Src2SS, em_lseg),
4764 F(DstMem | SrcReg | ModRM | BitOp | Lock, em_btr),
4765 I(DstReg | SrcMemFAddr | ModRM | Src2FS, em_lseg),
4766 I(DstReg | SrcMemFAddr | ModRM | Src2GS, em_lseg),
4767 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4771 F(DstMem | SrcReg | ModRM | BitOp | Lock | PageTable, em_btc),
4772 I(DstReg | SrcMem | ModRM, em_bsf_c),
4773 I(DstReg | SrcMem | ModRM, em_bsr_c),
4774 D(DstReg | SrcMem8 | ModRM | Mov), D(DstReg | SrcMem16 | ModRM | Mov),
4776 F2bv(DstMem | SrcReg | ModRM | SrcWrite | Lock, em_xadd),
4777 N, ID(0, &instr_dual_0f_c3),
4778 N, N, N, GD(0, &group9),
4780 X8(I(DstReg, em_bswap)),
4782 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N,
4784 N, N, N, N, N, N, N, GP(SrcReg | DstMem | ModRM | Mov, &pfx_0f_e7),
4785 N, N, N, N, N, N, N, N,
4787 N, N, N, N, N, N, N, N, N, N, N, N, N, N, N, N
4790 static const struct instr_dual instr_dual_0f_38_f0 = {
4791 I(DstReg | SrcMem | Mov, em_movbe), N
4794 static const struct instr_dual instr_dual_0f_38_f1 = {
4795 I(DstMem | SrcReg | Mov, em_movbe), N
4798 static const struct gprefix three_byte_0f_38_f0 = {
4799 ID(0, &instr_dual_0f_38_f0), N, N, N
4802 static const struct gprefix three_byte_0f_38_f1 = {
4803 ID(0, &instr_dual_0f_38_f1), N, N, N
4807 * Insns below are selected by the prefix which indexed by the third opcode
4810 static const struct opcode opcode_map_0f_38[256] = {
4812 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4814 X16(N), X16(N), X16(N), X16(N), X16(N), X16(N), X16(N),
4816 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f0),
4817 GP(EmulateOnUD | ModRM, &three_byte_0f_38_f1),
4838 static unsigned imm_size(struct x86_emulate_ctxt *ctxt)
4842 size = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4848 static int decode_imm(struct x86_emulate_ctxt *ctxt, struct operand *op,
4849 unsigned size, bool sign_extension)
4851 int rc = X86EMUL_CONTINUE;
4855 op->addr.mem.ea = ctxt->_eip;
4856 /* NB. Immediates are sign-extended as necessary. */
4857 switch (op->bytes) {
4859 op->val = insn_fetch(s8, ctxt);
4862 op->val = insn_fetch(s16, ctxt);
4865 op->val = insn_fetch(s32, ctxt);
4868 op->val = insn_fetch(s64, ctxt);
4871 if (!sign_extension) {
4872 switch (op->bytes) {
4880 op->val &= 0xffffffff;
4888 static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
4891 int rc = X86EMUL_CONTINUE;
4895 decode_register_operand(ctxt, op);
4898 rc = decode_imm(ctxt, op, 1, false);
4901 ctxt->memop.bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4905 if (ctxt->d & BitOp)
4906 fetch_bit_operand(ctxt);
4907 op->orig_val = op->val;
4910 ctxt->memop.bytes = (ctxt->op_bytes == 8) ? 16 : 8;
4914 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4915 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4916 fetch_register_operand(op);
4917 op->orig_val = op->val;
4921 op->bytes = (ctxt->d & ByteOp) ? 2 : ctxt->op_bytes;
4922 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
4923 fetch_register_operand(op);
4924 op->orig_val = op->val;
4927 if (ctxt->d & ByteOp) {
4932 op->bytes = ctxt->op_bytes;
4933 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4934 fetch_register_operand(op);
4935 op->orig_val = op->val;
4939 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4941 register_address(ctxt, VCPU_REGS_RDI);
4942 op->addr.mem.seg = VCPU_SREG_ES;
4949 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
4950 fetch_register_operand(op);
4955 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
4958 rc = decode_imm(ctxt, op, 1, true);
4966 rc = decode_imm(ctxt, op, imm_size(ctxt), true);
4969 rc = decode_imm(ctxt, op, ctxt->op_bytes, true);
4972 ctxt->memop.bytes = 1;
4973 if (ctxt->memop.type == OP_REG) {
4974 ctxt->memop.addr.reg = decode_register(ctxt,
4975 ctxt->modrm_rm, true);
4976 fetch_register_operand(&ctxt->memop);
4980 ctxt->memop.bytes = 2;
4983 ctxt->memop.bytes = 4;
4986 rc = decode_imm(ctxt, op, 2, false);
4989 rc = decode_imm(ctxt, op, imm_size(ctxt), false);
4993 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
4995 register_address(ctxt, VCPU_REGS_RSI);
4996 op->addr.mem.seg = ctxt->seg_override;
5002 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
5005 reg_read(ctxt, VCPU_REGS_RBX) +
5006 (reg_read(ctxt, VCPU_REGS_RAX) & 0xff));
5007 op->addr.mem.seg = ctxt->seg_override;
5012 op->addr.mem.ea = ctxt->_eip;
5013 op->bytes = ctxt->op_bytes + 2;
5014 insn_fetch_arr(op->valptr, op->bytes, ctxt);
5017 ctxt->memop.bytes = ctxt->op_bytes + 2;
5021 op->val = VCPU_SREG_ES;
5025 op->val = VCPU_SREG_CS;
5029 op->val = VCPU_SREG_SS;
5033 op->val = VCPU_SREG_DS;
5037 op->val = VCPU_SREG_FS;
5041 op->val = VCPU_SREG_GS;
5044 /* Special instructions do their own operand decoding. */
5046 op->type = OP_NONE; /* Disable writeback. */
5054 int x86_decode_insn(struct x86_emulate_ctxt *ctxt, void *insn, int insn_len)
5056 int rc = X86EMUL_CONTINUE;
5057 int mode = ctxt->mode;
5058 int def_op_bytes, def_ad_bytes, goffset, simd_prefix;
5059 bool op_prefix = false;
5060 bool has_seg_override = false;
5061 struct opcode opcode;
5063 struct desc_struct desc;
5065 ctxt->memop.type = OP_NONE;
5066 ctxt->memopp = NULL;
5067 ctxt->_eip = ctxt->eip;
5068 ctxt->fetch.ptr = ctxt->fetch.data;
5069 ctxt->fetch.end = ctxt->fetch.data + insn_len;
5070 ctxt->opcode_len = 1;
5071 ctxt->intercept = x86_intercept_none;
5073 memcpy(ctxt->fetch.data, insn, insn_len);
5075 rc = __do_insn_fetch_bytes(ctxt, 1);
5076 if (rc != X86EMUL_CONTINUE)
5081 case X86EMUL_MODE_REAL:
5082 case X86EMUL_MODE_VM86:
5083 def_op_bytes = def_ad_bytes = 2;
5084 ctxt->ops->get_segment(ctxt, &dummy, &desc, NULL, VCPU_SREG_CS);
5086 def_op_bytes = def_ad_bytes = 4;
5088 case X86EMUL_MODE_PROT16:
5089 def_op_bytes = def_ad_bytes = 2;
5091 case X86EMUL_MODE_PROT32:
5092 def_op_bytes = def_ad_bytes = 4;
5094 #ifdef CONFIG_X86_64
5095 case X86EMUL_MODE_PROT64:
5101 return EMULATION_FAILED;
5104 ctxt->op_bytes = def_op_bytes;
5105 ctxt->ad_bytes = def_ad_bytes;
5107 /* Legacy prefixes. */
5109 switch (ctxt->b = insn_fetch(u8, ctxt)) {
5110 case 0x66: /* operand-size override */
5112 /* switch between 2/4 bytes */
5113 ctxt->op_bytes = def_op_bytes ^ 6;
5115 case 0x67: /* address-size override */
5116 if (mode == X86EMUL_MODE_PROT64)
5117 /* switch between 4/8 bytes */
5118 ctxt->ad_bytes = def_ad_bytes ^ 12;
5120 /* switch between 2/4 bytes */
5121 ctxt->ad_bytes = def_ad_bytes ^ 6;
5123 case 0x26: /* ES override */
5124 has_seg_override = true;
5125 ctxt->seg_override = VCPU_SREG_ES;
5127 case 0x2e: /* CS override */
5128 has_seg_override = true;
5129 ctxt->seg_override = VCPU_SREG_CS;
5131 case 0x36: /* SS override */
5132 has_seg_override = true;
5133 ctxt->seg_override = VCPU_SREG_SS;
5135 case 0x3e: /* DS override */
5136 has_seg_override = true;
5137 ctxt->seg_override = VCPU_SREG_DS;
5139 case 0x64: /* FS override */
5140 has_seg_override = true;
5141 ctxt->seg_override = VCPU_SREG_FS;
5143 case 0x65: /* GS override */
5144 has_seg_override = true;
5145 ctxt->seg_override = VCPU_SREG_GS;
5147 case 0x40 ... 0x4f: /* REX */
5148 if (mode != X86EMUL_MODE_PROT64)
5150 ctxt->rex_prefix = ctxt->b;
5152 case 0xf0: /* LOCK */
5153 ctxt->lock_prefix = 1;
5155 case 0xf2: /* REPNE/REPNZ */
5156 case 0xf3: /* REP/REPE/REPZ */
5157 ctxt->rep_prefix = ctxt->b;
5163 /* Any legacy prefix after a REX prefix nullifies its effect. */
5165 ctxt->rex_prefix = 0;
5171 if (ctxt->rex_prefix & 8)
5172 ctxt->op_bytes = 8; /* REX.W */
5174 /* Opcode byte(s). */
5175 opcode = opcode_table[ctxt->b];
5176 /* Two-byte opcode? */
5177 if (ctxt->b == 0x0f) {
5178 ctxt->opcode_len = 2;
5179 ctxt->b = insn_fetch(u8, ctxt);
5180 opcode = twobyte_table[ctxt->b];
5182 /* 0F_38 opcode map */
5183 if (ctxt->b == 0x38) {
5184 ctxt->opcode_len = 3;
5185 ctxt->b = insn_fetch(u8, ctxt);
5186 opcode = opcode_map_0f_38[ctxt->b];
5189 ctxt->d = opcode.flags;
5191 if (ctxt->d & ModRM)
5192 ctxt->modrm = insn_fetch(u8, ctxt);
5194 /* vex-prefix instructions are not implemented */
5195 if (ctxt->opcode_len == 1 && (ctxt->b == 0xc5 || ctxt->b == 0xc4) &&
5196 (mode == X86EMUL_MODE_PROT64 || (ctxt->modrm & 0xc0) == 0xc0)) {
5200 while (ctxt->d & GroupMask) {
5201 switch (ctxt->d & GroupMask) {
5203 goffset = (ctxt->modrm >> 3) & 7;
5204 opcode = opcode.u.group[goffset];
5207 goffset = (ctxt->modrm >> 3) & 7;
5208 if ((ctxt->modrm >> 6) == 3)
5209 opcode = opcode.u.gdual->mod3[goffset];
5211 opcode = opcode.u.gdual->mod012[goffset];
5214 goffset = ctxt->modrm & 7;
5215 opcode = opcode.u.group[goffset];
5218 if (ctxt->rep_prefix && op_prefix)
5219 return EMULATION_FAILED;
5220 simd_prefix = op_prefix ? 0x66 : ctxt->rep_prefix;
5221 switch (simd_prefix) {
5222 case 0x00: opcode = opcode.u.gprefix->pfx_no; break;
5223 case 0x66: opcode = opcode.u.gprefix->pfx_66; break;
5224 case 0xf2: opcode = opcode.u.gprefix->pfx_f2; break;
5225 case 0xf3: opcode = opcode.u.gprefix->pfx_f3; break;
5229 if (ctxt->modrm > 0xbf) {
5230 size_t size = ARRAY_SIZE(opcode.u.esc->high);
5231 u32 index = array_index_nospec(
5232 ctxt->modrm - 0xc0, size);
5234 opcode = opcode.u.esc->high[index];
5236 opcode = opcode.u.esc->op[(ctxt->modrm >> 3) & 7];
5240 if ((ctxt->modrm >> 6) == 3)
5241 opcode = opcode.u.idual->mod3;
5243 opcode = opcode.u.idual->mod012;
5246 if (ctxt->mode == X86EMUL_MODE_PROT64)
5247 opcode = opcode.u.mdual->mode64;
5249 opcode = opcode.u.mdual->mode32;
5252 return EMULATION_FAILED;
5255 ctxt->d &= ~(u64)GroupMask;
5256 ctxt->d |= opcode.flags;
5261 return EMULATION_FAILED;
5263 ctxt->execute = opcode.u.execute;
5265 if (unlikely(ctxt->ud) && likely(!(ctxt->d & EmulateOnUD)))
5266 return EMULATION_FAILED;
5268 if (unlikely(ctxt->d &
5269 (NotImpl|Stack|Op3264|Sse|Mmx|Intercept|CheckPerm|NearBranch|
5272 * These are copied unconditionally here, and checked unconditionally
5273 * in x86_emulate_insn.
5275 ctxt->check_perm = opcode.check_perm;
5276 ctxt->intercept = opcode.intercept;
5278 if (ctxt->d & NotImpl)
5279 return EMULATION_FAILED;
5281 if (mode == X86EMUL_MODE_PROT64) {
5282 if (ctxt->op_bytes == 4 && (ctxt->d & Stack))
5284 else if (ctxt->d & NearBranch)
5288 if (ctxt->d & Op3264) {
5289 if (mode == X86EMUL_MODE_PROT64)
5295 if ((ctxt->d & No16) && ctxt->op_bytes == 2)
5299 ctxt->op_bytes = 16;
5300 else if (ctxt->d & Mmx)
5304 /* ModRM and SIB bytes. */
5305 if (ctxt->d & ModRM) {
5306 rc = decode_modrm(ctxt, &ctxt->memop);
5307 if (!has_seg_override) {
5308 has_seg_override = true;
5309 ctxt->seg_override = ctxt->modrm_seg;
5311 } else if (ctxt->d & MemAbs)
5312 rc = decode_abs(ctxt, &ctxt->memop);
5313 if (rc != X86EMUL_CONTINUE)
5316 if (!has_seg_override)
5317 ctxt->seg_override = VCPU_SREG_DS;
5319 ctxt->memop.addr.mem.seg = ctxt->seg_override;
5322 * Decode and fetch the source operand: register, memory
5325 rc = decode_operand(ctxt, &ctxt->src, (ctxt->d >> SrcShift) & OpMask);
5326 if (rc != X86EMUL_CONTINUE)
5330 * Decode and fetch the second source operand: register, memory
5333 rc = decode_operand(ctxt, &ctxt->src2, (ctxt->d >> Src2Shift) & OpMask);
5334 if (rc != X86EMUL_CONTINUE)
5337 /* Decode and fetch the destination operand: register or memory. */
5338 rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask);
5340 if (ctxt->rip_relative && likely(ctxt->memopp))
5341 ctxt->memopp->addr.mem.ea = address_mask(ctxt,
5342 ctxt->memopp->addr.mem.ea + ctxt->_eip);
5345 if (rc == X86EMUL_PROPAGATE_FAULT)
5346 ctxt->have_exception = true;
5347 return (rc != X86EMUL_CONTINUE) ? EMULATION_FAILED : EMULATION_OK;
5350 bool x86_page_table_writing_insn(struct x86_emulate_ctxt *ctxt)
5352 return ctxt->d & PageTable;
5355 static bool string_insn_completed(struct x86_emulate_ctxt *ctxt)
5357 /* The second termination condition only applies for REPE
5358 * and REPNE. Test if the repeat string operation prefix is
5359 * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the
5360 * corresponding termination condition according to:
5361 * - if REPE/REPZ and ZF = 0 then done
5362 * - if REPNE/REPNZ and ZF = 1 then done
5364 if (((ctxt->b == 0xa6) || (ctxt->b == 0xa7) ||
5365 (ctxt->b == 0xae) || (ctxt->b == 0xaf))
5366 && (((ctxt->rep_prefix == REPE_PREFIX) &&
5367 ((ctxt->eflags & X86_EFLAGS_ZF) == 0))
5368 || ((ctxt->rep_prefix == REPNE_PREFIX) &&
5369 ((ctxt->eflags & X86_EFLAGS_ZF) == X86_EFLAGS_ZF))))
5375 static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt)
5379 ctxt->ops->get_fpu(ctxt);
5380 rc = asm_safe("fwait");
5381 ctxt->ops->put_fpu(ctxt);
5383 if (unlikely(rc != X86EMUL_CONTINUE))
5384 return emulate_exception(ctxt, MF_VECTOR, 0, false);
5386 return X86EMUL_CONTINUE;
5389 static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
5392 if (op->type == OP_MM)
5393 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
5396 static int fastop(struct x86_emulate_ctxt *ctxt, void (*fop)(struct fastop *))
5398 ulong flags = (ctxt->eflags & EFLAGS_MASK) | X86_EFLAGS_IF;
5400 if (!(ctxt->d & ByteOp))
5401 fop += __ffs(ctxt->dst.bytes) * FASTOP_SIZE;
5403 asm("push %[flags]; popf; " CALL_NOSPEC " ; pushf; pop %[flags]\n"
5404 : "+a"(ctxt->dst.val), "+d"(ctxt->src.val), [flags]"+D"(flags),
5405 [thunk_target]"+S"(fop), ASM_CALL_CONSTRAINT
5406 : "c"(ctxt->src2.val));
5408 ctxt->eflags = (ctxt->eflags & ~EFLAGS_MASK) | (flags & EFLAGS_MASK);
5409 if (!fop) /* exception is returned in fop variable */
5410 return emulate_de(ctxt);
5411 return X86EMUL_CONTINUE;
5414 void init_decode_cache(struct x86_emulate_ctxt *ctxt)
5416 memset(&ctxt->rip_relative, 0,
5417 (void *)&ctxt->modrm - (void *)&ctxt->rip_relative);
5419 ctxt->io_read.pos = 0;
5420 ctxt->io_read.end = 0;
5421 ctxt->mem_read.end = 0;
5424 int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
5426 const struct x86_emulate_ops *ops = ctxt->ops;
5427 int rc = X86EMUL_CONTINUE;
5428 int saved_dst_type = ctxt->dst.type;
5429 unsigned emul_flags;
5431 ctxt->mem_read.pos = 0;
5433 /* LOCK prefix is allowed only with some instructions */
5434 if (ctxt->lock_prefix && (!(ctxt->d & Lock) || ctxt->dst.type != OP_MEM)) {
5435 rc = emulate_ud(ctxt);
5439 if ((ctxt->d & SrcMask) == SrcMemFAddr && ctxt->src.type != OP_MEM) {
5440 rc = emulate_ud(ctxt);
5444 emul_flags = ctxt->ops->get_hflags(ctxt);
5445 if (unlikely(ctxt->d &
5446 (No64|Undefined|Sse|Mmx|Intercept|CheckPerm|Priv|Prot|String))) {
5447 if ((ctxt->mode == X86EMUL_MODE_PROT64 && (ctxt->d & No64)) ||
5448 (ctxt->d & Undefined)) {
5449 rc = emulate_ud(ctxt);
5453 if (((ctxt->d & (Sse|Mmx)) && ((ops->get_cr(ctxt, 0) & X86_CR0_EM)))
5454 || ((ctxt->d & Sse) && !(ops->get_cr(ctxt, 4) & X86_CR4_OSFXSR))) {
5455 rc = emulate_ud(ctxt);
5459 if ((ctxt->d & (Sse|Mmx)) && (ops->get_cr(ctxt, 0) & X86_CR0_TS)) {
5460 rc = emulate_nm(ctxt);
5464 if (ctxt->d & Mmx) {
5465 rc = flush_pending_x87_faults(ctxt);
5466 if (rc != X86EMUL_CONTINUE)
5469 * Now that we know the fpu is exception safe, we can fetch
5472 fetch_possible_mmx_operand(ctxt, &ctxt->src);
5473 fetch_possible_mmx_operand(ctxt, &ctxt->src2);
5474 if (!(ctxt->d & Mov))
5475 fetch_possible_mmx_operand(ctxt, &ctxt->dst);
5478 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && ctxt->intercept) {
5479 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5480 X86_ICPT_PRE_EXCEPT);
5481 if (rc != X86EMUL_CONTINUE)
5485 /* Instruction can only be executed in protected mode */
5486 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
5487 rc = emulate_ud(ctxt);
5491 /* Privileged instruction can be executed only in CPL=0 */
5492 if ((ctxt->d & Priv) && ops->cpl(ctxt)) {
5493 if (ctxt->d & PrivUD)
5494 rc = emulate_ud(ctxt);
5496 rc = emulate_gp(ctxt, 0);
5500 /* Do instruction specific permission checks */
5501 if (ctxt->d & CheckPerm) {
5502 rc = ctxt->check_perm(ctxt);
5503 if (rc != X86EMUL_CONTINUE)
5507 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5508 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5509 X86_ICPT_POST_EXCEPT);
5510 if (rc != X86EMUL_CONTINUE)
5514 if (ctxt->rep_prefix && (ctxt->d & String)) {
5515 /* All REP prefixes have the same first termination condition */
5516 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
5517 string_registers_quirk(ctxt);
5518 ctxt->eip = ctxt->_eip;
5519 ctxt->eflags &= ~X86_EFLAGS_RF;
5525 if ((ctxt->src.type == OP_MEM) && !(ctxt->d & NoAccess)) {
5526 rc = segmented_read(ctxt, ctxt->src.addr.mem,
5527 ctxt->src.valptr, ctxt->src.bytes);
5528 if (rc != X86EMUL_CONTINUE)
5530 ctxt->src.orig_val64 = ctxt->src.val64;
5533 if (ctxt->src2.type == OP_MEM) {
5534 rc = segmented_read(ctxt, ctxt->src2.addr.mem,
5535 &ctxt->src2.val, ctxt->src2.bytes);
5536 if (rc != X86EMUL_CONTINUE)
5540 if ((ctxt->d & DstMask) == ImplicitOps)
5544 if ((ctxt->dst.type == OP_MEM) && !(ctxt->d & Mov)) {
5545 /* optimisation - avoid slow emulated read if Mov */
5546 rc = segmented_read(ctxt, ctxt->dst.addr.mem,
5547 &ctxt->dst.val, ctxt->dst.bytes);
5548 if (rc != X86EMUL_CONTINUE) {
5549 if (!(ctxt->d & NoWrite) &&
5550 rc == X86EMUL_PROPAGATE_FAULT &&
5551 ctxt->exception.vector == PF_VECTOR)
5552 ctxt->exception.error_code |= PFERR_WRITE_MASK;
5556 /* Copy full 64-bit value for CMPXCHG8B. */
5557 ctxt->dst.orig_val64 = ctxt->dst.val64;
5561 if (unlikely(emul_flags & X86EMUL_GUEST_MASK) && (ctxt->d & Intercept)) {
5562 rc = emulator_check_intercept(ctxt, ctxt->intercept,
5563 X86_ICPT_POST_MEMACCESS);
5564 if (rc != X86EMUL_CONTINUE)
5568 if (ctxt->rep_prefix && (ctxt->d & String))
5569 ctxt->eflags |= X86_EFLAGS_RF;
5571 ctxt->eflags &= ~X86_EFLAGS_RF;
5573 if (ctxt->execute) {
5574 if (ctxt->d & Fastop) {
5575 void (*fop)(struct fastop *) = (void *)ctxt->execute;
5576 rc = fastop(ctxt, fop);
5577 if (rc != X86EMUL_CONTINUE)
5581 rc = ctxt->execute(ctxt);
5582 if (rc != X86EMUL_CONTINUE)
5587 if (ctxt->opcode_len == 2)
5589 else if (ctxt->opcode_len == 3)
5590 goto threebyte_insn;
5593 case 0x70 ... 0x7f: /* jcc (short) */
5594 if (test_cc(ctxt->b, ctxt->eflags))
5595 rc = jmp_rel(ctxt, ctxt->src.val);
5597 case 0x8d: /* lea r16/r32, m */
5598 ctxt->dst.val = ctxt->src.addr.mem.ea;
5600 case 0x90 ... 0x97: /* nop / xchg reg, rax */
5601 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
5602 ctxt->dst.type = OP_NONE;
5606 case 0x98: /* cbw/cwde/cdqe */
5607 switch (ctxt->op_bytes) {
5608 case 2: ctxt->dst.val = (s8)ctxt->dst.val; break;
5609 case 4: ctxt->dst.val = (s16)ctxt->dst.val; break;
5610 case 8: ctxt->dst.val = (s32)ctxt->dst.val; break;
5613 case 0xcc: /* int3 */
5614 rc = emulate_int(ctxt, 3);
5616 case 0xcd: /* int n */
5617 rc = emulate_int(ctxt, ctxt->src.val);
5619 case 0xce: /* into */
5620 if (ctxt->eflags & X86_EFLAGS_OF)
5621 rc = emulate_int(ctxt, 4);
5623 case 0xe9: /* jmp rel */
5624 case 0xeb: /* jmp rel short */
5625 rc = jmp_rel(ctxt, ctxt->src.val);
5626 ctxt->dst.type = OP_NONE; /* Disable writeback. */
5628 case 0xf4: /* hlt */
5629 ctxt->ops->halt(ctxt);
5631 case 0xf5: /* cmc */
5632 /* complement carry flag from eflags reg */
5633 ctxt->eflags ^= X86_EFLAGS_CF;
5635 case 0xf8: /* clc */
5636 ctxt->eflags &= ~X86_EFLAGS_CF;
5638 case 0xf9: /* stc */
5639 ctxt->eflags |= X86_EFLAGS_CF;
5641 case 0xfc: /* cld */
5642 ctxt->eflags &= ~X86_EFLAGS_DF;
5644 case 0xfd: /* std */
5645 ctxt->eflags |= X86_EFLAGS_DF;
5648 goto cannot_emulate;
5651 if (rc != X86EMUL_CONTINUE)
5655 if (ctxt->d & SrcWrite) {
5656 BUG_ON(ctxt->src.type == OP_MEM || ctxt->src.type == OP_MEM_STR);
5657 rc = writeback(ctxt, &ctxt->src);
5658 if (rc != X86EMUL_CONTINUE)
5661 if (!(ctxt->d & NoWrite)) {
5662 rc = writeback(ctxt, &ctxt->dst);
5663 if (rc != X86EMUL_CONTINUE)
5668 * restore dst type in case the decoding will be reused
5669 * (happens for string instruction )
5671 ctxt->dst.type = saved_dst_type;
5673 if ((ctxt->d & SrcMask) == SrcSI)
5674 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
5676 if ((ctxt->d & DstMask) == DstDI)
5677 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
5679 if (ctxt->rep_prefix && (ctxt->d & String)) {
5681 struct read_cache *r = &ctxt->io_read;
5682 if ((ctxt->d & SrcMask) == SrcSI)
5683 count = ctxt->src.count;
5685 count = ctxt->dst.count;
5686 register_address_increment(ctxt, VCPU_REGS_RCX, -count);
5688 if (!string_insn_completed(ctxt)) {
5690 * Re-enter guest when pio read ahead buffer is empty
5691 * or, if it is not used, after each 1024 iteration.
5693 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
5694 (r->end == 0 || r->end != r->pos)) {
5696 * Reset read cache. Usually happens before
5697 * decode, but since instruction is restarted
5698 * we have to do it here.
5700 ctxt->mem_read.end = 0;
5701 writeback_registers(ctxt);
5702 return EMULATION_RESTART;
5704 goto done; /* skip rip writeback */
5706 ctxt->eflags &= ~X86_EFLAGS_RF;
5709 ctxt->eip = ctxt->_eip;
5712 if (rc == X86EMUL_PROPAGATE_FAULT) {
5713 WARN_ON(ctxt->exception.vector > 0x1f);
5714 ctxt->have_exception = true;
5716 if (rc == X86EMUL_INTERCEPTED)
5717 return EMULATION_INTERCEPTED;
5719 if (rc == X86EMUL_CONTINUE)
5720 writeback_registers(ctxt);
5722 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
5726 case 0x09: /* wbinvd */
5727 (ctxt->ops->wbinvd)(ctxt);
5729 case 0x08: /* invd */
5730 case 0x0d: /* GrpP (prefetch) */
5731 case 0x18: /* Grp16 (prefetch/nop) */
5732 case 0x1f: /* nop */
5734 case 0x20: /* mov cr, reg */
5735 ctxt->dst.val = ops->get_cr(ctxt, ctxt->modrm_reg);
5737 case 0x21: /* mov from dr to reg */
5738 ops->get_dr(ctxt, ctxt->modrm_reg, &ctxt->dst.val);
5740 case 0x40 ... 0x4f: /* cmov */
5741 if (test_cc(ctxt->b, ctxt->eflags))
5742 ctxt->dst.val = ctxt->src.val;
5743 else if (ctxt->op_bytes != 4)
5744 ctxt->dst.type = OP_NONE; /* no writeback */
5746 case 0x80 ... 0x8f: /* jnz rel, etc*/
5747 if (test_cc(ctxt->b, ctxt->eflags))
5748 rc = jmp_rel(ctxt, ctxt->src.val);
5750 case 0x90 ... 0x9f: /* setcc r/m8 */
5751 ctxt->dst.val = test_cc(ctxt->b, ctxt->eflags);
5753 case 0xb6 ... 0xb7: /* movzx */
5754 ctxt->dst.bytes = ctxt->op_bytes;
5755 ctxt->dst.val = (ctxt->src.bytes == 1) ? (u8) ctxt->src.val
5756 : (u16) ctxt->src.val;
5758 case 0xbe ... 0xbf: /* movsx */
5759 ctxt->dst.bytes = ctxt->op_bytes;
5760 ctxt->dst.val = (ctxt->src.bytes == 1) ? (s8) ctxt->src.val :
5761 (s16) ctxt->src.val;
5764 goto cannot_emulate;
5769 if (rc != X86EMUL_CONTINUE)
5775 return EMULATION_FAILED;
5778 void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
5780 invalidate_registers(ctxt);
5783 void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
5785 writeback_registers(ctxt);
5788 bool emulator_can_use_gpa(struct x86_emulate_ctxt *ctxt)
5790 if (ctxt->rep_prefix && (ctxt->d & String))
5793 if (ctxt->d & TwoMemOp)