1 // SPDX-License-Identifier: GPL-2.0+
3 * Kernel Probes (KProbes)
5 * Copyright IBM Corp. 2002, 2006
7 * s390 port, used ppc64 as template. Mike Grundy <grundym@us.ibm.com>
10 #include <linux/kprobes.h>
11 #include <linux/ptrace.h>
12 #include <linux/preempt.h>
13 #include <linux/stop_machine.h>
14 #include <linux/kdebug.h>
15 #include <linux/uaccess.h>
16 #include <linux/extable.h>
17 #include <linux/module.h>
18 #include <linux/slab.h>
19 #include <linux/hardirq.h>
20 #include <linux/ftrace.h>
21 #include <asm/set_memory.h>
22 #include <asm/sections.h>
25 DEFINE_PER_CPU(struct kprobe *, current_kprobe);
26 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
28 struct kretprobe_blackpoint kretprobe_blacklist[] = { };
30 DEFINE_INSN_CACHE_OPS(dmainsn);
32 static void *alloc_dmainsn_page(void)
36 page = (void *) __get_free_page(GFP_KERNEL | GFP_DMA);
38 set_memory_x((unsigned long) page, 1);
42 static void free_dmainsn_page(void *page)
44 set_memory_nx((unsigned long) page, 1);
45 free_page((unsigned long)page);
48 struct kprobe_insn_cache kprobe_dmainsn_slots = {
49 .mutex = __MUTEX_INITIALIZER(kprobe_dmainsn_slots.mutex),
50 .alloc = alloc_dmainsn_page,
51 .free = free_dmainsn_page,
52 .pages = LIST_HEAD_INIT(kprobe_dmainsn_slots.pages),
53 .insn_size = MAX_INSN_SIZE,
56 static void copy_instruction(struct kprobe *p)
58 unsigned long ip = (unsigned long) p->addr;
62 if (ftrace_location(ip) == ip) {
64 * If kprobes patches the instruction that is morphed by
65 * ftrace make sure that kprobes always sees the branch
66 * "jg .+24" that skips the mcount block or the "brcl 0,0"
67 * in case of hotpatch.
69 ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn);
70 p->ainsn.is_ftrace_insn = 1;
72 memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8));
73 p->opcode = p->ainsn.insn[0];
74 if (!probe_is_insn_relative_long(p->ainsn.insn))
77 * For pc-relative instructions in RIL-b or RIL-c format patch the
78 * RI2 displacement field. We have already made sure that the insn
79 * slot for the patched instruction is within the same 2GB area
80 * as the original instruction (either kernel image or module area).
81 * Therefore the new displacement will always fit.
83 disp = *(s32 *)&p->ainsn.insn[1];
84 addr = (u64)(unsigned long)p->addr;
85 new_addr = (u64)(unsigned long)p->ainsn.insn;
86 new_disp = ((addr + (disp * 2)) - new_addr) / 2;
87 *(s32 *)&p->ainsn.insn[1] = new_disp;
89 NOKPROBE_SYMBOL(copy_instruction);
91 static inline int is_kernel_addr(void *addr)
93 return addr < (void *)_end;
96 static int s390_get_insn_slot(struct kprobe *p)
99 * Get an insn slot that is within the same 2GB area like the original
100 * instruction. That way instructions with a 32bit signed displacement
101 * field can be patched and executed within the insn slot.
103 p->ainsn.insn = NULL;
104 if (is_kernel_addr(p->addr))
105 p->ainsn.insn = get_dmainsn_slot();
106 else if (is_module_addr(p->addr))
107 p->ainsn.insn = get_insn_slot();
108 return p->ainsn.insn ? 0 : -ENOMEM;
110 NOKPROBE_SYMBOL(s390_get_insn_slot);
112 static void s390_free_insn_slot(struct kprobe *p)
116 if (is_kernel_addr(p->addr))
117 free_dmainsn_slot(p->ainsn.insn, 0);
119 free_insn_slot(p->ainsn.insn, 0);
120 p->ainsn.insn = NULL;
122 NOKPROBE_SYMBOL(s390_free_insn_slot);
124 int arch_prepare_kprobe(struct kprobe *p)
126 if ((unsigned long) p->addr & 0x01)
128 /* Make sure the probe isn't going on a difficult instruction */
129 if (probe_is_prohibited_opcode(p->addr))
131 if (s390_get_insn_slot(p))
136 NOKPROBE_SYMBOL(arch_prepare_kprobe);
138 int arch_check_ftrace_location(struct kprobe *p)
143 struct swap_insn_args {
145 unsigned int arm_kprobe : 1;
148 static int swap_instruction(void *data)
150 struct swap_insn_args *args = data;
151 struct ftrace_insn new_insn, *insn;
152 struct kprobe *p = args->p;
155 new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
156 len = sizeof(new_insn.opc);
157 if (!p->ainsn.is_ftrace_insn)
159 len = sizeof(new_insn);
160 insn = (struct ftrace_insn *) p->addr;
161 if (args->arm_kprobe) {
162 if (is_ftrace_nop(insn))
163 new_insn.disp = KPROBE_ON_FTRACE_NOP;
165 new_insn.disp = KPROBE_ON_FTRACE_CALL;
167 ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr);
168 if (insn->disp == KPROBE_ON_FTRACE_NOP)
169 ftrace_generate_nop_insn(&new_insn);
172 s390_kernel_write(p->addr, &new_insn, len);
175 NOKPROBE_SYMBOL(swap_instruction);
177 void arch_arm_kprobe(struct kprobe *p)
179 struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
181 stop_machine_cpuslocked(swap_instruction, &args, NULL);
183 NOKPROBE_SYMBOL(arch_arm_kprobe);
185 void arch_disarm_kprobe(struct kprobe *p)
187 struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
189 stop_machine_cpuslocked(swap_instruction, &args, NULL);
191 NOKPROBE_SYMBOL(arch_disarm_kprobe);
193 void arch_remove_kprobe(struct kprobe *p)
195 s390_free_insn_slot(p);
197 NOKPROBE_SYMBOL(arch_remove_kprobe);
199 static void enable_singlestep(struct kprobe_ctlblk *kcb,
200 struct pt_regs *regs,
203 struct per_regs per_kprobe;
205 /* Set up the PER control registers %cr9-%cr11 */
206 per_kprobe.control = PER_EVENT_IFETCH;
207 per_kprobe.start = ip;
210 /* Save control regs and psw mask */
211 __ctl_store(kcb->kprobe_saved_ctl, 9, 11);
212 kcb->kprobe_saved_imask = regs->psw.mask &
213 (PSW_MASK_PER | PSW_MASK_IO | PSW_MASK_EXT);
215 /* Set PER control regs, turns on single step for the given address */
216 __ctl_load(per_kprobe, 9, 11);
217 regs->psw.mask |= PSW_MASK_PER;
218 regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
221 NOKPROBE_SYMBOL(enable_singlestep);
223 static void disable_singlestep(struct kprobe_ctlblk *kcb,
224 struct pt_regs *regs,
227 /* Restore control regs and psw mask, set new psw address */
228 __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
229 regs->psw.mask &= ~PSW_MASK_PER;
230 regs->psw.mask |= kcb->kprobe_saved_imask;
233 NOKPROBE_SYMBOL(disable_singlestep);
236 * Activate a kprobe by storing its pointer to current_kprobe. The
237 * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
238 * two kprobes can be active, see KPROBE_REENTER.
240 static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
242 kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
243 kcb->prev_kprobe.status = kcb->kprobe_status;
244 __this_cpu_write(current_kprobe, p);
246 NOKPROBE_SYMBOL(push_kprobe);
249 * Deactivate a kprobe by backing up to the previous state. If the
250 * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
251 * for any other state prev_kprobe.kp will be NULL.
253 static void pop_kprobe(struct kprobe_ctlblk *kcb)
255 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
256 kcb->kprobe_status = kcb->prev_kprobe.status;
257 kcb->prev_kprobe.kp = NULL;
259 NOKPROBE_SYMBOL(pop_kprobe);
261 void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
263 ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
265 /* Replace the return addr with trampoline addr */
266 regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
268 NOKPROBE_SYMBOL(arch_prepare_kretprobe);
270 static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
272 switch (kcb->kprobe_status) {
273 case KPROBE_HIT_SSDONE:
274 case KPROBE_HIT_ACTIVE:
275 kprobes_inc_nmissed_count(p);
281 * A kprobe on the code path to single step an instruction
282 * is a BUG. The code path resides in the .kprobes.text
283 * section and is executed with interrupts disabled.
285 pr_err("Invalid kprobe detected.\n");
290 NOKPROBE_SYMBOL(kprobe_reenter_check);
292 static int kprobe_handler(struct pt_regs *regs)
294 struct kprobe_ctlblk *kcb;
298 * We want to disable preemption for the entire duration of kprobe
299 * processing. That includes the calls to the pre/post handlers
300 * and single stepping the kprobe instruction.
303 kcb = get_kprobe_ctlblk();
304 p = get_kprobe((void *)(regs->psw.addr - 2));
307 if (kprobe_running()) {
309 * We have hit a kprobe while another is still
310 * active. This can happen in the pre and post
311 * handler. Single step the instruction of the
312 * new probe but do not call any handler function
313 * of this secondary kprobe.
314 * push_kprobe and pop_kprobe saves and restores
315 * the currently active kprobe.
317 kprobe_reenter_check(kcb, p);
319 kcb->kprobe_status = KPROBE_REENTER;
322 * If we have no pre-handler or it returned 0, we
323 * continue with single stepping. If we have a
324 * pre-handler and it returned non-zero, it prepped
325 * for changing execution path, so get out doing
329 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
330 if (p->pre_handler && p->pre_handler(p, regs)) {
332 preempt_enable_no_resched();
335 kcb->kprobe_status = KPROBE_HIT_SS;
337 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
340 * No kprobe at this address and no active kprobe. The trap has
341 * not been caused by a kprobe breakpoint. The race of breakpoint
342 * vs. kprobe remove does not exist because on s390 as we use
343 * stop_machine to arm/disarm the breakpoints.
345 preempt_enable_no_resched();
348 NOKPROBE_SYMBOL(kprobe_handler);
351 * Function return probe trampoline:
352 * - init_kprobes() establishes a probepoint here
353 * - When the probed function returns, this probe
354 * causes the handlers to fire
356 static void __used kretprobe_trampoline_holder(void)
358 asm volatile(".global kretprobe_trampoline\n"
359 "kretprobe_trampoline: bcr 0,0\n");
363 * Called when the probe at kretprobe trampoline is hit
365 static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
367 struct kretprobe_instance *ri;
368 struct hlist_head *head, empty_rp;
369 struct hlist_node *tmp;
370 unsigned long flags, orig_ret_address;
371 unsigned long trampoline_address;
372 kprobe_opcode_t *correct_ret_addr;
374 INIT_HLIST_HEAD(&empty_rp);
375 kretprobe_hash_lock(current, &head, &flags);
378 * It is possible to have multiple instances associated with a given
379 * task either because an multiple functions in the call path
380 * have a return probe installed on them, and/or more than one return
381 * return probe was registered for a target function.
383 * We can handle this because:
384 * - instances are always inserted at the head of the list
385 * - when multiple return probes are registered for the same
386 * function, the first instance's ret_addr will point to the
387 * real return address, and all the rest will point to
388 * kretprobe_trampoline
391 orig_ret_address = 0;
392 correct_ret_addr = NULL;
393 trampoline_address = (unsigned long) &kretprobe_trampoline;
394 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
395 if (ri->task != current)
396 /* another task is sharing our hash bucket */
399 orig_ret_address = (unsigned long) ri->ret_addr;
401 if (orig_ret_address != trampoline_address)
403 * This is the real return address. Any other
404 * instances associated with this task are for
405 * other calls deeper on the call stack
410 kretprobe_assert(ri, orig_ret_address, trampoline_address);
412 correct_ret_addr = ri->ret_addr;
413 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
414 if (ri->task != current)
415 /* another task is sharing our hash bucket */
418 orig_ret_address = (unsigned long) ri->ret_addr;
420 if (ri->rp && ri->rp->handler) {
421 ri->ret_addr = correct_ret_addr;
422 ri->rp->handler(ri, regs);
425 recycle_rp_inst(ri, &empty_rp);
427 if (orig_ret_address != trampoline_address)
429 * This is the real return address. Any other
430 * instances associated with this task are for
431 * other calls deeper on the call stack
436 regs->psw.addr = orig_ret_address;
438 kretprobe_hash_unlock(current, &flags);
440 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
441 hlist_del(&ri->hlist);
445 * By returning a non-zero value, we are telling
446 * kprobe_handler() that we don't want the post_handler
447 * to run (and have re-enabled preemption)
451 NOKPROBE_SYMBOL(trampoline_probe_handler);
454 * Called after single-stepping. p->addr is the address of the
455 * instruction whose first byte has been replaced by the "breakpoint"
456 * instruction. To avoid the SMP problems that can occur when we
457 * temporarily put back the original opcode to single-step, we
458 * single-stepped a copy of the instruction. The address of this
459 * copy is p->ainsn.insn.
461 static void resume_execution(struct kprobe *p, struct pt_regs *regs)
463 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
464 unsigned long ip = regs->psw.addr;
465 int fixup = probe_get_fixup_type(p->ainsn.insn);
467 /* Check if the kprobes location is an enabled ftrace caller */
468 if (p->ainsn.is_ftrace_insn) {
469 struct ftrace_insn *insn = (struct ftrace_insn *) p->addr;
470 struct ftrace_insn call_insn;
472 ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr);
474 * A kprobe on an enabled ftrace call site actually single
475 * stepped an unconditional branch (ftrace nop equivalent).
476 * Now we need to fixup things and pretend that a brasl r0,...
477 * was executed instead.
479 if (insn->disp == KPROBE_ON_FTRACE_CALL) {
480 ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE;
481 regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn);
485 if (fixup & FIXUP_PSW_NORMAL)
486 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
488 if (fixup & FIXUP_BRANCH_NOT_TAKEN) {
489 int ilen = insn_length(p->ainsn.insn[0] >> 8);
490 if (ip - (unsigned long) p->ainsn.insn == ilen)
491 ip = (unsigned long) p->addr + ilen;
494 if (fixup & FIXUP_RETURN_REGISTER) {
495 int reg = (p->ainsn.insn[0] & 0xf0) >> 4;
496 regs->gprs[reg] += (unsigned long) p->addr -
497 (unsigned long) p->ainsn.insn;
500 disable_singlestep(kcb, regs, ip);
502 NOKPROBE_SYMBOL(resume_execution);
504 static int post_kprobe_handler(struct pt_regs *regs)
506 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
507 struct kprobe *p = kprobe_running();
512 resume_execution(p, regs);
513 if (kcb->kprobe_status != KPROBE_REENTER && p->post_handler) {
514 kcb->kprobe_status = KPROBE_HIT_SSDONE;
515 p->post_handler(p, regs, 0);
518 preempt_enable_no_resched();
521 * if somebody else is singlestepping across a probe point, psw mask
522 * will have PER set, in which case, continue the remaining processing
523 * of do_single_step, as if this is not a probe hit.
525 if (regs->psw.mask & PSW_MASK_PER)
530 NOKPROBE_SYMBOL(post_kprobe_handler);
532 static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
534 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
535 struct kprobe *p = kprobe_running();
536 const struct exception_table_entry *entry;
538 switch(kcb->kprobe_status) {
542 * We are here because the instruction being single
543 * stepped caused a page fault. We reset the current
544 * kprobe and the nip points back to the probe address
545 * and allow the page fault handler to continue as a
548 disable_singlestep(kcb, regs, (unsigned long) p->addr);
550 preempt_enable_no_resched();
552 case KPROBE_HIT_ACTIVE:
553 case KPROBE_HIT_SSDONE:
555 * We increment the nmissed count for accounting,
556 * we can also use npre/npostfault count for accounting
557 * these specific fault cases.
559 kprobes_inc_nmissed_count(p);
562 * We come here because instructions in the pre/post
563 * handler caused the page_fault, this could happen
564 * if handler tries to access user space by
565 * copy_from_user(), get_user() etc. Let the
566 * user-specified handler try to fix it first.
568 if (p->fault_handler && p->fault_handler(p, regs, trapnr))
572 * In case the user-specified fault handler returned
573 * zero, try to fix up.
575 entry = search_exception_tables(regs->psw.addr);
577 regs->psw.addr = extable_fixup(entry);
582 * fixup_exception() could not handle it,
583 * Let do_page_fault() fix it.
591 NOKPROBE_SYMBOL(kprobe_trap_handler);
593 int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
597 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
599 ret = kprobe_trap_handler(regs, trapnr);
600 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
601 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
604 NOKPROBE_SYMBOL(kprobe_fault_handler);
607 * Wrapper routine to for handling exceptions.
609 int kprobe_exceptions_notify(struct notifier_block *self,
610 unsigned long val, void *data)
612 struct die_args *args = (struct die_args *) data;
613 struct pt_regs *regs = args->regs;
614 int ret = NOTIFY_DONE;
616 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
621 if (kprobe_handler(regs))
625 if (post_kprobe_handler(regs))
629 if (!preemptible() && kprobe_running() &&
630 kprobe_trap_handler(regs, args->trapnr))
637 if (regs->psw.mask & (PSW_MASK_IO | PSW_MASK_EXT))
638 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
642 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
644 static struct kprobe trampoline = {
645 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
646 .pre_handler = trampoline_probe_handler
649 int __init arch_init_kprobes(void)
651 return register_kprobe(&trampoline);
654 int arch_trampoline_kprobe(struct kprobe *p)
656 return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
658 NOKPROBE_SYMBOL(arch_trampoline_kprobe);