2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
78 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled = 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head {
125 unsigned long length;
128 union trace_enum_map_item;
130 struct trace_enum_map_tail {
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item *next;
136 const char *end; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item {
149 struct trace_enum_map map;
150 struct trace_enum_map_head head;
151 struct trace_enum_map_tail tail;
154 static union trace_enum_map_item *trace_enum_maps;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161 static char *default_bootup_tracer;
163 static bool allocate_snapshot;
165 static int __init set_cmdline_ftrace(char *str)
167 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168 default_bootup_tracer = bootup_tracer_buf;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded = true;
173 __setup("ftrace=", set_cmdline_ftrace);
175 static int __init set_ftrace_dump_on_oops(char *str)
177 if (*str++ != '=' || !*str) {
178 ftrace_dump_on_oops = DUMP_ALL;
182 if (!strcmp("orig_cpu", str)) {
183 ftrace_dump_on_oops = DUMP_ORIG;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
191 static int __init stop_trace_on_warning(char *str)
193 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194 __disable_trace_on_warning = 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning);
199 static int __init boot_alloc_snapshot(char *str)
201 allocate_snapshot = true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded = true;
206 __setup("alloc_snapshot", boot_alloc_snapshot);
209 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
211 static int __init set_trace_boot_options(char *str)
213 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
216 __setup("trace_options=", set_trace_boot_options);
218 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219 static char *trace_boot_clock __initdata;
221 static int __init set_trace_boot_clock(char *str)
223 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224 trace_boot_clock = trace_boot_clock_buf;
227 __setup("trace_clock=", set_trace_boot_clock);
229 static int __init set_tracepoint_printk(char *str)
231 /* Ignore the "tp_printk_stop_on_boot" param */
235 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
236 tracepoint_printk = 1;
239 __setup("tp_printk", set_tracepoint_printk);
241 unsigned long long ns2usecs(cycle_t nsec)
248 /* trace_flags holds trace_options default values */
249 #define TRACE_DEFAULT_FLAGS \
250 (FUNCTION_DEFAULT_FLAGS | \
251 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
252 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
253 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
254 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
256 /* trace_options that are only supported by global_trace */
257 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
258 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
260 /* trace_flags that are default zero for instances */
261 #define ZEROED_TRACE_FLAGS \
262 TRACE_ITER_EVENT_FORK
265 * The global_trace is the descriptor that holds the tracing
266 * buffers for the live tracing. For each CPU, it contains
267 * a link list of pages that will store trace entries. The
268 * page descriptor of the pages in the memory is used to hold
269 * the link list by linking the lru item in the page descriptor
270 * to each of the pages in the buffer per CPU.
272 * For each active CPU there is a data field that holds the
273 * pages for the buffer for that CPU. Each CPU has the same number
274 * of pages allocated for its buffer.
276 static struct trace_array global_trace = {
277 .trace_flags = TRACE_DEFAULT_FLAGS,
280 LIST_HEAD(ftrace_trace_arrays);
282 int trace_array_get(struct trace_array *this_tr)
284 struct trace_array *tr;
287 mutex_lock(&trace_types_lock);
288 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
295 mutex_unlock(&trace_types_lock);
300 static void __trace_array_put(struct trace_array *this_tr)
302 WARN_ON(!this_tr->ref);
306 void trace_array_put(struct trace_array *this_tr)
308 mutex_lock(&trace_types_lock);
309 __trace_array_put(this_tr);
310 mutex_unlock(&trace_types_lock);
313 int call_filter_check_discard(struct trace_event_call *call, void *rec,
314 struct ring_buffer *buffer,
315 struct ring_buffer_event *event)
317 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
318 !filter_match_preds(call->filter, rec)) {
319 __trace_event_discard_commit(buffer, event);
326 void trace_free_pid_list(struct trace_pid_list *pid_list)
328 vfree(pid_list->pids);
333 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
334 * @filtered_pids: The list of pids to check
335 * @search_pid: The PID to find in @filtered_pids
337 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
340 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
343 * If pid_max changed after filtered_pids was created, we
344 * by default ignore all pids greater than the previous pid_max.
346 if (search_pid >= filtered_pids->pid_max)
349 return test_bit(search_pid, filtered_pids->pids);
353 * trace_ignore_this_task - should a task be ignored for tracing
354 * @filtered_pids: The list of pids to check
355 * @task: The task that should be ignored if not filtered
357 * Checks if @task should be traced or not from @filtered_pids.
358 * Returns true if @task should *NOT* be traced.
359 * Returns false if @task should be traced.
362 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
365 * Return false, because if filtered_pids does not exist,
366 * all pids are good to trace.
371 return !trace_find_filtered_pid(filtered_pids, task->pid);
375 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
376 * @pid_list: The list to modify
377 * @self: The current task for fork or NULL for exit
378 * @task: The task to add or remove
380 * If adding a task, if @self is defined, the task is only added if @self
381 * is also included in @pid_list. This happens on fork and tasks should
382 * only be added when the parent is listed. If @self is NULL, then the
383 * @task pid will be removed from the list, which would happen on exit
386 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
387 struct task_struct *self,
388 struct task_struct *task)
393 /* For forks, we only add if the forking task is listed */
395 if (!trace_find_filtered_pid(pid_list, self->pid))
399 /* Sorry, but we don't support pid_max changing after setting */
400 if (task->pid >= pid_list->pid_max)
403 /* "self" is set for forks, and NULL for exits */
405 set_bit(task->pid, pid_list->pids);
407 clear_bit(task->pid, pid_list->pids);
411 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
412 * @pid_list: The pid list to show
413 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
414 * @pos: The position of the file
416 * This is used by the seq_file "next" operation to iterate the pids
417 * listed in a trace_pid_list structure.
419 * Returns the pid+1 as we want to display pid of zero, but NULL would
420 * stop the iteration.
422 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
424 unsigned long pid = (unsigned long)v;
428 /* pid already is +1 of the actual prevous bit */
429 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
431 /* Return pid + 1 to allow zero to be represented */
432 if (pid < pid_list->pid_max)
433 return (void *)(pid + 1);
439 * trace_pid_start - Used for seq_file to start reading pid lists
440 * @pid_list: The pid list to show
441 * @pos: The position of the file
443 * This is used by seq_file "start" operation to start the iteration
446 * Returns the pid+1 as we want to display pid of zero, but NULL would
447 * stop the iteration.
449 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
454 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
455 if (pid >= pid_list->pid_max)
458 /* Return pid + 1 so that zero can be the exit value */
459 for (pid++; pid && l < *pos;
460 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
466 * trace_pid_show - show the current pid in seq_file processing
467 * @m: The seq_file structure to write into
468 * @v: A void pointer of the pid (+1) value to display
470 * Can be directly used by seq_file operations to display the current
473 int trace_pid_show(struct seq_file *m, void *v)
475 unsigned long pid = (unsigned long)v - 1;
477 seq_printf(m, "%lu\n", pid);
481 /* 128 should be much more than enough */
482 #define PID_BUF_SIZE 127
484 int trace_pid_write(struct trace_pid_list *filtered_pids,
485 struct trace_pid_list **new_pid_list,
486 const char __user *ubuf, size_t cnt)
488 struct trace_pid_list *pid_list;
489 struct trace_parser parser;
497 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
501 * Always recreate a new array. The write is an all or nothing
502 * operation. Always create a new array when adding new pids by
503 * the user. If the operation fails, then the current list is
506 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
508 trace_parser_put(&parser);
512 pid_list->pid_max = READ_ONCE(pid_max);
514 /* Only truncating will shrink pid_max */
515 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
516 pid_list->pid_max = filtered_pids->pid_max;
518 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
519 if (!pid_list->pids) {
520 trace_parser_put(&parser);
526 /* copy the current bits to the new max */
527 for_each_set_bit(pid, filtered_pids->pids,
528 filtered_pids->pid_max) {
529 set_bit(pid, pid_list->pids);
538 ret = trace_get_user(&parser, ubuf, cnt, &pos);
539 if (ret < 0 || !trace_parser_loaded(&parser))
546 parser.buffer[parser.idx] = 0;
549 if (kstrtoul(parser.buffer, 0, &val))
551 if (val >= pid_list->pid_max)
556 set_bit(pid, pid_list->pids);
559 trace_parser_clear(&parser);
562 trace_parser_put(&parser);
565 trace_free_pid_list(pid_list);
570 /* Cleared the list of pids */
571 trace_free_pid_list(pid_list);
576 *new_pid_list = pid_list;
581 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
585 /* Early boot up does not have a buffer yet */
587 return trace_clock_local();
589 ts = ring_buffer_time_stamp(buf->buffer, cpu);
590 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
595 cycle_t ftrace_now(int cpu)
597 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
601 * tracing_is_enabled - Show if global_trace has been disabled
603 * Shows if the global trace has been enabled or not. It uses the
604 * mirror flag "buffer_disabled" to be used in fast paths such as for
605 * the irqsoff tracer. But it may be inaccurate due to races. If you
606 * need to know the accurate state, use tracing_is_on() which is a little
607 * slower, but accurate.
609 int tracing_is_enabled(void)
612 * For quick access (irqsoff uses this in fast path), just
613 * return the mirror variable of the state of the ring buffer.
614 * It's a little racy, but we don't really care.
617 return !global_trace.buffer_disabled;
621 * trace_buf_size is the size in bytes that is allocated
622 * for a buffer. Note, the number of bytes is always rounded
625 * This number is purposely set to a low number of 16384.
626 * If the dump on oops happens, it will be much appreciated
627 * to not have to wait for all that output. Anyway this can be
628 * boot time and run time configurable.
630 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
632 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
634 /* trace_types holds a link list of available tracers. */
635 static struct tracer *trace_types __read_mostly;
638 * trace_types_lock is used to protect the trace_types list.
640 DEFINE_MUTEX(trace_types_lock);
643 * serialize the access of the ring buffer
645 * ring buffer serializes readers, but it is low level protection.
646 * The validity of the events (which returns by ring_buffer_peek() ..etc)
647 * are not protected by ring buffer.
649 * The content of events may become garbage if we allow other process consumes
650 * these events concurrently:
651 * A) the page of the consumed events may become a normal page
652 * (not reader page) in ring buffer, and this page will be rewrited
653 * by events producer.
654 * B) The page of the consumed events may become a page for splice_read,
655 * and this page will be returned to system.
657 * These primitives allow multi process access to different cpu ring buffer
660 * These primitives don't distinguish read-only and read-consume access.
661 * Multi read-only access are also serialized.
665 static DECLARE_RWSEM(all_cpu_access_lock);
666 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
668 static inline void trace_access_lock(int cpu)
670 if (cpu == RING_BUFFER_ALL_CPUS) {
671 /* gain it for accessing the whole ring buffer. */
672 down_write(&all_cpu_access_lock);
674 /* gain it for accessing a cpu ring buffer. */
676 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
677 down_read(&all_cpu_access_lock);
679 /* Secondly block other access to this @cpu ring buffer. */
680 mutex_lock(&per_cpu(cpu_access_lock, cpu));
684 static inline void trace_access_unlock(int cpu)
686 if (cpu == RING_BUFFER_ALL_CPUS) {
687 up_write(&all_cpu_access_lock);
689 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
690 up_read(&all_cpu_access_lock);
694 static inline void trace_access_lock_init(void)
698 for_each_possible_cpu(cpu)
699 mutex_init(&per_cpu(cpu_access_lock, cpu));
704 static DEFINE_MUTEX(access_lock);
706 static inline void trace_access_lock(int cpu)
709 mutex_lock(&access_lock);
712 static inline void trace_access_unlock(int cpu)
715 mutex_unlock(&access_lock);
718 static inline void trace_access_lock_init(void)
724 #ifdef CONFIG_STACKTRACE
725 static void __ftrace_trace_stack(struct ring_buffer *buffer,
727 int skip, int pc, struct pt_regs *regs);
728 static inline void ftrace_trace_stack(struct trace_array *tr,
729 struct ring_buffer *buffer,
731 int skip, int pc, struct pt_regs *regs);
734 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
736 int skip, int pc, struct pt_regs *regs)
739 static inline void ftrace_trace_stack(struct trace_array *tr,
740 struct ring_buffer *buffer,
742 int skip, int pc, struct pt_regs *regs)
748 static void tracer_tracing_on(struct trace_array *tr)
750 if (tr->trace_buffer.buffer)
751 ring_buffer_record_on(tr->trace_buffer.buffer);
753 * This flag is looked at when buffers haven't been allocated
754 * yet, or by some tracers (like irqsoff), that just want to
755 * know if the ring buffer has been disabled, but it can handle
756 * races of where it gets disabled but we still do a record.
757 * As the check is in the fast path of the tracers, it is more
758 * important to be fast than accurate.
760 tr->buffer_disabled = 0;
761 /* Make the flag seen by readers */
766 * tracing_on - enable tracing buffers
768 * This function enables tracing buffers that may have been
769 * disabled with tracing_off.
771 void tracing_on(void)
773 tracer_tracing_on(&global_trace);
775 EXPORT_SYMBOL_GPL(tracing_on);
778 * __trace_puts - write a constant string into the trace buffer.
779 * @ip: The address of the caller
780 * @str: The constant string to write
781 * @size: The size of the string.
783 int __trace_puts(unsigned long ip, const char *str, int size)
785 struct ring_buffer_event *event;
786 struct ring_buffer *buffer;
787 struct print_entry *entry;
788 unsigned long irq_flags;
792 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
795 pc = preempt_count();
797 if (unlikely(tracing_selftest_running || tracing_disabled))
800 alloc = sizeof(*entry) + size + 2; /* possible \n added */
802 local_save_flags(irq_flags);
803 buffer = global_trace.trace_buffer.buffer;
804 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
809 entry = ring_buffer_event_data(event);
812 memcpy(&entry->buf, str, size);
814 /* Add a newline if necessary */
815 if (entry->buf[size - 1] != '\n') {
816 entry->buf[size] = '\n';
817 entry->buf[size + 1] = '\0';
819 entry->buf[size] = '\0';
821 __buffer_unlock_commit(buffer, event);
822 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
826 EXPORT_SYMBOL_GPL(__trace_puts);
829 * __trace_bputs - write the pointer to a constant string into trace buffer
830 * @ip: The address of the caller
831 * @str: The constant string to write to the buffer to
833 int __trace_bputs(unsigned long ip, const char *str)
835 struct ring_buffer_event *event;
836 struct ring_buffer *buffer;
837 struct bputs_entry *entry;
838 unsigned long irq_flags;
839 int size = sizeof(struct bputs_entry);
842 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
845 pc = preempt_count();
847 if (unlikely(tracing_selftest_running || tracing_disabled))
850 local_save_flags(irq_flags);
851 buffer = global_trace.trace_buffer.buffer;
852 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
857 entry = ring_buffer_event_data(event);
861 __buffer_unlock_commit(buffer, event);
862 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
866 EXPORT_SYMBOL_GPL(__trace_bputs);
868 #ifdef CONFIG_TRACER_SNAPSHOT
870 * trace_snapshot - take a snapshot of the current buffer.
872 * This causes a swap between the snapshot buffer and the current live
873 * tracing buffer. You can use this to take snapshots of the live
874 * trace when some condition is triggered, but continue to trace.
876 * Note, make sure to allocate the snapshot with either
877 * a tracing_snapshot_alloc(), or by doing it manually
878 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
880 * If the snapshot buffer is not allocated, it will stop tracing.
881 * Basically making a permanent snapshot.
883 void tracing_snapshot(void)
885 struct trace_array *tr = &global_trace;
886 struct tracer *tracer = tr->current_trace;
890 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
891 internal_trace_puts("*** snapshot is being ignored ***\n");
895 if (!tr->allocated_snapshot) {
896 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
897 internal_trace_puts("*** stopping trace here! ***\n");
902 /* Note, snapshot can not be used when the tracer uses it */
903 if (tracer->use_max_tr) {
904 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
905 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
909 local_irq_save(flags);
910 update_max_tr(tr, current, smp_processor_id());
911 local_irq_restore(flags);
913 EXPORT_SYMBOL_GPL(tracing_snapshot);
915 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
916 struct trace_buffer *size_buf, int cpu_id);
917 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
919 static int alloc_snapshot(struct trace_array *tr)
923 if (!tr->allocated_snapshot) {
925 /* allocate spare buffer */
926 ret = resize_buffer_duplicate_size(&tr->max_buffer,
927 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
931 tr->allocated_snapshot = true;
937 static void free_snapshot(struct trace_array *tr)
940 * We don't free the ring buffer. instead, resize it because
941 * The max_tr ring buffer has some state (e.g. ring->clock) and
942 * we want preserve it.
944 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
945 set_buffer_entries(&tr->max_buffer, 1);
946 tracing_reset_online_cpus(&tr->max_buffer);
947 tr->allocated_snapshot = false;
951 * tracing_alloc_snapshot - allocate snapshot buffer.
953 * This only allocates the snapshot buffer if it isn't already
954 * allocated - it doesn't also take a snapshot.
956 * This is meant to be used in cases where the snapshot buffer needs
957 * to be set up for events that can't sleep but need to be able to
958 * trigger a snapshot.
960 int tracing_alloc_snapshot(void)
962 struct trace_array *tr = &global_trace;
965 ret = alloc_snapshot(tr);
970 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
973 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
975 * This is similar to trace_snapshot(), but it will allocate the
976 * snapshot buffer if it isn't already allocated. Use this only
977 * where it is safe to sleep, as the allocation may sleep.
979 * This causes a swap between the snapshot buffer and the current live
980 * tracing buffer. You can use this to take snapshots of the live
981 * trace when some condition is triggered, but continue to trace.
983 void tracing_snapshot_alloc(void)
987 ret = tracing_alloc_snapshot();
993 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
995 void tracing_snapshot(void)
997 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
999 EXPORT_SYMBOL_GPL(tracing_snapshot);
1000 int tracing_alloc_snapshot(void)
1002 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1005 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1006 void tracing_snapshot_alloc(void)
1011 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1012 #endif /* CONFIG_TRACER_SNAPSHOT */
1014 static void tracer_tracing_off(struct trace_array *tr)
1016 if (tr->trace_buffer.buffer)
1017 ring_buffer_record_off(tr->trace_buffer.buffer);
1019 * This flag is looked at when buffers haven't been allocated
1020 * yet, or by some tracers (like irqsoff), that just want to
1021 * know if the ring buffer has been disabled, but it can handle
1022 * races of where it gets disabled but we still do a record.
1023 * As the check is in the fast path of the tracers, it is more
1024 * important to be fast than accurate.
1026 tr->buffer_disabled = 1;
1027 /* Make the flag seen by readers */
1032 * tracing_off - turn off tracing buffers
1034 * This function stops the tracing buffers from recording data.
1035 * It does not disable any overhead the tracers themselves may
1036 * be causing. This function simply causes all recording to
1037 * the ring buffers to fail.
1039 void tracing_off(void)
1041 tracer_tracing_off(&global_trace);
1043 EXPORT_SYMBOL_GPL(tracing_off);
1045 void disable_trace_on_warning(void)
1047 if (__disable_trace_on_warning)
1052 * tracer_tracing_is_on - show real state of ring buffer enabled
1053 * @tr : the trace array to know if ring buffer is enabled
1055 * Shows real state of the ring buffer if it is enabled or not.
1057 int tracer_tracing_is_on(struct trace_array *tr)
1059 if (tr->trace_buffer.buffer)
1060 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1061 return !tr->buffer_disabled;
1065 * tracing_is_on - show state of ring buffers enabled
1067 int tracing_is_on(void)
1069 return tracer_tracing_is_on(&global_trace);
1071 EXPORT_SYMBOL_GPL(tracing_is_on);
1073 static int __init set_buf_size(char *str)
1075 unsigned long buf_size;
1079 buf_size = memparse(str, &str);
1081 * nr_entries can not be zero and the startup
1082 * tests require some buffer space. Therefore
1083 * ensure we have at least 4096 bytes of buffer.
1085 trace_buf_size = max(4096UL, buf_size);
1088 __setup("trace_buf_size=", set_buf_size);
1090 static int __init set_tracing_thresh(char *str)
1092 unsigned long threshold;
1097 ret = kstrtoul(str, 0, &threshold);
1100 tracing_thresh = threshold * 1000;
1103 __setup("tracing_thresh=", set_tracing_thresh);
1105 unsigned long nsecs_to_usecs(unsigned long nsecs)
1107 return nsecs / 1000;
1111 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1112 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1113 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1114 * of strings in the order that the enums were defined.
1119 /* These must match the bit postions in trace_iterator_flags */
1120 static const char *trace_options[] = {
1128 int in_ns; /* is this clock in nanoseconds? */
1129 } trace_clocks[] = {
1130 { trace_clock_local, "local", 1 },
1131 { trace_clock_global, "global", 1 },
1132 { trace_clock_counter, "counter", 0 },
1133 { trace_clock_jiffies, "uptime", 0 },
1134 { trace_clock, "perf", 1 },
1135 { ktime_get_mono_fast_ns, "mono", 1 },
1136 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1141 * trace_parser_get_init - gets the buffer for trace parser
1143 int trace_parser_get_init(struct trace_parser *parser, int size)
1145 memset(parser, 0, sizeof(*parser));
1147 parser->buffer = kmalloc(size, GFP_KERNEL);
1148 if (!parser->buffer)
1151 parser->size = size;
1156 * trace_parser_put - frees the buffer for trace parser
1158 void trace_parser_put(struct trace_parser *parser)
1160 kfree(parser->buffer);
1164 * trace_get_user - reads the user input string separated by space
1165 * (matched by isspace(ch))
1167 * For each string found the 'struct trace_parser' is updated,
1168 * and the function returns.
1170 * Returns number of bytes read.
1172 * See kernel/trace/trace.h for 'struct trace_parser' details.
1174 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1175 size_t cnt, loff_t *ppos)
1182 trace_parser_clear(parser);
1184 ret = get_user(ch, ubuf++);
1192 * The parser is not finished with the last write,
1193 * continue reading the user input without skipping spaces.
1195 if (!parser->cont) {
1196 /* skip white space */
1197 while (cnt && isspace(ch)) {
1198 ret = get_user(ch, ubuf++);
1205 /* only spaces were written */
1215 /* read the non-space input */
1216 while (cnt && !isspace(ch)) {
1217 if (parser->idx < parser->size - 1)
1218 parser->buffer[parser->idx++] = ch;
1223 ret = get_user(ch, ubuf++);
1230 /* We either got finished input or we have to wait for another call. */
1232 parser->buffer[parser->idx] = 0;
1233 parser->cont = false;
1234 } else if (parser->idx < parser->size - 1) {
1235 parser->cont = true;
1236 parser->buffer[parser->idx++] = ch;
1249 /* TODO add a seq_buf_to_buffer() */
1250 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1254 if (trace_seq_used(s) <= s->seq.readpos)
1257 len = trace_seq_used(s) - s->seq.readpos;
1260 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1262 s->seq.readpos += cnt;
1266 unsigned long __read_mostly tracing_thresh;
1268 #ifdef CONFIG_TRACER_MAX_TRACE
1270 * Copy the new maximum trace into the separate maximum-trace
1271 * structure. (this way the maximum trace is permanently saved,
1272 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1275 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1277 struct trace_buffer *trace_buf = &tr->trace_buffer;
1278 struct trace_buffer *max_buf = &tr->max_buffer;
1279 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1280 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1283 max_buf->time_start = data->preempt_timestamp;
1285 max_data->saved_latency = tr->max_latency;
1286 max_data->critical_start = data->critical_start;
1287 max_data->critical_end = data->critical_end;
1289 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1290 max_data->pid = tsk->pid;
1292 * If tsk == current, then use current_uid(), as that does not use
1293 * RCU. The irq tracer can be called out of RCU scope.
1296 max_data->uid = current_uid();
1298 max_data->uid = task_uid(tsk);
1300 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1301 max_data->policy = tsk->policy;
1302 max_data->rt_priority = tsk->rt_priority;
1304 /* record this tasks comm */
1305 tracing_record_cmdline(tsk);
1309 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1311 * @tsk: the task with the latency
1312 * @cpu: The cpu that initiated the trace.
1314 * Flip the buffers between the @tr and the max_tr and record information
1315 * about which task was the cause of this latency.
1318 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1320 struct ring_buffer *buf;
1325 WARN_ON_ONCE(!irqs_disabled());
1327 if (!tr->allocated_snapshot) {
1328 /* Only the nop tracer should hit this when disabling */
1329 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1333 arch_spin_lock(&tr->max_lock);
1335 /* Inherit the recordable setting from trace_buffer */
1336 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1337 ring_buffer_record_on(tr->max_buffer.buffer);
1339 ring_buffer_record_off(tr->max_buffer.buffer);
1341 buf = tr->trace_buffer.buffer;
1342 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1343 tr->max_buffer.buffer = buf;
1345 __update_max_tr(tr, tsk, cpu);
1346 arch_spin_unlock(&tr->max_lock);
1350 * update_max_tr_single - only copy one trace over, and reset the rest
1352 * @tsk - task with the latency
1353 * @cpu - the cpu of the buffer to copy.
1355 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1358 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1365 WARN_ON_ONCE(!irqs_disabled());
1366 if (!tr->allocated_snapshot) {
1367 /* Only the nop tracer should hit this when disabling */
1368 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1372 arch_spin_lock(&tr->max_lock);
1374 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1376 if (ret == -EBUSY) {
1378 * We failed to swap the buffer due to a commit taking
1379 * place on this CPU. We fail to record, but we reset
1380 * the max trace buffer (no one writes directly to it)
1381 * and flag that it failed.
1383 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1384 "Failed to swap buffers due to commit in progress\n");
1387 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1389 __update_max_tr(tr, tsk, cpu);
1390 arch_spin_unlock(&tr->max_lock);
1392 #endif /* CONFIG_TRACER_MAX_TRACE */
1394 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1396 /* Iterators are static, they should be filled or empty */
1397 if (trace_buffer_iter(iter, iter->cpu_file))
1400 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1404 #ifdef CONFIG_FTRACE_STARTUP_TEST
1405 static int run_tracer_selftest(struct tracer *type)
1407 struct trace_array *tr = &global_trace;
1408 struct tracer *saved_tracer = tr->current_trace;
1411 if (!type->selftest || tracing_selftest_disabled)
1415 * Run a selftest on this tracer.
1416 * Here we reset the trace buffer, and set the current
1417 * tracer to be this tracer. The tracer can then run some
1418 * internal tracing to verify that everything is in order.
1419 * If we fail, we do not register this tracer.
1421 tracing_reset_online_cpus(&tr->trace_buffer);
1423 tr->current_trace = type;
1425 #ifdef CONFIG_TRACER_MAX_TRACE
1426 if (type->use_max_tr) {
1427 /* If we expanded the buffers, make sure the max is expanded too */
1428 if (ring_buffer_expanded)
1429 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1430 RING_BUFFER_ALL_CPUS);
1431 tr->allocated_snapshot = true;
1435 /* the test is responsible for initializing and enabling */
1436 pr_info("Testing tracer %s: ", type->name);
1437 ret = type->selftest(type, tr);
1438 /* the test is responsible for resetting too */
1439 tr->current_trace = saved_tracer;
1441 printk(KERN_CONT "FAILED!\n");
1442 /* Add the warning after printing 'FAILED' */
1446 /* Only reset on passing, to avoid touching corrupted buffers */
1447 tracing_reset_online_cpus(&tr->trace_buffer);
1449 #ifdef CONFIG_TRACER_MAX_TRACE
1450 if (type->use_max_tr) {
1451 tr->allocated_snapshot = false;
1453 /* Shrink the max buffer again */
1454 if (ring_buffer_expanded)
1455 ring_buffer_resize(tr->max_buffer.buffer, 1,
1456 RING_BUFFER_ALL_CPUS);
1460 printk(KERN_CONT "PASSED\n");
1464 static inline int run_tracer_selftest(struct tracer *type)
1468 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1470 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1472 static void __init apply_trace_boot_options(void);
1475 * register_tracer - register a tracer with the ftrace system.
1476 * @type - the plugin for the tracer
1478 * Register a new plugin tracer.
1480 int __init register_tracer(struct tracer *type)
1486 pr_info("Tracer must have a name\n");
1490 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1491 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1495 mutex_lock(&trace_types_lock);
1497 tracing_selftest_running = true;
1499 for (t = trace_types; t; t = t->next) {
1500 if (strcmp(type->name, t->name) == 0) {
1502 pr_info("Tracer %s already registered\n",
1509 if (!type->set_flag)
1510 type->set_flag = &dummy_set_flag;
1512 /*allocate a dummy tracer_flags*/
1513 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1518 type->flags->val = 0;
1519 type->flags->opts = dummy_tracer_opt;
1521 if (!type->flags->opts)
1522 type->flags->opts = dummy_tracer_opt;
1524 /* store the tracer for __set_tracer_option */
1525 type->flags->trace = type;
1527 ret = run_tracer_selftest(type);
1531 type->next = trace_types;
1533 add_tracer_options(&global_trace, type);
1536 tracing_selftest_running = false;
1537 mutex_unlock(&trace_types_lock);
1539 if (ret || !default_bootup_tracer)
1542 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1545 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1546 /* Do we want this tracer to start on bootup? */
1547 tracing_set_tracer(&global_trace, type->name);
1548 default_bootup_tracer = NULL;
1550 apply_trace_boot_options();
1552 /* disable other selftests, since this will break it. */
1553 tracing_selftest_disabled = true;
1554 #ifdef CONFIG_FTRACE_STARTUP_TEST
1555 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1563 void tracing_reset(struct trace_buffer *buf, int cpu)
1565 struct ring_buffer *buffer = buf->buffer;
1570 ring_buffer_record_disable(buffer);
1572 /* Make sure all commits have finished */
1573 synchronize_sched();
1574 ring_buffer_reset_cpu(buffer, cpu);
1576 ring_buffer_record_enable(buffer);
1579 void tracing_reset_online_cpus(struct trace_buffer *buf)
1581 struct ring_buffer *buffer = buf->buffer;
1587 ring_buffer_record_disable(buffer);
1589 /* Make sure all commits have finished */
1590 synchronize_sched();
1592 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1594 for_each_online_cpu(cpu)
1595 ring_buffer_reset_cpu(buffer, cpu);
1597 ring_buffer_record_enable(buffer);
1600 /* Must have trace_types_lock held */
1601 void tracing_reset_all_online_cpus(void)
1603 struct trace_array *tr;
1605 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1606 tracing_reset_online_cpus(&tr->trace_buffer);
1607 #ifdef CONFIG_TRACER_MAX_TRACE
1608 tracing_reset_online_cpus(&tr->max_buffer);
1613 #define SAVED_CMDLINES_DEFAULT 128
1614 #define NO_CMDLINE_MAP UINT_MAX
1615 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1616 struct saved_cmdlines_buffer {
1617 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1618 unsigned *map_cmdline_to_pid;
1619 unsigned cmdline_num;
1621 char *saved_cmdlines;
1623 static struct saved_cmdlines_buffer *savedcmd;
1625 static inline char *get_saved_cmdlines(int idx)
1627 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1630 static inline void set_cmdline(int idx, const char *cmdline)
1632 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1635 static int allocate_cmdlines_buffer(unsigned int val,
1636 struct saved_cmdlines_buffer *s)
1638 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1640 if (!s->map_cmdline_to_pid)
1643 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1644 if (!s->saved_cmdlines) {
1645 kfree(s->map_cmdline_to_pid);
1650 s->cmdline_num = val;
1651 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1652 sizeof(s->map_pid_to_cmdline));
1653 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1654 val * sizeof(*s->map_cmdline_to_pid));
1659 static int trace_create_savedcmd(void)
1663 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1667 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1677 int is_tracing_stopped(void)
1679 return global_trace.stop_count;
1683 * tracing_start - quick start of the tracer
1685 * If tracing is enabled but was stopped by tracing_stop,
1686 * this will start the tracer back up.
1688 void tracing_start(void)
1690 struct ring_buffer *buffer;
1691 unsigned long flags;
1693 if (tracing_disabled)
1696 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1697 if (--global_trace.stop_count) {
1698 if (global_trace.stop_count < 0) {
1699 /* Someone screwed up their debugging */
1701 global_trace.stop_count = 0;
1706 /* Prevent the buffers from switching */
1707 arch_spin_lock(&global_trace.max_lock);
1709 buffer = global_trace.trace_buffer.buffer;
1711 ring_buffer_record_enable(buffer);
1713 #ifdef CONFIG_TRACER_MAX_TRACE
1714 buffer = global_trace.max_buffer.buffer;
1716 ring_buffer_record_enable(buffer);
1719 arch_spin_unlock(&global_trace.max_lock);
1722 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1725 static void tracing_start_tr(struct trace_array *tr)
1727 struct ring_buffer *buffer;
1728 unsigned long flags;
1730 if (tracing_disabled)
1733 /* If global, we need to also start the max tracer */
1734 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1735 return tracing_start();
1737 raw_spin_lock_irqsave(&tr->start_lock, flags);
1739 if (--tr->stop_count) {
1740 if (tr->stop_count < 0) {
1741 /* Someone screwed up their debugging */
1748 buffer = tr->trace_buffer.buffer;
1750 ring_buffer_record_enable(buffer);
1753 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1757 * tracing_stop - quick stop of the tracer
1759 * Light weight way to stop tracing. Use in conjunction with
1762 void tracing_stop(void)
1764 struct ring_buffer *buffer;
1765 unsigned long flags;
1767 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1768 if (global_trace.stop_count++)
1771 /* Prevent the buffers from switching */
1772 arch_spin_lock(&global_trace.max_lock);
1774 buffer = global_trace.trace_buffer.buffer;
1776 ring_buffer_record_disable(buffer);
1778 #ifdef CONFIG_TRACER_MAX_TRACE
1779 buffer = global_trace.max_buffer.buffer;
1781 ring_buffer_record_disable(buffer);
1784 arch_spin_unlock(&global_trace.max_lock);
1787 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1790 static void tracing_stop_tr(struct trace_array *tr)
1792 struct ring_buffer *buffer;
1793 unsigned long flags;
1795 /* If global, we need to also stop the max tracer */
1796 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1797 return tracing_stop();
1799 raw_spin_lock_irqsave(&tr->start_lock, flags);
1800 if (tr->stop_count++)
1803 buffer = tr->trace_buffer.buffer;
1805 ring_buffer_record_disable(buffer);
1808 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1811 void trace_stop_cmdline_recording(void);
1813 static int trace_save_cmdline(struct task_struct *tsk)
1817 /* treat recording of idle task as a success */
1821 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
1824 * It's not the end of the world if we don't get
1825 * the lock, but we also don't want to spin
1826 * nor do we want to disable interrupts,
1827 * so if we miss here, then better luck next time.
1829 if (!arch_spin_trylock(&trace_cmdline_lock))
1832 idx = savedcmd->map_pid_to_cmdline[tpid];
1833 if (idx == NO_CMDLINE_MAP) {
1834 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1836 savedcmd->map_pid_to_cmdline[tpid] = idx;
1837 savedcmd->cmdline_idx = idx;
1840 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1841 set_cmdline(idx, tsk->comm);
1843 arch_spin_unlock(&trace_cmdline_lock);
1848 static void __trace_find_cmdline(int pid, char comm[])
1854 strcpy(comm, "<idle>");
1858 if (WARN_ON_ONCE(pid < 0)) {
1859 strcpy(comm, "<XXX>");
1863 tpid = pid & (PID_MAX_DEFAULT - 1);
1864 map = savedcmd->map_pid_to_cmdline[tpid];
1865 if (map != NO_CMDLINE_MAP) {
1866 tpid = savedcmd->map_cmdline_to_pid[map];
1868 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
1872 strcpy(comm, "<...>");
1875 void trace_find_cmdline(int pid, char comm[])
1878 arch_spin_lock(&trace_cmdline_lock);
1880 __trace_find_cmdline(pid, comm);
1882 arch_spin_unlock(&trace_cmdline_lock);
1886 void tracing_record_cmdline(struct task_struct *tsk)
1888 if (!__this_cpu_read(trace_cmdline_save))
1891 if (trace_save_cmdline(tsk))
1892 __this_cpu_write(trace_cmdline_save, false);
1896 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1899 struct task_struct *tsk = current;
1901 entry->preempt_count = pc & 0xff;
1902 entry->pid = (tsk) ? tsk->pid : 0;
1904 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1905 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1907 TRACE_FLAG_IRQS_NOSUPPORT |
1909 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
1910 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1911 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
1912 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1913 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1915 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1917 static __always_inline void
1918 trace_event_setup(struct ring_buffer_event *event,
1919 int type, unsigned long flags, int pc)
1921 struct trace_entry *ent = ring_buffer_event_data(event);
1923 tracing_generic_entry_update(ent, flags, pc);
1927 struct ring_buffer_event *
1928 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1931 unsigned long flags, int pc)
1933 struct ring_buffer_event *event;
1935 event = ring_buffer_lock_reserve(buffer, len);
1937 trace_event_setup(event, type, flags, pc);
1942 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1943 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1944 static int trace_buffered_event_ref;
1947 * trace_buffered_event_enable - enable buffering events
1949 * When events are being filtered, it is quicker to use a temporary
1950 * buffer to write the event data into if there's a likely chance
1951 * that it will not be committed. The discard of the ring buffer
1952 * is not as fast as committing, and is much slower than copying
1955 * When an event is to be filtered, allocate per cpu buffers to
1956 * write the event data into, and if the event is filtered and discarded
1957 * it is simply dropped, otherwise, the entire data is to be committed
1960 void trace_buffered_event_enable(void)
1962 struct ring_buffer_event *event;
1966 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1968 if (trace_buffered_event_ref++)
1971 for_each_tracing_cpu(cpu) {
1972 page = alloc_pages_node(cpu_to_node(cpu),
1973 GFP_KERNEL | __GFP_NORETRY, 0);
1977 event = page_address(page);
1978 memset(event, 0, sizeof(*event));
1980 per_cpu(trace_buffered_event, cpu) = event;
1983 if (cpu == smp_processor_id() &&
1984 this_cpu_read(trace_buffered_event) !=
1985 per_cpu(trace_buffered_event, cpu))
1992 trace_buffered_event_disable();
1995 static void enable_trace_buffered_event(void *data)
1997 /* Probably not needed, but do it anyway */
1999 this_cpu_dec(trace_buffered_event_cnt);
2002 static void disable_trace_buffered_event(void *data)
2004 this_cpu_inc(trace_buffered_event_cnt);
2008 * trace_buffered_event_disable - disable buffering events
2010 * When a filter is removed, it is faster to not use the buffered
2011 * events, and to commit directly into the ring buffer. Free up
2012 * the temp buffers when there are no more users. This requires
2013 * special synchronization with current events.
2015 void trace_buffered_event_disable(void)
2019 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2021 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2024 if (--trace_buffered_event_ref)
2028 /* For each CPU, set the buffer as used. */
2029 smp_call_function_many(tracing_buffer_mask,
2030 disable_trace_buffered_event, NULL, 1);
2033 /* Wait for all current users to finish */
2034 synchronize_sched();
2036 for_each_tracing_cpu(cpu) {
2037 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2038 per_cpu(trace_buffered_event, cpu) = NULL;
2041 * Make sure trace_buffered_event is NULL before clearing
2042 * trace_buffered_event_cnt.
2047 /* Do the work on each cpu */
2048 smp_call_function_many(tracing_buffer_mask,
2049 enable_trace_buffered_event, NULL, 1);
2054 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
2056 __this_cpu_write(trace_cmdline_save, true);
2058 /* If this is the temp buffer, we need to commit fully */
2059 if (this_cpu_read(trace_buffered_event) == event) {
2060 /* Length is in event->array[0] */
2061 ring_buffer_write(buffer, event->array[0], &event->array[1]);
2062 /* Release the temp buffer */
2063 this_cpu_dec(trace_buffered_event_cnt);
2065 ring_buffer_unlock_commit(buffer, event);
2068 static struct ring_buffer *temp_buffer;
2070 struct ring_buffer_event *
2071 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2072 struct trace_event_file *trace_file,
2073 int type, unsigned long len,
2074 unsigned long flags, int pc)
2076 struct ring_buffer_event *entry;
2079 *current_rb = trace_file->tr->trace_buffer.buffer;
2081 if ((trace_file->flags &
2082 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2083 (entry = this_cpu_read(trace_buffered_event))) {
2084 /* Try to use the per cpu buffer first */
2085 val = this_cpu_inc_return(trace_buffered_event_cnt);
2086 if ((len < (PAGE_SIZE - sizeof(*entry) - sizeof(entry->array[0]))) && val == 1) {
2087 trace_event_setup(entry, type, flags, pc);
2088 entry->array[0] = len;
2091 this_cpu_dec(trace_buffered_event_cnt);
2094 entry = trace_buffer_lock_reserve(*current_rb,
2095 type, len, flags, pc);
2097 * If tracing is off, but we have triggers enabled
2098 * we still need to look at the event data. Use the temp_buffer
2099 * to store the trace event for the tigger to use. It's recusive
2100 * safe and will not be recorded anywhere.
2102 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2103 *current_rb = temp_buffer;
2104 entry = trace_buffer_lock_reserve(*current_rb,
2105 type, len, flags, pc);
2109 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2111 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2112 struct ring_buffer *buffer,
2113 struct ring_buffer_event *event,
2114 unsigned long flags, int pc,
2115 struct pt_regs *regs)
2117 __buffer_unlock_commit(buffer, event);
2120 * If regs is not set, then skip the following callers:
2121 * trace_buffer_unlock_commit_regs
2122 * event_trigger_unlock_commit
2123 * trace_event_buffer_commit
2124 * trace_event_raw_event_sched_switch
2125 * Note, we can still get here via blktrace, wakeup tracer
2126 * and mmiotrace, but that's ok if they lose a function or
2127 * two. They are that meaningful.
2129 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
2130 ftrace_trace_userstack(tr, buffer, flags, pc);
2134 trace_function(struct trace_array *tr,
2135 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2138 struct trace_event_call *call = &event_function;
2139 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2140 struct ring_buffer_event *event;
2141 struct ftrace_entry *entry;
2143 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2147 entry = ring_buffer_event_data(event);
2149 entry->parent_ip = parent_ip;
2151 if (!call_filter_check_discard(call, entry, buffer, event))
2152 __buffer_unlock_commit(buffer, event);
2155 #ifdef CONFIG_STACKTRACE
2157 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2158 struct ftrace_stack {
2159 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2162 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2163 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2165 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2166 unsigned long flags,
2167 int skip, int pc, struct pt_regs *regs)
2169 struct trace_event_call *call = &event_kernel_stack;
2170 struct ring_buffer_event *event;
2171 struct stack_entry *entry;
2172 struct stack_trace trace;
2174 int size = FTRACE_STACK_ENTRIES;
2176 trace.nr_entries = 0;
2180 * Add two, for this function and the call to save_stack_trace()
2181 * If regs is set, then these functions will not be in the way.
2187 * Since events can happen in NMIs there's no safe way to
2188 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2189 * or NMI comes in, it will just have to use the default
2190 * FTRACE_STACK_SIZE.
2192 preempt_disable_notrace();
2194 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2196 * We don't need any atomic variables, just a barrier.
2197 * If an interrupt comes in, we don't care, because it would
2198 * have exited and put the counter back to what we want.
2199 * We just need a barrier to keep gcc from moving things
2203 if (use_stack == 1) {
2204 trace.entries = this_cpu_ptr(ftrace_stack.calls);
2205 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2208 save_stack_trace_regs(regs, &trace);
2210 save_stack_trace(&trace);
2212 if (trace.nr_entries > size)
2213 size = trace.nr_entries;
2215 /* From now on, use_stack is a boolean */
2218 size *= sizeof(unsigned long);
2220 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
2221 (sizeof(*entry) - sizeof(entry->caller)) + size,
2225 entry = ring_buffer_event_data(event);
2227 memset(&entry->caller, 0, size);
2230 memcpy(&entry->caller, trace.entries,
2231 trace.nr_entries * sizeof(unsigned long));
2233 trace.max_entries = FTRACE_STACK_ENTRIES;
2234 trace.entries = entry->caller;
2236 save_stack_trace_regs(regs, &trace);
2238 save_stack_trace(&trace);
2241 entry->size = trace.nr_entries;
2243 if (!call_filter_check_discard(call, entry, buffer, event))
2244 __buffer_unlock_commit(buffer, event);
2247 /* Again, don't let gcc optimize things here */
2249 __this_cpu_dec(ftrace_stack_reserve);
2250 preempt_enable_notrace();
2254 static inline void ftrace_trace_stack(struct trace_array *tr,
2255 struct ring_buffer *buffer,
2256 unsigned long flags,
2257 int skip, int pc, struct pt_regs *regs)
2259 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2262 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2265 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2268 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
2272 * trace_dump_stack - record a stack back trace in the trace buffer
2273 * @skip: Number of functions to skip (helper handlers)
2275 void trace_dump_stack(int skip)
2277 unsigned long flags;
2279 if (tracing_disabled || tracing_selftest_running)
2282 local_save_flags(flags);
2285 * Skip 3 more, seems to get us at the caller of
2289 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2290 flags, skip, preempt_count(), NULL);
2293 static DEFINE_PER_CPU(int, user_stack_count);
2296 ftrace_trace_userstack(struct trace_array *tr,
2297 struct ring_buffer *buffer, unsigned long flags, int pc)
2299 struct trace_event_call *call = &event_user_stack;
2300 struct ring_buffer_event *event;
2301 struct userstack_entry *entry;
2302 struct stack_trace trace;
2304 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
2308 * NMIs can not handle page faults, even with fix ups.
2309 * The save user stack can (and often does) fault.
2311 if (unlikely(in_nmi()))
2315 * prevent recursion, since the user stack tracing may
2316 * trigger other kernel events.
2319 if (__this_cpu_read(user_stack_count))
2322 __this_cpu_inc(user_stack_count);
2324 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2325 sizeof(*entry), flags, pc);
2327 goto out_drop_count;
2328 entry = ring_buffer_event_data(event);
2330 entry->tgid = current->tgid;
2331 memset(&entry->caller, 0, sizeof(entry->caller));
2333 trace.nr_entries = 0;
2334 trace.max_entries = FTRACE_STACK_ENTRIES;
2336 trace.entries = entry->caller;
2338 save_stack_trace_user(&trace);
2339 if (!call_filter_check_discard(call, entry, buffer, event))
2340 __buffer_unlock_commit(buffer, event);
2343 __this_cpu_dec(user_stack_count);
2349 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2351 ftrace_trace_userstack(tr, flags, preempt_count());
2355 #endif /* CONFIG_STACKTRACE */
2357 /* created for use with alloc_percpu */
2358 struct trace_buffer_struct {
2360 char buffer[4][TRACE_BUF_SIZE];
2363 static struct trace_buffer_struct __percpu *trace_percpu_buffer;
2366 * Thise allows for lockless recording. If we're nested too deeply, then
2367 * this returns NULL.
2369 static char *get_trace_buf(void)
2371 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
2373 if (!trace_percpu_buffer || buffer->nesting >= 4)
2378 /* Interrupts must see nesting incremented before we use the buffer */
2380 return &buffer->buffer[buffer->nesting - 1][0];
2383 static void put_trace_buf(void)
2385 /* Don't let the decrement of nesting leak before this */
2387 this_cpu_dec(trace_percpu_buffer->nesting);
2390 static int alloc_percpu_trace_buffer(void)
2392 struct trace_buffer_struct __percpu *buffers;
2394 buffers = alloc_percpu(struct trace_buffer_struct);
2395 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2398 trace_percpu_buffer = buffers;
2402 static int buffers_allocated;
2404 void trace_printk_init_buffers(void)
2406 if (buffers_allocated)
2409 if (alloc_percpu_trace_buffer())
2412 /* trace_printk() is for debug use only. Don't use it in production. */
2415 pr_warn("**********************************************************\n");
2416 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2418 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2420 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2421 pr_warn("** unsafe for production use. **\n");
2423 pr_warn("** If you see this message and you are not debugging **\n");
2424 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2426 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2427 pr_warn("**********************************************************\n");
2429 /* Expand the buffers to set size */
2430 tracing_update_buffers();
2432 buffers_allocated = 1;
2435 * trace_printk_init_buffers() can be called by modules.
2436 * If that happens, then we need to start cmdline recording
2437 * directly here. If the global_trace.buffer is already
2438 * allocated here, then this was called by module code.
2440 if (global_trace.trace_buffer.buffer)
2441 tracing_start_cmdline_record();
2444 void trace_printk_start_comm(void)
2446 /* Start tracing comms if trace printk is set */
2447 if (!buffers_allocated)
2449 tracing_start_cmdline_record();
2452 static void trace_printk_start_stop_comm(int enabled)
2454 if (!buffers_allocated)
2458 tracing_start_cmdline_record();
2460 tracing_stop_cmdline_record();
2464 * trace_vbprintk - write binary msg to tracing buffer
2467 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2469 struct trace_event_call *call = &event_bprint;
2470 struct ring_buffer_event *event;
2471 struct ring_buffer *buffer;
2472 struct trace_array *tr = &global_trace;
2473 struct bprint_entry *entry;
2474 unsigned long flags;
2476 int len = 0, size, pc;
2478 if (unlikely(tracing_selftest_running || tracing_disabled))
2481 /* Don't pollute graph traces with trace_vprintk internals */
2482 pause_graph_tracing();
2484 pc = preempt_count();
2485 preempt_disable_notrace();
2487 tbuffer = get_trace_buf();
2493 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2495 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2498 local_save_flags(flags);
2499 size = sizeof(*entry) + sizeof(u32) * len;
2500 buffer = tr->trace_buffer.buffer;
2501 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2505 entry = ring_buffer_event_data(event);
2509 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2510 if (!call_filter_check_discard(call, entry, buffer, event)) {
2511 __buffer_unlock_commit(buffer, event);
2512 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2519 preempt_enable_notrace();
2520 unpause_graph_tracing();
2524 EXPORT_SYMBOL_GPL(trace_vbprintk);
2528 __trace_array_vprintk(struct ring_buffer *buffer,
2529 unsigned long ip, const char *fmt, va_list args)
2531 struct trace_event_call *call = &event_print;
2532 struct ring_buffer_event *event;
2533 int len = 0, size, pc;
2534 struct print_entry *entry;
2535 unsigned long flags;
2538 if (tracing_disabled || tracing_selftest_running)
2541 /* Don't pollute graph traces with trace_vprintk internals */
2542 pause_graph_tracing();
2544 pc = preempt_count();
2545 preempt_disable_notrace();
2548 tbuffer = get_trace_buf();
2554 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2556 local_save_flags(flags);
2557 size = sizeof(*entry) + len + 1;
2558 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2562 entry = ring_buffer_event_data(event);
2565 memcpy(&entry->buf, tbuffer, len + 1);
2566 if (!call_filter_check_discard(call, entry, buffer, event)) {
2567 __buffer_unlock_commit(buffer, event);
2568 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2575 preempt_enable_notrace();
2576 unpause_graph_tracing();
2582 int trace_array_vprintk(struct trace_array *tr,
2583 unsigned long ip, const char *fmt, va_list args)
2585 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2589 int trace_array_printk(struct trace_array *tr,
2590 unsigned long ip, const char *fmt, ...)
2595 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2602 ret = trace_array_vprintk(tr, ip, fmt, ap);
2608 int trace_array_printk_buf(struct ring_buffer *buffer,
2609 unsigned long ip, const char *fmt, ...)
2614 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2618 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2624 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2626 return trace_array_vprintk(&global_trace, ip, fmt, args);
2628 EXPORT_SYMBOL_GPL(trace_vprintk);
2630 static void trace_iterator_increment(struct trace_iterator *iter)
2632 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2636 ring_buffer_read(buf_iter, NULL);
2639 static struct trace_entry *
2640 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2641 unsigned long *lost_events)
2643 struct ring_buffer_event *event;
2644 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2647 event = ring_buffer_iter_peek(buf_iter, ts);
2649 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2653 iter->ent_size = ring_buffer_event_length(event);
2654 return ring_buffer_event_data(event);
2660 static struct trace_entry *
2661 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2662 unsigned long *missing_events, u64 *ent_ts)
2664 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2665 struct trace_entry *ent, *next = NULL;
2666 unsigned long lost_events = 0, next_lost = 0;
2667 int cpu_file = iter->cpu_file;
2668 u64 next_ts = 0, ts;
2674 * If we are in a per_cpu trace file, don't bother by iterating over
2675 * all cpu and peek directly.
2677 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2678 if (ring_buffer_empty_cpu(buffer, cpu_file))
2680 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2682 *ent_cpu = cpu_file;
2687 for_each_tracing_cpu(cpu) {
2689 if (ring_buffer_empty_cpu(buffer, cpu))
2692 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2695 * Pick the entry with the smallest timestamp:
2697 if (ent && (!next || ts < next_ts)) {
2701 next_lost = lost_events;
2702 next_size = iter->ent_size;
2706 iter->ent_size = next_size;
2709 *ent_cpu = next_cpu;
2715 *missing_events = next_lost;
2720 /* Find the next real entry, without updating the iterator itself */
2721 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2722 int *ent_cpu, u64 *ent_ts)
2724 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2727 /* Find the next real entry, and increment the iterator to the next entry */
2728 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2730 iter->ent = __find_next_entry(iter, &iter->cpu,
2731 &iter->lost_events, &iter->ts);
2734 trace_iterator_increment(iter);
2736 return iter->ent ? iter : NULL;
2739 static void trace_consume(struct trace_iterator *iter)
2741 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2742 &iter->lost_events);
2745 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2747 struct trace_iterator *iter = m->private;
2751 WARN_ON_ONCE(iter->leftover);
2755 /* can't go backwards */
2760 ent = trace_find_next_entry_inc(iter);
2764 while (ent && iter->idx < i)
2765 ent = trace_find_next_entry_inc(iter);
2772 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2774 struct ring_buffer_event *event;
2775 struct ring_buffer_iter *buf_iter;
2776 unsigned long entries = 0;
2779 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2781 buf_iter = trace_buffer_iter(iter, cpu);
2785 ring_buffer_iter_reset(buf_iter);
2788 * We could have the case with the max latency tracers
2789 * that a reset never took place on a cpu. This is evident
2790 * by the timestamp being before the start of the buffer.
2792 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2793 if (ts >= iter->trace_buffer->time_start)
2796 ring_buffer_read(buf_iter, NULL);
2799 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2803 * The current tracer is copied to avoid a global locking
2806 static void *s_start(struct seq_file *m, loff_t *pos)
2808 struct trace_iterator *iter = m->private;
2809 struct trace_array *tr = iter->tr;
2810 int cpu_file = iter->cpu_file;
2816 * copy the tracer to avoid using a global lock all around.
2817 * iter->trace is a copy of current_trace, the pointer to the
2818 * name may be used instead of a strcmp(), as iter->trace->name
2819 * will point to the same string as current_trace->name.
2821 mutex_lock(&trace_types_lock);
2822 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2823 *iter->trace = *tr->current_trace;
2824 mutex_unlock(&trace_types_lock);
2826 #ifdef CONFIG_TRACER_MAX_TRACE
2827 if (iter->snapshot && iter->trace->use_max_tr)
2828 return ERR_PTR(-EBUSY);
2831 if (*pos != iter->pos) {
2836 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2837 for_each_tracing_cpu(cpu)
2838 tracing_iter_reset(iter, cpu);
2840 tracing_iter_reset(iter, cpu_file);
2843 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2848 * If we overflowed the seq_file before, then we want
2849 * to just reuse the trace_seq buffer again.
2855 p = s_next(m, p, &l);
2859 trace_event_read_lock();
2860 trace_access_lock(cpu_file);
2864 static void s_stop(struct seq_file *m, void *p)
2866 struct trace_iterator *iter = m->private;
2868 #ifdef CONFIG_TRACER_MAX_TRACE
2869 if (iter->snapshot && iter->trace->use_max_tr)
2873 trace_access_unlock(iter->cpu_file);
2874 trace_event_read_unlock();
2878 get_total_entries(struct trace_buffer *buf,
2879 unsigned long *total, unsigned long *entries)
2881 unsigned long count;
2887 for_each_tracing_cpu(cpu) {
2888 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2890 * If this buffer has skipped entries, then we hold all
2891 * entries for the trace and we need to ignore the
2892 * ones before the time stamp.
2894 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2895 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2896 /* total is the same as the entries */
2900 ring_buffer_overrun_cpu(buf->buffer, cpu);
2905 static void print_lat_help_header(struct seq_file *m)
2907 seq_puts(m, "# _------=> CPU# \n"
2908 "# / _-----=> irqs-off \n"
2909 "# | / _----=> need-resched \n"
2910 "# || / _---=> hardirq/softirq \n"
2911 "# ||| / _--=> preempt-depth \n"
2913 "# cmd pid ||||| time | caller \n"
2914 "# \\ / ||||| \\ | / \n");
2917 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2919 unsigned long total;
2920 unsigned long entries;
2922 get_total_entries(buf, &total, &entries);
2923 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2924 entries, total, num_online_cpus());
2928 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2930 print_event_info(buf, m);
2931 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2935 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2937 print_event_info(buf, m);
2938 seq_puts(m, "# _-----=> irqs-off\n"
2939 "# / _----=> need-resched\n"
2940 "# | / _---=> hardirq/softirq\n"
2941 "# || / _--=> preempt-depth\n"
2943 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2944 "# | | | |||| | |\n");
2948 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2950 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2951 struct trace_buffer *buf = iter->trace_buffer;
2952 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2953 struct tracer *type = iter->trace;
2954 unsigned long entries;
2955 unsigned long total;
2956 const char *name = "preemption";
2960 get_total_entries(buf, &total, &entries);
2962 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2964 seq_puts(m, "# -----------------------------------"
2965 "---------------------------------\n");
2966 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2967 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2968 nsecs_to_usecs(data->saved_latency),
2972 #if defined(CONFIG_PREEMPT_NONE)
2974 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2976 #elif defined(CONFIG_PREEMPT)
2981 /* These are reserved for later use */
2984 seq_printf(m, " #P:%d)\n", num_online_cpus());
2988 seq_puts(m, "# -----------------\n");
2989 seq_printf(m, "# | task: %.16s-%d "
2990 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2991 data->comm, data->pid,
2992 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2993 data->policy, data->rt_priority);
2994 seq_puts(m, "# -----------------\n");
2996 if (data->critical_start) {
2997 seq_puts(m, "# => started at: ");
2998 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2999 trace_print_seq(m, &iter->seq);
3000 seq_puts(m, "\n# => ended at: ");
3001 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3002 trace_print_seq(m, &iter->seq);
3003 seq_puts(m, "\n#\n");
3009 static void test_cpu_buff_start(struct trace_iterator *iter)
3011 struct trace_seq *s = &iter->seq;
3012 struct trace_array *tr = iter->tr;
3014 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3017 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3020 if (cpumask_available(iter->started) &&
3021 cpumask_test_cpu(iter->cpu, iter->started))
3024 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3027 if (cpumask_available(iter->started))
3028 cpumask_set_cpu(iter->cpu, iter->started);
3030 /* Don't print started cpu buffer for the first entry of the trace */
3032 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3036 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3038 struct trace_array *tr = iter->tr;
3039 struct trace_seq *s = &iter->seq;
3040 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3041 struct trace_entry *entry;
3042 struct trace_event *event;
3046 test_cpu_buff_start(iter);
3048 event = ftrace_find_event(entry->type);
3050 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3051 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3052 trace_print_lat_context(iter);
3054 trace_print_context(iter);
3057 if (trace_seq_has_overflowed(s))
3058 return TRACE_TYPE_PARTIAL_LINE;
3061 return event->funcs->trace(iter, sym_flags, event);
3063 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3065 return trace_handle_return(s);
3068 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3070 struct trace_array *tr = iter->tr;
3071 struct trace_seq *s = &iter->seq;
3072 struct trace_entry *entry;
3073 struct trace_event *event;
3077 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3078 trace_seq_printf(s, "%d %d %llu ",
3079 entry->pid, iter->cpu, iter->ts);
3081 if (trace_seq_has_overflowed(s))
3082 return TRACE_TYPE_PARTIAL_LINE;
3084 event = ftrace_find_event(entry->type);
3086 return event->funcs->raw(iter, 0, event);
3088 trace_seq_printf(s, "%d ?\n", entry->type);
3090 return trace_handle_return(s);
3093 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3095 struct trace_array *tr = iter->tr;
3096 struct trace_seq *s = &iter->seq;
3097 unsigned char newline = '\n';
3098 struct trace_entry *entry;
3099 struct trace_event *event;
3103 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3104 SEQ_PUT_HEX_FIELD(s, entry->pid);
3105 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3106 SEQ_PUT_HEX_FIELD(s, iter->ts);
3107 if (trace_seq_has_overflowed(s))
3108 return TRACE_TYPE_PARTIAL_LINE;
3111 event = ftrace_find_event(entry->type);
3113 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3114 if (ret != TRACE_TYPE_HANDLED)
3118 SEQ_PUT_FIELD(s, newline);
3120 return trace_handle_return(s);
3123 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3125 struct trace_array *tr = iter->tr;
3126 struct trace_seq *s = &iter->seq;
3127 struct trace_entry *entry;
3128 struct trace_event *event;
3132 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3133 SEQ_PUT_FIELD(s, entry->pid);
3134 SEQ_PUT_FIELD(s, iter->cpu);
3135 SEQ_PUT_FIELD(s, iter->ts);
3136 if (trace_seq_has_overflowed(s))
3137 return TRACE_TYPE_PARTIAL_LINE;
3140 event = ftrace_find_event(entry->type);
3141 return event ? event->funcs->binary(iter, 0, event) :
3145 int trace_empty(struct trace_iterator *iter)
3147 struct ring_buffer_iter *buf_iter;
3150 /* If we are looking at one CPU buffer, only check that one */
3151 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3152 cpu = iter->cpu_file;
3153 buf_iter = trace_buffer_iter(iter, cpu);
3155 if (!ring_buffer_iter_empty(buf_iter))
3158 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3164 for_each_tracing_cpu(cpu) {
3165 buf_iter = trace_buffer_iter(iter, cpu);
3167 if (!ring_buffer_iter_empty(buf_iter))
3170 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3178 /* Called with trace_event_read_lock() held. */
3179 enum print_line_t print_trace_line(struct trace_iterator *iter)
3181 struct trace_array *tr = iter->tr;
3182 unsigned long trace_flags = tr->trace_flags;
3183 enum print_line_t ret;
3185 if (iter->lost_events) {
3186 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3187 iter->cpu, iter->lost_events);
3188 if (trace_seq_has_overflowed(&iter->seq))
3189 return TRACE_TYPE_PARTIAL_LINE;
3192 if (iter->trace && iter->trace->print_line) {
3193 ret = iter->trace->print_line(iter);
3194 if (ret != TRACE_TYPE_UNHANDLED)
3198 if (iter->ent->type == TRACE_BPUTS &&
3199 trace_flags & TRACE_ITER_PRINTK &&
3200 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3201 return trace_print_bputs_msg_only(iter);
3203 if (iter->ent->type == TRACE_BPRINT &&
3204 trace_flags & TRACE_ITER_PRINTK &&
3205 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3206 return trace_print_bprintk_msg_only(iter);
3208 if (iter->ent->type == TRACE_PRINT &&
3209 trace_flags & TRACE_ITER_PRINTK &&
3210 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3211 return trace_print_printk_msg_only(iter);
3213 if (trace_flags & TRACE_ITER_BIN)
3214 return print_bin_fmt(iter);
3216 if (trace_flags & TRACE_ITER_HEX)
3217 return print_hex_fmt(iter);
3219 if (trace_flags & TRACE_ITER_RAW)
3220 return print_raw_fmt(iter);
3222 return print_trace_fmt(iter);
3225 void trace_latency_header(struct seq_file *m)
3227 struct trace_iterator *iter = m->private;
3228 struct trace_array *tr = iter->tr;
3230 /* print nothing if the buffers are empty */
3231 if (trace_empty(iter))
3234 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3235 print_trace_header(m, iter);
3237 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3238 print_lat_help_header(m);
3241 void trace_default_header(struct seq_file *m)
3243 struct trace_iterator *iter = m->private;
3244 struct trace_array *tr = iter->tr;
3245 unsigned long trace_flags = tr->trace_flags;
3247 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3250 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3251 /* print nothing if the buffers are empty */
3252 if (trace_empty(iter))
3254 print_trace_header(m, iter);
3255 if (!(trace_flags & TRACE_ITER_VERBOSE))
3256 print_lat_help_header(m);
3258 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3259 if (trace_flags & TRACE_ITER_IRQ_INFO)
3260 print_func_help_header_irq(iter->trace_buffer, m);
3262 print_func_help_header(iter->trace_buffer, m);
3267 static void test_ftrace_alive(struct seq_file *m)
3269 if (!ftrace_is_dead())
3271 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3272 "# MAY BE MISSING FUNCTION EVENTS\n");
3275 #ifdef CONFIG_TRACER_MAX_TRACE
3276 static void show_snapshot_main_help(struct seq_file *m)
3278 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3279 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3280 "# Takes a snapshot of the main buffer.\n"
3281 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3282 "# (Doesn't have to be '2' works with any number that\n"
3283 "# is not a '0' or '1')\n");
3286 static void show_snapshot_percpu_help(struct seq_file *m)
3288 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3289 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3290 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3291 "# Takes a snapshot of the main buffer for this cpu.\n");
3293 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3294 "# Must use main snapshot file to allocate.\n");
3296 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3297 "# (Doesn't have to be '2' works with any number that\n"
3298 "# is not a '0' or '1')\n");
3301 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3303 if (iter->tr->allocated_snapshot)
3304 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3306 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3308 seq_puts(m, "# Snapshot commands:\n");
3309 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3310 show_snapshot_main_help(m);
3312 show_snapshot_percpu_help(m);
3315 /* Should never be called */
3316 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3319 static int s_show(struct seq_file *m, void *v)
3321 struct trace_iterator *iter = v;
3324 if (iter->ent == NULL) {
3326 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3328 test_ftrace_alive(m);
3330 if (iter->snapshot && trace_empty(iter))
3331 print_snapshot_help(m, iter);
3332 else if (iter->trace && iter->trace->print_header)
3333 iter->trace->print_header(m);
3335 trace_default_header(m);
3337 } else if (iter->leftover) {
3339 * If we filled the seq_file buffer earlier, we
3340 * want to just show it now.
3342 ret = trace_print_seq(m, &iter->seq);
3344 /* ret should this time be zero, but you never know */
3345 iter->leftover = ret;
3348 print_trace_line(iter);
3349 ret = trace_print_seq(m, &iter->seq);
3351 * If we overflow the seq_file buffer, then it will
3352 * ask us for this data again at start up.
3354 * ret is 0 if seq_file write succeeded.
3357 iter->leftover = ret;
3364 * Should be used after trace_array_get(), trace_types_lock
3365 * ensures that i_cdev was already initialized.
3367 static inline int tracing_get_cpu(struct inode *inode)
3369 if (inode->i_cdev) /* See trace_create_cpu_file() */
3370 return (long)inode->i_cdev - 1;
3371 return RING_BUFFER_ALL_CPUS;
3374 static const struct seq_operations tracer_seq_ops = {
3381 static struct trace_iterator *
3382 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3384 struct trace_array *tr = inode->i_private;
3385 struct trace_iterator *iter;
3388 if (tracing_disabled)
3389 return ERR_PTR(-ENODEV);
3391 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3393 return ERR_PTR(-ENOMEM);
3395 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3397 if (!iter->buffer_iter)
3401 * We make a copy of the current tracer to avoid concurrent
3402 * changes on it while we are reading.
3404 mutex_lock(&trace_types_lock);
3405 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3409 *iter->trace = *tr->current_trace;
3411 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3416 #ifdef CONFIG_TRACER_MAX_TRACE
3417 /* Currently only the top directory has a snapshot */
3418 if (tr->current_trace->print_max || snapshot)
3419 iter->trace_buffer = &tr->max_buffer;
3422 iter->trace_buffer = &tr->trace_buffer;
3423 iter->snapshot = snapshot;
3425 iter->cpu_file = tracing_get_cpu(inode);
3426 mutex_init(&iter->mutex);
3428 /* Notify the tracer early; before we stop tracing. */
3429 if (iter->trace && iter->trace->open)
3430 iter->trace->open(iter);
3432 /* Annotate start of buffers if we had overruns */
3433 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3434 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3436 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3437 if (trace_clocks[tr->clock_id].in_ns)
3438 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3440 /* stop the trace while dumping if we are not opening "snapshot" */
3441 if (!iter->snapshot)
3442 tracing_stop_tr(tr);
3444 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3445 for_each_tracing_cpu(cpu) {
3446 iter->buffer_iter[cpu] =
3447 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3450 ring_buffer_read_prepare_sync();
3451 for_each_tracing_cpu(cpu) {
3452 ring_buffer_read_start(iter->buffer_iter[cpu]);
3453 tracing_iter_reset(iter, cpu);
3456 cpu = iter->cpu_file;
3457 iter->buffer_iter[cpu] =
3458 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3460 ring_buffer_read_prepare_sync();
3461 ring_buffer_read_start(iter->buffer_iter[cpu]);
3462 tracing_iter_reset(iter, cpu);
3465 mutex_unlock(&trace_types_lock);
3470 mutex_unlock(&trace_types_lock);
3472 kfree(iter->buffer_iter);
3474 seq_release_private(inode, file);
3475 return ERR_PTR(-ENOMEM);
3478 int tracing_open_generic(struct inode *inode, struct file *filp)
3480 if (tracing_disabled)
3483 filp->private_data = inode->i_private;
3487 bool tracing_is_disabled(void)
3489 return (tracing_disabled) ? true: false;
3493 * Open and update trace_array ref count.
3494 * Must have the current trace_array passed to it.
3496 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3498 struct trace_array *tr = inode->i_private;
3500 if (tracing_disabled)
3503 if (trace_array_get(tr) < 0)
3506 filp->private_data = inode->i_private;
3511 static int tracing_release(struct inode *inode, struct file *file)
3513 struct trace_array *tr = inode->i_private;
3514 struct seq_file *m = file->private_data;
3515 struct trace_iterator *iter;
3518 if (!(file->f_mode & FMODE_READ)) {
3519 trace_array_put(tr);
3523 /* Writes do not use seq_file */
3525 mutex_lock(&trace_types_lock);
3527 for_each_tracing_cpu(cpu) {
3528 if (iter->buffer_iter[cpu])
3529 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3532 if (iter->trace && iter->trace->close)
3533 iter->trace->close(iter);
3535 if (!iter->snapshot)
3536 /* reenable tracing if it was previously enabled */
3537 tracing_start_tr(tr);
3539 __trace_array_put(tr);
3541 mutex_unlock(&trace_types_lock);
3543 mutex_destroy(&iter->mutex);
3544 free_cpumask_var(iter->started);
3546 kfree(iter->buffer_iter);
3547 seq_release_private(inode, file);
3552 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3554 struct trace_array *tr = inode->i_private;
3556 trace_array_put(tr);
3560 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3562 struct trace_array *tr = inode->i_private;
3564 trace_array_put(tr);
3566 return single_release(inode, file);
3569 static int tracing_open(struct inode *inode, struct file *file)
3571 struct trace_array *tr = inode->i_private;
3572 struct trace_iterator *iter;
3575 if (trace_array_get(tr) < 0)
3578 /* If this file was open for write, then erase contents */
3579 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3580 int cpu = tracing_get_cpu(inode);
3581 struct trace_buffer *trace_buf = &tr->trace_buffer;
3583 #ifdef CONFIG_TRACER_MAX_TRACE
3584 if (tr->current_trace->print_max)
3585 trace_buf = &tr->max_buffer;
3588 if (cpu == RING_BUFFER_ALL_CPUS)
3589 tracing_reset_online_cpus(trace_buf);
3591 tracing_reset(trace_buf, cpu);
3594 if (file->f_mode & FMODE_READ) {
3595 iter = __tracing_open(inode, file, false);
3597 ret = PTR_ERR(iter);
3598 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3599 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3603 trace_array_put(tr);
3609 * Some tracers are not suitable for instance buffers.
3610 * A tracer is always available for the global array (toplevel)
3611 * or if it explicitly states that it is.
3614 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3616 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3619 /* Find the next tracer that this trace array may use */
3620 static struct tracer *
3621 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3623 while (t && !trace_ok_for_array(t, tr))
3630 t_next(struct seq_file *m, void *v, loff_t *pos)
3632 struct trace_array *tr = m->private;
3633 struct tracer *t = v;
3638 t = get_tracer_for_array(tr, t->next);
3643 static void *t_start(struct seq_file *m, loff_t *pos)
3645 struct trace_array *tr = m->private;
3649 mutex_lock(&trace_types_lock);
3651 t = get_tracer_for_array(tr, trace_types);
3652 for (; t && l < *pos; t = t_next(m, t, &l))
3658 static void t_stop(struct seq_file *m, void *p)
3660 mutex_unlock(&trace_types_lock);
3663 static int t_show(struct seq_file *m, void *v)
3665 struct tracer *t = v;
3670 seq_puts(m, t->name);
3679 static const struct seq_operations show_traces_seq_ops = {
3686 static int show_traces_open(struct inode *inode, struct file *file)
3688 struct trace_array *tr = inode->i_private;
3692 if (tracing_disabled)
3695 if (trace_array_get(tr) < 0)
3698 ret = seq_open(file, &show_traces_seq_ops);
3700 trace_array_put(tr);
3704 m = file->private_data;
3710 static int show_traces_release(struct inode *inode, struct file *file)
3712 struct trace_array *tr = inode->i_private;
3714 trace_array_put(tr);
3715 return seq_release(inode, file);
3719 tracing_write_stub(struct file *filp, const char __user *ubuf,
3720 size_t count, loff_t *ppos)
3725 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3729 if (file->f_mode & FMODE_READ)
3730 ret = seq_lseek(file, offset, whence);
3732 file->f_pos = ret = 0;
3737 static const struct file_operations tracing_fops = {
3738 .open = tracing_open,
3740 .write = tracing_write_stub,
3741 .llseek = tracing_lseek,
3742 .release = tracing_release,
3745 static const struct file_operations show_traces_fops = {
3746 .open = show_traces_open,
3748 .llseek = seq_lseek,
3749 .release = show_traces_release,
3753 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3754 size_t count, loff_t *ppos)
3756 struct trace_array *tr = file_inode(filp)->i_private;
3760 len = snprintf(NULL, 0, "%*pb\n",
3761 cpumask_pr_args(tr->tracing_cpumask)) + 1;
3762 mask_str = kmalloc(len, GFP_KERNEL);
3766 len = snprintf(mask_str, len, "%*pb\n",
3767 cpumask_pr_args(tr->tracing_cpumask));
3772 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
3781 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3782 size_t count, loff_t *ppos)
3784 struct trace_array *tr = file_inode(filp)->i_private;
3785 cpumask_var_t tracing_cpumask_new;
3788 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3791 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3795 local_irq_disable();
3796 arch_spin_lock(&tr->max_lock);
3797 for_each_tracing_cpu(cpu) {
3799 * Increase/decrease the disabled counter if we are
3800 * about to flip a bit in the cpumask:
3802 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3803 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3804 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3805 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3807 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3808 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3809 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3810 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3813 arch_spin_unlock(&tr->max_lock);
3816 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3817 free_cpumask_var(tracing_cpumask_new);
3822 free_cpumask_var(tracing_cpumask_new);
3827 static const struct file_operations tracing_cpumask_fops = {
3828 .open = tracing_open_generic_tr,
3829 .read = tracing_cpumask_read,
3830 .write = tracing_cpumask_write,
3831 .release = tracing_release_generic_tr,
3832 .llseek = generic_file_llseek,
3835 static int tracing_trace_options_show(struct seq_file *m, void *v)
3837 struct tracer_opt *trace_opts;
3838 struct trace_array *tr = m->private;
3842 mutex_lock(&trace_types_lock);
3843 tracer_flags = tr->current_trace->flags->val;
3844 trace_opts = tr->current_trace->flags->opts;
3846 for (i = 0; trace_options[i]; i++) {
3847 if (tr->trace_flags & (1 << i))
3848 seq_printf(m, "%s\n", trace_options[i]);
3850 seq_printf(m, "no%s\n", trace_options[i]);
3853 for (i = 0; trace_opts[i].name; i++) {
3854 if (tracer_flags & trace_opts[i].bit)
3855 seq_printf(m, "%s\n", trace_opts[i].name);
3857 seq_printf(m, "no%s\n", trace_opts[i].name);
3859 mutex_unlock(&trace_types_lock);
3864 static int __set_tracer_option(struct trace_array *tr,
3865 struct tracer_flags *tracer_flags,
3866 struct tracer_opt *opts, int neg)
3868 struct tracer *trace = tracer_flags->trace;
3871 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3876 tracer_flags->val &= ~opts->bit;
3878 tracer_flags->val |= opts->bit;
3882 /* Try to assign a tracer specific option */
3883 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3885 struct tracer *trace = tr->current_trace;
3886 struct tracer_flags *tracer_flags = trace->flags;
3887 struct tracer_opt *opts = NULL;
3890 for (i = 0; tracer_flags->opts[i].name; i++) {
3891 opts = &tracer_flags->opts[i];
3893 if (strcmp(cmp, opts->name) == 0)
3894 return __set_tracer_option(tr, trace->flags, opts, neg);
3900 /* Some tracers require overwrite to stay enabled */
3901 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3903 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3909 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3911 /* do nothing if flag is already set */
3912 if (!!(tr->trace_flags & mask) == !!enabled)
3915 /* Give the tracer a chance to approve the change */
3916 if (tr->current_trace->flag_changed)
3917 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3921 tr->trace_flags |= mask;
3923 tr->trace_flags &= ~mask;
3925 if (mask == TRACE_ITER_RECORD_CMD)
3926 trace_event_enable_cmd_record(enabled);
3928 if (mask == TRACE_ITER_EVENT_FORK)
3929 trace_event_follow_fork(tr, enabled);
3931 if (mask == TRACE_ITER_OVERWRITE) {
3932 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3933 #ifdef CONFIG_TRACER_MAX_TRACE
3934 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3938 if (mask == TRACE_ITER_PRINTK) {
3939 trace_printk_start_stop_comm(enabled);
3940 trace_printk_control(enabled);
3946 static int trace_set_options(struct trace_array *tr, char *option)
3952 size_t orig_len = strlen(option);
3954 cmp = strstrip(option);
3956 if (strncmp(cmp, "no", 2) == 0) {
3961 mutex_lock(&trace_types_lock);
3963 for (i = 0; trace_options[i]; i++) {
3964 if (strcmp(cmp, trace_options[i]) == 0) {
3965 ret = set_tracer_flag(tr, 1 << i, !neg);
3970 /* If no option could be set, test the specific tracer options */
3971 if (!trace_options[i])
3972 ret = set_tracer_option(tr, cmp, neg);
3974 mutex_unlock(&trace_types_lock);
3977 * If the first trailing whitespace is replaced with '\0' by strstrip,
3978 * turn it back into a space.
3980 if (orig_len > strlen(option))
3981 option[strlen(option)] = ' ';
3986 static void __init apply_trace_boot_options(void)
3988 char *buf = trace_boot_options_buf;
3992 option = strsep(&buf, ",");
3998 trace_set_options(&global_trace, option);
4000 /* Put back the comma to allow this to be called again */
4007 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4008 size_t cnt, loff_t *ppos)
4010 struct seq_file *m = filp->private_data;
4011 struct trace_array *tr = m->private;
4015 if (cnt >= sizeof(buf))
4018 if (copy_from_user(buf, ubuf, cnt))
4023 ret = trace_set_options(tr, buf);
4032 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4034 struct trace_array *tr = inode->i_private;
4037 if (tracing_disabled)
4040 if (trace_array_get(tr) < 0)
4043 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4045 trace_array_put(tr);
4050 static const struct file_operations tracing_iter_fops = {
4051 .open = tracing_trace_options_open,
4053 .llseek = seq_lseek,
4054 .release = tracing_single_release_tr,
4055 .write = tracing_trace_options_write,
4058 static const char readme_msg[] =
4059 "tracing mini-HOWTO:\n\n"
4060 "# echo 0 > tracing_on : quick way to disable tracing\n"
4061 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4062 " Important files:\n"
4063 " trace\t\t\t- The static contents of the buffer\n"
4064 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4065 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4066 " current_tracer\t- function and latency tracers\n"
4067 " available_tracers\t- list of configured tracers for current_tracer\n"
4068 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4069 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4070 " trace_clock\t\t-change the clock used to order events\n"
4071 " local: Per cpu clock but may not be synced across CPUs\n"
4072 " global: Synced across CPUs but slows tracing down.\n"
4073 " counter: Not a clock, but just an increment\n"
4074 " uptime: Jiffy counter from time of boot\n"
4075 " perf: Same clock that perf events use\n"
4076 #ifdef CONFIG_X86_64
4077 " x86-tsc: TSC cycle counter\n"
4079 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4080 " tracing_cpumask\t- Limit which CPUs to trace\n"
4081 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4082 "\t\t\t Remove sub-buffer with rmdir\n"
4083 " trace_options\t\t- Set format or modify how tracing happens\n"
4084 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4085 "\t\t\t option name\n"
4086 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4087 #ifdef CONFIG_DYNAMIC_FTRACE
4088 "\n available_filter_functions - list of functions that can be filtered on\n"
4089 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4090 "\t\t\t functions\n"
4091 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4092 "\t modules: Can select a group via module\n"
4093 "\t Format: :mod:<module-name>\n"
4094 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4095 "\t triggers: a command to perform when function is hit\n"
4096 "\t Format: <function>:<trigger>[:count]\n"
4097 "\t trigger: traceon, traceoff\n"
4098 "\t\t enable_event:<system>:<event>\n"
4099 "\t\t disable_event:<system>:<event>\n"
4100 #ifdef CONFIG_STACKTRACE
4103 #ifdef CONFIG_TRACER_SNAPSHOT
4108 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4109 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4110 "\t The first one will disable tracing every time do_fault is hit\n"
4111 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4112 "\t The first time do trap is hit and it disables tracing, the\n"
4113 "\t counter will decrement to 2. If tracing is already disabled,\n"
4114 "\t the counter will not decrement. It only decrements when the\n"
4115 "\t trigger did work\n"
4116 "\t To remove trigger without count:\n"
4117 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4118 "\t To remove trigger with a count:\n"
4119 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4120 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4121 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4122 "\t modules: Can select a group via module command :mod:\n"
4123 "\t Does not accept triggers\n"
4124 #endif /* CONFIG_DYNAMIC_FTRACE */
4125 #ifdef CONFIG_FUNCTION_TRACER
4126 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4129 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4130 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4131 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4132 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4134 #ifdef CONFIG_TRACER_SNAPSHOT
4135 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4136 "\t\t\t snapshot buffer. Read the contents for more\n"
4137 "\t\t\t information\n"
4139 #ifdef CONFIG_STACK_TRACER
4140 " stack_trace\t\t- Shows the max stack trace when active\n"
4141 " stack_max_size\t- Shows current max stack size that was traced\n"
4142 "\t\t\t Write into this file to reset the max size (trigger a\n"
4143 "\t\t\t new trace)\n"
4144 #ifdef CONFIG_DYNAMIC_FTRACE
4145 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4148 #endif /* CONFIG_STACK_TRACER */
4149 #ifdef CONFIG_KPROBE_EVENT
4150 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4151 "\t\t\t Write into this file to define/undefine new trace events.\n"
4153 #ifdef CONFIG_UPROBE_EVENT
4154 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4155 "\t\t\t Write into this file to define/undefine new trace events.\n"
4157 #if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
4158 "\t accepts: event-definitions (one definition per line)\n"
4159 "\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
4160 "\t -:[<group>/]<event>\n"
4161 #ifdef CONFIG_KPROBE_EVENT
4162 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4164 #ifdef CONFIG_UPROBE_EVENT
4165 "\t place: <path>:<offset>\n"
4167 "\t args: <name>=fetcharg[:type]\n"
4168 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4169 "\t $stack<index>, $stack, $retval, $comm\n"
4170 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4171 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4173 " events/\t\t- Directory containing all trace event subsystems:\n"
4174 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4175 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4176 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4178 " filter\t\t- If set, only events passing filter are traced\n"
4179 " events/<system>/<event>/\t- Directory containing control files for\n"
4181 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4182 " filter\t\t- If set, only events passing filter are traced\n"
4183 " trigger\t\t- If set, a command to perform when event is hit\n"
4184 "\t Format: <trigger>[:count][if <filter>]\n"
4185 "\t trigger: traceon, traceoff\n"
4186 "\t enable_event:<system>:<event>\n"
4187 "\t disable_event:<system>:<event>\n"
4188 #ifdef CONFIG_HIST_TRIGGERS
4189 "\t enable_hist:<system>:<event>\n"
4190 "\t disable_hist:<system>:<event>\n"
4192 #ifdef CONFIG_STACKTRACE
4195 #ifdef CONFIG_TRACER_SNAPSHOT
4198 #ifdef CONFIG_HIST_TRIGGERS
4199 "\t\t hist (see below)\n"
4201 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4202 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4203 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4204 "\t events/block/block_unplug/trigger\n"
4205 "\t The first disables tracing every time block_unplug is hit.\n"
4206 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4207 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4208 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4209 "\t Like function triggers, the counter is only decremented if it\n"
4210 "\t enabled or disabled tracing.\n"
4211 "\t To remove a trigger without a count:\n"
4212 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4213 "\t To remove a trigger with a count:\n"
4214 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4215 "\t Filters can be ignored when removing a trigger.\n"
4216 #ifdef CONFIG_HIST_TRIGGERS
4217 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4218 "\t Format: hist:keys=<field1[,field2,...]>\n"
4219 "\t [:values=<field1[,field2,...]>]\n"
4220 "\t [:sort=<field1[,field2,...]>]\n"
4221 "\t [:size=#entries]\n"
4222 "\t [:pause][:continue][:clear]\n"
4223 "\t [:name=histname1]\n"
4224 "\t [if <filter>]\n\n"
4225 "\t When a matching event is hit, an entry is added to a hash\n"
4226 "\t table using the key(s) and value(s) named, and the value of a\n"
4227 "\t sum called 'hitcount' is incremented. Keys and values\n"
4228 "\t correspond to fields in the event's format description. Keys\n"
4229 "\t can be any field, or the special string 'stacktrace'.\n"
4230 "\t Compound keys consisting of up to two fields can be specified\n"
4231 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4232 "\t fields. Sort keys consisting of up to two fields can be\n"
4233 "\t specified using the 'sort' keyword. The sort direction can\n"
4234 "\t be modified by appending '.descending' or '.ascending' to a\n"
4235 "\t sort field. The 'size' parameter can be used to specify more\n"
4236 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4237 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4238 "\t its histogram data will be shared with other triggers of the\n"
4239 "\t same name, and trigger hits will update this common data.\n\n"
4240 "\t Reading the 'hist' file for the event will dump the hash\n"
4241 "\t table in its entirety to stdout. If there are multiple hist\n"
4242 "\t triggers attached to an event, there will be a table for each\n"
4243 "\t trigger in the output. The table displayed for a named\n"
4244 "\t trigger will be the same as any other instance having the\n"
4245 "\t same name. The default format used to display a given field\n"
4246 "\t can be modified by appending any of the following modifiers\n"
4247 "\t to the field name, as applicable:\n\n"
4248 "\t .hex display a number as a hex value\n"
4249 "\t .sym display an address as a symbol\n"
4250 "\t .sym-offset display an address as a symbol and offset\n"
4251 "\t .execname display a common_pid as a program name\n"
4252 "\t .syscall display a syscall id as a syscall name\n\n"
4253 "\t .log2 display log2 value rather than raw number\n\n"
4254 "\t The 'pause' parameter can be used to pause an existing hist\n"
4255 "\t trigger or to start a hist trigger but not log any events\n"
4256 "\t until told to do so. 'continue' can be used to start or\n"
4257 "\t restart a paused hist trigger.\n\n"
4258 "\t The 'clear' parameter will clear the contents of a running\n"
4259 "\t hist trigger and leave its current paused/active state\n"
4261 "\t The enable_hist and disable_hist triggers can be used to\n"
4262 "\t have one event conditionally start and stop another event's\n"
4263 "\t already-attached hist trigger. The syntax is analagous to\n"
4264 "\t the enable_event and disable_event triggers.\n"
4269 tracing_readme_read(struct file *filp, char __user *ubuf,
4270 size_t cnt, loff_t *ppos)
4272 return simple_read_from_buffer(ubuf, cnt, ppos,
4273 readme_msg, strlen(readme_msg));
4276 static const struct file_operations tracing_readme_fops = {
4277 .open = tracing_open_generic,
4278 .read = tracing_readme_read,
4279 .llseek = generic_file_llseek,
4282 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4284 unsigned int *ptr = v;
4286 if (*pos || m->count)
4291 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4293 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4302 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4308 arch_spin_lock(&trace_cmdline_lock);
4310 v = &savedcmd->map_cmdline_to_pid[0];
4312 v = saved_cmdlines_next(m, v, &l);
4320 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4322 arch_spin_unlock(&trace_cmdline_lock);
4326 static int saved_cmdlines_show(struct seq_file *m, void *v)
4328 char buf[TASK_COMM_LEN];
4329 unsigned int *pid = v;
4331 __trace_find_cmdline(*pid, buf);
4332 seq_printf(m, "%d %s\n", *pid, buf);
4336 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4337 .start = saved_cmdlines_start,
4338 .next = saved_cmdlines_next,
4339 .stop = saved_cmdlines_stop,
4340 .show = saved_cmdlines_show,
4343 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4345 if (tracing_disabled)
4348 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4351 static const struct file_operations tracing_saved_cmdlines_fops = {
4352 .open = tracing_saved_cmdlines_open,
4354 .llseek = seq_lseek,
4355 .release = seq_release,
4359 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4360 size_t cnt, loff_t *ppos)
4365 arch_spin_lock(&trace_cmdline_lock);
4366 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4367 arch_spin_unlock(&trace_cmdline_lock);
4369 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4372 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4374 kfree(s->saved_cmdlines);
4375 kfree(s->map_cmdline_to_pid);
4379 static int tracing_resize_saved_cmdlines(unsigned int val)
4381 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4383 s = kmalloc(sizeof(*s), GFP_KERNEL);
4387 if (allocate_cmdlines_buffer(val, s) < 0) {
4392 arch_spin_lock(&trace_cmdline_lock);
4393 savedcmd_temp = savedcmd;
4395 arch_spin_unlock(&trace_cmdline_lock);
4396 free_saved_cmdlines_buffer(savedcmd_temp);
4402 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4403 size_t cnt, loff_t *ppos)
4408 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4412 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4413 if (!val || val > PID_MAX_DEFAULT)
4416 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4425 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4426 .open = tracing_open_generic,
4427 .read = tracing_saved_cmdlines_size_read,
4428 .write = tracing_saved_cmdlines_size_write,
4431 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4432 static union trace_enum_map_item *
4433 update_enum_map(union trace_enum_map_item *ptr)
4435 if (!ptr->map.enum_string) {
4436 if (ptr->tail.next) {
4437 ptr = ptr->tail.next;
4438 /* Set ptr to the next real item (skip head) */
4446 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4448 union trace_enum_map_item *ptr = v;
4451 * Paranoid! If ptr points to end, we don't want to increment past it.
4452 * This really should never happen.
4454 ptr = update_enum_map(ptr);
4455 if (WARN_ON_ONCE(!ptr))
4462 ptr = update_enum_map(ptr);
4467 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4469 union trace_enum_map_item *v;
4472 mutex_lock(&trace_enum_mutex);
4474 v = trace_enum_maps;
4478 while (v && l < *pos) {
4479 v = enum_map_next(m, v, &l);
4485 static void enum_map_stop(struct seq_file *m, void *v)
4487 mutex_unlock(&trace_enum_mutex);
4490 static int enum_map_show(struct seq_file *m, void *v)
4492 union trace_enum_map_item *ptr = v;
4494 seq_printf(m, "%s %ld (%s)\n",
4495 ptr->map.enum_string, ptr->map.enum_value,
4501 static const struct seq_operations tracing_enum_map_seq_ops = {
4502 .start = enum_map_start,
4503 .next = enum_map_next,
4504 .stop = enum_map_stop,
4505 .show = enum_map_show,
4508 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4510 if (tracing_disabled)
4513 return seq_open(filp, &tracing_enum_map_seq_ops);
4516 static const struct file_operations tracing_enum_map_fops = {
4517 .open = tracing_enum_map_open,
4519 .llseek = seq_lseek,
4520 .release = seq_release,
4523 static inline union trace_enum_map_item *
4524 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4526 /* Return tail of array given the head */
4527 return ptr + ptr->head.length + 1;
4531 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4534 struct trace_enum_map **stop;
4535 struct trace_enum_map **map;
4536 union trace_enum_map_item *map_array;
4537 union trace_enum_map_item *ptr;
4542 * The trace_enum_maps contains the map plus a head and tail item,
4543 * where the head holds the module and length of array, and the
4544 * tail holds a pointer to the next list.
4546 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4548 pr_warn("Unable to allocate trace enum mapping\n");
4552 mutex_lock(&trace_enum_mutex);
4554 if (!trace_enum_maps)
4555 trace_enum_maps = map_array;
4557 ptr = trace_enum_maps;
4559 ptr = trace_enum_jmp_to_tail(ptr);
4560 if (!ptr->tail.next)
4562 ptr = ptr->tail.next;
4565 ptr->tail.next = map_array;
4567 map_array->head.mod = mod;
4568 map_array->head.length = len;
4571 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4572 map_array->map = **map;
4575 memset(map_array, 0, sizeof(*map_array));
4577 mutex_unlock(&trace_enum_mutex);
4580 static void trace_create_enum_file(struct dentry *d_tracer)
4582 trace_create_file("enum_map", 0444, d_tracer,
4583 NULL, &tracing_enum_map_fops);
4586 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4587 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4588 static inline void trace_insert_enum_map_file(struct module *mod,
4589 struct trace_enum_map **start, int len) { }
4590 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4592 static void trace_insert_enum_map(struct module *mod,
4593 struct trace_enum_map **start, int len)
4595 struct trace_enum_map **map;
4602 trace_event_enum_update(map, len);
4604 trace_insert_enum_map_file(mod, start, len);
4608 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4609 size_t cnt, loff_t *ppos)
4611 struct trace_array *tr = filp->private_data;
4612 char buf[MAX_TRACER_SIZE+2];
4615 mutex_lock(&trace_types_lock);
4616 r = sprintf(buf, "%s\n", tr->current_trace->name);
4617 mutex_unlock(&trace_types_lock);
4619 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4622 int tracer_init(struct tracer *t, struct trace_array *tr)
4624 tracing_reset_online_cpus(&tr->trace_buffer);
4628 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4632 for_each_tracing_cpu(cpu)
4633 per_cpu_ptr(buf->data, cpu)->entries = val;
4636 #ifdef CONFIG_TRACER_MAX_TRACE
4637 /* resize @tr's buffer to the size of @size_tr's entries */
4638 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4639 struct trace_buffer *size_buf, int cpu_id)
4643 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4644 for_each_tracing_cpu(cpu) {
4645 ret = ring_buffer_resize(trace_buf->buffer,
4646 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4649 per_cpu_ptr(trace_buf->data, cpu)->entries =
4650 per_cpu_ptr(size_buf->data, cpu)->entries;
4653 ret = ring_buffer_resize(trace_buf->buffer,
4654 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4656 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4657 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4662 #endif /* CONFIG_TRACER_MAX_TRACE */
4664 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4665 unsigned long size, int cpu)
4670 * If kernel or user changes the size of the ring buffer
4671 * we use the size that was given, and we can forget about
4672 * expanding it later.
4674 ring_buffer_expanded = true;
4676 /* May be called before buffers are initialized */
4677 if (!tr->trace_buffer.buffer)
4680 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4684 #ifdef CONFIG_TRACER_MAX_TRACE
4685 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4686 !tr->current_trace->use_max_tr)
4689 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4691 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4692 &tr->trace_buffer, cpu);
4695 * AARGH! We are left with different
4696 * size max buffer!!!!
4697 * The max buffer is our "snapshot" buffer.
4698 * When a tracer needs a snapshot (one of the
4699 * latency tracers), it swaps the max buffer
4700 * with the saved snap shot. We succeeded to
4701 * update the size of the main buffer, but failed to
4702 * update the size of the max buffer. But when we tried
4703 * to reset the main buffer to the original size, we
4704 * failed there too. This is very unlikely to
4705 * happen, but if it does, warn and kill all
4709 tracing_disabled = 1;
4714 if (cpu == RING_BUFFER_ALL_CPUS)
4715 set_buffer_entries(&tr->max_buffer, size);
4717 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4720 #endif /* CONFIG_TRACER_MAX_TRACE */
4722 if (cpu == RING_BUFFER_ALL_CPUS)
4723 set_buffer_entries(&tr->trace_buffer, size);
4725 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4730 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4731 unsigned long size, int cpu_id)
4735 mutex_lock(&trace_types_lock);
4737 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4738 /* make sure, this cpu is enabled in the mask */
4739 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4745 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4750 mutex_unlock(&trace_types_lock);
4757 * tracing_update_buffers - used by tracing facility to expand ring buffers
4759 * To save on memory when the tracing is never used on a system with it
4760 * configured in. The ring buffers are set to a minimum size. But once
4761 * a user starts to use the tracing facility, then they need to grow
4762 * to their default size.
4764 * This function is to be called when a tracer is about to be used.
4766 int tracing_update_buffers(void)
4770 mutex_lock(&trace_types_lock);
4771 if (!ring_buffer_expanded)
4772 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4773 RING_BUFFER_ALL_CPUS);
4774 mutex_unlock(&trace_types_lock);
4779 struct trace_option_dentry;
4782 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4785 * Used to clear out the tracer before deletion of an instance.
4786 * Must have trace_types_lock held.
4788 static void tracing_set_nop(struct trace_array *tr)
4790 if (tr->current_trace == &nop_trace)
4793 tr->current_trace->enabled--;
4795 if (tr->current_trace->reset)
4796 tr->current_trace->reset(tr);
4798 tr->current_trace = &nop_trace;
4801 static bool tracer_options_updated;
4803 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4805 /* Only enable if the directory has been created already. */
4809 /* Only create trace option files after update_tracer_options finish */
4810 if (!tracer_options_updated)
4813 create_trace_option_files(tr, t);
4816 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4819 #ifdef CONFIG_TRACER_MAX_TRACE
4824 mutex_lock(&trace_types_lock);
4826 if (!ring_buffer_expanded) {
4827 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4828 RING_BUFFER_ALL_CPUS);
4834 for (t = trace_types; t; t = t->next) {
4835 if (strcmp(t->name, buf) == 0)
4842 if (t == tr->current_trace)
4845 /* Some tracers are only allowed for the top level buffer */
4846 if (!trace_ok_for_array(t, tr)) {
4851 /* If trace pipe files are being read, we can't change the tracer */
4852 if (tr->current_trace->ref) {
4857 trace_branch_disable();
4859 tr->current_trace->enabled--;
4861 if (tr->current_trace->reset)
4862 tr->current_trace->reset(tr);
4864 /* Current trace needs to be nop_trace before synchronize_sched */
4865 tr->current_trace = &nop_trace;
4867 #ifdef CONFIG_TRACER_MAX_TRACE
4868 had_max_tr = tr->allocated_snapshot;
4870 if (had_max_tr && !t->use_max_tr) {
4872 * We need to make sure that the update_max_tr sees that
4873 * current_trace changed to nop_trace to keep it from
4874 * swapping the buffers after we resize it.
4875 * The update_max_tr is called from interrupts disabled
4876 * so a synchronized_sched() is sufficient.
4878 synchronize_sched();
4883 #ifdef CONFIG_TRACER_MAX_TRACE
4884 if (t->use_max_tr && !had_max_tr) {
4885 ret = alloc_snapshot(tr);
4892 ret = tracer_init(t, tr);
4897 tr->current_trace = t;
4898 tr->current_trace->enabled++;
4899 trace_branch_enable(tr);
4901 mutex_unlock(&trace_types_lock);
4907 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4908 size_t cnt, loff_t *ppos)
4910 struct trace_array *tr = filp->private_data;
4911 char buf[MAX_TRACER_SIZE+1];
4918 if (cnt > MAX_TRACER_SIZE)
4919 cnt = MAX_TRACER_SIZE;
4921 if (copy_from_user(buf, ubuf, cnt))
4926 /* strip ending whitespace. */
4927 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4930 err = tracing_set_tracer(tr, buf);
4940 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4941 size_t cnt, loff_t *ppos)
4946 r = snprintf(buf, sizeof(buf), "%ld\n",
4947 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4948 if (r > sizeof(buf))
4950 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4954 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4955 size_t cnt, loff_t *ppos)
4960 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4970 tracing_thresh_read(struct file *filp, char __user *ubuf,
4971 size_t cnt, loff_t *ppos)
4973 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4977 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4978 size_t cnt, loff_t *ppos)
4980 struct trace_array *tr = filp->private_data;
4983 mutex_lock(&trace_types_lock);
4984 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4988 if (tr->current_trace->update_thresh) {
4989 ret = tr->current_trace->update_thresh(tr);
4996 mutex_unlock(&trace_types_lock);
5001 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5004 tracing_max_lat_read(struct file *filp, char __user *ubuf,
5005 size_t cnt, loff_t *ppos)
5007 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5011 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5012 size_t cnt, loff_t *ppos)
5014 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5019 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5021 struct trace_array *tr = inode->i_private;
5022 struct trace_iterator *iter;
5025 if (tracing_disabled)
5028 if (trace_array_get(tr) < 0)
5031 mutex_lock(&trace_types_lock);
5033 /* create a buffer to store the information to pass to userspace */
5034 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5037 __trace_array_put(tr);
5041 trace_seq_init(&iter->seq);
5042 iter->trace = tr->current_trace;
5044 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5049 /* trace pipe does not show start of buffer */
5050 cpumask_setall(iter->started);
5052 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5053 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5055 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5056 if (trace_clocks[tr->clock_id].in_ns)
5057 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5060 iter->trace_buffer = &tr->trace_buffer;
5061 iter->cpu_file = tracing_get_cpu(inode);
5062 mutex_init(&iter->mutex);
5063 filp->private_data = iter;
5065 if (iter->trace->pipe_open)
5066 iter->trace->pipe_open(iter);
5068 nonseekable_open(inode, filp);
5070 tr->current_trace->ref++;
5072 mutex_unlock(&trace_types_lock);
5077 __trace_array_put(tr);
5078 mutex_unlock(&trace_types_lock);
5082 static int tracing_release_pipe(struct inode *inode, struct file *file)
5084 struct trace_iterator *iter = file->private_data;
5085 struct trace_array *tr = inode->i_private;
5087 mutex_lock(&trace_types_lock);
5089 tr->current_trace->ref--;
5091 if (iter->trace->pipe_close)
5092 iter->trace->pipe_close(iter);
5094 mutex_unlock(&trace_types_lock);
5096 free_cpumask_var(iter->started);
5097 mutex_destroy(&iter->mutex);
5100 trace_array_put(tr);
5106 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5108 struct trace_array *tr = iter->tr;
5110 /* Iterators are static, they should be filled or empty */
5111 if (trace_buffer_iter(iter, iter->cpu_file))
5112 return POLLIN | POLLRDNORM;
5114 if (tr->trace_flags & TRACE_ITER_BLOCK)
5116 * Always select as readable when in blocking mode
5118 return POLLIN | POLLRDNORM;
5120 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5125 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5127 struct trace_iterator *iter = filp->private_data;
5129 return trace_poll(iter, filp, poll_table);
5132 /* Must be called with iter->mutex held. */
5133 static int tracing_wait_pipe(struct file *filp)
5135 struct trace_iterator *iter = filp->private_data;
5138 while (trace_empty(iter)) {
5140 if ((filp->f_flags & O_NONBLOCK)) {
5145 * We block until we read something and tracing is disabled.
5146 * We still block if tracing is disabled, but we have never
5147 * read anything. This allows a user to cat this file, and
5148 * then enable tracing. But after we have read something,
5149 * we give an EOF when tracing is again disabled.
5151 * iter->pos will be 0 if we haven't read anything.
5153 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5156 mutex_unlock(&iter->mutex);
5158 ret = wait_on_pipe(iter, false);
5160 mutex_lock(&iter->mutex);
5173 tracing_read_pipe(struct file *filp, char __user *ubuf,
5174 size_t cnt, loff_t *ppos)
5176 struct trace_iterator *iter = filp->private_data;
5180 * Avoid more than one consumer on a single file descriptor
5181 * This is just a matter of traces coherency, the ring buffer itself
5184 mutex_lock(&iter->mutex);
5186 /* return any leftover data */
5187 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5191 trace_seq_init(&iter->seq);
5193 if (iter->trace->read) {
5194 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5200 sret = tracing_wait_pipe(filp);
5204 /* stop when tracing is finished */
5205 if (trace_empty(iter)) {
5210 if (cnt >= PAGE_SIZE)
5211 cnt = PAGE_SIZE - 1;
5213 /* reset all but tr, trace, and overruns */
5214 memset(&iter->seq, 0,
5215 sizeof(struct trace_iterator) -
5216 offsetof(struct trace_iterator, seq));
5217 cpumask_clear(iter->started);
5218 trace_seq_init(&iter->seq);
5221 trace_event_read_lock();
5222 trace_access_lock(iter->cpu_file);
5223 while (trace_find_next_entry_inc(iter) != NULL) {
5224 enum print_line_t ret;
5225 int save_len = iter->seq.seq.len;
5227 ret = print_trace_line(iter);
5228 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5230 * If one print_trace_line() fills entire trace_seq in one shot,
5231 * trace_seq_to_user() will returns -EBUSY because save_len == 0,
5232 * In this case, we need to consume it, otherwise, loop will peek
5233 * this event next time, resulting in an infinite loop.
5235 if (save_len == 0) {
5237 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");
5238 trace_consume(iter);
5242 /* In other cases, don't print partial lines */
5243 iter->seq.seq.len = save_len;
5246 if (ret != TRACE_TYPE_NO_CONSUME)
5247 trace_consume(iter);
5249 if (trace_seq_used(&iter->seq) >= cnt)
5253 * Setting the full flag means we reached the trace_seq buffer
5254 * size and we should leave by partial output condition above.
5255 * One of the trace_seq_* functions is not used properly.
5257 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5260 trace_access_unlock(iter->cpu_file);
5261 trace_event_read_unlock();
5263 /* Now copy what we have to the user */
5264 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5265 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
5266 trace_seq_init(&iter->seq);
5269 * If there was nothing to send to user, in spite of consuming trace
5270 * entries, go back to wait for more entries.
5276 mutex_unlock(&iter->mutex);
5281 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5284 __free_page(spd->pages[idx]);
5287 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5289 .confirm = generic_pipe_buf_confirm,
5290 .release = generic_pipe_buf_release,
5291 .steal = generic_pipe_buf_steal,
5292 .get = generic_pipe_buf_get,
5296 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5302 /* Seq buffer is page-sized, exactly what we need. */
5304 save_len = iter->seq.seq.len;
5305 ret = print_trace_line(iter);
5307 if (trace_seq_has_overflowed(&iter->seq)) {
5308 iter->seq.seq.len = save_len;
5313 * This should not be hit, because it should only
5314 * be set if the iter->seq overflowed. But check it
5315 * anyway to be safe.
5317 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5318 iter->seq.seq.len = save_len;
5322 count = trace_seq_used(&iter->seq) - save_len;
5325 iter->seq.seq.len = save_len;
5329 if (ret != TRACE_TYPE_NO_CONSUME)
5330 trace_consume(iter);
5332 if (!trace_find_next_entry_inc(iter)) {
5342 static ssize_t tracing_splice_read_pipe(struct file *filp,
5344 struct pipe_inode_info *pipe,
5348 struct page *pages_def[PIPE_DEF_BUFFERS];
5349 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5350 struct trace_iterator *iter = filp->private_data;
5351 struct splice_pipe_desc spd = {
5353 .partial = partial_def,
5354 .nr_pages = 0, /* This gets updated below. */
5355 .nr_pages_max = PIPE_DEF_BUFFERS,
5357 .ops = &tracing_pipe_buf_ops,
5358 .spd_release = tracing_spd_release_pipe,
5364 if (splice_grow_spd(pipe, &spd))
5367 mutex_lock(&iter->mutex);
5369 if (iter->trace->splice_read) {
5370 ret = iter->trace->splice_read(iter, filp,
5371 ppos, pipe, len, flags);
5376 ret = tracing_wait_pipe(filp);
5380 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5385 trace_event_read_lock();
5386 trace_access_lock(iter->cpu_file);
5388 /* Fill as many pages as possible. */
5389 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5390 spd.pages[i] = alloc_page(GFP_KERNEL);
5394 rem = tracing_fill_pipe_page(rem, iter);
5396 /* Copy the data into the page, so we can start over. */
5397 ret = trace_seq_to_buffer(&iter->seq,
5398 page_address(spd.pages[i]),
5399 trace_seq_used(&iter->seq));
5401 __free_page(spd.pages[i]);
5404 spd.partial[i].offset = 0;
5405 spd.partial[i].len = trace_seq_used(&iter->seq);
5407 trace_seq_init(&iter->seq);
5410 trace_access_unlock(iter->cpu_file);
5411 trace_event_read_unlock();
5412 mutex_unlock(&iter->mutex);
5417 ret = splice_to_pipe(pipe, &spd);
5421 splice_shrink_spd(&spd);
5425 mutex_unlock(&iter->mutex);
5430 tracing_entries_read(struct file *filp, char __user *ubuf,
5431 size_t cnt, loff_t *ppos)
5433 struct inode *inode = file_inode(filp);
5434 struct trace_array *tr = inode->i_private;
5435 int cpu = tracing_get_cpu(inode);
5440 mutex_lock(&trace_types_lock);
5442 if (cpu == RING_BUFFER_ALL_CPUS) {
5443 int cpu, buf_size_same;
5448 /* check if all cpu sizes are same */
5449 for_each_tracing_cpu(cpu) {
5450 /* fill in the size from first enabled cpu */
5452 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5453 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5459 if (buf_size_same) {
5460 if (!ring_buffer_expanded)
5461 r = sprintf(buf, "%lu (expanded: %lu)\n",
5463 trace_buf_size >> 10);
5465 r = sprintf(buf, "%lu\n", size >> 10);
5467 r = sprintf(buf, "X\n");
5469 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5471 mutex_unlock(&trace_types_lock);
5473 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5478 tracing_entries_write(struct file *filp, const char __user *ubuf,
5479 size_t cnt, loff_t *ppos)
5481 struct inode *inode = file_inode(filp);
5482 struct trace_array *tr = inode->i_private;
5486 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5490 /* must have at least 1 entry */
5494 /* value is in KB */
5496 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5506 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5507 size_t cnt, loff_t *ppos)
5509 struct trace_array *tr = filp->private_data;
5512 unsigned long size = 0, expanded_size = 0;
5514 mutex_lock(&trace_types_lock);
5515 for_each_tracing_cpu(cpu) {
5516 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5517 if (!ring_buffer_expanded)
5518 expanded_size += trace_buf_size >> 10;
5520 if (ring_buffer_expanded)
5521 r = sprintf(buf, "%lu\n", size);
5523 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5524 mutex_unlock(&trace_types_lock);
5526 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5530 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5531 size_t cnt, loff_t *ppos)
5534 * There is no need to read what the user has written, this function
5535 * is just to make sure that there is no error when "echo" is used
5544 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5546 struct trace_array *tr = inode->i_private;
5548 /* disable tracing ? */
5549 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5550 tracer_tracing_off(tr);
5551 /* resize the ring buffer to 0 */
5552 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5554 trace_array_put(tr);
5560 tracing_mark_write(struct file *filp, const char __user *ubuf,
5561 size_t cnt, loff_t *fpos)
5563 unsigned long addr = (unsigned long)ubuf;
5564 struct trace_array *tr = filp->private_data;
5565 struct ring_buffer_event *event;
5566 struct ring_buffer *buffer;
5567 struct print_entry *entry;
5568 unsigned long irq_flags;
5569 struct page *pages[2];
5579 if (tracing_disabled)
5582 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5585 if (cnt > TRACE_BUF_SIZE)
5586 cnt = TRACE_BUF_SIZE;
5589 * Userspace is injecting traces into the kernel trace buffer.
5590 * We want to be as non intrusive as possible.
5591 * To do so, we do not want to allocate any special buffers
5592 * or take any locks, but instead write the userspace data
5593 * straight into the ring buffer.
5595 * First we need to pin the userspace buffer into memory,
5596 * which, most likely it is, because it just referenced it.
5597 * But there's no guarantee that it is. By using get_user_pages_fast()
5598 * and kmap_atomic/kunmap_atomic() we can get access to the
5599 * pages directly. We then write the data directly into the
5602 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5604 /* check if we cross pages */
5605 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5608 offset = addr & (PAGE_SIZE - 1);
5611 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5612 if (ret < nr_pages) {
5614 put_page(pages[ret]);
5619 for (i = 0; i < nr_pages; i++)
5620 map_page[i] = kmap_atomic(pages[i]);
5622 local_save_flags(irq_flags);
5623 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5624 buffer = tr->trace_buffer.buffer;
5625 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5626 irq_flags, preempt_count());
5628 /* Ring buffer disabled, return as if not open for write */
5633 entry = ring_buffer_event_data(event);
5634 entry->ip = _THIS_IP_;
5636 if (nr_pages == 2) {
5637 len = PAGE_SIZE - offset;
5638 memcpy(&entry->buf, map_page[0] + offset, len);
5639 memcpy(&entry->buf[len], map_page[1], cnt - len);
5641 memcpy(&entry->buf, map_page[0] + offset, cnt);
5643 if (entry->buf[cnt - 1] != '\n') {
5644 entry->buf[cnt] = '\n';
5645 entry->buf[cnt + 1] = '\0';
5647 entry->buf[cnt] = '\0';
5649 __buffer_unlock_commit(buffer, event);
5656 for (i = nr_pages - 1; i >= 0; i--) {
5657 kunmap_atomic(map_page[i]);
5664 static int tracing_clock_show(struct seq_file *m, void *v)
5666 struct trace_array *tr = m->private;
5669 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5671 "%s%s%s%s", i ? " " : "",
5672 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5673 i == tr->clock_id ? "]" : "");
5679 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5683 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5684 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5687 if (i == ARRAY_SIZE(trace_clocks))
5690 mutex_lock(&trace_types_lock);
5694 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5697 * New clock may not be consistent with the previous clock.
5698 * Reset the buffer so that it doesn't have incomparable timestamps.
5700 tracing_reset_online_cpus(&tr->trace_buffer);
5702 #ifdef CONFIG_TRACER_MAX_TRACE
5703 if (tr->max_buffer.buffer)
5704 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5705 tracing_reset_online_cpus(&tr->max_buffer);
5708 mutex_unlock(&trace_types_lock);
5713 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5714 size_t cnt, loff_t *fpos)
5716 struct seq_file *m = filp->private_data;
5717 struct trace_array *tr = m->private;
5719 const char *clockstr;
5722 if (cnt >= sizeof(buf))
5725 if (copy_from_user(buf, ubuf, cnt))
5730 clockstr = strstrip(buf);
5732 ret = tracing_set_clock(tr, clockstr);
5741 static int tracing_clock_open(struct inode *inode, struct file *file)
5743 struct trace_array *tr = inode->i_private;
5746 if (tracing_disabled)
5749 if (trace_array_get(tr))
5752 ret = single_open(file, tracing_clock_show, inode->i_private);
5754 trace_array_put(tr);
5759 struct ftrace_buffer_info {
5760 struct trace_iterator iter;
5765 #ifdef CONFIG_TRACER_SNAPSHOT
5766 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5768 struct trace_array *tr = inode->i_private;
5769 struct trace_iterator *iter;
5773 if (trace_array_get(tr) < 0)
5776 if (file->f_mode & FMODE_READ) {
5777 iter = __tracing_open(inode, file, true);
5779 ret = PTR_ERR(iter);
5781 /* Writes still need the seq_file to hold the private data */
5783 m = kzalloc(sizeof(*m), GFP_KERNEL);
5786 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5794 iter->trace_buffer = &tr->max_buffer;
5795 iter->cpu_file = tracing_get_cpu(inode);
5797 file->private_data = m;
5801 trace_array_put(tr);
5807 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5810 struct seq_file *m = filp->private_data;
5811 struct trace_iterator *iter = m->private;
5812 struct trace_array *tr = iter->tr;
5816 ret = tracing_update_buffers();
5820 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5824 mutex_lock(&trace_types_lock);
5826 if (tr->current_trace->use_max_tr) {
5833 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5837 if (tr->allocated_snapshot)
5841 /* Only allow per-cpu swap if the ring buffer supports it */
5842 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5843 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5848 if (!tr->allocated_snapshot)
5849 ret = resize_buffer_duplicate_size(&tr->max_buffer,
5850 &tr->trace_buffer, iter->cpu_file);
5852 ret = alloc_snapshot(tr);
5857 local_irq_disable();
5858 /* Now, we're going to swap */
5859 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5860 update_max_tr(tr, current, smp_processor_id());
5862 update_max_tr_single(tr, current, iter->cpu_file);
5866 if (tr->allocated_snapshot) {
5867 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5868 tracing_reset_online_cpus(&tr->max_buffer);
5870 tracing_reset(&tr->max_buffer, iter->cpu_file);
5880 mutex_unlock(&trace_types_lock);
5884 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5886 struct seq_file *m = file->private_data;
5889 ret = tracing_release(inode, file);
5891 if (file->f_mode & FMODE_READ)
5894 /* If write only, the seq_file is just a stub */
5902 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5903 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5904 size_t count, loff_t *ppos);
5905 static int tracing_buffers_release(struct inode *inode, struct file *file);
5906 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5907 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5909 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5911 struct ftrace_buffer_info *info;
5914 ret = tracing_buffers_open(inode, filp);
5918 info = filp->private_data;
5920 if (info->iter.trace->use_max_tr) {
5921 tracing_buffers_release(inode, filp);
5925 info->iter.snapshot = true;
5926 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5931 #endif /* CONFIG_TRACER_SNAPSHOT */
5934 static const struct file_operations tracing_thresh_fops = {
5935 .open = tracing_open_generic,
5936 .read = tracing_thresh_read,
5937 .write = tracing_thresh_write,
5938 .llseek = generic_file_llseek,
5941 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5942 static const struct file_operations tracing_max_lat_fops = {
5943 .open = tracing_open_generic,
5944 .read = tracing_max_lat_read,
5945 .write = tracing_max_lat_write,
5946 .llseek = generic_file_llseek,
5950 static const struct file_operations set_tracer_fops = {
5951 .open = tracing_open_generic,
5952 .read = tracing_set_trace_read,
5953 .write = tracing_set_trace_write,
5954 .llseek = generic_file_llseek,
5957 static const struct file_operations tracing_pipe_fops = {
5958 .open = tracing_open_pipe,
5959 .poll = tracing_poll_pipe,
5960 .read = tracing_read_pipe,
5961 .splice_read = tracing_splice_read_pipe,
5962 .release = tracing_release_pipe,
5963 .llseek = no_llseek,
5966 static const struct file_operations tracing_entries_fops = {
5967 .open = tracing_open_generic_tr,
5968 .read = tracing_entries_read,
5969 .write = tracing_entries_write,
5970 .llseek = generic_file_llseek,
5971 .release = tracing_release_generic_tr,
5974 static const struct file_operations tracing_total_entries_fops = {
5975 .open = tracing_open_generic_tr,
5976 .read = tracing_total_entries_read,
5977 .llseek = generic_file_llseek,
5978 .release = tracing_release_generic_tr,
5981 static const struct file_operations tracing_free_buffer_fops = {
5982 .open = tracing_open_generic_tr,
5983 .write = tracing_free_buffer_write,
5984 .release = tracing_free_buffer_release,
5987 static const struct file_operations tracing_mark_fops = {
5988 .open = tracing_open_generic_tr,
5989 .write = tracing_mark_write,
5990 .llseek = generic_file_llseek,
5991 .release = tracing_release_generic_tr,
5994 static const struct file_operations trace_clock_fops = {
5995 .open = tracing_clock_open,
5997 .llseek = seq_lseek,
5998 .release = tracing_single_release_tr,
5999 .write = tracing_clock_write,
6002 #ifdef CONFIG_TRACER_SNAPSHOT
6003 static const struct file_operations snapshot_fops = {
6004 .open = tracing_snapshot_open,
6006 .write = tracing_snapshot_write,
6007 .llseek = tracing_lseek,
6008 .release = tracing_snapshot_release,
6011 static const struct file_operations snapshot_raw_fops = {
6012 .open = snapshot_raw_open,
6013 .read = tracing_buffers_read,
6014 .release = tracing_buffers_release,
6015 .splice_read = tracing_buffers_splice_read,
6016 .llseek = no_llseek,
6019 #endif /* CONFIG_TRACER_SNAPSHOT */
6021 static int tracing_buffers_open(struct inode *inode, struct file *filp)
6023 struct trace_array *tr = inode->i_private;
6024 struct ftrace_buffer_info *info;
6027 if (tracing_disabled)
6030 if (trace_array_get(tr) < 0)
6033 info = kzalloc(sizeof(*info), GFP_KERNEL);
6035 trace_array_put(tr);
6039 mutex_lock(&trace_types_lock);
6042 info->iter.cpu_file = tracing_get_cpu(inode);
6043 info->iter.trace = tr->current_trace;
6044 info->iter.trace_buffer = &tr->trace_buffer;
6046 /* Force reading ring buffer for first read */
6047 info->read = (unsigned int)-1;
6049 filp->private_data = info;
6051 tr->current_trace->ref++;
6053 mutex_unlock(&trace_types_lock);
6055 ret = nonseekable_open(inode, filp);
6057 trace_array_put(tr);
6063 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6065 struct ftrace_buffer_info *info = filp->private_data;
6066 struct trace_iterator *iter = &info->iter;
6068 return trace_poll(iter, filp, poll_table);
6072 tracing_buffers_read(struct file *filp, char __user *ubuf,
6073 size_t count, loff_t *ppos)
6075 struct ftrace_buffer_info *info = filp->private_data;
6076 struct trace_iterator *iter = &info->iter;
6083 #ifdef CONFIG_TRACER_MAX_TRACE
6084 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6089 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6094 /* Do we have previous read data to read? */
6095 if (info->read < PAGE_SIZE)
6099 trace_access_lock(iter->cpu_file);
6100 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6104 trace_access_unlock(iter->cpu_file);
6107 if (trace_empty(iter)) {
6108 if ((filp->f_flags & O_NONBLOCK))
6111 ret = wait_on_pipe(iter, false);
6122 size = PAGE_SIZE - info->read;
6126 ret = copy_to_user(ubuf, info->spare + info->read, size);
6138 static int tracing_buffers_release(struct inode *inode, struct file *file)
6140 struct ftrace_buffer_info *info = file->private_data;
6141 struct trace_iterator *iter = &info->iter;
6143 mutex_lock(&trace_types_lock);
6145 iter->tr->current_trace->ref--;
6147 __trace_array_put(iter->tr);
6150 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
6153 mutex_unlock(&trace_types_lock);
6159 struct ring_buffer *buffer;
6164 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6165 struct pipe_buffer *buf)
6167 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6172 ring_buffer_free_read_page(ref->buffer, ref->page);
6177 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6178 struct pipe_buffer *buf)
6180 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6182 if (ref->ref > INT_MAX/2)
6189 /* Pipe buffer operations for a buffer. */
6190 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6192 .confirm = generic_pipe_buf_confirm,
6193 .release = buffer_pipe_buf_release,
6194 .steal = generic_pipe_buf_steal,
6195 .get = buffer_pipe_buf_get,
6199 * Callback from splice_to_pipe(), if we need to release some pages
6200 * at the end of the spd in case we error'ed out in filling the pipe.
6202 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6204 struct buffer_ref *ref =
6205 (struct buffer_ref *)spd->partial[i].private;
6210 ring_buffer_free_read_page(ref->buffer, ref->page);
6212 spd->partial[i].private = 0;
6216 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6217 struct pipe_inode_info *pipe, size_t len,
6220 struct ftrace_buffer_info *info = file->private_data;
6221 struct trace_iterator *iter = &info->iter;
6222 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6223 struct page *pages_def[PIPE_DEF_BUFFERS];
6224 struct splice_pipe_desc spd = {
6226 .partial = partial_def,
6227 .nr_pages_max = PIPE_DEF_BUFFERS,
6229 .ops = &buffer_pipe_buf_ops,
6230 .spd_release = buffer_spd_release,
6232 struct buffer_ref *ref;
6236 #ifdef CONFIG_TRACER_MAX_TRACE
6237 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6241 if (*ppos & (PAGE_SIZE - 1))
6244 if (len & (PAGE_SIZE - 1)) {
6245 if (len < PAGE_SIZE)
6250 if (splice_grow_spd(pipe, &spd))
6254 trace_access_lock(iter->cpu_file);
6255 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6257 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6261 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
6268 ref->buffer = iter->trace_buffer->buffer;
6269 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6276 r = ring_buffer_read_page(ref->buffer, &ref->page,
6277 len, iter->cpu_file, 1);
6279 ring_buffer_free_read_page(ref->buffer, ref->page);
6284 page = virt_to_page(ref->page);
6286 spd.pages[i] = page;
6287 spd.partial[i].len = PAGE_SIZE;
6288 spd.partial[i].offset = 0;
6289 spd.partial[i].private = (unsigned long)ref;
6293 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6296 trace_access_unlock(iter->cpu_file);
6299 /* did we read anything? */
6300 if (!spd.nr_pages) {
6305 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6308 ret = wait_on_pipe(iter, true);
6315 ret = splice_to_pipe(pipe, &spd);
6317 splice_shrink_spd(&spd);
6322 static const struct file_operations tracing_buffers_fops = {
6323 .open = tracing_buffers_open,
6324 .read = tracing_buffers_read,
6325 .poll = tracing_buffers_poll,
6326 .release = tracing_buffers_release,
6327 .splice_read = tracing_buffers_splice_read,
6328 .llseek = no_llseek,
6332 tracing_stats_read(struct file *filp, char __user *ubuf,
6333 size_t count, loff_t *ppos)
6335 struct inode *inode = file_inode(filp);
6336 struct trace_array *tr = inode->i_private;
6337 struct trace_buffer *trace_buf = &tr->trace_buffer;
6338 int cpu = tracing_get_cpu(inode);
6339 struct trace_seq *s;
6341 unsigned long long t;
6342 unsigned long usec_rem;
6344 s = kmalloc(sizeof(*s), GFP_KERNEL);
6350 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6351 trace_seq_printf(s, "entries: %ld\n", cnt);
6353 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6354 trace_seq_printf(s, "overrun: %ld\n", cnt);
6356 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6357 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6359 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6360 trace_seq_printf(s, "bytes: %ld\n", cnt);
6362 if (trace_clocks[tr->clock_id].in_ns) {
6363 /* local or global for trace_clock */
6364 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6365 usec_rem = do_div(t, USEC_PER_SEC);
6366 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6369 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6370 usec_rem = do_div(t, USEC_PER_SEC);
6371 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6373 /* counter or tsc mode for trace_clock */
6374 trace_seq_printf(s, "oldest event ts: %llu\n",
6375 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6377 trace_seq_printf(s, "now ts: %llu\n",
6378 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6381 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6382 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6384 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6385 trace_seq_printf(s, "read events: %ld\n", cnt);
6387 count = simple_read_from_buffer(ubuf, count, ppos,
6388 s->buffer, trace_seq_used(s));
6395 static const struct file_operations tracing_stats_fops = {
6396 .open = tracing_open_generic_tr,
6397 .read = tracing_stats_read,
6398 .llseek = generic_file_llseek,
6399 .release = tracing_release_generic_tr,
6402 #ifdef CONFIG_DYNAMIC_FTRACE
6404 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6410 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6411 size_t cnt, loff_t *ppos)
6413 static char ftrace_dyn_info_buffer[1024];
6414 static DEFINE_MUTEX(dyn_info_mutex);
6415 unsigned long *p = filp->private_data;
6416 char *buf = ftrace_dyn_info_buffer;
6417 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
6420 mutex_lock(&dyn_info_mutex);
6421 r = sprintf(buf, "%ld ", *p);
6423 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
6426 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6428 mutex_unlock(&dyn_info_mutex);
6433 static const struct file_operations tracing_dyn_info_fops = {
6434 .open = tracing_open_generic,
6435 .read = tracing_read_dyn_info,
6436 .llseek = generic_file_llseek,
6438 #endif /* CONFIG_DYNAMIC_FTRACE */
6440 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6442 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6448 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6450 unsigned long *count = (long *)data;
6462 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6463 struct ftrace_probe_ops *ops, void *data)
6465 long count = (long)data;
6467 seq_printf(m, "%ps:", (void *)ip);
6469 seq_puts(m, "snapshot");
6472 seq_puts(m, ":unlimited\n");
6474 seq_printf(m, ":count=%ld\n", count);
6479 static struct ftrace_probe_ops snapshot_probe_ops = {
6480 .func = ftrace_snapshot,
6481 .print = ftrace_snapshot_print,
6484 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6485 .func = ftrace_count_snapshot,
6486 .print = ftrace_snapshot_print,
6490 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6491 char *glob, char *cmd, char *param, int enable)
6493 struct ftrace_probe_ops *ops;
6494 void *count = (void *)-1;
6498 /* hash funcs only work with set_ftrace_filter */
6502 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6504 if (glob[0] == '!') {
6505 unregister_ftrace_function_probe_func(glob+1, ops);
6512 number = strsep(¶m, ":");
6514 if (!strlen(number))
6518 * We use the callback data field (which is a pointer)
6521 ret = kstrtoul(number, 0, (unsigned long *)&count);
6526 ret = alloc_snapshot(&global_trace);
6530 ret = register_ftrace_function_probe(glob, ops, count);
6533 return ret < 0 ? ret : 0;
6536 static struct ftrace_func_command ftrace_snapshot_cmd = {
6538 .func = ftrace_trace_snapshot_callback,
6541 static __init int register_snapshot_cmd(void)
6543 return register_ftrace_command(&ftrace_snapshot_cmd);
6546 static inline __init int register_snapshot_cmd(void) { return 0; }
6547 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6549 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6551 if (WARN_ON(!tr->dir))
6552 return ERR_PTR(-ENODEV);
6554 /* Top directory uses NULL as the parent */
6555 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6558 /* All sub buffers have a descriptor */
6562 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6564 struct dentry *d_tracer;
6567 return tr->percpu_dir;
6569 d_tracer = tracing_get_dentry(tr);
6570 if (IS_ERR(d_tracer))
6573 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6575 WARN_ONCE(!tr->percpu_dir,
6576 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6578 return tr->percpu_dir;
6581 static struct dentry *
6582 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6583 void *data, long cpu, const struct file_operations *fops)
6585 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6587 if (ret) /* See tracing_get_cpu() */
6588 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6593 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6595 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6596 struct dentry *d_cpu;
6597 char cpu_dir[30]; /* 30 characters should be more than enough */
6602 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6603 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6605 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6609 /* per cpu trace_pipe */
6610 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6611 tr, cpu, &tracing_pipe_fops);
6614 trace_create_cpu_file("trace", 0644, d_cpu,
6615 tr, cpu, &tracing_fops);
6617 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6618 tr, cpu, &tracing_buffers_fops);
6620 trace_create_cpu_file("stats", 0444, d_cpu,
6621 tr, cpu, &tracing_stats_fops);
6623 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6624 tr, cpu, &tracing_entries_fops);
6626 #ifdef CONFIG_TRACER_SNAPSHOT
6627 trace_create_cpu_file("snapshot", 0644, d_cpu,
6628 tr, cpu, &snapshot_fops);
6630 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6631 tr, cpu, &snapshot_raw_fops);
6635 #ifdef CONFIG_FTRACE_SELFTEST
6636 /* Let selftest have access to static functions in this file */
6637 #include "trace_selftest.c"
6641 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6644 struct trace_option_dentry *topt = filp->private_data;
6647 if (topt->flags->val & topt->opt->bit)
6652 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6656 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6659 struct trace_option_dentry *topt = filp->private_data;
6663 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6667 if (val != 0 && val != 1)
6670 if (!!(topt->flags->val & topt->opt->bit) != val) {
6671 mutex_lock(&trace_types_lock);
6672 ret = __set_tracer_option(topt->tr, topt->flags,
6674 mutex_unlock(&trace_types_lock);
6685 static const struct file_operations trace_options_fops = {
6686 .open = tracing_open_generic,
6687 .read = trace_options_read,
6688 .write = trace_options_write,
6689 .llseek = generic_file_llseek,
6693 * In order to pass in both the trace_array descriptor as well as the index
6694 * to the flag that the trace option file represents, the trace_array
6695 * has a character array of trace_flags_index[], which holds the index
6696 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6697 * The address of this character array is passed to the flag option file
6698 * read/write callbacks.
6700 * In order to extract both the index and the trace_array descriptor,
6701 * get_tr_index() uses the following algorithm.
6705 * As the pointer itself contains the address of the index (remember
6708 * Then to get the trace_array descriptor, by subtracting that index
6709 * from the ptr, we get to the start of the index itself.
6711 * ptr - idx == &index[0]
6713 * Then a simple container_of() from that pointer gets us to the
6714 * trace_array descriptor.
6716 static void get_tr_index(void *data, struct trace_array **ptr,
6717 unsigned int *pindex)
6719 *pindex = *(unsigned char *)data;
6721 *ptr = container_of(data - *pindex, struct trace_array,
6726 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6729 void *tr_index = filp->private_data;
6730 struct trace_array *tr;
6734 get_tr_index(tr_index, &tr, &index);
6736 if (tr->trace_flags & (1 << index))
6741 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6745 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6748 void *tr_index = filp->private_data;
6749 struct trace_array *tr;
6754 get_tr_index(tr_index, &tr, &index);
6756 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6760 if (val != 0 && val != 1)
6763 mutex_lock(&trace_types_lock);
6764 ret = set_tracer_flag(tr, 1 << index, val);
6765 mutex_unlock(&trace_types_lock);
6775 static const struct file_operations trace_options_core_fops = {
6776 .open = tracing_open_generic,
6777 .read = trace_options_core_read,
6778 .write = trace_options_core_write,
6779 .llseek = generic_file_llseek,
6782 struct dentry *trace_create_file(const char *name,
6784 struct dentry *parent,
6786 const struct file_operations *fops)
6790 ret = tracefs_create_file(name, mode, parent, data, fops);
6792 pr_warn("Could not create tracefs '%s' entry\n", name);
6798 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6800 struct dentry *d_tracer;
6805 d_tracer = tracing_get_dentry(tr);
6806 if (IS_ERR(d_tracer))
6809 tr->options = tracefs_create_dir("options", d_tracer);
6811 pr_warn("Could not create tracefs directory 'options'\n");
6819 create_trace_option_file(struct trace_array *tr,
6820 struct trace_option_dentry *topt,
6821 struct tracer_flags *flags,
6822 struct tracer_opt *opt)
6824 struct dentry *t_options;
6826 t_options = trace_options_init_dentry(tr);
6830 topt->flags = flags;
6834 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6835 &trace_options_fops);
6840 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6842 struct trace_option_dentry *topts;
6843 struct trace_options *tr_topts;
6844 struct tracer_flags *flags;
6845 struct tracer_opt *opts;
6852 flags = tracer->flags;
6854 if (!flags || !flags->opts)
6858 * If this is an instance, only create flags for tracers
6859 * the instance may have.
6861 if (!trace_ok_for_array(tracer, tr))
6864 for (i = 0; i < tr->nr_topts; i++) {
6865 /* Make sure there's no duplicate flags. */
6866 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6872 for (cnt = 0; opts[cnt].name; cnt++)
6875 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6879 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6886 tr->topts = tr_topts;
6887 tr->topts[tr->nr_topts].tracer = tracer;
6888 tr->topts[tr->nr_topts].topts = topts;
6891 for (cnt = 0; opts[cnt].name; cnt++) {
6892 create_trace_option_file(tr, &topts[cnt], flags,
6894 WARN_ONCE(topts[cnt].entry == NULL,
6895 "Failed to create trace option: %s",
6900 static struct dentry *
6901 create_trace_option_core_file(struct trace_array *tr,
6902 const char *option, long index)
6904 struct dentry *t_options;
6906 t_options = trace_options_init_dentry(tr);
6910 return trace_create_file(option, 0644, t_options,
6911 (void *)&tr->trace_flags_index[index],
6912 &trace_options_core_fops);
6915 static void create_trace_options_dir(struct trace_array *tr)
6917 struct dentry *t_options;
6918 bool top_level = tr == &global_trace;
6921 t_options = trace_options_init_dentry(tr);
6925 for (i = 0; trace_options[i]; i++) {
6927 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6928 create_trace_option_core_file(tr, trace_options[i], i);
6933 rb_simple_read(struct file *filp, char __user *ubuf,
6934 size_t cnt, loff_t *ppos)
6936 struct trace_array *tr = filp->private_data;
6940 r = tracer_tracing_is_on(tr);
6941 r = sprintf(buf, "%d\n", r);
6943 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6947 rb_simple_write(struct file *filp, const char __user *ubuf,
6948 size_t cnt, loff_t *ppos)
6950 struct trace_array *tr = filp->private_data;
6951 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6955 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6960 mutex_lock(&trace_types_lock);
6961 if (!!val == tracer_tracing_is_on(tr)) {
6962 val = 0; /* do nothing */
6964 tracer_tracing_on(tr);
6965 if (tr->current_trace->start)
6966 tr->current_trace->start(tr);
6968 tracer_tracing_off(tr);
6969 if (tr->current_trace->stop)
6970 tr->current_trace->stop(tr);
6972 mutex_unlock(&trace_types_lock);
6980 static const struct file_operations rb_simple_fops = {
6981 .open = tracing_open_generic_tr,
6982 .read = rb_simple_read,
6983 .write = rb_simple_write,
6984 .release = tracing_release_generic_tr,
6985 .llseek = default_llseek,
6988 struct dentry *trace_instance_dir;
6991 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6994 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6996 enum ring_buffer_flags rb_flags;
6998 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
7002 buf->buffer = ring_buffer_alloc(size, rb_flags);
7006 buf->data = alloc_percpu(struct trace_array_cpu);
7008 ring_buffer_free(buf->buffer);
7013 /* Allocate the first page for all buffers */
7014 set_buffer_entries(&tr->trace_buffer,
7015 ring_buffer_size(tr->trace_buffer.buffer, 0));
7020 static int allocate_trace_buffers(struct trace_array *tr, int size)
7024 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7028 #ifdef CONFIG_TRACER_MAX_TRACE
7029 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7030 allocate_snapshot ? size : 1);
7032 ring_buffer_free(tr->trace_buffer.buffer);
7033 tr->trace_buffer.buffer = NULL;
7034 free_percpu(tr->trace_buffer.data);
7035 tr->trace_buffer.data = NULL;
7038 tr->allocated_snapshot = allocate_snapshot;
7041 * Only the top level trace array gets its snapshot allocated
7042 * from the kernel command line.
7044 allocate_snapshot = false;
7048 * Because of some magic with the way alloc_percpu() works on
7049 * x86_64, we need to synchronize the pgd of all the tables,
7050 * otherwise the trace events that happen in x86_64 page fault
7051 * handlers can't cope with accessing the chance that a
7052 * alloc_percpu()'d memory might be touched in the page fault trace
7053 * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
7054 * calls in tracing, because something might get triggered within a
7055 * page fault trace event!
7057 vmalloc_sync_mappings();
7062 static void free_trace_buffer(struct trace_buffer *buf)
7065 ring_buffer_free(buf->buffer);
7067 free_percpu(buf->data);
7072 static void free_trace_buffers(struct trace_array *tr)
7077 free_trace_buffer(&tr->trace_buffer);
7079 #ifdef CONFIG_TRACER_MAX_TRACE
7080 free_trace_buffer(&tr->max_buffer);
7084 static void init_trace_flags_index(struct trace_array *tr)
7088 /* Used by the trace options files */
7089 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7090 tr->trace_flags_index[i] = i;
7093 static void __update_tracer_options(struct trace_array *tr)
7097 for (t = trace_types; t; t = t->next)
7098 add_tracer_options(tr, t);
7101 static void update_tracer_options(struct trace_array *tr)
7103 mutex_lock(&trace_types_lock);
7104 tracer_options_updated = true;
7105 __update_tracer_options(tr);
7106 mutex_unlock(&trace_types_lock);
7109 static int instance_mkdir(const char *name)
7111 struct trace_array *tr;
7114 mutex_lock(&trace_types_lock);
7117 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7118 if (tr->name && strcmp(tr->name, name) == 0)
7123 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7127 tr->name = kstrdup(name, GFP_KERNEL);
7131 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7134 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
7136 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7138 raw_spin_lock_init(&tr->start_lock);
7140 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7142 tr->current_trace = &nop_trace;
7144 INIT_LIST_HEAD(&tr->systems);
7145 INIT_LIST_HEAD(&tr->events);
7147 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7150 tr->dir = tracefs_create_dir(name, trace_instance_dir);
7154 ret = event_trace_add_tracer(tr->dir, tr);
7156 tracefs_remove_recursive(tr->dir);
7160 init_tracer_tracefs(tr, tr->dir);
7161 init_trace_flags_index(tr);
7162 __update_tracer_options(tr);
7164 list_add(&tr->list, &ftrace_trace_arrays);
7166 mutex_unlock(&trace_types_lock);
7171 free_trace_buffers(tr);
7172 free_cpumask_var(tr->tracing_cpumask);
7177 mutex_unlock(&trace_types_lock);
7183 static int instance_rmdir(const char *name)
7185 struct trace_array *tr;
7190 mutex_lock(&trace_types_lock);
7193 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7194 if (tr->name && strcmp(tr->name, name) == 0) {
7203 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
7206 list_del(&tr->list);
7208 /* Disable all the flags that were enabled coming in */
7209 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7210 if ((1 << i) & ZEROED_TRACE_FLAGS)
7211 set_tracer_flag(tr, 1 << i, 0);
7214 tracing_set_nop(tr);
7215 event_trace_del_tracer(tr);
7216 ftrace_clear_pids(tr);
7217 ftrace_destroy_function_files(tr);
7218 tracefs_remove_recursive(tr->dir);
7219 free_trace_buffers(tr);
7221 for (i = 0; i < tr->nr_topts; i++) {
7222 kfree(tr->topts[i].topts);
7226 free_cpumask_var(tr->tracing_cpumask);
7233 mutex_unlock(&trace_types_lock);
7238 static __init void create_trace_instances(struct dentry *d_tracer)
7240 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7243 if (WARN_ON(!trace_instance_dir))
7248 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7252 trace_create_file("available_tracers", 0444, d_tracer,
7253 tr, &show_traces_fops);
7255 trace_create_file("current_tracer", 0644, d_tracer,
7256 tr, &set_tracer_fops);
7258 trace_create_file("tracing_cpumask", 0644, d_tracer,
7259 tr, &tracing_cpumask_fops);
7261 trace_create_file("trace_options", 0644, d_tracer,
7262 tr, &tracing_iter_fops);
7264 trace_create_file("trace", 0644, d_tracer,
7267 trace_create_file("trace_pipe", 0444, d_tracer,
7268 tr, &tracing_pipe_fops);
7270 trace_create_file("buffer_size_kb", 0644, d_tracer,
7271 tr, &tracing_entries_fops);
7273 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7274 tr, &tracing_total_entries_fops);
7276 trace_create_file("free_buffer", 0200, d_tracer,
7277 tr, &tracing_free_buffer_fops);
7279 trace_create_file("trace_marker", 0220, d_tracer,
7280 tr, &tracing_mark_fops);
7282 trace_create_file("trace_clock", 0644, d_tracer, tr,
7285 trace_create_file("tracing_on", 0644, d_tracer,
7286 tr, &rb_simple_fops);
7288 create_trace_options_dir(tr);
7290 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7291 trace_create_file("tracing_max_latency", 0644, d_tracer,
7292 &tr->max_latency, &tracing_max_lat_fops);
7295 if (ftrace_create_function_files(tr, d_tracer))
7296 WARN(1, "Could not allocate function filter files");
7298 #ifdef CONFIG_TRACER_SNAPSHOT
7299 trace_create_file("snapshot", 0644, d_tracer,
7300 tr, &snapshot_fops);
7303 for_each_tracing_cpu(cpu)
7304 tracing_init_tracefs_percpu(tr, cpu);
7306 ftrace_init_tracefs(tr, d_tracer);
7309 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
7311 struct vfsmount *mnt;
7312 struct file_system_type *type;
7315 * To maintain backward compatibility for tools that mount
7316 * debugfs to get to the tracing facility, tracefs is automatically
7317 * mounted to the debugfs/tracing directory.
7319 type = get_fs_type("tracefs");
7322 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
7323 put_filesystem(type);
7332 * tracing_init_dentry - initialize top level trace array
7334 * This is called when creating files or directories in the tracing
7335 * directory. It is called via fs_initcall() by any of the boot up code
7336 * and expects to return the dentry of the top level tracing directory.
7338 struct dentry *tracing_init_dentry(void)
7340 struct trace_array *tr = &global_trace;
7342 /* The top level trace array uses NULL as parent */
7346 if (WARN_ON(!tracefs_initialized()) ||
7347 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7348 WARN_ON(!debugfs_initialized())))
7349 return ERR_PTR(-ENODEV);
7352 * As there may still be users that expect the tracing
7353 * files to exist in debugfs/tracing, we must automount
7354 * the tracefs file system there, so older tools still
7355 * work with the newer kerenl.
7357 tr->dir = debugfs_create_automount("tracing", NULL,
7358 trace_automount, NULL);
7360 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7361 return ERR_PTR(-ENOMEM);
7367 extern struct trace_enum_map *__start_ftrace_enum_maps[];
7368 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7370 static void __init trace_enum_init(void)
7374 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
7375 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
7378 #ifdef CONFIG_MODULES
7379 static void trace_module_add_enums(struct module *mod)
7381 if (!mod->num_trace_enums)
7385 * Modules with bad taint do not have events created, do
7386 * not bother with enums either.
7388 if (trace_module_has_bad_taint(mod))
7391 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
7394 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7395 static void trace_module_remove_enums(struct module *mod)
7397 union trace_enum_map_item *map;
7398 union trace_enum_map_item **last = &trace_enum_maps;
7400 if (!mod->num_trace_enums)
7403 mutex_lock(&trace_enum_mutex);
7405 map = trace_enum_maps;
7408 if (map->head.mod == mod)
7410 map = trace_enum_jmp_to_tail(map);
7411 last = &map->tail.next;
7412 map = map->tail.next;
7417 *last = trace_enum_jmp_to_tail(map)->tail.next;
7420 mutex_unlock(&trace_enum_mutex);
7423 static inline void trace_module_remove_enums(struct module *mod) { }
7424 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7426 static int trace_module_notify(struct notifier_block *self,
7427 unsigned long val, void *data)
7429 struct module *mod = data;
7432 case MODULE_STATE_COMING:
7433 trace_module_add_enums(mod);
7435 case MODULE_STATE_GOING:
7436 trace_module_remove_enums(mod);
7443 static struct notifier_block trace_module_nb = {
7444 .notifier_call = trace_module_notify,
7447 #endif /* CONFIG_MODULES */
7449 static __init int tracer_init_tracefs(void)
7451 struct dentry *d_tracer;
7453 trace_access_lock_init();
7455 d_tracer = tracing_init_dentry();
7456 if (IS_ERR(d_tracer))
7459 init_tracer_tracefs(&global_trace, d_tracer);
7460 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
7462 trace_create_file("tracing_thresh", 0644, d_tracer,
7463 &global_trace, &tracing_thresh_fops);
7465 trace_create_file("README", 0444, d_tracer,
7466 NULL, &tracing_readme_fops);
7468 trace_create_file("saved_cmdlines", 0444, d_tracer,
7469 NULL, &tracing_saved_cmdlines_fops);
7471 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7472 NULL, &tracing_saved_cmdlines_size_fops);
7476 trace_create_enum_file(d_tracer);
7478 #ifdef CONFIG_MODULES
7479 register_module_notifier(&trace_module_nb);
7482 #ifdef CONFIG_DYNAMIC_FTRACE
7483 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7484 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7487 create_trace_instances(d_tracer);
7489 update_tracer_options(&global_trace);
7494 static int trace_panic_handler(struct notifier_block *this,
7495 unsigned long event, void *unused)
7497 if (ftrace_dump_on_oops)
7498 ftrace_dump(ftrace_dump_on_oops);
7502 static struct notifier_block trace_panic_notifier = {
7503 .notifier_call = trace_panic_handler,
7505 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7508 static int trace_die_handler(struct notifier_block *self,
7514 if (ftrace_dump_on_oops)
7515 ftrace_dump(ftrace_dump_on_oops);
7523 static struct notifier_block trace_die_notifier = {
7524 .notifier_call = trace_die_handler,
7529 * printk is set to max of 1024, we really don't need it that big.
7530 * Nothing should be printing 1000 characters anyway.
7532 #define TRACE_MAX_PRINT 1000
7535 * Define here KERN_TRACE so that we have one place to modify
7536 * it if we decide to change what log level the ftrace dump
7539 #define KERN_TRACE KERN_EMERG
7542 trace_printk_seq(struct trace_seq *s)
7544 /* Probably should print a warning here. */
7545 if (s->seq.len >= TRACE_MAX_PRINT)
7546 s->seq.len = TRACE_MAX_PRINT;
7549 * More paranoid code. Although the buffer size is set to
7550 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7551 * an extra layer of protection.
7553 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7554 s->seq.len = s->seq.size - 1;
7556 /* should be zero ended, but we are paranoid. */
7557 s->buffer[s->seq.len] = 0;
7559 printk(KERN_TRACE "%s", s->buffer);
7564 void trace_init_global_iter(struct trace_iterator *iter)
7566 iter->tr = &global_trace;
7567 iter->trace = iter->tr->current_trace;
7568 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7569 iter->trace_buffer = &global_trace.trace_buffer;
7571 if (iter->trace && iter->trace->open)
7572 iter->trace->open(iter);
7574 /* Annotate start of buffers if we had overruns */
7575 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7576 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7578 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7579 if (trace_clocks[iter->tr->clock_id].in_ns)
7580 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7583 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7585 /* use static because iter can be a bit big for the stack */
7586 static struct trace_iterator iter;
7587 static atomic_t dump_running;
7588 struct trace_array *tr = &global_trace;
7589 unsigned int old_userobj;
7590 unsigned long flags;
7593 /* Only allow one dump user at a time. */
7594 if (atomic_inc_return(&dump_running) != 1) {
7595 atomic_dec(&dump_running);
7600 * Always turn off tracing when we dump.
7601 * We don't need to show trace output of what happens
7602 * between multiple crashes.
7604 * If the user does a sysrq-z, then they can re-enable
7605 * tracing with echo 1 > tracing_on.
7609 local_irq_save(flags);
7611 /* Simulate the iterator */
7612 trace_init_global_iter(&iter);
7614 for_each_tracing_cpu(cpu) {
7615 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7618 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7620 /* don't look at user memory in panic mode */
7621 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7623 switch (oops_dump_mode) {
7625 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7628 iter.cpu_file = raw_smp_processor_id();
7633 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7634 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7637 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7639 /* Did function tracer already get disabled? */
7640 if (ftrace_is_dead()) {
7641 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7642 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7646 * We need to stop all tracing on all CPUS to read the
7647 * the next buffer. This is a bit expensive, but is
7648 * not done often. We fill all what we can read,
7649 * and then release the locks again.
7652 while (!trace_empty(&iter)) {
7655 printk(KERN_TRACE "---------------------------------\n");
7659 trace_iterator_reset(&iter);
7660 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7662 if (trace_find_next_entry_inc(&iter) != NULL) {
7665 ret = print_trace_line(&iter);
7666 if (ret != TRACE_TYPE_NO_CONSUME)
7667 trace_consume(&iter);
7669 touch_nmi_watchdog();
7671 trace_printk_seq(&iter.seq);
7675 printk(KERN_TRACE " (ftrace buffer empty)\n");
7677 printk(KERN_TRACE "---------------------------------\n");
7680 tr->trace_flags |= old_userobj;
7682 for_each_tracing_cpu(cpu) {
7683 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7685 atomic_dec(&dump_running);
7686 local_irq_restore(flags);
7688 EXPORT_SYMBOL_GPL(ftrace_dump);
7690 __init static int tracer_alloc_buffers(void)
7696 * Make sure we don't accidently add more trace options
7697 * than we have bits for.
7699 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7701 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7704 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7705 goto out_free_buffer_mask;
7707 /* Only allocate trace_printk buffers if a trace_printk exists */
7708 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
7709 /* Must be called before global_trace.buffer is allocated */
7710 trace_printk_init_buffers();
7712 /* To save memory, keep the ring buffer size to its minimum */
7713 if (ring_buffer_expanded)
7714 ring_buf_size = trace_buf_size;
7718 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7719 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7721 raw_spin_lock_init(&global_trace.start_lock);
7723 /* Used for event triggers */
7724 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7726 goto out_free_cpumask;
7728 if (trace_create_savedcmd() < 0)
7729 goto out_free_temp_buffer;
7731 /* TODO: make the number of buffers hot pluggable with CPUS */
7732 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7733 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7735 goto out_free_savedcmd;
7738 if (global_trace.buffer_disabled)
7741 if (trace_boot_clock) {
7742 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7744 pr_warn("Trace clock %s not defined, going back to default\n",
7749 * register_tracer() might reference current_trace, so it
7750 * needs to be set before we register anything. This is
7751 * just a bootstrap of current_trace anyway.
7753 global_trace.current_trace = &nop_trace;
7755 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7757 ftrace_init_global_array_ops(&global_trace);
7759 init_trace_flags_index(&global_trace);
7761 register_tracer(&nop_trace);
7763 /* All seems OK, enable tracing */
7764 tracing_disabled = 0;
7766 atomic_notifier_chain_register(&panic_notifier_list,
7767 &trace_panic_notifier);
7769 register_die_notifier(&trace_die_notifier);
7771 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7773 INIT_LIST_HEAD(&global_trace.systems);
7774 INIT_LIST_HEAD(&global_trace.events);
7775 list_add(&global_trace.list, &ftrace_trace_arrays);
7777 apply_trace_boot_options();
7779 register_snapshot_cmd();
7784 free_saved_cmdlines_buffer(savedcmd);
7785 out_free_temp_buffer:
7786 ring_buffer_free(temp_buffer);
7788 free_cpumask_var(global_trace.tracing_cpumask);
7789 out_free_buffer_mask:
7790 free_cpumask_var(tracing_buffer_mask);
7795 void __init trace_init(void)
7797 if (tracepoint_printk) {
7798 tracepoint_print_iter =
7799 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7800 if (WARN_ON(!tracepoint_print_iter))
7801 tracepoint_printk = 0;
7803 tracer_alloc_buffers();
7807 __init static int clear_boot_tracer(void)
7810 * The default tracer at boot buffer is an init section.
7811 * This function is called in lateinit. If we did not
7812 * find the boot tracer, then clear it out, to prevent
7813 * later registration from accessing the buffer that is
7814 * about to be freed.
7816 if (!default_bootup_tracer)
7819 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7820 default_bootup_tracer);
7821 default_bootup_tracer = NULL;
7826 fs_initcall(tracer_init_tracefs);
7827 late_initcall_sync(clear_boot_tracer);