kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18
  19 #define SYNTH_SYSTEM            "synthetic"
  20 #define SYNTH_FIELDS_MAX        16
  21
  22 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  23
  24 struct hist_field;
  25
  26 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  27                                 struct tracing_map_elt *elt,
  28                                 struct ring_buffer_event *rbe,
  29                                 void *event);
  30
  31 #define HIST_FIELD_OPERANDS_MAX 2
  32 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  33 #define HIST_ACTIONS_MAX        8
  34
  35 enum field_op_id {
  36         FIELD_OP_NONE,
  37         FIELD_OP_PLUS,
  38         FIELD_OP_MINUS,
  39         FIELD_OP_UNARY_MINUS,
  40 };
  41
  42 struct hist_var {
  43         char                            *name;
  44         struct hist_trigger_data        *hist_data;
  45         unsigned int                    idx;
  46 };
  47
  48 struct hist_field {
  49         struct ftrace_event_field       *field;
  50         unsigned long                   flags;
  51         hist_field_fn_t                 fn;
  52         unsigned int                    ref;
  53         unsigned int                    size;
  54         unsigned int                    offset;
  55         unsigned int                    is_signed;
  56         const char                      *type;
  57         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  58         struct hist_trigger_data        *hist_data;
  59         struct hist_var                 var;
  60         enum field_op_id                operator;
  61         char                            *system;
  62         char                            *event_name;
  63         char                            *name;
  64         unsigned int                    var_idx;
  65         unsigned int                    var_ref_idx;
  66         bool                            read_once;
  67 };
  68
  69 static u64 hist_field_none(struct hist_field *field,
  70                            struct tracing_map_elt *elt,
  71                            struct ring_buffer_event *rbe,
  72                            void *event)
  73 {
  74         return 0;
  75 }
  76
  77 static u64 hist_field_counter(struct hist_field *field,
  78                               struct tracing_map_elt *elt,
  79                               struct ring_buffer_event *rbe,
  80                               void *event)
  81 {
  82         return 1;
  83 }
  84
  85 static u64 hist_field_string(struct hist_field *hist_field,
  86                              struct tracing_map_elt *elt,
  87                              struct ring_buffer_event *rbe,
  88                              void *event)
  89 {
  90         char *addr = (char *)(event + hist_field->field->offset);
  91
  92         return (u64)(unsigned long)addr;
  93 }
  94
  95 static u64 hist_field_dynstring(struct hist_field *hist_field,
  96                                 struct tracing_map_elt *elt,
  97                                 struct ring_buffer_event *rbe,
  98                                 void *event)
  99 {
 100         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 101         int str_loc = str_item & 0xffff;
 102         char *addr = (char *)(event + str_loc);
 103
 104         return (u64)(unsigned long)addr;
 105 }
 106
 107 static u64 hist_field_pstring(struct hist_field *hist_field,
 108                               struct tracing_map_elt *elt,
 109                               struct ring_buffer_event *rbe,
 110                               void *event)
 111 {
 112         char **addr = (char **)(event + hist_field->field->offset);
 113
 114         return (u64)(unsigned long)*addr;
 115 }
 116
 117 static u64 hist_field_log2(struct hist_field *hist_field,
 118                            struct tracing_map_elt *elt,
 119                            struct ring_buffer_event *rbe,
 120                            void *event)
 121 {
 122         struct hist_field *operand = hist_field->operands[0];
 123
 124         u64 val = operand->fn(operand, elt, rbe, event);
 125
 126         return (u64) ilog2(roundup_pow_of_two(val));
 127 }
 128
 129 static u64 hist_field_plus(struct hist_field *hist_field,
 130                            struct tracing_map_elt *elt,
 131                            struct ring_buffer_event *rbe,
 132                            void *event)
 133 {
 134         struct hist_field *operand1 = hist_field->operands[0];
 135         struct hist_field *operand2 = hist_field->operands[1];
 136
 137         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 138         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 139
 140         return val1 + val2;
 141 }
 142
 143 static u64 hist_field_minus(struct hist_field *hist_field,
 144                             struct tracing_map_elt *elt,
 145                             struct ring_buffer_event *rbe,
 146                             void *event)
 147 {
 148         struct hist_field *operand1 = hist_field->operands[0];
 149         struct hist_field *operand2 = hist_field->operands[1];
 150
 151         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 152         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 153
 154         return val1 - val2;
 155 }
 156
 157 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 158                                   struct tracing_map_elt *elt,
 159                                   struct ring_buffer_event *rbe,
 160                                   void *event)
 161 {
 162         struct hist_field *operand = hist_field->operands[0];
 163
 164         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 165         u64 val = (u64)-sval;
 166
 167         return val;
 168 }
 169
 170 #define DEFINE_HIST_FIELD_FN(type)                                      \
 171         static u64 hist_field_##type(struct hist_field *hist_field,     \
 172                                      struct tracing_map_elt *elt,       \
 173                                      struct ring_buffer_event *rbe,     \
 174                                      void *event)                       \
 175 {                                                                       \
 176         type *addr = (type *)(event + hist_field->field->offset);       \
 177                                                                         \
 178         return (u64)(unsigned long)*addr;                               \
 179 }
 180
 181 DEFINE_HIST_FIELD_FN(s64);
 182 DEFINE_HIST_FIELD_FN(u64);
 183 DEFINE_HIST_FIELD_FN(s32);
 184 DEFINE_HIST_FIELD_FN(u32);
 185 DEFINE_HIST_FIELD_FN(s16);
 186 DEFINE_HIST_FIELD_FN(u16);
 187 DEFINE_HIST_FIELD_FN(s8);
 188 DEFINE_HIST_FIELD_FN(u8);
 189
 190 #define for_each_hist_field(i, hist_data)       \
 191         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 192
 193 #define for_each_hist_val_field(i, hist_data)   \
 194         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 195
 196 #define for_each_hist_key_field(i, hist_data)   \
 197         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 198
 199 #define HIST_STACKTRACE_DEPTH   16
 200 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 201 #define HIST_STACKTRACE_SKIP    5
 202
 203 #define HITCOUNT_IDX            0
 204 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 205
 206 enum hist_field_flags {
 207         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 208         HIST_FIELD_FL_KEY               = 1 << 1,
 209         HIST_FIELD_FL_STRING            = 1 << 2,
 210         HIST_FIELD_FL_HEX               = 1 << 3,
 211         HIST_FIELD_FL_SYM               = 1 << 4,
 212         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 213         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 214         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 215         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 216         HIST_FIELD_FL_LOG2              = 1 << 9,
 217         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 218         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 219         HIST_FIELD_FL_VAR               = 1 << 12,
 220         HIST_FIELD_FL_EXPR              = 1 << 13,
 221         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 222         HIST_FIELD_FL_CPU               = 1 << 15,
 223         HIST_FIELD_FL_ALIAS             = 1 << 16,
 224 };
 225
 226 struct var_defs {
 227         unsigned int    n_vars;
 228         char            *name[TRACING_MAP_VARS_MAX];
 229         char            *expr[TRACING_MAP_VARS_MAX];
 230 };
 231
 232 struct hist_trigger_attrs {
 233         char            *keys_str;
 234         char            *vals_str;
 235         char            *sort_key_str;
 236         char            *name;
 237         char            *clock;
 238         bool            pause;
 239         bool            cont;
 240         bool            clear;
 241         bool            ts_in_usecs;
 242         unsigned int    map_bits;
 243
 244         char            *assignment_str[TRACING_MAP_VARS_MAX];
 245         unsigned int    n_assignments;
 246
 247         char            *action_str[HIST_ACTIONS_MAX];
 248         unsigned int    n_actions;
 249
 250         struct var_defs var_defs;
 251 };
 252
 253 struct field_var {
 254         struct hist_field       *var;
 255         struct hist_field       *val;
 256 };
 257
 258 struct field_var_hist {
 259         struct hist_trigger_data        *hist_data;
 260         char                            *cmd;
 261 };
 262
 263 struct hist_trigger_data {
 264         struct hist_field               *fields[HIST_FIELDS_MAX];
 265         unsigned int                    n_vals;
 266         unsigned int                    n_keys;
 267         unsigned int                    n_fields;
 268         unsigned int                    n_vars;
 269         unsigned int                    key_size;
 270         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 271         unsigned int                    n_sort_keys;
 272         struct trace_event_file         *event_file;
 273         struct hist_trigger_attrs       *attrs;
 274         struct tracing_map              *map;
 275         bool                            enable_timestamps;
 276         bool                            remove;
 277         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 278         unsigned int                    n_var_refs;
 279
 280         struct action_data              *actions[HIST_ACTIONS_MAX];
 281         unsigned int                    n_actions;
 282
 283         struct hist_field               *synth_var_refs[SYNTH_FIELDS_MAX];
 284         unsigned int                    n_synth_var_refs;
 285         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 286         unsigned int                    n_field_vars;
 287         unsigned int                    n_field_var_str;
 288         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 289         unsigned int                    n_field_var_hists;
 290
 291         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
 292         unsigned int                    n_max_vars;
 293         unsigned int                    n_max_var_str;
 294 };
 295
 296 struct synth_field {
 297         char *type;
 298         char *name;
 299         size_t size;
 300         bool is_signed;
 301         bool is_string;
 302 };
 303
 304 struct synth_event {
 305         struct list_head                        list;
 306         int                                     ref;
 307         char                                    *name;
 308         struct synth_field                      **fields;
 309         unsigned int                            n_fields;
 310         unsigned int                            n_u64;
 311         struct trace_event_class                class;
 312         struct trace_event_call                 call;
 313         struct tracepoint                       *tp;
 314 };
 315
 316 struct action_data;
 317
 318 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 319                              struct tracing_map_elt *elt, void *rec,
 320                              struct ring_buffer_event *rbe,
 321                              struct action_data *data, u64 *var_ref_vals);
 322
 323 struct action_data {
 324         action_fn_t             fn;
 325         unsigned int            n_params;
 326         char                    *params[SYNTH_FIELDS_MAX];
 327
 328         union {
 329                 struct {
 330                         unsigned int            var_ref_idx;
 331                         char                    *match_event;
 332                         char                    *match_event_system;
 333                         char                    *synth_event_name;
 334                         struct synth_event      *synth_event;
 335                 } onmatch;
 336
 337                 struct {
 338                         char                    *var_str;
 339                         char                    *fn_name;
 340                         unsigned int            max_var_ref_idx;
 341                         struct hist_field       *max_var;
 342                         struct hist_field       *var;
 343                 } onmax;
 344         };
 345 };
 346
 347
 348 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 349 static char hist_err_str[MAX_FILTER_STR_VAL];
 350
 351 static void last_cmd_set(char *str)
 352 {
 353         if (!str)
 354                 return;
 355
 356         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 357 }
 358
 359 static void hist_err(char *str, char *var)
 360 {
 361         int maxlen = MAX_FILTER_STR_VAL - 1;
 362
 363         if (!str)
 364                 return;
 365
 366         if (strlen(hist_err_str))
 367                 return;
 368
 369         if (!var)
 370                 var = "";
 371
 372         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 373                 return;
 374
 375         strcat(hist_err_str, str);
 376         strcat(hist_err_str, var);
 377 }
 378
 379 static void hist_err_event(char *str, char *system, char *event, char *var)
 380 {
 381         char err[MAX_FILTER_STR_VAL];
 382
 383         if (system && var)
 384                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 385         else if (system)
 386                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 387         else
 388                 strscpy(err, var, MAX_FILTER_STR_VAL);
 389
 390         hist_err(str, err);
 391 }
 392
 393 static void hist_err_clear(void)
 394 {
 395         hist_err_str[0] = '\0';
 396 }
 397
 398 static bool have_hist_err(void)
 399 {
 400         if (strlen(hist_err_str))
 401                 return true;
 402
 403         return false;
 404 }
 405
 406 static LIST_HEAD(synth_event_list);
 407 static DEFINE_MUTEX(synth_event_mutex);
 408
 409 struct synth_trace_event {
 410         struct trace_entry      ent;
 411         u64                     fields[];
 412 };
 413
 414 static int synth_event_define_fields(struct trace_event_call *call)
 415 {
 416         struct synth_trace_event trace;
 417         int offset = offsetof(typeof(trace), fields);
 418         struct synth_event *event = call->data;
 419         unsigned int i, size, n_u64;
 420         char *name, *type;
 421         bool is_signed;
 422         int ret = 0;
 423
 424         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 425                 size = event->fields[i]->size;
 426                 is_signed = event->fields[i]->is_signed;
 427                 type = event->fields[i]->type;
 428                 name = event->fields[i]->name;
 429                 ret = trace_define_field(call, type, name, offset, size,
 430                                          is_signed, FILTER_OTHER);
 431                 if (ret)
 432                         break;
 433
 434                 if (event->fields[i]->is_string) {
 435                         offset += STR_VAR_LEN_MAX;
 436                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 437                 } else {
 438                         offset += sizeof(u64);
 439                         n_u64++;
 440                 }
 441         }
 442
 443         event->n_u64 = n_u64;
 444
 445         return ret;
 446 }
 447
 448 static bool synth_field_signed(char *type)
 449 {
 450         if (strncmp(type, "u", 1) == 0)
 451                 return false;
 452         if (strcmp(type, "gfp_t") == 0)
 453                 return false;
 454
 455         return true;
 456 }
 457
 458 static int synth_field_is_string(char *type)
 459 {
 460         if (strstr(type, "char[") != NULL)
 461                 return true;
 462
 463         return false;
 464 }
 465
 466 static int synth_field_string_size(char *type)
 467 {
 468         char buf[4], *end, *start;
 469         unsigned int len;
 470         int size, err;
 471
 472         start = strstr(type, "char[");
 473         if (start == NULL)
 474                 return -EINVAL;
 475         start += strlen("char[");
 476
 477         end = strchr(type, ']');
 478         if (!end || end < start)
 479                 return -EINVAL;
 480
 481         len = end - start;
 482         if (len > 3)
 483                 return -EINVAL;
 484
 485         strncpy(buf, start, len);
 486         buf[len] = '\0';
 487
 488         err = kstrtouint(buf, 0, &size);
 489         if (err)
 490                 return err;
 491
 492         if (size > STR_VAR_LEN_MAX)
 493                 return -EINVAL;
 494
 495         return size;
 496 }
 497
 498 static int synth_field_size(char *type)
 499 {
 500         int size = 0;
 501
 502         if (strcmp(type, "s64") == 0)
 503                 size = sizeof(s64);
 504         else if (strcmp(type, "u64") == 0)
 505                 size = sizeof(u64);
 506         else if (strcmp(type, "s32") == 0)
 507                 size = sizeof(s32);
 508         else if (strcmp(type, "u32") == 0)
 509                 size = sizeof(u32);
 510         else if (strcmp(type, "s16") == 0)
 511                 size = sizeof(s16);
 512         else if (strcmp(type, "u16") == 0)
 513                 size = sizeof(u16);
 514         else if (strcmp(type, "s8") == 0)
 515                 size = sizeof(s8);
 516         else if (strcmp(type, "u8") == 0)
 517                 size = sizeof(u8);
 518         else if (strcmp(type, "char") == 0)
 519                 size = sizeof(char);
 520         else if (strcmp(type, "unsigned char") == 0)
 521                 size = sizeof(unsigned char);
 522         else if (strcmp(type, "int") == 0)
 523                 size = sizeof(int);
 524         else if (strcmp(type, "unsigned int") == 0)
 525                 size = sizeof(unsigned int);
 526         else if (strcmp(type, "long") == 0)
 527                 size = sizeof(long);
 528         else if (strcmp(type, "unsigned long") == 0)
 529                 size = sizeof(unsigned long);
 530         else if (strcmp(type, "pid_t") == 0)
 531                 size = sizeof(pid_t);
 532         else if (synth_field_is_string(type))
 533                 size = synth_field_string_size(type);
 534
 535         return size;
 536 }
 537
 538 static const char *synth_field_fmt(char *type)
 539 {
 540         const char *fmt = "%llu";
 541
 542         if (strcmp(type, "s64") == 0)
 543                 fmt = "%lld";
 544         else if (strcmp(type, "u64") == 0)
 545                 fmt = "%llu";
 546         else if (strcmp(type, "s32") == 0)
 547                 fmt = "%d";
 548         else if (strcmp(type, "u32") == 0)
 549                 fmt = "%u";
 550         else if (strcmp(type, "s16") == 0)
 551                 fmt = "%d";
 552         else if (strcmp(type, "u16") == 0)
 553                 fmt = "%u";
 554         else if (strcmp(type, "s8") == 0)
 555                 fmt = "%d";
 556         else if (strcmp(type, "u8") == 0)
 557                 fmt = "%u";
 558         else if (strcmp(type, "char") == 0)
 559                 fmt = "%d";
 560         else if (strcmp(type, "unsigned char") == 0)
 561                 fmt = "%u";
 562         else if (strcmp(type, "int") == 0)
 563                 fmt = "%d";
 564         else if (strcmp(type, "unsigned int") == 0)
 565                 fmt = "%u";
 566         else if (strcmp(type, "long") == 0)
 567                 fmt = "%ld";
 568         else if (strcmp(type, "unsigned long") == 0)
 569                 fmt = "%lu";
 570         else if (strcmp(type, "pid_t") == 0)
 571                 fmt = "%d";
 572         else if (synth_field_is_string(type))
 573                 fmt = "%s";
 574
 575         return fmt;
 576 }
 577
 578 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 579                                            int flags,
 580                                            struct trace_event *event)
 581 {
 582         struct trace_array *tr = iter->tr;
 583         struct trace_seq *s = &iter->seq;
 584         struct synth_trace_event *entry;
 585         struct synth_event *se;
 586         unsigned int i, n_u64;
 587         char print_fmt[32];
 588         const char *fmt;
 589
 590         entry = (struct synth_trace_event *)iter->ent;
 591         se = container_of(event, struct synth_event, call.event);
 592
 593         trace_seq_printf(s, "%s: ", se->name);
 594
 595         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 596                 if (trace_seq_has_overflowed(s))
 597                         goto end;
 598
 599                 fmt = synth_field_fmt(se->fields[i]->type);
 600
 601                 /* parameter types */
 602                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 603                         trace_seq_printf(s, "%s ", fmt);
 604
 605                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 606
 607                 /* parameter values */
 608                 if (se->fields[i]->is_string) {
 609                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 610                                          (char *)&entry->fields[n_u64],
 611                                          i == se->n_fields - 1 ? "" : " ");
 612                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 613                 } else {
 614                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 615                                          entry->fields[n_u64],
 616                                          i == se->n_fields - 1 ? "" : " ");
 617                         n_u64++;
 618                 }
 619         }
 620 end:
 621         trace_seq_putc(s, '\n');
 622
 623         return trace_handle_return(s);
 624 }
 625
 626 static struct trace_event_functions synth_event_funcs = {
 627         .trace          = print_synth_event
 628 };
 629
 630 static notrace void trace_event_raw_event_synth(void *__data,
 631                                                 u64 *var_ref_vals,
 632                                                 unsigned int var_ref_idx)
 633 {
 634         struct trace_event_file *trace_file = __data;
 635         struct synth_trace_event *entry;
 636         struct trace_event_buffer fbuffer;
 637         struct ring_buffer *buffer;
 638         struct synth_event *event;
 639         unsigned int i, n_u64;
 640         int fields_size = 0;
 641
 642         event = trace_file->event_call->data;
 643
 644         if (trace_trigger_soft_disabled(trace_file))
 645                 return;
 646
 647         fields_size = event->n_u64 * sizeof(u64);
 648
 649         /*
 650          * Avoid ring buffer recursion detection, as this event
 651          * is being performed within another event.
 652          */
 653         buffer = trace_file->tr->trace_buffer.buffer;
 654         ring_buffer_nest_start(buffer);
 655
 656         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 657                                            sizeof(*entry) + fields_size);
 658         if (!entry)
 659                 goto out;
 660
 661         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 662                 if (event->fields[i]->is_string) {
 663                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 664                         char *str_field = (char *)&entry->fields[n_u64];
 665
 666                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 667                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 668                 } else {
 669                         struct synth_field *field = event->fields[i];
 670                         u64 val = var_ref_vals[var_ref_idx + i];
 671
 672                         switch (field->size) {
 673                         case 1:
 674                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 675                                 break;
 676
 677                         case 2:
 678                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 679                                 break;
 680
 681                         case 4:
 682                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 683                                 break;
 684
 685                         default:
 686                                 entry->fields[n_u64] = val;
 687                                 break;
 688                         }
 689                         n_u64++;
 690                 }
 691         }
 692
 693         trace_event_buffer_commit(&fbuffer);
 694 out:
 695         ring_buffer_nest_end(buffer);
 696 }
 697
 698 static void free_synth_event_print_fmt(struct trace_event_call *call)
 699 {
 700         if (call) {
 701                 kfree(call->print_fmt);
 702                 call->print_fmt = NULL;
 703         }
 704 }
 705
 706 static int __set_synth_event_print_fmt(struct synth_event *event,
 707                                        char *buf, int len)
 708 {
 709         const char *fmt;
 710         int pos = 0;
 711         int i;
 712
 713         /* When len=0, we just calculate the needed length */
 714 #define LEN_OR_ZERO (len ? len - pos : 0)
 715
 716         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 717         for (i = 0; i < event->n_fields; i++) {
 718                 fmt = synth_field_fmt(event->fields[i]->type);
 719                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 720                                 event->fields[i]->name, fmt,
 721                                 i == event->n_fields - 1 ? "" : ", ");
 722         }
 723         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 724
 725         for (i = 0; i < event->n_fields; i++) {
 726                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 727                                 ", REC->%s", event->fields[i]->name);
 728         }
 729
 730 #undef LEN_OR_ZERO
 731
 732         /* return the length of print_fmt */
 733         return pos;
 734 }
 735
 736 static int set_synth_event_print_fmt(struct trace_event_call *call)
 737 {
 738         struct synth_event *event = call->data;
 739         char *print_fmt;
 740         int len;
 741
 742         /* First: called with 0 length to calculate the needed length */
 743         len = __set_synth_event_print_fmt(event, NULL, 0);
 744
 745         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 746         if (!print_fmt)
 747                 return -ENOMEM;
 748
 749         /* Second: actually write the @print_fmt */
 750         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 751         call->print_fmt = print_fmt;
 752
 753         return 0;
 754 }
 755
 756 static void free_synth_field(struct synth_field *field)
 757 {
 758         kfree(field->type);
 759         kfree(field->name);
 760         kfree(field);
 761 }
 762
 763 static struct synth_field *parse_synth_field(int argc, char **argv,
 764                                              int *consumed)
 765 {
 766         struct synth_field *field;
 767         const char *prefix = NULL;
 768         char *field_type = argv[0], *field_name;
 769         int len, ret = 0;
 770         char *array;
 771
 772         if (field_type[0] == ';')
 773                 field_type++;
 774
 775         if (!strcmp(field_type, "unsigned")) {
 776                 if (argc < 3)
 777                         return ERR_PTR(-EINVAL);
 778                 prefix = "unsigned ";
 779                 field_type = argv[1];
 780                 field_name = argv[2];
 781                 *consumed = 3;
 782         } else {
 783                 field_name = argv[1];
 784                 *consumed = 2;
 785         }
 786
 787         len = strlen(field_name);
 788         if (field_name[len - 1] == ';')
 789                 field_name[len - 1] = '\0';
 790
 791         field = kzalloc(sizeof(*field), GFP_KERNEL);
 792         if (!field)
 793                 return ERR_PTR(-ENOMEM);
 794
 795         len = strlen(field_type) + 1;
 796         array = strchr(field_name, '[');
 797         if (array)
 798                 len += strlen(array);
 799         if (prefix)
 800                 len += strlen(prefix);
 801         field->type = kzalloc(len, GFP_KERNEL);
 802         if (!field->type) {
 803                 ret = -ENOMEM;
 804                 goto free;
 805         }
 806         if (prefix)
 807                 strcat(field->type, prefix);
 808         strcat(field->type, field_type);
 809         if (array) {
 810                 strcat(field->type, array);
 811                 *array = '\0';
 812         }
 813
 814         field->size = synth_field_size(field->type);
 815         if (!field->size) {
 816                 ret = -EINVAL;
 817                 goto free;
 818         }
 819
 820         if (synth_field_is_string(field->type))
 821                 field->is_string = true;
 822
 823         field->is_signed = synth_field_signed(field->type);
 824
 825         field->name = kstrdup(field_name, GFP_KERNEL);
 826         if (!field->name) {
 827                 ret = -ENOMEM;
 828                 goto free;
 829         }
 830  out:
 831         return field;
 832  free:
 833         free_synth_field(field);
 834         field = ERR_PTR(ret);
 835         goto out;
 836 }
 837
 838 static void free_synth_tracepoint(struct tracepoint *tp)
 839 {
 840         if (!tp)
 841                 return;
 842
 843         kfree(tp->name);
 844         kfree(tp);
 845 }
 846
 847 static struct tracepoint *alloc_synth_tracepoint(char *name)
 848 {
 849         struct tracepoint *tp;
 850
 851         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 852         if (!tp)
 853                 return ERR_PTR(-ENOMEM);
 854
 855         tp->name = kstrdup(name, GFP_KERNEL);
 856         if (!tp->name) {
 857                 kfree(tp);
 858                 return ERR_PTR(-ENOMEM);
 859         }
 860
 861         return tp;
 862 }
 863
 864 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 865                                     unsigned int var_ref_idx);
 866
 867 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 868                                unsigned int var_ref_idx)
 869 {
 870         struct tracepoint *tp = event->tp;
 871
 872         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 873                 struct tracepoint_func *probe_func_ptr;
 874                 synth_probe_func_t probe_func;
 875                 void *__data;
 876
 877                 if (!(cpu_online(raw_smp_processor_id())))
 878                         return;
 879
 880                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 881                 if (probe_func_ptr) {
 882                         do {
 883                                 probe_func = probe_func_ptr->func;
 884                                 __data = probe_func_ptr->data;
 885                                 probe_func(__data, var_ref_vals, var_ref_idx);
 886                         } while ((++probe_func_ptr)->func);
 887                 }
 888         }
 889 }
 890
 891 static struct synth_event *find_synth_event(const char *name)
 892 {
 893         struct synth_event *event;
 894
 895         list_for_each_entry(event, &synth_event_list, list) {
 896                 if (strcmp(event->name, name) == 0)
 897                         return event;
 898         }
 899
 900         return NULL;
 901 }
 902
 903 static int register_synth_event(struct synth_event *event)
 904 {
 905         struct trace_event_call *call = &event->call;
 906         int ret = 0;
 907
 908         event->call.class = &event->class;
 909         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 910         if (!event->class.system) {
 911                 ret = -ENOMEM;
 912                 goto out;
 913         }
 914
 915         event->tp = alloc_synth_tracepoint(event->name);
 916         if (IS_ERR(event->tp)) {
 917                 ret = PTR_ERR(event->tp);
 918                 event->tp = NULL;
 919                 goto out;
 920         }
 921
 922         INIT_LIST_HEAD(&call->class->fields);
 923         call->event.funcs = &synth_event_funcs;
 924         call->class->define_fields = synth_event_define_fields;
 925
 926         ret = register_trace_event(&call->event);
 927         if (!ret) {
 928                 ret = -ENODEV;
 929                 goto out;
 930         }
 931         call->flags = TRACE_EVENT_FL_TRACEPOINT;
 932         call->class->reg = trace_event_reg;
 933         call->class->probe = trace_event_raw_event_synth;
 934         call->data = event;
 935         call->tp = event->tp;
 936
 937         ret = trace_add_event_call_nolock(call);
 938         if (ret) {
 939                 pr_warn("Failed to register synthetic event: %s\n",
 940                         trace_event_name(call));
 941                 goto err;
 942         }
 943
 944         ret = set_synth_event_print_fmt(call);
 945         if (ret < 0) {
 946                 trace_remove_event_call(call);
 947                 goto err;
 948         }
 949  out:
 950         return ret;
 951  err:
 952         unregister_trace_event(&call->event);
 953         goto out;
 954 }
 955
 956 static int unregister_synth_event(struct synth_event *event)
 957 {
 958         struct trace_event_call *call = &event->call;
 959         int ret;
 960
 961         ret = trace_remove_event_call_nolock(call);
 962
 963         return ret;
 964 }
 965
 966 static void free_synth_event(struct synth_event *event)
 967 {
 968         unsigned int i;
 969
 970         if (!event)
 971                 return;
 972
 973         for (i = 0; i < event->n_fields; i++)
 974                 free_synth_field(event->fields[i]);
 975
 976         kfree(event->fields);
 977         kfree(event->name);
 978         kfree(event->class.system);
 979         free_synth_tracepoint(event->tp);
 980         free_synth_event_print_fmt(&event->call);
 981         kfree(event);
 982 }
 983
 984 static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
 985                                              struct synth_field **fields)
 986 {
 987         struct synth_event *event;
 988         unsigned int i;
 989
 990         event = kzalloc(sizeof(*event), GFP_KERNEL);
 991         if (!event) {
 992                 event = ERR_PTR(-ENOMEM);
 993                 goto out;
 994         }
 995
 996         event->name = kstrdup(event_name, GFP_KERNEL);
 997         if (!event->name) {
 998                 kfree(event);
 999                 event = ERR_PTR(-ENOMEM);
1000                 goto out;
1001         }
1002
1003         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1004         if (!event->fields) {
1005                 free_synth_event(event);
1006                 event = ERR_PTR(-ENOMEM);
1007                 goto out;
1008         }
1009
1010         for (i = 0; i < n_fields; i++)
1011                 event->fields[i] = fields[i];
1012
1013         event->n_fields = n_fields;
1014  out:
1015         return event;
1016 }
1017
1018 static void action_trace(struct hist_trigger_data *hist_data,
1019                          struct tracing_map_elt *elt, void *rec,
1020                          struct ring_buffer_event *rbe,
1021                          struct action_data *data, u64 *var_ref_vals)
1022 {
1023         struct synth_event *event = data->onmatch.synth_event;
1024
1025         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
1026 }
1027
1028 struct hist_var_data {
1029         struct list_head list;
1030         struct hist_trigger_data *hist_data;
1031 };
1032
1033 static void add_or_delete_synth_event(struct synth_event *event, int delete)
1034 {
1035         if (delete)
1036                 free_synth_event(event);
1037         else {
1038                 if (!find_synth_event(event->name))
1039                         list_add(&event->list, &synth_event_list);
1040                 else
1041                         free_synth_event(event);
1042         }
1043 }
1044
1045 static int create_synth_event(int argc, char **argv)
1046 {
1047         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1048         struct synth_event *event = NULL;
1049         bool delete_event = false;
1050         int i, consumed = 0, n_fields = 0, ret = 0;
1051         char *name;
1052
1053         mutex_lock(&event_mutex);
1054         mutex_lock(&synth_event_mutex);
1055
1056         /*
1057          * Argument syntax:
1058          *  - Add synthetic event: <event_name> field[;field] ...
1059          *  - Remove synthetic event: !<event_name> field[;field] ...
1060          *      where 'field' = type field_name
1061          */
1062         if (argc < 1) {
1063                 ret = -EINVAL;
1064                 goto out;
1065         }
1066
1067         name = argv[0];
1068         if (name[0] == '!') {
1069                 delete_event = true;
1070                 name++;
1071         }
1072
1073         event = find_synth_event(name);
1074         if (event) {
1075                 if (delete_event) {
1076                         if (event->ref) {
1077                                 event = NULL;
1078                                 ret = -EBUSY;
1079                                 goto out;
1080                         }
1081                         list_del(&event->list);
1082                         goto out;
1083                 }
1084                 event = NULL;
1085                 ret = -EEXIST;
1086                 goto out;
1087         } else if (delete_event) {
1088                 ret = -ENOENT;
1089                 goto out;
1090         }
1091
1092         if (argc < 2) {
1093                 ret = -EINVAL;
1094                 goto out;
1095         }
1096
1097         for (i = 1; i < argc - 1; i++) {
1098                 if (strcmp(argv[i], ";") == 0)
1099                         continue;
1100                 if (n_fields == SYNTH_FIELDS_MAX) {
1101                         ret = -EINVAL;
1102                         goto err;
1103                 }
1104
1105                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1106                 if (IS_ERR(field)) {
1107                         ret = PTR_ERR(field);
1108                         goto err;
1109                 }
1110                 fields[n_fields++] = field;
1111                 i += consumed - 1;
1112         }
1113
1114         if (i < argc && strcmp(argv[i], ";") != 0) {
1115                 ret = -EINVAL;
1116                 goto err;
1117         }
1118
1119         event = alloc_synth_event(name, n_fields, fields);
1120         if (IS_ERR(event)) {
1121                 ret = PTR_ERR(event);
1122                 event = NULL;
1123                 goto err;
1124         }
1125  out:
1126         if (event) {
1127                 if (delete_event) {
1128                         ret = unregister_synth_event(event);
1129                         add_or_delete_synth_event(event, !ret);
1130                 } else {
1131                         ret = register_synth_event(event);
1132                         add_or_delete_synth_event(event, ret);
1133                 }
1134         }
1135         mutex_unlock(&synth_event_mutex);
1136         mutex_unlock(&event_mutex);
1137
1138         return ret;
1139  err:
1140         mutex_unlock(&synth_event_mutex);
1141         mutex_unlock(&event_mutex);
1142
1143         for (i = 0; i < n_fields; i++)
1144                 free_synth_field(fields[i]);
1145         free_synth_event(event);
1146
1147         return ret;
1148 }
1149
1150 static int release_all_synth_events(void)
1151 {
1152         struct synth_event *event, *e;
1153         int ret = 0;
1154
1155         mutex_lock(&event_mutex);
1156         mutex_lock(&synth_event_mutex);
1157
1158         list_for_each_entry(event, &synth_event_list, list) {
1159                 if (event->ref) {
1160                         mutex_unlock(&synth_event_mutex);
1161                         return -EBUSY;
1162                 }
1163         }
1164
1165         list_for_each_entry_safe(event, e, &synth_event_list, list) {
1166                 list_del(&event->list);
1167
1168                 ret = unregister_synth_event(event);
1169                 add_or_delete_synth_event(event, !ret);
1170         }
1171         mutex_unlock(&synth_event_mutex);
1172         mutex_unlock(&event_mutex);
1173
1174         return ret;
1175 }
1176
1177
1178 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1179 {
1180         mutex_lock(&synth_event_mutex);
1181
1182         return seq_list_start(&synth_event_list, *pos);
1183 }
1184
1185 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1186 {
1187         return seq_list_next(v, &synth_event_list, pos);
1188 }
1189
1190 static void synth_events_seq_stop(struct seq_file *m, void *v)
1191 {
1192         mutex_unlock(&synth_event_mutex);
1193 }
1194
1195 static int synth_events_seq_show(struct seq_file *m, void *v)
1196 {
1197         struct synth_field *field;
1198         struct synth_event *event = v;
1199         unsigned int i;
1200
1201         seq_printf(m, "%s\t", event->name);
1202
1203         for (i = 0; i < event->n_fields; i++) {
1204                 field = event->fields[i];
1205
1206                 /* parameter values */
1207                 seq_printf(m, "%s %s%s", field->type, field->name,
1208                            i == event->n_fields - 1 ? "" : "; ");
1209         }
1210
1211         seq_putc(m, '\n');
1212
1213         return 0;
1214 }
1215
1216 static const struct seq_operations synth_events_seq_op = {
1217         .start  = synth_events_seq_start,
1218         .next   = synth_events_seq_next,
1219         .stop   = synth_events_seq_stop,
1220         .show   = synth_events_seq_show
1221 };
1222
1223 static int synth_events_open(struct inode *inode, struct file *file)
1224 {
1225         int ret;
1226
1227         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1228                 ret = release_all_synth_events();
1229                 if (ret < 0)
1230                         return ret;
1231         }
1232
1233         return seq_open(file, &synth_events_seq_op);
1234 }
1235
1236 static ssize_t synth_events_write(struct file *file,
1237                                   const char __user *buffer,
1238                                   size_t count, loff_t *ppos)
1239 {
1240         return trace_parse_run_command(file, buffer, count, ppos,
1241                                        create_synth_event);
1242 }
1243
1244 static const struct file_operations synth_events_fops = {
1245         .open           = synth_events_open,
1246         .write          = synth_events_write,
1247         .read           = seq_read,
1248         .llseek         = seq_lseek,
1249         .release        = seq_release,
1250 };
1251
1252 static u64 hist_field_timestamp(struct hist_field *hist_field,
1253                                 struct tracing_map_elt *elt,
1254                                 struct ring_buffer_event *rbe,
1255                                 void *event)
1256 {
1257         struct hist_trigger_data *hist_data = hist_field->hist_data;
1258         struct trace_array *tr = hist_data->event_file->tr;
1259
1260         u64 ts = ring_buffer_event_time_stamp(rbe);
1261
1262         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1263                 ts = ns2usecs(ts);
1264
1265         return ts;
1266 }
1267
1268 static u64 hist_field_cpu(struct hist_field *hist_field,
1269                           struct tracing_map_elt *elt,
1270                           struct ring_buffer_event *rbe,
1271                           void *event)
1272 {
1273         int cpu = smp_processor_id();
1274
1275         return cpu;
1276 }
1277
1278 /**
1279  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1280  * @hist_field: The VAR_REF field to check
1281  * @var_data: The hist trigger that owns the variable
1282  * @var_idx: The trigger variable identifier
1283  *
1284  * Check the given VAR_REF field to see whether or not it references
1285  * the given variable associated with the given trigger.
1286  *
1287  * Return: The VAR_REF field if it does reference the variable, NULL if not
1288  */
1289 static struct hist_field *
1290 check_field_for_var_ref(struct hist_field *hist_field,
1291                         struct hist_trigger_data *var_data,
1292                         unsigned int var_idx)
1293 {
1294         struct hist_field *found = NULL;
1295
1296         if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1297                 if (hist_field->var.idx == var_idx &&
1298                     hist_field->var.hist_data == var_data) {
1299                         found = hist_field;
1300                 }
1301         }
1302
1303         return found;
1304 }
1305
1306 static struct hist_field *
1307 check_field_for_var_refs(struct hist_trigger_data *hist_data,
1308                          struct hist_field *hist_field,
1309                          struct hist_trigger_data *var_data,
1310                          unsigned int var_idx,
1311                          unsigned int level)
1312 {
1313         struct hist_field *found = NULL;
1314         unsigned int i;
1315
1316         if (level > 3)
1317                 return found;
1318
1319         if (!hist_field)
1320                 return found;
1321
1322         found = check_field_for_var_ref(hist_field, var_data, var_idx);
1323         if (found)
1324                 return found;
1325
1326         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1327                 struct hist_field *operand;
1328
1329                 operand = hist_field->operands[i];
1330                 found = check_field_for_var_refs(hist_data, operand, var_data,
1331                                                  var_idx, level + 1);
1332                 if (found)
1333                         return found;
1334         }
1335
1336         return found;
1337 }
1338
1339 /**
1340  * find_var_ref - Check if a trigger has a reference to a trigger variable
1341  * @hist_data: The hist trigger that might have a reference to the variable
1342  * @var_data: The hist trigger that owns the variable
1343  * @var_idx: The trigger variable identifier
1344  *
1345  * Check the list of var_refs[] on the first hist trigger to see
1346  * whether any of them are references to the variable on the second
1347  * trigger.
1348  *
1349  * Return: The VAR_REF field referencing the variable if so, NULL if not
1350  */
1351 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1352                                        struct hist_trigger_data *var_data,
1353                                        unsigned int var_idx)
1354 {
1355         struct hist_field *hist_field, *found = NULL;
1356         unsigned int i;
1357
1358         for_each_hist_field(i, hist_data) {
1359                 hist_field = hist_data->fields[i];
1360                 found = check_field_for_var_refs(hist_data, hist_field,
1361                                                  var_data, var_idx, 0);
1362                 if (found)
1363                         return found;
1364         }
1365
1366         for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1367                 hist_field = hist_data->synth_var_refs[i];
1368                 found = check_field_for_var_refs(hist_data, hist_field,
1369                                                  var_data, var_idx, 0);
1370                 if (found)
1371                         return found;
1372         }
1373
1374         return found;
1375 }
1376
1377 /**
1378  * find_any_var_ref - Check if there is a reference to a given trigger variable
1379  * @hist_data: The hist trigger
1380  * @var_idx: The trigger variable identifier
1381  *
1382  * Check to see whether the given variable is currently referenced by
1383  * any other trigger.
1384  *
1385  * The trigger the variable is defined on is explicitly excluded - the
1386  * assumption being that a self-reference doesn't prevent a trigger
1387  * from being removed.
1388  *
1389  * Return: The VAR_REF field referencing the variable if so, NULL if not
1390  */
1391 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1392                                            unsigned int var_idx)
1393 {
1394         struct trace_array *tr = hist_data->event_file->tr;
1395         struct hist_field *found = NULL;
1396         struct hist_var_data *var_data;
1397
1398         list_for_each_entry(var_data, &tr->hist_vars, list) {
1399                 if (var_data->hist_data == hist_data)
1400                         continue;
1401                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1402                 if (found)
1403                         break;
1404         }
1405
1406         return found;
1407 }
1408
1409 /**
1410  * check_var_refs - Check if there is a reference to any of trigger's variables
1411  * @hist_data: The hist trigger
1412  *
1413  * A trigger can define one or more variables.  If any one of them is
1414  * currently referenced by any other trigger, this function will
1415  * determine that.
1416
1417  * Typically used to determine whether or not a trigger can be removed
1418  * - if there are any references to a trigger's variables, it cannot.
1419  *
1420  * Return: True if there is a reference to any of trigger's variables
1421  */
1422 static bool check_var_refs(struct hist_trigger_data *hist_data)
1423 {
1424         struct hist_field *field;
1425         bool found = false;
1426         int i;
1427
1428         for_each_hist_field(i, hist_data) {
1429                 field = hist_data->fields[i];
1430                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1431                         if (find_any_var_ref(hist_data, field->var.idx)) {
1432                                 found = true;
1433                                 break;
1434                         }
1435                 }
1436         }
1437
1438         return found;
1439 }
1440
1441 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1442 {
1443         struct trace_array *tr = hist_data->event_file->tr;
1444         struct hist_var_data *var_data, *found = NULL;
1445
1446         list_for_each_entry(var_data, &tr->hist_vars, list) {
1447                 if (var_data->hist_data == hist_data) {
1448                         found = var_data;
1449                         break;
1450                 }
1451         }
1452
1453         return found;
1454 }
1455
1456 static bool field_has_hist_vars(struct hist_field *hist_field,
1457                                 unsigned int level)
1458 {
1459         int i;
1460
1461         if (level > 3)
1462                 return false;
1463
1464         if (!hist_field)
1465                 return false;
1466
1467         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1468             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1469                 return true;
1470
1471         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1472                 struct hist_field *operand;
1473
1474                 operand = hist_field->operands[i];
1475                 if (field_has_hist_vars(operand, level + 1))
1476                         return true;
1477         }
1478
1479         return false;
1480 }
1481
1482 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1483 {
1484         struct hist_field *hist_field;
1485         int i;
1486
1487         for_each_hist_field(i, hist_data) {
1488                 hist_field = hist_data->fields[i];
1489                 if (field_has_hist_vars(hist_field, 0))
1490                         return true;
1491         }
1492
1493         return false;
1494 }
1495
1496 static int save_hist_vars(struct hist_trigger_data *hist_data)
1497 {
1498         struct trace_array *tr = hist_data->event_file->tr;
1499         struct hist_var_data *var_data;
1500
1501         var_data = find_hist_vars(hist_data);
1502         if (var_data)
1503                 return 0;
1504
1505         if (trace_array_get(tr) < 0)
1506                 return -ENODEV;
1507
1508         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1509         if (!var_data) {
1510                 trace_array_put(tr);
1511                 return -ENOMEM;
1512         }
1513
1514         var_data->hist_data = hist_data;
1515         list_add(&var_data->list, &tr->hist_vars);
1516
1517         return 0;
1518 }
1519
1520 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1521 {
1522         struct trace_array *tr = hist_data->event_file->tr;
1523         struct hist_var_data *var_data;
1524
1525         var_data = find_hist_vars(hist_data);
1526         if (!var_data)
1527                 return;
1528
1529         if (WARN_ON(check_var_refs(hist_data)))
1530                 return;
1531
1532         list_del(&var_data->list);
1533
1534         kfree(var_data);
1535
1536         trace_array_put(tr);
1537 }
1538
1539 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1540                                          const char *var_name)
1541 {
1542         struct hist_field *hist_field, *found = NULL;
1543         int i;
1544
1545         for_each_hist_field(i, hist_data) {
1546                 hist_field = hist_data->fields[i];
1547                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1548                     strcmp(hist_field->var.name, var_name) == 0) {
1549                         found = hist_field;
1550                         break;
1551                 }
1552         }
1553
1554         return found;
1555 }
1556
1557 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1558                                    struct trace_event_file *file,
1559                                    const char *var_name)
1560 {
1561         struct hist_trigger_data *test_data;
1562         struct event_trigger_data *test;
1563         struct hist_field *hist_field;
1564
1565         lockdep_assert_held(&event_mutex);
1566
1567         hist_field = find_var_field(hist_data, var_name);
1568         if (hist_field)
1569                 return hist_field;
1570
1571         list_for_each_entry(test, &file->triggers, list) {
1572                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1573                         test_data = test->private_data;
1574                         hist_field = find_var_field(test_data, var_name);
1575                         if (hist_field)
1576                                 return hist_field;
1577                 }
1578         }
1579
1580         return NULL;
1581 }
1582
1583 static struct trace_event_file *find_var_file(struct trace_array *tr,
1584                                               char *system,
1585                                               char *event_name,
1586                                               char *var_name)
1587 {
1588         struct hist_trigger_data *var_hist_data;
1589         struct hist_var_data *var_data;
1590         struct trace_event_file *file, *found = NULL;
1591
1592         if (system)
1593                 return find_event_file(tr, system, event_name);
1594
1595         list_for_each_entry(var_data, &tr->hist_vars, list) {
1596                 var_hist_data = var_data->hist_data;
1597                 file = var_hist_data->event_file;
1598                 if (file == found)
1599                         continue;
1600
1601                 if (find_var_field(var_hist_data, var_name)) {
1602                         if (found) {
1603                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1604                                 return NULL;
1605                         }
1606
1607                         found = file;
1608                 }
1609         }
1610
1611         return found;
1612 }
1613
1614 static struct hist_field *find_file_var(struct trace_event_file *file,
1615                                         const char *var_name)
1616 {
1617         struct hist_trigger_data *test_data;
1618         struct event_trigger_data *test;
1619         struct hist_field *hist_field;
1620
1621         lockdep_assert_held(&event_mutex);
1622
1623         list_for_each_entry(test, &file->triggers, list) {
1624                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1625                         test_data = test->private_data;
1626                         hist_field = find_var_field(test_data, var_name);
1627                         if (hist_field)
1628                                 return hist_field;
1629                 }
1630         }
1631
1632         return NULL;
1633 }
1634
1635 static struct hist_field *
1636 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1637 {
1638         struct trace_array *tr = hist_data->event_file->tr;
1639         struct hist_field *hist_field, *found = NULL;
1640         struct trace_event_file *file;
1641         unsigned int i;
1642
1643         for (i = 0; i < hist_data->n_actions; i++) {
1644                 struct action_data *data = hist_data->actions[i];
1645
1646                 if (data->fn == action_trace) {
1647                         char *system = data->onmatch.match_event_system;
1648                         char *event_name = data->onmatch.match_event;
1649
1650                         file = find_var_file(tr, system, event_name, var_name);
1651                         if (!file)
1652                                 continue;
1653                         hist_field = find_file_var(file, var_name);
1654                         if (hist_field) {
1655                                 if (found) {
1656                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1657                                         return ERR_PTR(-EINVAL);
1658                                 }
1659
1660                                 found = hist_field;
1661                         }
1662                 }
1663         }
1664         return found;
1665 }
1666
1667 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1668                                          char *system,
1669                                          char *event_name,
1670                                          char *var_name)
1671 {
1672         struct trace_array *tr = hist_data->event_file->tr;
1673         struct hist_field *hist_field = NULL;
1674         struct trace_event_file *file;
1675
1676         if (!system || !event_name) {
1677                 hist_field = find_match_var(hist_data, var_name);
1678                 if (IS_ERR(hist_field))
1679                         return NULL;
1680                 if (hist_field)
1681                         return hist_field;
1682         }
1683
1684         file = find_var_file(tr, system, event_name, var_name);
1685         if (!file)
1686                 return NULL;
1687
1688         hist_field = find_file_var(file, var_name);
1689
1690         return hist_field;
1691 }
1692
1693 struct hist_elt_data {
1694         char *comm;
1695         u64 *var_ref_vals;
1696         char *field_var_str[SYNTH_FIELDS_MAX];
1697 };
1698
1699 static u64 hist_field_var_ref(struct hist_field *hist_field,
1700                               struct tracing_map_elt *elt,
1701                               struct ring_buffer_event *rbe,
1702                               void *event)
1703 {
1704         struct hist_elt_data *elt_data;
1705         u64 var_val = 0;
1706
1707         if (WARN_ON_ONCE(!elt))
1708                 return var_val;
1709
1710         elt_data = elt->private_data;
1711         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1712
1713         return var_val;
1714 }
1715
1716 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1717                              u64 *var_ref_vals, bool self)
1718 {
1719         struct hist_trigger_data *var_data;
1720         struct tracing_map_elt *var_elt;
1721         struct hist_field *hist_field;
1722         unsigned int i, var_idx;
1723         bool resolved = true;
1724         u64 var_val = 0;
1725
1726         for (i = 0; i < hist_data->n_var_refs; i++) {
1727                 hist_field = hist_data->var_refs[i];
1728                 var_idx = hist_field->var.idx;
1729                 var_data = hist_field->var.hist_data;
1730
1731                 if (var_data == NULL) {
1732                         resolved = false;
1733                         break;
1734                 }
1735
1736                 if ((self && var_data != hist_data) ||
1737                     (!self && var_data == hist_data))
1738                         continue;
1739
1740                 var_elt = tracing_map_lookup(var_data->map, key);
1741                 if (!var_elt) {
1742                         resolved = false;
1743                         break;
1744                 }
1745
1746                 if (!tracing_map_var_set(var_elt, var_idx)) {
1747                         resolved = false;
1748                         break;
1749                 }
1750
1751                 if (self || !hist_field->read_once)
1752                         var_val = tracing_map_read_var(var_elt, var_idx);
1753                 else
1754                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1755
1756                 var_ref_vals[i] = var_val;
1757         }
1758
1759         return resolved;
1760 }
1761
1762 static const char *hist_field_name(struct hist_field *field,
1763                                    unsigned int level)
1764 {
1765         const char *field_name = "";
1766
1767         if (level > 1)
1768                 return field_name;
1769
1770         if (field->field)
1771                 field_name = field->field->name;
1772         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1773                  field->flags & HIST_FIELD_FL_ALIAS)
1774                 field_name = hist_field_name(field->operands[0], ++level);
1775         else if (field->flags & HIST_FIELD_FL_CPU)
1776                 field_name = "common_cpu";
1777         else if (field->flags & HIST_FIELD_FL_EXPR ||
1778                  field->flags & HIST_FIELD_FL_VAR_REF) {
1779                 if (field->system) {
1780                         static char full_name[MAX_FILTER_STR_VAL];
1781
1782                         strcat(full_name, field->system);
1783                         strcat(full_name, ".");
1784                         strcat(full_name, field->event_name);
1785                         strcat(full_name, ".");
1786                         strcat(full_name, field->name);
1787                         field_name = full_name;
1788                 } else
1789                         field_name = field->name;
1790         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1791                 field_name = "common_timestamp";
1792
1793         if (field_name == NULL)
1794                 field_name = "";
1795
1796         return field_name;
1797 }
1798
1799 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1800 {
1801         hist_field_fn_t fn = NULL;
1802
1803         switch (field_size) {
1804         case 8:
1805                 if (field_is_signed)
1806                         fn = hist_field_s64;
1807                 else
1808                         fn = hist_field_u64;
1809                 break;
1810         case 4:
1811                 if (field_is_signed)
1812                         fn = hist_field_s32;
1813                 else
1814                         fn = hist_field_u32;
1815                 break;
1816         case 2:
1817                 if (field_is_signed)
1818                         fn = hist_field_s16;
1819                 else
1820                         fn = hist_field_u16;
1821                 break;
1822         case 1:
1823                 if (field_is_signed)
1824                         fn = hist_field_s8;
1825                 else
1826                         fn = hist_field_u8;
1827                 break;
1828         }
1829
1830         return fn;
1831 }
1832
1833 static int parse_map_size(char *str)
1834 {
1835         unsigned long size, map_bits;
1836         int ret;
1837
1838         strsep(&str, "=");
1839         if (!str) {
1840                 ret = -EINVAL;
1841                 goto out;
1842         }
1843
1844         ret = kstrtoul(str, 0, &size);
1845         if (ret)
1846                 goto out;
1847
1848         map_bits = ilog2(roundup_pow_of_two(size));
1849         if (map_bits < TRACING_MAP_BITS_MIN ||
1850             map_bits > TRACING_MAP_BITS_MAX)
1851                 ret = -EINVAL;
1852         else
1853                 ret = map_bits;
1854  out:
1855         return ret;
1856 }
1857
1858 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1859 {
1860         unsigned int i;
1861
1862         if (!attrs)
1863                 return;
1864
1865         for (i = 0; i < attrs->n_assignments; i++)
1866                 kfree(attrs->assignment_str[i]);
1867
1868         for (i = 0; i < attrs->n_actions; i++)
1869                 kfree(attrs->action_str[i]);
1870
1871         kfree(attrs->name);
1872         kfree(attrs->sort_key_str);
1873         kfree(attrs->keys_str);
1874         kfree(attrs->vals_str);
1875         kfree(attrs->clock);
1876         kfree(attrs);
1877 }
1878
1879 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1880 {
1881         int ret = -EINVAL;
1882
1883         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1884                 return ret;
1885
1886         if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1887             (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
1888                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1889                 if (!attrs->action_str[attrs->n_actions]) {
1890                         ret = -ENOMEM;
1891                         return ret;
1892                 }
1893                 attrs->n_actions++;
1894                 ret = 0;
1895         }
1896
1897         return ret;
1898 }
1899
1900 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1901 {
1902         int ret = 0;
1903
1904         if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1905             (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1906                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1907                 if (!attrs->keys_str) {
1908                         ret = -ENOMEM;
1909                         goto out;
1910                 }
1911         } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1912                  (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1913                  (strncmp(str, "values=", strlen("values=")) == 0)) {
1914                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1915                 if (!attrs->vals_str) {
1916                         ret = -ENOMEM;
1917                         goto out;
1918                 }
1919         } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1920                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1921                 if (!attrs->sort_key_str) {
1922                         ret = -ENOMEM;
1923                         goto out;
1924                 }
1925         } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1926                 attrs->name = kstrdup(str, GFP_KERNEL);
1927                 if (!attrs->name) {
1928                         ret = -ENOMEM;
1929                         goto out;
1930                 }
1931         } else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1932                 strsep(&str, "=");
1933                 if (!str) {
1934                         ret = -EINVAL;
1935                         goto out;
1936                 }
1937
1938                 str = strstrip(str);
1939                 attrs->clock = kstrdup(str, GFP_KERNEL);
1940                 if (!attrs->clock) {
1941                         ret = -ENOMEM;
1942                         goto out;
1943                 }
1944         } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1945                 int map_bits = parse_map_size(str);
1946
1947                 if (map_bits < 0) {
1948                         ret = map_bits;
1949                         goto out;
1950                 }
1951                 attrs->map_bits = map_bits;
1952         } else {
1953                 char *assignment;
1954
1955                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1956                         hist_err("Too many variables defined: ", str);
1957                         ret = -EINVAL;
1958                         goto out;
1959                 }
1960
1961                 assignment = kstrdup(str, GFP_KERNEL);
1962                 if (!assignment) {
1963                         ret = -ENOMEM;
1964                         goto out;
1965                 }
1966
1967                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1968         }
1969  out:
1970         return ret;
1971 }
1972
1973 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1974 {
1975         struct hist_trigger_attrs *attrs;
1976         int ret = 0;
1977
1978         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1979         if (!attrs)
1980                 return ERR_PTR(-ENOMEM);
1981
1982         while (trigger_str) {
1983                 char *str = strsep(&trigger_str, ":");
1984
1985                 if (strchr(str, '=')) {
1986                         ret = parse_assignment(str, attrs);
1987                         if (ret)
1988                                 goto free;
1989                 } else if (strcmp(str, "pause") == 0)
1990                         attrs->pause = true;
1991                 else if ((strcmp(str, "cont") == 0) ||
1992                          (strcmp(str, "continue") == 0))
1993                         attrs->cont = true;
1994                 else if (strcmp(str, "clear") == 0)
1995                         attrs->clear = true;
1996                 else {
1997                         ret = parse_action(str, attrs);
1998                         if (ret)
1999                                 goto free;
2000                 }
2001         }
2002
2003         if (!attrs->keys_str) {
2004                 ret = -EINVAL;
2005                 goto free;
2006         }
2007
2008         if (!attrs->clock) {
2009                 attrs->clock = kstrdup("global", GFP_KERNEL);
2010                 if (!attrs->clock) {
2011                         ret = -ENOMEM;
2012                         goto free;
2013                 }
2014         }
2015
2016         return attrs;
2017  free:
2018         destroy_hist_trigger_attrs(attrs);
2019
2020         return ERR_PTR(ret);
2021 }
2022
2023 static inline void save_comm(char *comm, struct task_struct *task)
2024 {
2025         if (!task->pid) {
2026                 strcpy(comm, "<idle>");
2027                 return;
2028         }
2029
2030         if (WARN_ON_ONCE(task->pid < 0)) {
2031                 strcpy(comm, "<XXX>");
2032                 return;
2033         }
2034
2035         memcpy(comm, task->comm, TASK_COMM_LEN);
2036 }
2037
2038 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2039 {
2040         unsigned int i;
2041
2042         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2043                 kfree(elt_data->field_var_str[i]);
2044
2045         kfree(elt_data->comm);
2046         kfree(elt_data);
2047 }
2048
2049 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2050 {
2051         struct hist_elt_data *elt_data = elt->private_data;
2052
2053         hist_elt_data_free(elt_data);
2054 }
2055
2056 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2057 {
2058         struct hist_trigger_data *hist_data = elt->map->private_data;
2059         unsigned int size = TASK_COMM_LEN;
2060         struct hist_elt_data *elt_data;
2061         struct hist_field *key_field;
2062         unsigned int i, n_str;
2063
2064         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2065         if (!elt_data)
2066                 return -ENOMEM;
2067
2068         for_each_hist_key_field(i, hist_data) {
2069                 key_field = hist_data->fields[i];
2070
2071                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2072                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2073                         if (!elt_data->comm) {
2074                                 kfree(elt_data);
2075                                 return -ENOMEM;
2076                         }
2077                         break;
2078                 }
2079         }
2080
2081         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
2082
2083         size = STR_VAR_LEN_MAX;
2084
2085         for (i = 0; i < n_str; i++) {
2086                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2087                 if (!elt_data->field_var_str[i]) {
2088                         hist_elt_data_free(elt_data);
2089                         return -ENOMEM;
2090                 }
2091         }
2092
2093         elt->private_data = elt_data;
2094
2095         return 0;
2096 }
2097
2098 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2099 {
2100         struct hist_elt_data *elt_data = elt->private_data;
2101
2102         if (elt_data->comm)
2103                 save_comm(elt_data->comm, current);
2104 }
2105
2106 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2107         .elt_alloc      = hist_trigger_elt_data_alloc,
2108         .elt_free       = hist_trigger_elt_data_free,
2109         .elt_init       = hist_trigger_elt_data_init,
2110 };
2111
2112 static const char *get_hist_field_flags(struct hist_field *hist_field)
2113 {
2114         const char *flags_str = NULL;
2115
2116         if (hist_field->flags & HIST_FIELD_FL_HEX)
2117                 flags_str = "hex";
2118         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2119                 flags_str = "sym";
2120         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2121                 flags_str = "sym-offset";
2122         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2123                 flags_str = "execname";
2124         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2125                 flags_str = "syscall";
2126         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2127                 flags_str = "log2";
2128         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2129                 flags_str = "usecs";
2130
2131         return flags_str;
2132 }
2133
2134 static void expr_field_str(struct hist_field *field, char *expr)
2135 {
2136         if (field->flags & HIST_FIELD_FL_VAR_REF)
2137                 strcat(expr, "$");
2138
2139         strcat(expr, hist_field_name(field, 0));
2140
2141         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2142                 const char *flags_str = get_hist_field_flags(field);
2143
2144                 if (flags_str) {
2145                         strcat(expr, ".");
2146                         strcat(expr, flags_str);
2147                 }
2148         }
2149 }
2150
2151 static char *expr_str(struct hist_field *field, unsigned int level)
2152 {
2153         char *expr;
2154
2155         if (level > 1)
2156                 return NULL;
2157
2158         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2159         if (!expr)
2160                 return NULL;
2161
2162         if (!field->operands[0]) {
2163                 expr_field_str(field, expr);
2164                 return expr;
2165         }
2166
2167         if (field->operator == FIELD_OP_UNARY_MINUS) {
2168                 char *subexpr;
2169
2170                 strcat(expr, "-(");
2171                 subexpr = expr_str(field->operands[0], ++level);
2172                 if (!subexpr) {
2173                         kfree(expr);
2174                         return NULL;
2175                 }
2176                 strcat(expr, subexpr);
2177                 strcat(expr, ")");
2178
2179                 kfree(subexpr);
2180
2181                 return expr;
2182         }
2183
2184         expr_field_str(field->operands[0], expr);
2185
2186         switch (field->operator) {
2187         case FIELD_OP_MINUS:
2188                 strcat(expr, "-");
2189                 break;
2190         case FIELD_OP_PLUS:
2191                 strcat(expr, "+");
2192                 break;
2193         default:
2194                 kfree(expr);
2195                 return NULL;
2196         }
2197
2198         expr_field_str(field->operands[1], expr);
2199
2200         return expr;
2201 }
2202
2203 static int contains_operator(char *str)
2204 {
2205         enum field_op_id field_op = FIELD_OP_NONE;
2206         char *op;
2207
2208         op = strpbrk(str, "+-");
2209         if (!op)
2210                 return FIELD_OP_NONE;
2211
2212         switch (*op) {
2213         case '-':
2214                 /*
2215                  * Unfortunately, the modifier ".sym-offset"
2216                  * can confuse things.
2217                  */
2218                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
2219                         return FIELD_OP_NONE;
2220
2221                 if (*str == '-')
2222                         field_op = FIELD_OP_UNARY_MINUS;
2223                 else
2224                         field_op = FIELD_OP_MINUS;
2225                 break;
2226         case '+':
2227                 field_op = FIELD_OP_PLUS;
2228                 break;
2229         default:
2230                 break;
2231         }
2232
2233         return field_op;
2234 }
2235
2236 static void get_hist_field(struct hist_field *hist_field)
2237 {
2238         hist_field->ref++;
2239 }
2240
2241 static void __destroy_hist_field(struct hist_field *hist_field)
2242 {
2243         if (--hist_field->ref > 1)
2244                 return;
2245
2246         kfree(hist_field->var.name);
2247         kfree(hist_field->name);
2248         kfree(hist_field->type);
2249
2250         kfree(hist_field);
2251 }
2252
2253 static void destroy_hist_field(struct hist_field *hist_field,
2254                                unsigned int level)
2255 {
2256         unsigned int i;
2257
2258         if (level > 3)
2259                 return;
2260
2261         if (!hist_field)
2262                 return;
2263
2264         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2265                 return; /* var refs will be destroyed separately */
2266
2267         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2268                 destroy_hist_field(hist_field->operands[i], level + 1);
2269
2270         __destroy_hist_field(hist_field);
2271 }
2272
2273 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2274                                             struct ftrace_event_field *field,
2275                                             unsigned long flags,
2276                                             char *var_name)
2277 {
2278         struct hist_field *hist_field;
2279
2280         if (field && is_function_field(field))
2281                 return NULL;
2282
2283         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2284         if (!hist_field)
2285                 return NULL;
2286
2287         hist_field->ref = 1;
2288
2289         hist_field->hist_data = hist_data;
2290
2291         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2292                 goto out; /* caller will populate */
2293
2294         if (flags & HIST_FIELD_FL_VAR_REF) {
2295                 hist_field->fn = hist_field_var_ref;
2296                 goto out;
2297         }
2298
2299         if (flags & HIST_FIELD_FL_HITCOUNT) {
2300                 hist_field->fn = hist_field_counter;
2301                 hist_field->size = sizeof(u64);
2302                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2303                 if (!hist_field->type)
2304                         goto free;
2305                 goto out;
2306         }
2307
2308         if (flags & HIST_FIELD_FL_STACKTRACE) {
2309                 hist_field->fn = hist_field_none;
2310                 goto out;
2311         }
2312
2313         if (flags & HIST_FIELD_FL_LOG2) {
2314                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2315                 hist_field->fn = hist_field_log2;
2316                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2317                 hist_field->size = hist_field->operands[0]->size;
2318                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2319                 if (!hist_field->type)
2320                         goto free;
2321                 goto out;
2322         }
2323
2324         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2325                 hist_field->fn = hist_field_timestamp;
2326                 hist_field->size = sizeof(u64);
2327                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2328                 if (!hist_field->type)
2329                         goto free;
2330                 goto out;
2331         }
2332
2333         if (flags & HIST_FIELD_FL_CPU) {
2334                 hist_field->fn = hist_field_cpu;
2335                 hist_field->size = sizeof(int);
2336                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2337                 if (!hist_field->type)
2338                         goto free;
2339                 goto out;
2340         }
2341
2342         if (WARN_ON_ONCE(!field))
2343                 goto out;
2344
2345         /* Pointers to strings are just pointers and dangerous to dereference */
2346         if (is_string_field(field) &&
2347             (field->filter_type != FILTER_PTR_STRING)) {
2348                 flags |= HIST_FIELD_FL_STRING;
2349
2350                 hist_field->size = MAX_FILTER_STR_VAL;
2351                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2352                 if (!hist_field->type)
2353                         goto free;
2354
2355                 if (field->filter_type == FILTER_STATIC_STRING)
2356                         hist_field->fn = hist_field_string;
2357                 else if (field->filter_type == FILTER_DYN_STRING)
2358                         hist_field->fn = hist_field_dynstring;
2359                 else
2360                         hist_field->fn = hist_field_pstring;
2361         } else {
2362                 hist_field->size = field->size;
2363                 hist_field->is_signed = field->is_signed;
2364                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2365                 if (!hist_field->type)
2366                         goto free;
2367
2368                 hist_field->fn = select_value_fn(field->size,
2369                                                  field->is_signed);
2370                 if (!hist_field->fn) {
2371                         destroy_hist_field(hist_field, 0);
2372                         return NULL;
2373                 }
2374         }
2375  out:
2376         hist_field->field = field;
2377         hist_field->flags = flags;
2378
2379         if (var_name) {
2380                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2381                 if (!hist_field->var.name)
2382                         goto free;
2383         }
2384
2385         return hist_field;
2386  free:
2387         destroy_hist_field(hist_field, 0);
2388         return NULL;
2389 }
2390
2391 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2392 {
2393         unsigned int i;
2394
2395         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2396                 if (hist_data->fields[i]) {
2397                         destroy_hist_field(hist_data->fields[i], 0);
2398                         hist_data->fields[i] = NULL;
2399                 }
2400         }
2401
2402         for (i = 0; i < hist_data->n_var_refs; i++) {
2403                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2404                 __destroy_hist_field(hist_data->var_refs[i]);
2405                 hist_data->var_refs[i] = NULL;
2406         }
2407 }
2408
2409 static int init_var_ref(struct hist_field *ref_field,
2410                         struct hist_field *var_field,
2411                         char *system, char *event_name)
2412 {
2413         int err = 0;
2414
2415         ref_field->var.idx = var_field->var.idx;
2416         ref_field->var.hist_data = var_field->hist_data;
2417         ref_field->size = var_field->size;
2418         ref_field->is_signed = var_field->is_signed;
2419         ref_field->flags |= var_field->flags &
2420                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2421
2422         if (system) {
2423                 ref_field->system = kstrdup(system, GFP_KERNEL);
2424                 if (!ref_field->system)
2425                         return -ENOMEM;
2426         }
2427
2428         if (event_name) {
2429                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2430                 if (!ref_field->event_name) {
2431                         err = -ENOMEM;
2432                         goto free;
2433                 }
2434         }
2435
2436         if (var_field->var.name) {
2437                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2438                 if (!ref_field->name) {
2439                         err = -ENOMEM;
2440                         goto free;
2441                 }
2442         } else if (var_field->name) {
2443                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2444                 if (!ref_field->name) {
2445                         err = -ENOMEM;
2446                         goto free;
2447                 }
2448         }
2449
2450         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2451         if (!ref_field->type) {
2452                 err = -ENOMEM;
2453                 goto free;
2454         }
2455  out:
2456         return err;
2457  free:
2458         kfree(ref_field->system);
2459         ref_field->system = NULL;
2460         kfree(ref_field->event_name);
2461         ref_field->event_name = NULL;
2462         kfree(ref_field->name);
2463         ref_field->name = NULL;
2464
2465         goto out;
2466 }
2467
2468 /**
2469  * create_var_ref - Create a variable reference and attach it to trigger
2470  * @hist_data: The trigger that will be referencing the variable
2471  * @var_field: The VAR field to create a reference to
2472  * @system: The optional system string
2473  * @event_name: The optional event_name string
2474  *
2475  * Given a variable hist_field, create a VAR_REF hist_field that
2476  * represents a reference to it.
2477  *
2478  * This function also adds the reference to the trigger that
2479  * now references the variable.
2480  *
2481  * Return: The VAR_REF field if successful, NULL if not
2482  */
2483 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2484                                          struct hist_field *var_field,
2485                                          char *system, char *event_name)
2486 {
2487         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2488         struct hist_field *ref_field;
2489         int i;
2490
2491         /* Check if the variable already exists */
2492         for (i = 0; i < hist_data->n_var_refs; i++) {
2493                 ref_field = hist_data->var_refs[i];
2494                 if (ref_field->var.idx == var_field->var.idx &&
2495                     ref_field->var.hist_data == var_field->hist_data) {
2496                         get_hist_field(ref_field);
2497                         return ref_field;
2498                 }
2499         }
2500
2501         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2502         if (ref_field) {
2503                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2504                         destroy_hist_field(ref_field, 0);
2505                         return NULL;
2506                 }
2507
2508                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2509                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2510         }
2511
2512         return ref_field;
2513 }
2514
2515 static bool is_var_ref(char *var_name)
2516 {
2517         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2518                 return false;
2519
2520         return true;
2521 }
2522
2523 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2524                                  char *var_name)
2525 {
2526         char *name, *field;
2527         unsigned int i;
2528
2529         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2530                 name = hist_data->attrs->var_defs.name[i];
2531
2532                 if (strcmp(var_name, name) == 0) {
2533                         field = hist_data->attrs->var_defs.expr[i];
2534                         if (contains_operator(field) || is_var_ref(field))
2535                                 continue;
2536                         return field;
2537                 }
2538         }
2539
2540         return NULL;
2541 }
2542
2543 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2544                                  char *system, char *event_name,
2545                                  char *var_name)
2546 {
2547         struct trace_event_call *call;
2548
2549         if (system && event_name) {
2550                 call = hist_data->event_file->event_call;
2551
2552                 if (strcmp(system, call->class->system) != 0)
2553                         return NULL;
2554
2555                 if (strcmp(event_name, trace_event_name(call)) != 0)
2556                         return NULL;
2557         }
2558
2559         if (!!system != !!event_name)
2560                 return NULL;
2561
2562         if (!is_var_ref(var_name))
2563                 return NULL;
2564
2565         var_name++;
2566
2567         return field_name_from_var(hist_data, var_name);
2568 }
2569
2570 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2571                                         char *system, char *event_name,
2572                                         char *var_name)
2573 {
2574         struct hist_field *var_field = NULL, *ref_field = NULL;
2575
2576         if (!is_var_ref(var_name))
2577                 return NULL;
2578
2579         var_name++;
2580
2581         var_field = find_event_var(hist_data, system, event_name, var_name);
2582         if (var_field)
2583                 ref_field = create_var_ref(hist_data, var_field,
2584                                            system, event_name);
2585
2586         if (!ref_field)
2587                 hist_err_event("Couldn't find variable: $",
2588                                system, event_name, var_name);
2589
2590         return ref_field;
2591 }
2592
2593 static struct ftrace_event_field *
2594 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2595             char *field_str, unsigned long *flags)
2596 {
2597         struct ftrace_event_field *field = NULL;
2598         char *field_name, *modifier, *str;
2599
2600         modifier = str = kstrdup(field_str, GFP_KERNEL);
2601         if (!modifier)
2602                 return ERR_PTR(-ENOMEM);
2603
2604         field_name = strsep(&modifier, ".");
2605         if (modifier) {
2606                 if (strcmp(modifier, "hex") == 0)
2607                         *flags |= HIST_FIELD_FL_HEX;
2608                 else if (strcmp(modifier, "sym") == 0)
2609                         *flags |= HIST_FIELD_FL_SYM;
2610                 else if (strcmp(modifier, "sym-offset") == 0)
2611                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2612                 else if ((strcmp(modifier, "execname") == 0) &&
2613                          (strcmp(field_name, "common_pid") == 0))
2614                         *flags |= HIST_FIELD_FL_EXECNAME;
2615                 else if (strcmp(modifier, "syscall") == 0)
2616                         *flags |= HIST_FIELD_FL_SYSCALL;
2617                 else if (strcmp(modifier, "log2") == 0)
2618                         *flags |= HIST_FIELD_FL_LOG2;
2619                 else if (strcmp(modifier, "usecs") == 0)
2620                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2621                 else {
2622                         hist_err("Invalid field modifier: ", modifier);
2623                         field = ERR_PTR(-EINVAL);
2624                         goto out;
2625                 }
2626         }
2627
2628         if (strcmp(field_name, "common_timestamp") == 0) {
2629                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2630                 hist_data->enable_timestamps = true;
2631                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2632                         hist_data->attrs->ts_in_usecs = true;
2633         } else if (strcmp(field_name, "common_cpu") == 0)
2634                 *flags |= HIST_FIELD_FL_CPU;
2635         else {
2636                 field = trace_find_event_field(file->event_call, field_name);
2637                 if (!field || !field->size) {
2638                         /*
2639                          * For backward compatibility, if field_name
2640                          * was "cpu", then we treat this the same as
2641                          * common_cpu. This also works for "CPU".
2642                          */
2643                         if (field && field->filter_type == FILTER_CPU) {
2644                                 *flags |= HIST_FIELD_FL_CPU;
2645                         } else {
2646                                 hist_err("Couldn't find field: ", field_name);
2647                                 field = ERR_PTR(-EINVAL);
2648                                 goto out;
2649                         }
2650                 }
2651         }
2652  out:
2653         kfree(str);
2654
2655         return field;
2656 }
2657
2658 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2659                                        struct hist_field *var_ref,
2660                                        char *var_name)
2661 {
2662         struct hist_field *alias = NULL;
2663         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2664
2665         alias = create_hist_field(hist_data, NULL, flags, var_name);
2666         if (!alias)
2667                 return NULL;
2668
2669         alias->fn = var_ref->fn;
2670         alias->operands[0] = var_ref;
2671
2672         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2673                 destroy_hist_field(alias, 0);
2674                 return NULL;
2675         }
2676
2677         alias->var_ref_idx = var_ref->var_ref_idx;
2678
2679         return alias;
2680 }
2681
2682 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2683                                      struct trace_event_file *file, char *str,
2684                                      unsigned long *flags, char *var_name)
2685 {
2686         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2687         struct ftrace_event_field *field = NULL;
2688         struct hist_field *hist_field = NULL;
2689         int ret = 0;
2690
2691         s = strchr(str, '.');
2692         if (s) {
2693                 s = strchr(++s, '.');
2694                 if (s) {
2695                         ref_system = strsep(&str, ".");
2696                         if (!str) {
2697                                 ret = -EINVAL;
2698                                 goto out;
2699                         }
2700                         ref_event = strsep(&str, ".");
2701                         if (!str) {
2702                                 ret = -EINVAL;
2703                                 goto out;
2704                         }
2705                         ref_var = str;
2706                 }
2707         }
2708
2709         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2710         if (!s) {
2711                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2712                 if (hist_field) {
2713                         if (var_name) {
2714                                 hist_field = create_alias(hist_data, hist_field, var_name);
2715                                 if (!hist_field) {
2716                                         ret = -ENOMEM;
2717                                         goto out;
2718                                 }
2719                         }
2720                         return hist_field;
2721                 }
2722         } else
2723                 str = s;
2724
2725         field = parse_field(hist_data, file, str, flags);
2726         if (IS_ERR(field)) {
2727                 ret = PTR_ERR(field);
2728                 goto out;
2729         }
2730
2731         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2732         if (!hist_field) {
2733                 ret = -ENOMEM;
2734                 goto out;
2735         }
2736
2737         return hist_field;
2738  out:
2739         return ERR_PTR(ret);
2740 }
2741
2742 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2743                                      struct trace_event_file *file,
2744                                      char *str, unsigned long flags,
2745                                      char *var_name, unsigned int level);
2746
2747 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2748                                       struct trace_event_file *file,
2749                                       char *str, unsigned long flags,
2750                                       char *var_name, unsigned int level)
2751 {
2752         struct hist_field *operand1, *expr = NULL;
2753         unsigned long operand_flags;
2754         int ret = 0;
2755         char *s;
2756
2757         /* we support only -(xxx) i.e. explicit parens required */
2758
2759         if (level > 3) {
2760                 hist_err("Too many subexpressions (3 max): ", str);
2761                 ret = -EINVAL;
2762                 goto free;
2763         }
2764
2765         str++; /* skip leading '-' */
2766
2767         s = strchr(str, '(');
2768         if (s)
2769                 str++;
2770         else {
2771                 ret = -EINVAL;
2772                 goto free;
2773         }
2774
2775         s = strrchr(str, ')');
2776         if (s)
2777                 *s = '\0';
2778         else {
2779                 ret = -EINVAL; /* no closing ')' */
2780                 goto free;
2781         }
2782
2783         flags |= HIST_FIELD_FL_EXPR;
2784         expr = create_hist_field(hist_data, NULL, flags, var_name);
2785         if (!expr) {
2786                 ret = -ENOMEM;
2787                 goto free;
2788         }
2789
2790         operand_flags = 0;
2791         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2792         if (IS_ERR(operand1)) {
2793                 ret = PTR_ERR(operand1);
2794                 goto free;
2795         }
2796         if (operand1->flags & HIST_FIELD_FL_STRING) {
2797                 /* String type can not be the operand of unary operator. */
2798                 destroy_hist_field(operand1, 0);
2799                 ret = -EINVAL;
2800                 goto free;
2801         }
2802
2803         expr->flags |= operand1->flags &
2804                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2805         expr->fn = hist_field_unary_minus;
2806         expr->operands[0] = operand1;
2807         expr->operator = FIELD_OP_UNARY_MINUS;
2808         expr->name = expr_str(expr, 0);
2809         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2810         if (!expr->type) {
2811                 ret = -ENOMEM;
2812                 goto free;
2813         }
2814
2815         return expr;
2816  free:
2817         destroy_hist_field(expr, 0);
2818         return ERR_PTR(ret);
2819 }
2820
2821 static int check_expr_operands(struct hist_field *operand1,
2822                                struct hist_field *operand2)
2823 {
2824         unsigned long operand1_flags = operand1->flags;
2825         unsigned long operand2_flags = operand2->flags;
2826
2827         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2828             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2829                 struct hist_field *var;
2830
2831                 var = find_var_field(operand1->var.hist_data, operand1->name);
2832                 if (!var)
2833                         return -EINVAL;
2834                 operand1_flags = var->flags;
2835         }
2836
2837         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2838             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2839                 struct hist_field *var;
2840
2841                 var = find_var_field(operand2->var.hist_data, operand2->name);
2842                 if (!var)
2843                         return -EINVAL;
2844                 operand2_flags = var->flags;
2845         }
2846
2847         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2848             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2849                 hist_err("Timestamp units in expression don't match", NULL);
2850                 return -EINVAL;
2851         }
2852
2853         return 0;
2854 }
2855
2856 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2857                                      struct trace_event_file *file,
2858                                      char *str, unsigned long flags,
2859                                      char *var_name, unsigned int level)
2860 {
2861         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2862         unsigned long operand_flags;
2863         int field_op, ret = -EINVAL;
2864         char *sep, *operand1_str;
2865
2866         if (level > 3) {
2867                 hist_err("Too many subexpressions (3 max): ", str);
2868                 return ERR_PTR(-EINVAL);
2869         }
2870
2871         field_op = contains_operator(str);
2872
2873         if (field_op == FIELD_OP_NONE)
2874                 return parse_atom(hist_data, file, str, &flags, var_name);
2875
2876         if (field_op == FIELD_OP_UNARY_MINUS)
2877                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2878
2879         switch (field_op) {
2880         case FIELD_OP_MINUS:
2881                 sep = "-";
2882                 break;
2883         case FIELD_OP_PLUS:
2884                 sep = "+";
2885                 break;
2886         default:
2887                 goto free;
2888         }
2889
2890         operand1_str = strsep(&str, sep);
2891         if (!operand1_str || !str)
2892                 goto free;
2893
2894         operand_flags = 0;
2895         operand1 = parse_atom(hist_data, file, operand1_str,
2896                               &operand_flags, NULL);
2897         if (IS_ERR(operand1)) {
2898                 ret = PTR_ERR(operand1);
2899                 operand1 = NULL;
2900                 goto free;
2901         }
2902         if (operand1->flags & HIST_FIELD_FL_STRING) {
2903                 ret = -EINVAL;
2904                 goto free;
2905         }
2906
2907         /* rest of string could be another expression e.g. b+c in a+b+c */
2908         operand_flags = 0;
2909         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2910         if (IS_ERR(operand2)) {
2911                 ret = PTR_ERR(operand2);
2912                 operand2 = NULL;
2913                 goto free;
2914         }
2915         if (operand2->flags & HIST_FIELD_FL_STRING) {
2916                 ret = -EINVAL;
2917                 goto free;
2918         }
2919
2920         ret = check_expr_operands(operand1, operand2);
2921         if (ret)
2922                 goto free;
2923
2924         flags |= HIST_FIELD_FL_EXPR;
2925
2926         flags |= operand1->flags &
2927                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2928
2929         expr = create_hist_field(hist_data, NULL, flags, var_name);
2930         if (!expr) {
2931                 ret = -ENOMEM;
2932                 goto free;
2933         }
2934
2935         operand1->read_once = true;
2936         operand2->read_once = true;
2937
2938         expr->operands[0] = operand1;
2939         expr->operands[1] = operand2;
2940
2941         /* The operand sizes should be the same, so just pick one */
2942         expr->size = operand1->size;
2943
2944         expr->operator = field_op;
2945         expr->name = expr_str(expr, 0);
2946         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2947         if (!expr->type) {
2948                 ret = -ENOMEM;
2949                 goto free;
2950         }
2951
2952         switch (field_op) {
2953         case FIELD_OP_MINUS:
2954                 expr->fn = hist_field_minus;
2955                 break;
2956         case FIELD_OP_PLUS:
2957                 expr->fn = hist_field_plus;
2958                 break;
2959         default:
2960                 ret = -EINVAL;
2961                 goto free;
2962         }
2963
2964         return expr;
2965  free:
2966         destroy_hist_field(operand1, 0);
2967         destroy_hist_field(operand2, 0);
2968         destroy_hist_field(expr, 0);
2969
2970         return ERR_PTR(ret);
2971 }
2972
2973 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2974                                  struct trace_event_file *file)
2975 {
2976         struct event_trigger_data *test;
2977
2978         lockdep_assert_held(&event_mutex);
2979
2980         list_for_each_entry(test, &file->triggers, list) {
2981                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2982                         if (test->private_data == hist_data)
2983                                 return test->filter_str;
2984                 }
2985         }
2986
2987         return NULL;
2988 }
2989
2990 static struct event_command trigger_hist_cmd;
2991 static int event_hist_trigger_func(struct event_command *cmd_ops,
2992                                    struct trace_event_file *file,
2993                                    char *glob, char *cmd, char *param);
2994
2995 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2996                             struct hist_trigger_data *hist_data,
2997                             unsigned int n_keys)
2998 {
2999         struct hist_field *target_hist_field, *hist_field;
3000         unsigned int n, i, j;
3001
3002         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3003                 return false;
3004
3005         i = hist_data->n_vals;
3006         j = target_hist_data->n_vals;
3007
3008         for (n = 0; n < n_keys; n++) {
3009                 hist_field = hist_data->fields[i + n];
3010                 target_hist_field = target_hist_data->fields[j + n];
3011
3012                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3013                         return false;
3014                 if (hist_field->size != target_hist_field->size)
3015                         return false;
3016                 if (hist_field->is_signed != target_hist_field->is_signed)
3017                         return false;
3018         }
3019
3020         return true;
3021 }
3022
3023 static struct hist_trigger_data *
3024 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3025                      struct trace_event_file *file)
3026 {
3027         struct hist_trigger_data *hist_data;
3028         struct event_trigger_data *test;
3029         unsigned int n_keys;
3030
3031         lockdep_assert_held(&event_mutex);
3032
3033         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3034
3035         list_for_each_entry(test, &file->triggers, list) {
3036                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3037                         hist_data = test->private_data;
3038
3039                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3040                                 return hist_data;
3041                 }
3042         }
3043
3044         return NULL;
3045 }
3046
3047 static struct trace_event_file *event_file(struct trace_array *tr,
3048                                            char *system, char *event_name)
3049 {
3050         struct trace_event_file *file;
3051
3052         file = __find_event_file(tr, system, event_name);
3053         if (!file)
3054                 return ERR_PTR(-EINVAL);
3055
3056         return file;
3057 }
3058
3059 static struct hist_field *
3060 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3061                          char *system, char *event_name, char *field_name)
3062 {
3063         struct hist_field *event_var;
3064         char *synthetic_name;
3065
3066         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3067         if (!synthetic_name)
3068                 return ERR_PTR(-ENOMEM);
3069
3070         strcpy(synthetic_name, "synthetic_");
3071         strcat(synthetic_name, field_name);
3072
3073         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3074
3075         kfree(synthetic_name);
3076
3077         return event_var;
3078 }
3079
3080 /**
3081  * create_field_var_hist - Automatically create a histogram and var for a field
3082  * @target_hist_data: The target hist trigger
3083  * @subsys_name: Optional subsystem name
3084  * @event_name: Optional event name
3085  * @field_name: The name of the field (and the resulting variable)
3086  *
3087  * Hist trigger actions fetch data from variables, not directly from
3088  * events.  However, for convenience, users are allowed to directly
3089  * specify an event field in an action, which will be automatically
3090  * converted into a variable on their behalf.
3091
3092  * If a user specifies a field on an event that isn't the event the
3093  * histogram currently being defined (the target event histogram), the
3094  * only way that can be accomplished is if a new hist trigger is
3095  * created and the field variable defined on that.
3096  *
3097  * This function creates a new histogram compatible with the target
3098  * event (meaning a histogram with the same key as the target
3099  * histogram), and creates a variable for the specified field, but
3100  * with 'synthetic_' prepended to the variable name in order to avoid
3101  * collision with normal field variables.
3102  *
3103  * Return: The variable created for the field.
3104  */
3105 static struct hist_field *
3106 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3107                       char *subsys_name, char *event_name, char *field_name)
3108 {
3109         struct trace_array *tr = target_hist_data->event_file->tr;
3110         struct hist_field *event_var = ERR_PTR(-EINVAL);
3111         struct hist_trigger_data *hist_data;
3112         unsigned int i, n, first = true;
3113         struct field_var_hist *var_hist;
3114         struct trace_event_file *file;
3115         struct hist_field *key_field;
3116         char *saved_filter;
3117         char *cmd;
3118         int ret;
3119
3120         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3121                 hist_err_event("onmatch: Too many field variables defined: ",
3122                                subsys_name, event_name, field_name);
3123                 return ERR_PTR(-EINVAL);
3124         }
3125
3126         file = event_file(tr, subsys_name, event_name);
3127
3128         if (IS_ERR(file)) {
3129                 hist_err_event("onmatch: Event file not found: ",
3130                                subsys_name, event_name, field_name);
3131                 ret = PTR_ERR(file);
3132                 return ERR_PTR(ret);
3133         }
3134
3135         /*
3136          * Look for a histogram compatible with target.  We'll use the
3137          * found histogram specification to create a new matching
3138          * histogram with our variable on it.  target_hist_data is not
3139          * yet a registered histogram so we can't use that.
3140          */
3141         hist_data = find_compatible_hist(target_hist_data, file);
3142         if (!hist_data) {
3143                 hist_err_event("onmatch: Matching event histogram not found: ",
3144                                subsys_name, event_name, field_name);
3145                 return ERR_PTR(-EINVAL);
3146         }
3147
3148         /* See if a synthetic field variable has already been created */
3149         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3150                                              event_name, field_name);
3151         if (!IS_ERR_OR_NULL(event_var))
3152                 return event_var;
3153
3154         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3155         if (!var_hist)
3156                 return ERR_PTR(-ENOMEM);
3157
3158         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3159         if (!cmd) {
3160                 kfree(var_hist);
3161                 return ERR_PTR(-ENOMEM);
3162         }
3163
3164         /* Use the same keys as the compatible histogram */
3165         strcat(cmd, "keys=");
3166
3167         for_each_hist_key_field(i, hist_data) {
3168                 key_field = hist_data->fields[i];
3169                 if (!first)
3170                         strcat(cmd, ",");
3171                 strcat(cmd, key_field->field->name);
3172                 first = false;
3173         }
3174
3175         /* Create the synthetic field variable specification */
3176         strcat(cmd, ":synthetic_");
3177         strcat(cmd, field_name);
3178         strcat(cmd, "=");
3179         strcat(cmd, field_name);
3180
3181         /* Use the same filter as the compatible histogram */
3182         saved_filter = find_trigger_filter(hist_data, file);
3183         if (saved_filter) {
3184                 strcat(cmd, " if ");
3185                 strcat(cmd, saved_filter);
3186         }
3187
3188         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3189         if (!var_hist->cmd) {
3190                 kfree(cmd);
3191                 kfree(var_hist);
3192                 return ERR_PTR(-ENOMEM);
3193         }
3194
3195         /* Save the compatible histogram information */
3196         var_hist->hist_data = hist_data;
3197
3198         /* Create the new histogram with our variable */
3199         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3200                                       "", "hist", cmd);
3201         if (ret) {
3202                 kfree(cmd);
3203                 kfree(var_hist->cmd);
3204                 kfree(var_hist);
3205                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3206                                subsys_name, event_name, field_name);
3207                 return ERR_PTR(ret);
3208         }
3209
3210         kfree(cmd);
3211
3212         /* If we can't find the variable, something went wrong */
3213         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3214                                              event_name, field_name);
3215         if (IS_ERR_OR_NULL(event_var)) {
3216                 kfree(var_hist->cmd);
3217                 kfree(var_hist);
3218                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3219                                subsys_name, event_name, field_name);
3220                 return ERR_PTR(-EINVAL);
3221         }
3222
3223         n = target_hist_data->n_field_var_hists;
3224         target_hist_data->field_var_hists[n] = var_hist;
3225         target_hist_data->n_field_var_hists++;
3226
3227         return event_var;
3228 }
3229
3230 static struct hist_field *
3231 find_target_event_var(struct hist_trigger_data *hist_data,
3232                       char *subsys_name, char *event_name, char *var_name)
3233 {
3234         struct trace_event_file *file = hist_data->event_file;
3235         struct hist_field *hist_field = NULL;
3236
3237         if (subsys_name) {
3238                 struct trace_event_call *call;
3239
3240                 if (!event_name)
3241                         return NULL;
3242
3243                 call = file->event_call;
3244
3245                 if (strcmp(subsys_name, call->class->system) != 0)
3246                         return NULL;
3247
3248                 if (strcmp(event_name, trace_event_name(call)) != 0)
3249                         return NULL;
3250         }
3251
3252         hist_field = find_var_field(hist_data, var_name);
3253
3254         return hist_field;
3255 }
3256
3257 static inline void __update_field_vars(struct tracing_map_elt *elt,
3258                                        struct ring_buffer_event *rbe,
3259                                        void *rec,
3260                                        struct field_var **field_vars,
3261                                        unsigned int n_field_vars,
3262                                        unsigned int field_var_str_start)
3263 {
3264         struct hist_elt_data *elt_data = elt->private_data;
3265         unsigned int i, j, var_idx;
3266         u64 var_val;
3267
3268         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3269                 struct field_var *field_var = field_vars[i];
3270                 struct hist_field *var = field_var->var;
3271                 struct hist_field *val = field_var->val;
3272
3273                 var_val = val->fn(val, elt, rbe, rec);
3274                 var_idx = var->var.idx;
3275
3276                 if (val->flags & HIST_FIELD_FL_STRING) {
3277                         char *str = elt_data->field_var_str[j++];
3278                         char *val_str = (char *)(uintptr_t)var_val;
3279
3280                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3281                         var_val = (u64)(uintptr_t)str;
3282                 }
3283                 tracing_map_set_var(elt, var_idx, var_val);
3284         }
3285 }
3286
3287 static void update_field_vars(struct hist_trigger_data *hist_data,
3288                               struct tracing_map_elt *elt,
3289                               struct ring_buffer_event *rbe,
3290                               void *rec)
3291 {
3292         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3293                             hist_data->n_field_vars, 0);
3294 }
3295
3296 static void update_max_vars(struct hist_trigger_data *hist_data,
3297                             struct tracing_map_elt *elt,
3298                             struct ring_buffer_event *rbe,
3299                             void *rec)
3300 {
3301         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3302                             hist_data->n_max_vars, hist_data->n_field_var_str);
3303 }
3304
3305 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3306                                      struct trace_event_file *file,
3307                                      char *name, int size, const char *type)
3308 {
3309         struct hist_field *var;
3310         int idx;
3311
3312         if (find_var(hist_data, file, name) && !hist_data->remove) {
3313                 var = ERR_PTR(-EINVAL);
3314                 goto out;
3315         }
3316
3317         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3318         if (!var) {
3319                 var = ERR_PTR(-ENOMEM);
3320                 goto out;
3321         }
3322
3323         idx = tracing_map_add_var(hist_data->map);
3324         if (idx < 0) {
3325                 kfree(var);
3326                 var = ERR_PTR(-EINVAL);
3327                 goto out;
3328         }
3329
3330         var->flags = HIST_FIELD_FL_VAR;
3331         var->var.idx = idx;
3332         var->var.hist_data = var->hist_data = hist_data;
3333         var->size = size;
3334         var->var.name = kstrdup(name, GFP_KERNEL);
3335         var->type = kstrdup(type, GFP_KERNEL);
3336         if (!var->var.name || !var->type) {
3337                 kfree(var->var.name);
3338                 kfree(var->type);
3339                 kfree(var);
3340                 var = ERR_PTR(-ENOMEM);
3341         }
3342  out:
3343         return var;
3344 }
3345
3346 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3347                                           struct trace_event_file *file,
3348                                           char *field_name)
3349 {
3350         struct hist_field *val = NULL, *var = NULL;
3351         unsigned long flags = HIST_FIELD_FL_VAR;
3352         struct field_var *field_var;
3353         int ret = 0;
3354
3355         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3356                 hist_err("Too many field variables defined: ", field_name);
3357                 ret = -EINVAL;
3358                 goto err;
3359         }
3360
3361         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3362         if (IS_ERR(val)) {
3363                 hist_err("Couldn't parse field variable: ", field_name);
3364                 ret = PTR_ERR(val);
3365                 goto err;
3366         }
3367
3368         var = create_var(hist_data, file, field_name, val->size, val->type);
3369         if (IS_ERR(var)) {
3370                 hist_err("Couldn't create or find variable: ", field_name);
3371                 kfree(val);
3372                 ret = PTR_ERR(var);
3373                 goto err;
3374         }
3375
3376         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3377         if (!field_var) {
3378                 kfree(val);
3379                 kfree(var);
3380                 ret =  -ENOMEM;
3381                 goto err;
3382         }
3383
3384         field_var->var = var;
3385         field_var->val = val;
3386  out:
3387         return field_var;
3388  err:
3389         field_var = ERR_PTR(ret);
3390         goto out;
3391 }
3392
3393 /**
3394  * create_target_field_var - Automatically create a variable for a field
3395  * @target_hist_data: The target hist trigger
3396  * @subsys_name: Optional subsystem name
3397  * @event_name: Optional event name
3398  * @var_name: The name of the field (and the resulting variable)
3399  *
3400  * Hist trigger actions fetch data from variables, not directly from
3401  * events.  However, for convenience, users are allowed to directly
3402  * specify an event field in an action, which will be automatically
3403  * converted into a variable on their behalf.
3404
3405  * This function creates a field variable with the name var_name on
3406  * the hist trigger currently being defined on the target event.  If
3407  * subsys_name and event_name are specified, this function simply
3408  * verifies that they do in fact match the target event subsystem and
3409  * event name.
3410  *
3411  * Return: The variable created for the field.
3412  */
3413 static struct field_var *
3414 create_target_field_var(struct hist_trigger_data *target_hist_data,
3415                         char *subsys_name, char *event_name, char *var_name)
3416 {
3417         struct trace_event_file *file = target_hist_data->event_file;
3418
3419         if (subsys_name) {
3420                 struct trace_event_call *call;
3421
3422                 if (!event_name)
3423                         return NULL;
3424
3425                 call = file->event_call;
3426
3427                 if (strcmp(subsys_name, call->class->system) != 0)
3428                         return NULL;
3429
3430                 if (strcmp(event_name, trace_event_name(call)) != 0)
3431                         return NULL;
3432         }
3433
3434         return create_field_var(target_hist_data, file, var_name);
3435 }
3436
3437 static void onmax_print(struct seq_file *m,
3438                         struct hist_trigger_data *hist_data,
3439                         struct tracing_map_elt *elt,
3440                         struct action_data *data)
3441 {
3442         unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3443
3444         seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3445
3446         for (i = 0; i < hist_data->n_max_vars; i++) {
3447                 struct hist_field *save_val = hist_data->max_vars[i]->val;
3448                 struct hist_field *save_var = hist_data->max_vars[i]->var;
3449                 u64 val;
3450
3451                 save_var_idx = save_var->var.idx;
3452
3453                 val = tracing_map_read_var(elt, save_var_idx);
3454
3455                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3456                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3457                                    (char *)(uintptr_t)(val));
3458                 } else
3459                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3460         }
3461 }
3462
3463 static void onmax_save(struct hist_trigger_data *hist_data,
3464                        struct tracing_map_elt *elt, void *rec,
3465                        struct ring_buffer_event *rbe,
3466                        struct action_data *data, u64 *var_ref_vals)
3467 {
3468         unsigned int max_idx = data->onmax.max_var->var.idx;
3469         unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3470
3471         u64 var_val, max_val;
3472
3473         var_val = var_ref_vals[max_var_ref_idx];
3474         max_val = tracing_map_read_var(elt, max_idx);
3475
3476         if (var_val <= max_val)
3477                 return;
3478
3479         tracing_map_set_var(elt, max_idx, var_val);
3480
3481         update_max_vars(hist_data, elt, rbe, rec);
3482 }
3483
3484 static void onmax_destroy(struct action_data *data)
3485 {
3486         unsigned int i;
3487
3488         destroy_hist_field(data->onmax.max_var, 0);
3489         destroy_hist_field(data->onmax.var, 0);
3490
3491         kfree(data->onmax.var_str);
3492         kfree(data->onmax.fn_name);
3493
3494         for (i = 0; i < data->n_params; i++)
3495                 kfree(data->params[i]);
3496
3497         kfree(data);
3498 }
3499
3500 static int onmax_create(struct hist_trigger_data *hist_data,
3501                         struct action_data *data)
3502 {
3503         struct trace_event_file *file = hist_data->event_file;
3504         struct hist_field *var_field, *ref_field, *max_var;
3505         unsigned int var_ref_idx = hist_data->n_var_refs;
3506         struct field_var *field_var;
3507         char *onmax_var_str, *param;
3508         unsigned int i;
3509         int ret = 0;
3510
3511         onmax_var_str = data->onmax.var_str;
3512         if (onmax_var_str[0] != '$') {
3513                 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3514                 return -EINVAL;
3515         }
3516         onmax_var_str++;
3517
3518         var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3519         if (!var_field) {
3520                 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3521                 return -EINVAL;
3522         }
3523
3524         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3525         if (!ref_field)
3526                 return -ENOMEM;
3527
3528         data->onmax.var = ref_field;
3529
3530         data->fn = onmax_save;
3531         data->onmax.max_var_ref_idx = var_ref_idx;
3532         max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3533         if (IS_ERR(max_var)) {
3534                 hist_err("onmax: Couldn't create onmax variable: ", "max");
3535                 ret = PTR_ERR(max_var);
3536                 goto out;
3537         }
3538         data->onmax.max_var = max_var;
3539
3540         for (i = 0; i < data->n_params; i++) {
3541                 param = kstrdup(data->params[i], GFP_KERNEL);
3542                 if (!param) {
3543                         ret = -ENOMEM;
3544                         goto out;
3545                 }
3546
3547                 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3548                 if (IS_ERR(field_var)) {
3549                         hist_err("onmax: Couldn't create field variable: ", param);
3550                         ret = PTR_ERR(field_var);
3551                         kfree(param);
3552                         goto out;
3553                 }
3554
3555                 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3556                 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3557                         hist_data->n_max_var_str++;
3558
3559                 kfree(param);
3560         }
3561  out:
3562         return ret;
3563 }
3564
3565 static int parse_action_params(char *params, struct action_data *data)
3566 {
3567         char *param, *saved_param;
3568         int ret = 0;
3569
3570         while (params) {
3571                 if (data->n_params >= SYNTH_FIELDS_MAX)
3572                         goto out;
3573
3574                 param = strsep(&params, ",");
3575                 if (!param) {
3576                         ret = -EINVAL;
3577                         goto out;
3578                 }
3579
3580                 param = strstrip(param);
3581                 if (strlen(param) < 2) {
3582                         hist_err("Invalid action param: ", param);
3583                         ret = -EINVAL;
3584                         goto out;
3585                 }
3586
3587                 saved_param = kstrdup(param, GFP_KERNEL);
3588                 if (!saved_param) {
3589                         ret = -ENOMEM;
3590                         goto out;
3591                 }
3592
3593                 data->params[data->n_params++] = saved_param;
3594         }
3595  out:
3596         return ret;
3597 }
3598
3599 static struct action_data *onmax_parse(char *str)
3600 {
3601         char *onmax_fn_name, *onmax_var_str;
3602         struct action_data *data;
3603         int ret = -EINVAL;
3604
3605         data = kzalloc(sizeof(*data), GFP_KERNEL);
3606         if (!data)
3607                 return ERR_PTR(-ENOMEM);
3608
3609         onmax_var_str = strsep(&str, ")");
3610         if (!onmax_var_str || !str) {
3611                 ret = -EINVAL;
3612                 goto free;
3613         }
3614
3615         data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3616         if (!data->onmax.var_str) {
3617                 ret = -ENOMEM;
3618                 goto free;
3619         }
3620
3621         strsep(&str, ".");
3622         if (!str)
3623                 goto free;
3624
3625         onmax_fn_name = strsep(&str, "(");
3626         if (!onmax_fn_name || !str)
3627                 goto free;
3628
3629         if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3630                 char *params = strsep(&str, ")");
3631
3632                 if (!params) {
3633                         ret = -EINVAL;
3634                         goto free;
3635                 }
3636
3637                 ret = parse_action_params(params, data);
3638                 if (ret)
3639                         goto free;
3640         } else
3641                 goto free;
3642
3643         data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3644         if (!data->onmax.fn_name) {
3645                 ret = -ENOMEM;
3646                 goto free;
3647         }
3648  out:
3649         return data;
3650  free:
3651         onmax_destroy(data);
3652         data = ERR_PTR(ret);
3653         goto out;
3654 }
3655
3656 static void onmatch_destroy(struct action_data *data)
3657 {
3658         unsigned int i;
3659
3660         mutex_lock(&synth_event_mutex);
3661
3662         kfree(data->onmatch.match_event);
3663         kfree(data->onmatch.match_event_system);
3664         kfree(data->onmatch.synth_event_name);
3665
3666         for (i = 0; i < data->n_params; i++)
3667                 kfree(data->params[i]);
3668
3669         if (data->onmatch.synth_event)
3670                 data->onmatch.synth_event->ref--;
3671
3672         kfree(data);
3673
3674         mutex_unlock(&synth_event_mutex);
3675 }
3676
3677 static void destroy_field_var(struct field_var *field_var)
3678 {
3679         if (!field_var)
3680                 return;
3681
3682         destroy_hist_field(field_var->var, 0);
3683         destroy_hist_field(field_var->val, 0);
3684
3685         kfree(field_var);
3686 }
3687
3688 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3689 {
3690         unsigned int i;
3691
3692         for (i = 0; i < hist_data->n_field_vars; i++)
3693                 destroy_field_var(hist_data->field_vars[i]);
3694 }
3695
3696 static void save_field_var(struct hist_trigger_data *hist_data,
3697                            struct field_var *field_var)
3698 {
3699         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3700
3701         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3702                 hist_data->n_field_var_str++;
3703 }
3704
3705
3706 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3707 {
3708         unsigned int i;
3709
3710         for (i = 0; i < hist_data->n_synth_var_refs; i++)
3711                 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3712 }
3713
3714 static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3715                          struct hist_field *var_ref)
3716 {
3717         hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3718 }
3719
3720 static int check_synth_field(struct synth_event *event,
3721                              struct hist_field *hist_field,
3722                              unsigned int field_pos)
3723 {
3724         struct synth_field *field;
3725
3726         if (field_pos >= event->n_fields)
3727                 return -EINVAL;
3728
3729         field = event->fields[field_pos];
3730
3731         if (strcmp(field->type, hist_field->type) != 0)
3732                 return -EINVAL;
3733
3734         return 0;
3735 }
3736
3737 static struct hist_field *
3738 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3739                  char *system, char *event, char *var)
3740 {
3741         struct hist_field *hist_field;
3742
3743         var++; /* skip '$' */
3744
3745         hist_field = find_target_event_var(hist_data, system, event, var);
3746         if (!hist_field) {
3747                 if (!system) {
3748                         system = data->onmatch.match_event_system;
3749                         event = data->onmatch.match_event;
3750                 }
3751
3752                 hist_field = find_event_var(hist_data, system, event, var);
3753         }
3754
3755         if (!hist_field)
3756                 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3757
3758         return hist_field;
3759 }
3760
3761 static struct hist_field *
3762 onmatch_create_field_var(struct hist_trigger_data *hist_data,
3763                          struct action_data *data, char *system,
3764                          char *event, char *var)
3765 {
3766         struct hist_field *hist_field = NULL;
3767         struct field_var *field_var;
3768
3769         /*
3770          * First try to create a field var on the target event (the
3771          * currently being defined).  This will create a variable for
3772          * unqualified fields on the target event, or if qualified,
3773          * target fields that have qualified names matching the target.
3774          */
3775         field_var = create_target_field_var(hist_data, system, event, var);
3776
3777         if (field_var && !IS_ERR(field_var)) {
3778                 save_field_var(hist_data, field_var);
3779                 hist_field = field_var->var;
3780         } else {
3781                 field_var = NULL;
3782                 /*
3783                  * If no explicit system.event is specfied, default to
3784                  * looking for fields on the onmatch(system.event.xxx)
3785                  * event.
3786                  */
3787                 if (!system) {
3788                         system = data->onmatch.match_event_system;
3789                         event = data->onmatch.match_event;
3790                 }
3791
3792                 if (!event)
3793                         goto free;
3794                 /*
3795                  * At this point, we're looking at a field on another
3796                  * event.  Because we can't modify a hist trigger on
3797                  * another event to add a variable for a field, we need
3798                  * to create a new trigger on that event and create the
3799                  * variable at the same time.
3800                  */
3801                 hist_field = create_field_var_hist(hist_data, system, event, var);
3802                 if (IS_ERR(hist_field))
3803                         goto free;
3804         }
3805  out:
3806         return hist_field;
3807  free:
3808         destroy_field_var(field_var);
3809         hist_field = NULL;
3810         goto out;
3811 }
3812
3813 static int onmatch_create(struct hist_trigger_data *hist_data,
3814                           struct trace_event_file *file,
3815                           struct action_data *data)
3816 {
3817         char *event_name, *param, *system = NULL;
3818         struct hist_field *hist_field, *var_ref;
3819         unsigned int i, var_ref_idx;
3820         unsigned int field_pos = 0;
3821         struct synth_event *event;
3822         int ret = 0;
3823
3824         mutex_lock(&synth_event_mutex);
3825         event = find_synth_event(data->onmatch.synth_event_name);
3826         if (!event) {
3827                 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3828                 mutex_unlock(&synth_event_mutex);
3829                 return -EINVAL;
3830         }
3831         event->ref++;
3832         mutex_unlock(&synth_event_mutex);
3833
3834         var_ref_idx = hist_data->n_var_refs;
3835
3836         for (i = 0; i < data->n_params; i++) {
3837                 char *p;
3838
3839                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3840                 if (!param) {
3841                         ret = -ENOMEM;
3842                         goto err;
3843                 }
3844
3845                 system = strsep(&param, ".");
3846                 if (!param) {
3847                         param = (char *)system;
3848                         system = event_name = NULL;
3849                 } else {
3850                         event_name = strsep(&param, ".");
3851                         if (!param) {
3852                                 kfree(p);
3853                                 ret = -EINVAL;
3854                                 goto err;
3855                         }
3856                 }
3857
3858                 if (param[0] == '$')
3859                         hist_field = onmatch_find_var(hist_data, data, system,
3860                                                       event_name, param);
3861                 else
3862                         hist_field = onmatch_create_field_var(hist_data, data,
3863                                                               system,
3864                                                               event_name,
3865                                                               param);
3866
3867                 if (!hist_field) {
3868                         kfree(p);
3869                         ret = -EINVAL;
3870                         goto err;
3871                 }
3872
3873                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3874                         var_ref = create_var_ref(hist_data, hist_field,
3875                                                  system, event_name);
3876                         if (!var_ref) {
3877                                 kfree(p);
3878                                 ret = -ENOMEM;
3879                                 goto err;
3880                         }
3881
3882                         save_synth_var_ref(hist_data, var_ref);
3883                         field_pos++;
3884                         kfree(p);
3885                         continue;
3886                 }
3887
3888                 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3889                                system, event_name, param);
3890                 kfree(p);
3891                 ret = -EINVAL;
3892                 goto err;
3893         }
3894
3895         if (field_pos != event->n_fields) {
3896                 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3897                 ret = -EINVAL;
3898                 goto err;
3899         }
3900
3901         data->fn = action_trace;
3902         data->onmatch.synth_event = event;
3903         data->onmatch.var_ref_idx = var_ref_idx;
3904  out:
3905         return ret;
3906  err:
3907         mutex_lock(&synth_event_mutex);
3908         event->ref--;
3909         mutex_unlock(&synth_event_mutex);
3910
3911         goto out;
3912 }
3913
3914 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3915 {
3916         char *match_event, *match_event_system;
3917         char *synth_event_name, *params;
3918         struct action_data *data;
3919         int ret = -EINVAL;
3920
3921         data = kzalloc(sizeof(*data), GFP_KERNEL);
3922         if (!data)
3923                 return ERR_PTR(-ENOMEM);
3924
3925         match_event = strsep(&str, ")");
3926         if (!match_event || !str) {
3927                 hist_err("onmatch: Missing closing paren: ", match_event);
3928                 goto free;
3929         }
3930
3931         match_event_system = strsep(&match_event, ".");
3932         if (!match_event) {
3933                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3934                 goto free;
3935         }
3936
3937         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3938                 hist_err_event("onmatch: Invalid subsystem or event name: ",
3939                                match_event_system, match_event, NULL);
3940                 goto free;
3941         }
3942
3943         data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3944         if (!data->onmatch.match_event) {
3945                 ret = -ENOMEM;
3946                 goto free;
3947         }
3948
3949         data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3950         if (!data->onmatch.match_event_system) {
3951                 ret = -ENOMEM;
3952                 goto free;
3953         }
3954
3955         strsep(&str, ".");
3956         if (!str) {
3957                 hist_err("onmatch: Missing . after onmatch(): ", str);
3958                 goto free;
3959         }
3960
3961         synth_event_name = strsep(&str, "(");
3962         if (!synth_event_name || !str) {
3963                 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3964                 goto free;
3965         }
3966
3967         data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3968         if (!data->onmatch.synth_event_name) {
3969                 ret = -ENOMEM;
3970                 goto free;
3971         }
3972
3973         params = strsep(&str, ")");
3974         if (!params || !str || (str && strlen(str))) {
3975                 hist_err("onmatch: Missing closing paramlist paren: ", params);
3976                 goto free;
3977         }
3978
3979         ret = parse_action_params(params, data);
3980         if (ret)
3981                 goto free;
3982  out:
3983         return data;
3984  free:
3985         onmatch_destroy(data);
3986         data = ERR_PTR(ret);
3987         goto out;
3988 }
3989
3990 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3991 {
3992         hist_data->fields[HITCOUNT_IDX] =
3993                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3994         if (!hist_data->fields[HITCOUNT_IDX])
3995                 return -ENOMEM;
3996
3997         hist_data->n_vals++;
3998         hist_data->n_fields++;
3999
4000         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4001                 return -EINVAL;
4002
4003         return 0;
4004 }
4005
4006 static int __create_val_field(struct hist_trigger_data *hist_data,
4007                               unsigned int val_idx,
4008                               struct trace_event_file *file,
4009                               char *var_name, char *field_str,
4010                               unsigned long flags)
4011 {
4012         struct hist_field *hist_field;
4013         int ret = 0;
4014
4015         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4016         if (IS_ERR(hist_field)) {
4017                 ret = PTR_ERR(hist_field);
4018                 goto out;
4019         }
4020
4021         hist_data->fields[val_idx] = hist_field;
4022
4023         ++hist_data->n_vals;
4024         ++hist_data->n_fields;
4025
4026         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4027                 ret = -EINVAL;
4028  out:
4029         return ret;
4030 }
4031
4032 static int create_val_field(struct hist_trigger_data *hist_data,
4033                             unsigned int val_idx,
4034                             struct trace_event_file *file,
4035                             char *field_str)
4036 {
4037         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4038                 return -EINVAL;
4039
4040         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4041 }
4042
4043 static int create_var_field(struct hist_trigger_data *hist_data,
4044                             unsigned int val_idx,
4045                             struct trace_event_file *file,
4046                             char *var_name, char *expr_str)
4047 {
4048         unsigned long flags = 0;
4049
4050         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4051                 return -EINVAL;
4052
4053         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4054                 hist_err("Variable already defined: ", var_name);
4055                 return -EINVAL;
4056         }
4057
4058         flags |= HIST_FIELD_FL_VAR;
4059         hist_data->n_vars++;
4060         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4061                 return -EINVAL;
4062
4063         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4064 }
4065
4066 static int create_val_fields(struct hist_trigger_data *hist_data,
4067                              struct trace_event_file *file)
4068 {
4069         char *fields_str, *field_str;
4070         unsigned int i, j = 1;
4071         int ret;
4072
4073         ret = create_hitcount_val(hist_data);
4074         if (ret)
4075                 goto out;
4076
4077         fields_str = hist_data->attrs->vals_str;
4078         if (!fields_str)
4079                 goto out;
4080
4081         strsep(&fields_str, "=");
4082         if (!fields_str)
4083                 goto out;
4084
4085         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4086                      j < TRACING_MAP_VALS_MAX; i++) {
4087                 field_str = strsep(&fields_str, ",");
4088                 if (!field_str)
4089                         break;
4090
4091                 if (strcmp(field_str, "hitcount") == 0)
4092                         continue;
4093
4094                 ret = create_val_field(hist_data, j++, file, field_str);
4095                 if (ret)
4096                         goto out;
4097         }
4098
4099         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4100                 ret = -EINVAL;
4101  out:
4102         return ret;
4103 }
4104
4105 static int create_key_field(struct hist_trigger_data *hist_data,
4106                             unsigned int key_idx,
4107                             unsigned int key_offset,
4108                             struct trace_event_file *file,
4109                             char *field_str)
4110 {
4111         struct hist_field *hist_field = NULL;
4112
4113         unsigned long flags = 0;
4114         unsigned int key_size;
4115         int ret = 0;
4116
4117         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4118                 return -EINVAL;
4119
4120         flags |= HIST_FIELD_FL_KEY;
4121
4122         if (strcmp(field_str, "stacktrace") == 0) {
4123                 flags |= HIST_FIELD_FL_STACKTRACE;
4124                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4125                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4126         } else {
4127                 hist_field = parse_expr(hist_data, file, field_str, flags,
4128                                         NULL, 0);
4129                 if (IS_ERR(hist_field)) {
4130                         ret = PTR_ERR(hist_field);
4131                         goto out;
4132                 }
4133
4134                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4135                         hist_err("Using variable references as keys not supported: ", field_str);
4136                         destroy_hist_field(hist_field, 0);
4137                         ret = -EINVAL;
4138                         goto out;
4139                 }
4140
4141                 key_size = hist_field->size;
4142         }
4143
4144         hist_data->fields[key_idx] = hist_field;
4145
4146         key_size = ALIGN(key_size, sizeof(u64));
4147         hist_data->fields[key_idx]->size = key_size;
4148         hist_data->fields[key_idx]->offset = key_offset;
4149
4150         hist_data->key_size += key_size;
4151
4152         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4153                 ret = -EINVAL;
4154                 goto out;
4155         }
4156
4157         hist_data->n_keys++;
4158         hist_data->n_fields++;
4159
4160         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4161                 return -EINVAL;
4162
4163         ret = key_size;
4164  out:
4165         return ret;
4166 }
4167
4168 static int create_key_fields(struct hist_trigger_data *hist_data,
4169                              struct trace_event_file *file)
4170 {
4171         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4172         char *fields_str, *field_str;
4173         int ret = -EINVAL;
4174
4175         fields_str = hist_data->attrs->keys_str;
4176         if (!fields_str)
4177                 goto out;
4178
4179         strsep(&fields_str, "=");
4180         if (!fields_str)
4181                 goto out;
4182
4183         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4184                 field_str = strsep(&fields_str, ",");
4185                 if (!field_str)
4186                         break;
4187                 ret = create_key_field(hist_data, i, key_offset,
4188                                        file, field_str);
4189                 if (ret < 0)
4190                         goto out;
4191                 key_offset += ret;
4192         }
4193         if (fields_str) {
4194                 ret = -EINVAL;
4195                 goto out;
4196         }
4197         ret = 0;
4198  out:
4199         return ret;
4200 }
4201
4202 static int create_var_fields(struct hist_trigger_data *hist_data,
4203                              struct trace_event_file *file)
4204 {
4205         unsigned int i, j = hist_data->n_vals;
4206         int ret = 0;
4207
4208         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4209
4210         for (i = 0; i < n_vars; i++) {
4211                 char *var_name = hist_data->attrs->var_defs.name[i];
4212                 char *expr = hist_data->attrs->var_defs.expr[i];
4213
4214                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4215                 if (ret)
4216                         goto out;
4217         }
4218  out:
4219         return ret;
4220 }
4221
4222 static void free_var_defs(struct hist_trigger_data *hist_data)
4223 {
4224         unsigned int i;
4225
4226         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4227                 kfree(hist_data->attrs->var_defs.name[i]);
4228                 kfree(hist_data->attrs->var_defs.expr[i]);
4229         }
4230
4231         hist_data->attrs->var_defs.n_vars = 0;
4232 }
4233
4234 static int parse_var_defs(struct hist_trigger_data *hist_data)
4235 {
4236         char *s, *str, *var_name, *field_str;
4237         unsigned int i, j, n_vars = 0;
4238         int ret = 0;
4239
4240         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4241                 str = hist_data->attrs->assignment_str[i];
4242                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4243                         field_str = strsep(&str, ",");
4244                         if (!field_str)
4245                                 break;
4246
4247                         var_name = strsep(&field_str, "=");
4248                         if (!var_name || !field_str) {
4249                                 hist_err("Malformed assignment: ", var_name);
4250                                 ret = -EINVAL;
4251                                 goto free;
4252                         }
4253
4254                         if (n_vars == TRACING_MAP_VARS_MAX) {
4255                                 hist_err("Too many variables defined: ", var_name);
4256                                 ret = -EINVAL;
4257                                 goto free;
4258                         }
4259
4260                         s = kstrdup(var_name, GFP_KERNEL);
4261                         if (!s) {
4262                                 ret = -ENOMEM;
4263                                 goto free;
4264                         }
4265                         hist_data->attrs->var_defs.name[n_vars] = s;
4266
4267                         s = kstrdup(field_str, GFP_KERNEL);
4268                         if (!s) {
4269                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4270                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4271                                 ret = -ENOMEM;
4272                                 goto free;
4273                         }
4274                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4275
4276                         hist_data->attrs->var_defs.n_vars = n_vars;
4277                 }
4278         }
4279
4280         return ret;
4281  free:
4282         free_var_defs(hist_data);
4283
4284         return ret;
4285 }
4286
4287 static int create_hist_fields(struct hist_trigger_data *hist_data,
4288                               struct trace_event_file *file)
4289 {
4290         int ret;
4291
4292         ret = parse_var_defs(hist_data);
4293         if (ret)
4294                 goto out;
4295
4296         ret = create_val_fields(hist_data, file);
4297         if (ret)
4298                 goto out;
4299
4300         ret = create_var_fields(hist_data, file);
4301         if (ret)
4302                 goto out;
4303
4304         ret = create_key_fields(hist_data, file);
4305         if (ret)
4306                 goto out;
4307  out:
4308         free_var_defs(hist_data);
4309
4310         return ret;
4311 }
4312
4313 static int is_descending(const char *str)
4314 {
4315         if (!str)
4316                 return 0;
4317
4318         if (strcmp(str, "descending") == 0)
4319                 return 1;
4320
4321         if (strcmp(str, "ascending") == 0)
4322                 return 0;
4323
4324         return -EINVAL;
4325 }
4326
4327 static int create_sort_keys(struct hist_trigger_data *hist_data)
4328 {
4329         char *fields_str = hist_data->attrs->sort_key_str;
4330         struct tracing_map_sort_key *sort_key;
4331         int descending, ret = 0;
4332         unsigned int i, j, k;
4333
4334         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4335
4336         if (!fields_str)
4337                 goto out;
4338
4339         strsep(&fields_str, "=");
4340         if (!fields_str) {
4341                 ret = -EINVAL;
4342                 goto out;
4343         }
4344
4345         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4346                 struct hist_field *hist_field;
4347                 char *field_str, *field_name;
4348                 const char *test_name;
4349
4350                 sort_key = &hist_data->sort_keys[i];
4351
4352                 field_str = strsep(&fields_str, ",");
4353                 if (!field_str) {
4354                         if (i == 0)
4355                                 ret = -EINVAL;
4356                         break;
4357                 }
4358
4359                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4360                         ret = -EINVAL;
4361                         break;
4362                 }
4363
4364                 field_name = strsep(&field_str, ".");
4365                 if (!field_name) {
4366                         ret = -EINVAL;
4367                         break;
4368                 }
4369
4370                 if (strcmp(field_name, "hitcount") == 0) {
4371                         descending = is_descending(field_str);
4372                         if (descending < 0) {
4373                                 ret = descending;
4374                                 break;
4375                         }
4376                         sort_key->descending = descending;
4377                         continue;
4378                 }
4379
4380                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4381                         unsigned int idx;
4382
4383                         hist_field = hist_data->fields[j];
4384                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4385                                 continue;
4386
4387                         idx = k++;
4388
4389                         test_name = hist_field_name(hist_field, 0);
4390
4391                         if (strcmp(field_name, test_name) == 0) {
4392                                 sort_key->field_idx = idx;
4393                                 descending = is_descending(field_str);
4394                                 if (descending < 0) {
4395                                         ret = descending;
4396                                         goto out;
4397                                 }
4398                                 sort_key->descending = descending;
4399                                 break;
4400                         }
4401                 }
4402                 if (j == hist_data->n_fields) {
4403                         ret = -EINVAL;
4404                         break;
4405                 }
4406         }
4407
4408         hist_data->n_sort_keys = i;
4409  out:
4410         return ret;
4411 }
4412
4413 static void destroy_actions(struct hist_trigger_data *hist_data)
4414 {
4415         unsigned int i;
4416
4417         for (i = 0; i < hist_data->n_actions; i++) {
4418                 struct action_data *data = hist_data->actions[i];
4419
4420                 if (data->fn == action_trace)
4421                         onmatch_destroy(data);
4422                 else if (data->fn == onmax_save)
4423                         onmax_destroy(data);
4424                 else
4425                         kfree(data);
4426         }
4427 }
4428
4429 static int parse_actions(struct hist_trigger_data *hist_data)
4430 {
4431         struct trace_array *tr = hist_data->event_file->tr;
4432         struct action_data *data;
4433         unsigned int i;
4434         int ret = 0;
4435         char *str;
4436
4437         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4438                 str = hist_data->attrs->action_str[i];
4439
4440                 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4441                         char *action_str = str + strlen("onmatch(");
4442
4443                         data = onmatch_parse(tr, action_str);
4444                         if (IS_ERR(data)) {
4445                                 ret = PTR_ERR(data);
4446                                 break;
4447                         }
4448                         data->fn = action_trace;
4449                 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4450                         char *action_str = str + strlen("onmax(");
4451
4452                         data = onmax_parse(action_str);
4453                         if (IS_ERR(data)) {
4454                                 ret = PTR_ERR(data);
4455                                 break;
4456                         }
4457                         data->fn = onmax_save;
4458                 } else {
4459                         ret = -EINVAL;
4460                         break;
4461                 }
4462
4463                 hist_data->actions[hist_data->n_actions++] = data;
4464         }
4465
4466         return ret;
4467 }
4468
4469 static int create_actions(struct hist_trigger_data *hist_data,
4470                           struct trace_event_file *file)
4471 {
4472         struct action_data *data;
4473         unsigned int i;
4474         int ret = 0;
4475
4476         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4477                 data = hist_data->actions[i];
4478
4479                 if (data->fn == action_trace) {
4480                         ret = onmatch_create(hist_data, file, data);
4481                         if (ret)
4482                                 return ret;
4483                 } else if (data->fn == onmax_save) {
4484                         ret = onmax_create(hist_data, data);
4485                         if (ret)
4486                                 return ret;
4487                 }
4488         }
4489
4490         return ret;
4491 }
4492
4493 static void print_actions(struct seq_file *m,
4494                           struct hist_trigger_data *hist_data,
4495                           struct tracing_map_elt *elt)
4496 {
4497         unsigned int i;
4498
4499         for (i = 0; i < hist_data->n_actions; i++) {
4500                 struct action_data *data = hist_data->actions[i];
4501
4502                 if (data->fn == onmax_save)
4503                         onmax_print(m, hist_data, elt, data);
4504         }
4505 }
4506
4507 static void print_onmax_spec(struct seq_file *m,
4508                              struct hist_trigger_data *hist_data,
4509                              struct action_data *data)
4510 {
4511         unsigned int i;
4512
4513         seq_puts(m, ":onmax(");
4514         seq_printf(m, "%s", data->onmax.var_str);
4515         seq_printf(m, ").%s(", data->onmax.fn_name);
4516
4517         for (i = 0; i < hist_data->n_max_vars; i++) {
4518                 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4519                 if (i < hist_data->n_max_vars - 1)
4520                         seq_puts(m, ",");
4521         }
4522         seq_puts(m, ")");
4523 }
4524
4525 static void print_onmatch_spec(struct seq_file *m,
4526                                struct hist_trigger_data *hist_data,
4527                                struct action_data *data)
4528 {
4529         unsigned int i;
4530
4531         seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4532                    data->onmatch.match_event);
4533
4534         seq_printf(m, "%s(", data->onmatch.synth_event->name);
4535
4536         for (i = 0; i < data->n_params; i++) {
4537                 if (i)
4538                         seq_puts(m, ",");
4539                 seq_printf(m, "%s", data->params[i]);
4540         }
4541
4542         seq_puts(m, ")");
4543 }
4544
4545 static bool actions_match(struct hist_trigger_data *hist_data,
4546                           struct hist_trigger_data *hist_data_test)
4547 {
4548         unsigned int i, j;
4549
4550         if (hist_data->n_actions != hist_data_test->n_actions)
4551                 return false;
4552
4553         for (i = 0; i < hist_data->n_actions; i++) {
4554                 struct action_data *data = hist_data->actions[i];
4555                 struct action_data *data_test = hist_data_test->actions[i];
4556
4557                 if (data->fn != data_test->fn)
4558                         return false;
4559
4560                 if (data->n_params != data_test->n_params)
4561                         return false;
4562
4563                 for (j = 0; j < data->n_params; j++) {
4564                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4565                                 return false;
4566                 }
4567
4568                 if (data->fn == action_trace) {
4569                         if (strcmp(data->onmatch.synth_event_name,
4570                                    data_test->onmatch.synth_event_name) != 0)
4571                                 return false;
4572                         if (strcmp(data->onmatch.match_event_system,
4573                                    data_test->onmatch.match_event_system) != 0)
4574                                 return false;
4575                         if (strcmp(data->onmatch.match_event,
4576                                    data_test->onmatch.match_event) != 0)
4577                                 return false;
4578                 } else if (data->fn == onmax_save) {
4579                         if (strcmp(data->onmax.var_str,
4580                                    data_test->onmax.var_str) != 0)
4581                                 return false;
4582                         if (strcmp(data->onmax.fn_name,
4583                                    data_test->onmax.fn_name) != 0)
4584                                 return false;
4585                 }
4586         }
4587
4588         return true;
4589 }
4590
4591
4592 static void print_actions_spec(struct seq_file *m,
4593                                struct hist_trigger_data *hist_data)
4594 {
4595         unsigned int i;
4596
4597         for (i = 0; i < hist_data->n_actions; i++) {
4598                 struct action_data *data = hist_data->actions[i];
4599
4600                 if (data->fn == action_trace)
4601                         print_onmatch_spec(m, hist_data, data);
4602                 else if (data->fn == onmax_save)
4603                         print_onmax_spec(m, hist_data, data);
4604         }
4605 }
4606
4607 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4608 {
4609         unsigned int i;
4610
4611         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4612                 kfree(hist_data->field_var_hists[i]->cmd);
4613                 kfree(hist_data->field_var_hists[i]);
4614         }
4615 }
4616
4617 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4618 {
4619         if (!hist_data)
4620                 return;
4621
4622         destroy_hist_trigger_attrs(hist_data->attrs);
4623         destroy_hist_fields(hist_data);
4624         tracing_map_destroy(hist_data->map);
4625
4626         destroy_actions(hist_data);
4627         destroy_field_vars(hist_data);
4628         destroy_field_var_hists(hist_data);
4629         destroy_synth_var_refs(hist_data);
4630
4631         kfree(hist_data);
4632 }
4633
4634 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4635 {
4636         struct tracing_map *map = hist_data->map;
4637         struct ftrace_event_field *field;
4638         struct hist_field *hist_field;
4639         int i, idx = 0;
4640
4641         for_each_hist_field(i, hist_data) {
4642                 hist_field = hist_data->fields[i];
4643                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4644                         tracing_map_cmp_fn_t cmp_fn;
4645
4646                         field = hist_field->field;
4647
4648                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4649                                 cmp_fn = tracing_map_cmp_none;
4650                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4651                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4652                                                              hist_field->is_signed);
4653                         else if (is_string_field(field))
4654                                 cmp_fn = tracing_map_cmp_string;
4655                         else
4656                                 cmp_fn = tracing_map_cmp_num(field->size,
4657                                                              field->is_signed);
4658                         idx = tracing_map_add_key_field(map,
4659                                                         hist_field->offset,
4660                                                         cmp_fn);
4661                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4662                         idx = tracing_map_add_sum_field(map);
4663
4664                 if (idx < 0)
4665                         return idx;
4666
4667                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4668                         idx = tracing_map_add_var(map);
4669                         if (idx < 0)
4670                                 return idx;
4671                         hist_field->var.idx = idx;
4672                         hist_field->var.hist_data = hist_data;
4673                 }
4674         }
4675
4676         return 0;
4677 }
4678
4679 static struct hist_trigger_data *
4680 create_hist_data(unsigned int map_bits,
4681                  struct hist_trigger_attrs *attrs,
4682                  struct trace_event_file *file,
4683                  bool remove)
4684 {
4685         const struct tracing_map_ops *map_ops = NULL;
4686         struct hist_trigger_data *hist_data;
4687         int ret = 0;
4688
4689         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4690         if (!hist_data)
4691                 return ERR_PTR(-ENOMEM);
4692
4693         hist_data->attrs = attrs;
4694         hist_data->remove = remove;
4695         hist_data->event_file = file;
4696
4697         ret = parse_actions(hist_data);
4698         if (ret)
4699                 goto free;
4700
4701         ret = create_hist_fields(hist_data, file);
4702         if (ret)
4703                 goto free;
4704
4705         ret = create_sort_keys(hist_data);
4706         if (ret)
4707                 goto free;
4708
4709         map_ops = &hist_trigger_elt_data_ops;
4710
4711         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4712                                             map_ops, hist_data);
4713         if (IS_ERR(hist_data->map)) {
4714                 ret = PTR_ERR(hist_data->map);
4715                 hist_data->map = NULL;
4716                 goto free;
4717         }
4718
4719         ret = create_tracing_map_fields(hist_data);
4720         if (ret)
4721                 goto free;
4722  out:
4723         return hist_data;
4724  free:
4725         hist_data->attrs = NULL;
4726
4727         destroy_hist_data(hist_data);
4728
4729         hist_data = ERR_PTR(ret);
4730
4731         goto out;
4732 }
4733
4734 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4735                                     struct tracing_map_elt *elt, void *rec,
4736                                     struct ring_buffer_event *rbe,
4737                                     u64 *var_ref_vals)
4738 {
4739         struct hist_elt_data *elt_data;
4740         struct hist_field *hist_field;
4741         unsigned int i, var_idx;
4742         u64 hist_val;
4743
4744         elt_data = elt->private_data;
4745         elt_data->var_ref_vals = var_ref_vals;
4746
4747         for_each_hist_val_field(i, hist_data) {
4748                 hist_field = hist_data->fields[i];
4749                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4750                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4751                         var_idx = hist_field->var.idx;
4752                         tracing_map_set_var(elt, var_idx, hist_val);
4753                         continue;
4754                 }
4755                 tracing_map_update_sum(elt, i, hist_val);
4756         }
4757
4758         for_each_hist_key_field(i, hist_data) {
4759                 hist_field = hist_data->fields[i];
4760                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4761                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4762                         var_idx = hist_field->var.idx;
4763                         tracing_map_set_var(elt, var_idx, hist_val);
4764                 }
4765         }
4766
4767         update_field_vars(hist_data, elt, rbe, rec);
4768 }
4769
4770 static inline void add_to_key(char *compound_key, void *key,
4771                               struct hist_field *key_field, void *rec)
4772 {
4773         size_t size = key_field->size;
4774
4775         if (key_field->flags & HIST_FIELD_FL_STRING) {
4776                 struct ftrace_event_field *field;
4777
4778                 field = key_field->field;
4779                 if (field->filter_type == FILTER_DYN_STRING)
4780                         size = *(u32 *)(rec + field->offset) >> 16;
4781                 else if (field->filter_type == FILTER_STATIC_STRING)
4782                         size = field->size;
4783
4784                 /* ensure NULL-termination */
4785                 if (size > key_field->size - 1)
4786                         size = key_field->size - 1;
4787
4788                 strncpy(compound_key + key_field->offset, (char *)key, size);
4789         } else
4790                 memcpy(compound_key + key_field->offset, key, size);
4791 }
4792
4793 static void
4794 hist_trigger_actions(struct hist_trigger_data *hist_data,
4795                      struct tracing_map_elt *elt, void *rec,
4796                      struct ring_buffer_event *rbe, u64 *var_ref_vals)
4797 {
4798         struct action_data *data;
4799         unsigned int i;
4800
4801         for (i = 0; i < hist_data->n_actions; i++) {
4802                 data = hist_data->actions[i];
4803                 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4804         }
4805 }
4806
4807 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4808                                struct ring_buffer_event *rbe)
4809 {
4810         struct hist_trigger_data *hist_data = data->private_data;
4811         bool use_compound_key = (hist_data->n_keys > 1);
4812         unsigned long entries[HIST_STACKTRACE_DEPTH];
4813         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4814         char compound_key[HIST_KEY_SIZE_MAX];
4815         struct tracing_map_elt *elt = NULL;
4816         struct stack_trace stacktrace;
4817         struct hist_field *key_field;
4818         u64 field_contents;
4819         void *key = NULL;
4820         unsigned int i;
4821
4822         memset(compound_key, 0, hist_data->key_size);
4823
4824         for_each_hist_key_field(i, hist_data) {
4825                 key_field = hist_data->fields[i];
4826
4827                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4828                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4829                         stacktrace.entries = entries;
4830                         stacktrace.nr_entries = 0;
4831                         stacktrace.skip = HIST_STACKTRACE_SKIP;
4832
4833                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4834                         save_stack_trace(&stacktrace);
4835
4836                         key = entries;
4837                 } else {
4838                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4839                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4840                                 key = (void *)(unsigned long)field_contents;
4841                                 use_compound_key = true;
4842                         } else
4843                                 key = (void *)&field_contents;
4844                 }
4845
4846                 if (use_compound_key)
4847                         add_to_key(compound_key, key, key_field, rec);
4848         }
4849
4850         if (use_compound_key)
4851                 key = compound_key;
4852
4853         if (hist_data->n_var_refs &&
4854             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4855                 return;
4856
4857         elt = tracing_map_insert(hist_data->map, key);
4858         if (!elt)
4859                 return;
4860
4861         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4862
4863         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4864                 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4865 }
4866
4867 static void hist_trigger_stacktrace_print(struct seq_file *m,
4868                                           unsigned long *stacktrace_entries,
4869                                           unsigned int max_entries)
4870 {
4871         char str[KSYM_SYMBOL_LEN];
4872         unsigned int spaces = 8;
4873         unsigned int i;
4874
4875         for (i = 0; i < max_entries; i++) {
4876                 if (stacktrace_entries[i] == ULONG_MAX)
4877                         return;
4878
4879                 seq_printf(m, "%*c", 1 + spaces, ' ');
4880                 sprint_symbol(str, stacktrace_entries[i]);
4881                 seq_printf(m, "%s\n", str);
4882         }
4883 }
4884
4885 static void
4886 hist_trigger_entry_print(struct seq_file *m,
4887                          struct hist_trigger_data *hist_data, void *key,
4888                          struct tracing_map_elt *elt)
4889 {
4890         struct hist_field *key_field;
4891         char str[KSYM_SYMBOL_LEN];
4892         bool multiline = false;
4893         const char *field_name;
4894         unsigned int i;
4895         u64 uval;
4896
4897         seq_puts(m, "{ ");
4898
4899         for_each_hist_key_field(i, hist_data) {
4900                 key_field = hist_data->fields[i];
4901
4902                 if (i > hist_data->n_vals)
4903                         seq_puts(m, ", ");
4904
4905                 field_name = hist_field_name(key_field, 0);
4906
4907                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4908                         uval = *(u64 *)(key + key_field->offset);
4909                         seq_printf(m, "%s: %llx", field_name, uval);
4910                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4911                         uval = *(u64 *)(key + key_field->offset);
4912                         sprint_symbol_no_offset(str, uval);
4913                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4914                                    uval, str);
4915                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4916                         uval = *(u64 *)(key + key_field->offset);
4917                         sprint_symbol(str, uval);
4918                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4919                                    uval, str);
4920                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4921                         struct hist_elt_data *elt_data = elt->private_data;
4922                         char *comm;
4923
4924                         if (WARN_ON_ONCE(!elt_data))
4925                                 return;
4926
4927                         comm = elt_data->comm;
4928
4929                         uval = *(u64 *)(key + key_field->offset);
4930                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4931                                    comm, uval);
4932                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4933                         const char *syscall_name;
4934
4935                         uval = *(u64 *)(key + key_field->offset);
4936                         syscall_name = get_syscall_name(uval);
4937                         if (!syscall_name)
4938                                 syscall_name = "unknown_syscall";
4939
4940                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4941                                    syscall_name, uval);
4942                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4943                         seq_puts(m, "stacktrace:\n");
4944                         hist_trigger_stacktrace_print(m,
4945                                                       key + key_field->offset,
4946                                                       HIST_STACKTRACE_DEPTH);
4947                         multiline = true;
4948                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4949                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4950                                    *(u64 *)(key + key_field->offset));
4951                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4952                         seq_printf(m, "%s: %-50s", field_name,
4953                                    (char *)(key + key_field->offset));
4954                 } else {
4955                         uval = *(u64 *)(key + key_field->offset);
4956                         seq_printf(m, "%s: %10llu", field_name, uval);
4957                 }
4958         }
4959
4960         if (!multiline)
4961                 seq_puts(m, " ");
4962
4963         seq_puts(m, "}");
4964
4965         seq_printf(m, " hitcount: %10llu",
4966                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4967
4968         for (i = 1; i < hist_data->n_vals; i++) {
4969                 field_name = hist_field_name(hist_data->fields[i], 0);
4970
4971                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4972                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4973                         continue;
4974
4975                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4976                         seq_printf(m, "  %s: %10llx", field_name,
4977                                    tracing_map_read_sum(elt, i));
4978                 } else {
4979                         seq_printf(m, "  %s: %10llu", field_name,
4980                                    tracing_map_read_sum(elt, i));
4981                 }
4982         }
4983
4984         print_actions(m, hist_data, elt);
4985
4986         seq_puts(m, "\n");
4987 }
4988
4989 static int print_entries(struct seq_file *m,
4990                          struct hist_trigger_data *hist_data)
4991 {
4992         struct tracing_map_sort_entry **sort_entries = NULL;
4993         struct tracing_map *map = hist_data->map;
4994         int i, n_entries;
4995
4996         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4997                                              hist_data->n_sort_keys,
4998                                              &sort_entries);
4999         if (n_entries < 0)
5000                 return n_entries;
5001
5002         for (i = 0; i < n_entries; i++)
5003                 hist_trigger_entry_print(m, hist_data,
5004                                          sort_entries[i]->key,
5005                                          sort_entries[i]->elt);
5006
5007         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5008
5009         return n_entries;
5010 }
5011
5012 static void hist_trigger_show(struct seq_file *m,
5013                               struct event_trigger_data *data, int n)
5014 {
5015         struct hist_trigger_data *hist_data;
5016         int n_entries;
5017
5018         if (n > 0)
5019                 seq_puts(m, "\n\n");
5020
5021         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5022         data->ops->print(m, data->ops, data);
5023         seq_puts(m, "#\n\n");
5024
5025         hist_data = data->private_data;
5026         n_entries = print_entries(m, hist_data);
5027         if (n_entries < 0)
5028                 n_entries = 0;
5029
5030         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5031                    (u64)atomic64_read(&hist_data->map->hits),
5032                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5033 }
5034
5035 static int hist_show(struct seq_file *m, void *v)
5036 {
5037         struct event_trigger_data *data;
5038         struct trace_event_file *event_file;
5039         int n = 0, ret = 0;
5040
5041         mutex_lock(&event_mutex);
5042
5043         event_file = event_file_data(m->private);
5044         if (unlikely(!event_file)) {
5045                 ret = -ENODEV;
5046                 goto out_unlock;
5047         }
5048
5049         list_for_each_entry(data, &event_file->triggers, list) {
5050                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5051                         hist_trigger_show(m, data, n++);
5052         }
5053
5054         if (have_hist_err()) {
5055                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
5056                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
5057         }
5058
5059  out_unlock:
5060         mutex_unlock(&event_mutex);
5061
5062         return ret;
5063 }
5064
5065 static int event_hist_open(struct inode *inode, struct file *file)
5066 {
5067         return single_open(file, hist_show, file);
5068 }
5069
5070 const struct file_operations event_hist_fops = {
5071         .open = event_hist_open,
5072         .read = seq_read,
5073         .llseek = seq_lseek,
5074         .release = single_release,
5075 };
5076
5077 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5078 {
5079         const char *field_name = hist_field_name(hist_field, 0);
5080
5081         if (hist_field->var.name)
5082                 seq_printf(m, "%s=", hist_field->var.name);
5083
5084         if (hist_field->flags & HIST_FIELD_FL_CPU)
5085                 seq_puts(m, "common_cpu");
5086         else if (field_name) {
5087                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5088                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5089                         seq_putc(m, '$');
5090                 seq_printf(m, "%s", field_name);
5091         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5092                 seq_puts(m, "common_timestamp");
5093
5094         if (hist_field->flags) {
5095                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5096                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5097                         const char *flags = get_hist_field_flags(hist_field);
5098
5099                         if (flags)
5100                                 seq_printf(m, ".%s", flags);
5101                 }
5102         }
5103 }
5104
5105 static int event_hist_trigger_print(struct seq_file *m,
5106                                     struct event_trigger_ops *ops,
5107                                     struct event_trigger_data *data)
5108 {
5109         struct hist_trigger_data *hist_data = data->private_data;
5110         struct hist_field *field;
5111         bool have_var = false;
5112         unsigned int i;
5113
5114         seq_puts(m, "hist:");
5115
5116         if (data->name)
5117                 seq_printf(m, "%s:", data->name);
5118
5119         seq_puts(m, "keys=");
5120
5121         for_each_hist_key_field(i, hist_data) {
5122                 field = hist_data->fields[i];
5123
5124                 if (i > hist_data->n_vals)
5125                         seq_puts(m, ",");
5126
5127                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5128                         seq_puts(m, "stacktrace");
5129                 else
5130                         hist_field_print(m, field);
5131         }
5132
5133         seq_puts(m, ":vals=");
5134
5135         for_each_hist_val_field(i, hist_data) {
5136                 field = hist_data->fields[i];
5137                 if (field->flags & HIST_FIELD_FL_VAR) {
5138                         have_var = true;
5139                         continue;
5140                 }
5141
5142                 if (i == HITCOUNT_IDX)
5143                         seq_puts(m, "hitcount");
5144                 else {
5145                         seq_puts(m, ",");
5146                         hist_field_print(m, field);
5147                 }
5148         }
5149
5150         if (have_var) {
5151                 unsigned int n = 0;
5152
5153                 seq_puts(m, ":");
5154
5155                 for_each_hist_val_field(i, hist_data) {
5156                         field = hist_data->fields[i];
5157
5158                         if (field->flags & HIST_FIELD_FL_VAR) {
5159                                 if (n++)
5160                                         seq_puts(m, ",");
5161                                 hist_field_print(m, field);
5162                         }
5163                 }
5164         }
5165
5166         seq_puts(m, ":sort=");
5167
5168         for (i = 0; i < hist_data->n_sort_keys; i++) {
5169                 struct tracing_map_sort_key *sort_key;
5170                 unsigned int idx, first_key_idx;
5171
5172                 /* skip VAR vals */
5173                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5174
5175                 sort_key = &hist_data->sort_keys[i];
5176                 idx = sort_key->field_idx;
5177
5178                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5179                         return -EINVAL;
5180
5181                 if (i > 0)
5182                         seq_puts(m, ",");
5183
5184                 if (idx == HITCOUNT_IDX)
5185                         seq_puts(m, "hitcount");
5186                 else {
5187                         if (idx >= first_key_idx)
5188                                 idx += hist_data->n_vars;
5189                         hist_field_print(m, hist_data->fields[idx]);
5190                 }
5191
5192                 if (sort_key->descending)
5193                         seq_puts(m, ".descending");
5194         }
5195         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5196         if (hist_data->enable_timestamps)
5197                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5198
5199         print_actions_spec(m, hist_data);
5200
5201         if (data->filter_str)
5202                 seq_printf(m, " if %s", data->filter_str);
5203
5204         if (data->paused)
5205                 seq_puts(m, " [paused]");
5206         else
5207                 seq_puts(m, " [active]");
5208
5209         seq_putc(m, '\n');
5210
5211         return 0;
5212 }
5213
5214 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5215                                    struct event_trigger_data *data)
5216 {
5217         struct hist_trigger_data *hist_data = data->private_data;
5218
5219         if (!data->ref && hist_data->attrs->name)
5220                 save_named_trigger(hist_data->attrs->name, data);
5221
5222         data->ref++;
5223
5224         return 0;
5225 }
5226
5227 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5228 {
5229         struct trace_event_file *file;
5230         unsigned int i;
5231         char *cmd;
5232         int ret;
5233
5234         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5235                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5236                 cmd = hist_data->field_var_hists[i]->cmd;
5237                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5238                                               "!hist", "hist", cmd);
5239         }
5240 }
5241
5242 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5243                                     struct event_trigger_data *data)
5244 {
5245         struct hist_trigger_data *hist_data = data->private_data;
5246
5247         if (WARN_ON_ONCE(data->ref <= 0))
5248                 return;
5249
5250         data->ref--;
5251         if (!data->ref) {
5252                 if (data->name)
5253                         del_named_trigger(data);
5254
5255                 trigger_data_free(data);
5256
5257                 remove_hist_vars(hist_data);
5258
5259                 unregister_field_var_hists(hist_data);
5260
5261                 destroy_hist_data(hist_data);
5262         }
5263 }
5264
5265 static struct event_trigger_ops event_hist_trigger_ops = {
5266         .func                   = event_hist_trigger,
5267         .print                  = event_hist_trigger_print,
5268         .init                   = event_hist_trigger_init,
5269         .free                   = event_hist_trigger_free,
5270 };
5271
5272 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5273                                          struct event_trigger_data *data)
5274 {
5275         data->ref++;
5276
5277         save_named_trigger(data->named_data->name, data);
5278
5279         event_hist_trigger_init(ops, data->named_data);
5280
5281         return 0;
5282 }
5283
5284 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5285                                           struct event_trigger_data *data)
5286 {
5287         if (WARN_ON_ONCE(data->ref <= 0))
5288                 return;
5289
5290         event_hist_trigger_free(ops, data->named_data);
5291
5292         data->ref--;
5293         if (!data->ref) {
5294                 del_named_trigger(data);
5295                 trigger_data_free(data);
5296         }
5297 }
5298
5299 static struct event_trigger_ops event_hist_trigger_named_ops = {
5300         .func                   = event_hist_trigger,
5301         .print                  = event_hist_trigger_print,
5302         .init                   = event_hist_trigger_named_init,
5303         .free                   = event_hist_trigger_named_free,
5304 };
5305
5306 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5307                                                             char *param)
5308 {
5309         return &event_hist_trigger_ops;
5310 }
5311
5312 static void hist_clear(struct event_trigger_data *data)
5313 {
5314         struct hist_trigger_data *hist_data = data->private_data;
5315
5316         if (data->name)
5317                 pause_named_trigger(data);
5318
5319         tracepoint_synchronize_unregister();
5320
5321         tracing_map_clear(hist_data->map);
5322
5323         if (data->name)
5324                 unpause_named_trigger(data);
5325 }
5326
5327 static bool compatible_field(struct ftrace_event_field *field,
5328                              struct ftrace_event_field *test_field)
5329 {
5330         if (field == test_field)
5331                 return true;
5332         if (field == NULL || test_field == NULL)
5333                 return false;
5334         if (strcmp(field->name, test_field->name) != 0)
5335                 return false;
5336         if (strcmp(field->type, test_field->type) != 0)
5337                 return false;
5338         if (field->size != test_field->size)
5339                 return false;
5340         if (field->is_signed != test_field->is_signed)
5341                 return false;
5342
5343         return true;
5344 }
5345
5346 static bool hist_trigger_match(struct event_trigger_data *data,
5347                                struct event_trigger_data *data_test,
5348                                struct event_trigger_data *named_data,
5349                                bool ignore_filter)
5350 {
5351         struct tracing_map_sort_key *sort_key, *sort_key_test;
5352         struct hist_trigger_data *hist_data, *hist_data_test;
5353         struct hist_field *key_field, *key_field_test;
5354         unsigned int i;
5355
5356         if (named_data && (named_data != data_test) &&
5357             (named_data != data_test->named_data))
5358                 return false;
5359
5360         if (!named_data && is_named_trigger(data_test))
5361                 return false;
5362
5363         hist_data = data->private_data;
5364         hist_data_test = data_test->private_data;
5365
5366         if (hist_data->n_vals != hist_data_test->n_vals ||
5367             hist_data->n_fields != hist_data_test->n_fields ||
5368             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5369                 return false;
5370
5371         if (!ignore_filter) {
5372                 if ((data->filter_str && !data_test->filter_str) ||
5373                    (!data->filter_str && data_test->filter_str))
5374                         return false;
5375         }
5376
5377         for_each_hist_field(i, hist_data) {
5378                 key_field = hist_data->fields[i];
5379                 key_field_test = hist_data_test->fields[i];
5380
5381                 if (key_field->flags != key_field_test->flags)
5382                         return false;
5383                 if (!compatible_field(key_field->field, key_field_test->field))
5384                         return false;
5385                 if (key_field->offset != key_field_test->offset)
5386                         return false;
5387                 if (key_field->size != key_field_test->size)
5388                         return false;
5389                 if (key_field->is_signed != key_field_test->is_signed)
5390                         return false;
5391                 if (!!key_field->var.name != !!key_field_test->var.name)
5392                         return false;
5393                 if (key_field->var.name &&
5394                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5395                         return false;
5396         }
5397
5398         for (i = 0; i < hist_data->n_sort_keys; i++) {
5399                 sort_key = &hist_data->sort_keys[i];
5400                 sort_key_test = &hist_data_test->sort_keys[i];
5401
5402                 if (sort_key->field_idx != sort_key_test->field_idx ||
5403                     sort_key->descending != sort_key_test->descending)
5404                         return false;
5405         }
5406
5407         if (!ignore_filter && data->filter_str &&
5408             (strcmp(data->filter_str, data_test->filter_str) != 0))
5409                 return false;
5410
5411         if (!actions_match(hist_data, hist_data_test))
5412                 return false;
5413
5414         return true;
5415 }
5416
5417 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5418                                  struct event_trigger_data *data,
5419                                  struct trace_event_file *file)
5420 {
5421         struct hist_trigger_data *hist_data = data->private_data;
5422         struct event_trigger_data *test, *named_data = NULL;
5423         int ret = 0;
5424
5425         if (hist_data->attrs->name) {
5426                 named_data = find_named_trigger(hist_data->attrs->name);
5427                 if (named_data) {
5428                         if (!hist_trigger_match(data, named_data, named_data,
5429                                                 true)) {
5430                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5431                                 ret = -EINVAL;
5432                                 goto out;
5433                         }
5434                 }
5435         }
5436
5437         if (hist_data->attrs->name && !named_data)
5438                 goto new;
5439
5440         lockdep_assert_held(&event_mutex);
5441
5442         list_for_each_entry(test, &file->triggers, list) {
5443                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5444                         if (!hist_trigger_match(data, test, named_data, false))
5445                                 continue;
5446                         if (hist_data->attrs->pause)
5447                                 test->paused = true;
5448                         else if (hist_data->attrs->cont)
5449                                 test->paused = false;
5450                         else if (hist_data->attrs->clear)
5451                                 hist_clear(test);
5452                         else {
5453                                 hist_err("Hist trigger already exists", NULL);
5454                                 ret = -EEXIST;
5455                         }
5456                         goto out;
5457                 }
5458         }
5459  new:
5460         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5461                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5462                 ret = -ENOENT;
5463                 goto out;
5464         }
5465
5466         if (hist_data->attrs->pause)
5467                 data->paused = true;
5468
5469         if (named_data) {
5470                 data->private_data = named_data->private_data;
5471                 set_named_trigger_data(data, named_data);
5472                 data->ops = &event_hist_trigger_named_ops;
5473         }
5474
5475         if (data->ops->init) {
5476                 ret = data->ops->init(data->ops, data);
5477                 if (ret < 0)
5478                         goto out;
5479         }
5480
5481         if (hist_data->enable_timestamps) {
5482                 char *clock = hist_data->attrs->clock;
5483
5484                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5485                 if (ret) {
5486                         hist_err("Couldn't set trace_clock: ", clock);
5487                         goto out;
5488                 }
5489
5490                 tracing_set_time_stamp_abs(file->tr, true);
5491         }
5492
5493         if (named_data)
5494                 destroy_hist_data(hist_data);
5495
5496         ret++;
5497  out:
5498         return ret;
5499 }
5500
5501 static int hist_trigger_enable(struct event_trigger_data *data,
5502                                struct trace_event_file *file)
5503 {
5504         int ret = 0;
5505
5506         list_add_tail_rcu(&data->list, &file->triggers);
5507
5508         update_cond_flag(file);
5509
5510         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5511                 list_del_rcu(&data->list);
5512                 update_cond_flag(file);
5513                 ret--;
5514         }
5515
5516         return ret;
5517 }
5518
5519 static bool have_hist_trigger_match(struct event_trigger_data *data,
5520                                     struct trace_event_file *file)
5521 {
5522         struct hist_trigger_data *hist_data = data->private_data;
5523         struct event_trigger_data *test, *named_data = NULL;
5524         bool match = false;
5525
5526         lockdep_assert_held(&event_mutex);
5527
5528         if (hist_data->attrs->name)
5529                 named_data = find_named_trigger(hist_data->attrs->name);
5530
5531         list_for_each_entry(test, &file->triggers, list) {
5532                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5533                         if (hist_trigger_match(data, test, named_data, false)) {
5534                                 match = true;
5535                                 break;
5536                         }
5537                 }
5538         }
5539
5540         return match;
5541 }
5542
5543 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5544                                     struct trace_event_file *file)
5545 {
5546         struct hist_trigger_data *hist_data = data->private_data;
5547         struct event_trigger_data *test, *named_data = NULL;
5548
5549         lockdep_assert_held(&event_mutex);
5550
5551         if (hist_data->attrs->name)
5552                 named_data = find_named_trigger(hist_data->attrs->name);
5553
5554         list_for_each_entry(test, &file->triggers, list) {
5555                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5556                         if (!hist_trigger_match(data, test, named_data, false))
5557                                 continue;
5558                         hist_data = test->private_data;
5559                         if (check_var_refs(hist_data))
5560                                 return true;
5561                         break;
5562                 }
5563         }
5564
5565         return false;
5566 }
5567
5568 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5569                                     struct event_trigger_data *data,
5570                                     struct trace_event_file *file)
5571 {
5572         struct hist_trigger_data *hist_data = data->private_data;
5573         struct event_trigger_data *test, *named_data = NULL;
5574         bool unregistered = false;
5575
5576         lockdep_assert_held(&event_mutex);
5577
5578         if (hist_data->attrs->name)
5579                 named_data = find_named_trigger(hist_data->attrs->name);
5580
5581         list_for_each_entry(test, &file->triggers, list) {
5582                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5583                         if (!hist_trigger_match(data, test, named_data, false))
5584                                 continue;
5585                         unregistered = true;
5586                         list_del_rcu(&test->list);
5587                         trace_event_trigger_enable_disable(file, 0);
5588                         update_cond_flag(file);
5589                         break;
5590                 }
5591         }
5592
5593         if (unregistered && test->ops->free)
5594                 test->ops->free(test->ops, test);
5595
5596         if (hist_data->enable_timestamps) {
5597                 if (!hist_data->remove || unregistered)
5598                         tracing_set_time_stamp_abs(file->tr, false);
5599         }
5600 }
5601
5602 static bool hist_file_check_refs(struct trace_event_file *file)
5603 {
5604         struct hist_trigger_data *hist_data;
5605         struct event_trigger_data *test;
5606
5607         lockdep_assert_held(&event_mutex);
5608
5609         list_for_each_entry(test, &file->triggers, list) {
5610                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5611                         hist_data = test->private_data;
5612                         if (check_var_refs(hist_data))
5613                                 return true;
5614                 }
5615         }
5616
5617         return false;
5618 }
5619
5620 static void hist_unreg_all(struct trace_event_file *file)
5621 {
5622         struct event_trigger_data *test, *n;
5623         struct hist_trigger_data *hist_data;
5624         struct synth_event *se;
5625         const char *se_name;
5626
5627         if (hist_file_check_refs(file))
5628                 return;
5629
5630         list_for_each_entry_safe(test, n, &file->triggers, list) {
5631                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5632                         hist_data = test->private_data;
5633                         list_del_rcu(&test->list);
5634                         trace_event_trigger_enable_disable(file, 0);
5635
5636                         mutex_lock(&synth_event_mutex);
5637                         se_name = trace_event_name(file->event_call);
5638                         se = find_synth_event(se_name);
5639                         if (se)
5640                                 se->ref--;
5641                         mutex_unlock(&synth_event_mutex);
5642
5643                         update_cond_flag(file);
5644                         if (hist_data->enable_timestamps)
5645                                 tracing_set_time_stamp_abs(file->tr, false);
5646                         if (test->ops->free)
5647                                 test->ops->free(test->ops, test);
5648                 }
5649         }
5650 }
5651
5652 static int event_hist_trigger_func(struct event_command *cmd_ops,
5653                                    struct trace_event_file *file,
5654                                    char *glob, char *cmd, char *param)
5655 {
5656         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5657         struct event_trigger_data *trigger_data;
5658         struct hist_trigger_attrs *attrs;
5659         struct event_trigger_ops *trigger_ops;
5660         struct hist_trigger_data *hist_data;
5661         struct synth_event *se;
5662         const char *se_name;
5663         bool remove = false;
5664         char *trigger, *p;
5665         int ret = 0;
5666
5667         if (glob && strlen(glob)) {
5668                 last_cmd_set(param);
5669                 hist_err_clear();
5670         }
5671
5672         if (!param)
5673                 return -EINVAL;
5674
5675         if (glob[0] == '!')
5676                 remove = true;
5677
5678         /*
5679          * separate the trigger from the filter (k:v [if filter])
5680          * allowing for whitespace in the trigger
5681          */
5682         p = trigger = param;
5683         do {
5684                 p = strstr(p, "if");
5685                 if (!p)
5686                         break;
5687                 if (p == param)
5688                         return -EINVAL;
5689                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5690                         p++;
5691                         continue;
5692                 }
5693                 if (p >= param + strlen(param) - strlen("if") - 1)
5694                         return -EINVAL;
5695                 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5696                         p++;
5697                         continue;
5698                 }
5699                 break;
5700         } while (p);
5701
5702         if (!p)
5703                 param = NULL;
5704         else {
5705                 *(p - 1) = '\0';
5706                 param = strstrip(p);
5707                 trigger = strstrip(trigger);
5708         }
5709
5710         attrs = parse_hist_trigger_attrs(trigger);
5711         if (IS_ERR(attrs))
5712                 return PTR_ERR(attrs);
5713
5714         if (attrs->map_bits)
5715                 hist_trigger_bits = attrs->map_bits;
5716
5717         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5718         if (IS_ERR(hist_data)) {
5719                 destroy_hist_trigger_attrs(attrs);
5720                 return PTR_ERR(hist_data);
5721         }
5722
5723         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5724
5725         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5726         if (!trigger_data) {
5727                 ret = -ENOMEM;
5728                 goto out_free;
5729         }
5730
5731         trigger_data->count = -1;
5732         trigger_data->ops = trigger_ops;
5733         trigger_data->cmd_ops = cmd_ops;
5734
5735         INIT_LIST_HEAD(&trigger_data->list);
5736         RCU_INIT_POINTER(trigger_data->filter, NULL);
5737
5738         trigger_data->private_data = hist_data;
5739
5740         /* if param is non-empty, it's supposed to be a filter */
5741         if (param && cmd_ops->set_filter) {
5742                 ret = cmd_ops->set_filter(param, trigger_data, file);
5743                 if (ret < 0)
5744                         goto out_free;
5745         }
5746
5747         if (remove) {
5748                 if (!have_hist_trigger_match(trigger_data, file))
5749                         goto out_free;
5750
5751                 if (hist_trigger_check_refs(trigger_data, file)) {
5752                         ret = -EBUSY;
5753                         goto out_free;
5754                 }
5755
5756                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5757
5758                 mutex_lock(&synth_event_mutex);
5759                 se_name = trace_event_name(file->event_call);
5760                 se = find_synth_event(se_name);
5761                 if (se)
5762                         se->ref--;
5763                 mutex_unlock(&synth_event_mutex);
5764
5765                 ret = 0;
5766                 goto out_free;
5767         }
5768
5769         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5770         /*
5771          * The above returns on success the # of triggers registered,
5772          * but if it didn't register any it returns zero.  Consider no
5773          * triggers registered a failure too.
5774          */
5775         if (!ret) {
5776                 if (!(attrs->pause || attrs->cont || attrs->clear))
5777                         ret = -ENOENT;
5778                 goto out_free;
5779         } else if (ret < 0)
5780                 goto out_free;
5781
5782         if (get_named_trigger_data(trigger_data))
5783                 goto enable;
5784
5785         if (has_hist_vars(hist_data))
5786                 save_hist_vars(hist_data);
5787
5788         ret = create_actions(hist_data, file);
5789         if (ret)
5790                 goto out_unreg;
5791
5792         ret = tracing_map_init(hist_data->map);
5793         if (ret)
5794                 goto out_unreg;
5795 enable:
5796         ret = hist_trigger_enable(trigger_data, file);
5797         if (ret)
5798                 goto out_unreg;
5799
5800         mutex_lock(&synth_event_mutex);
5801         se_name = trace_event_name(file->event_call);
5802         se = find_synth_event(se_name);
5803         if (se)
5804                 se->ref++;
5805         mutex_unlock(&synth_event_mutex);
5806
5807         /* Just return zero, not the number of registered triggers */
5808         ret = 0;
5809  out:
5810         if (ret == 0)
5811                 hist_err_clear();
5812
5813         return ret;
5814  out_unreg:
5815         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5816  out_free:
5817         if (cmd_ops->set_filter)
5818                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5819
5820         remove_hist_vars(hist_data);
5821
5822         kfree(trigger_data);
5823
5824         destroy_hist_data(hist_data);
5825         goto out;
5826 }
5827
5828 static struct event_command trigger_hist_cmd = {
5829         .name                   = "hist",
5830         .trigger_type           = ETT_EVENT_HIST,
5831         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5832         .func                   = event_hist_trigger_func,
5833         .reg                    = hist_register_trigger,
5834         .unreg                  = hist_unregister_trigger,
5835         .unreg_all              = hist_unreg_all,
5836         .get_trigger_ops        = event_hist_get_trigger_ops,
5837         .set_filter             = set_trigger_filter,
5838 };
5839
5840 __init int register_trigger_hist_cmd(void)
5841 {
5842         int ret;
5843
5844         ret = register_event_command(&trigger_hist_cmd);
5845         WARN_ON(ret < 0);
5846
5847         return ret;
5848 }
5849
5850 static void
5851 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5852                     struct ring_buffer_event *event)
5853 {
5854         struct enable_trigger_data *enable_data = data->private_data;
5855         struct event_trigger_data *test;
5856
5857         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5858                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5859                         if (enable_data->enable)
5860                                 test->paused = false;
5861                         else
5862                                 test->paused = true;
5863                 }
5864         }
5865 }
5866
5867 static void
5868 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5869                           struct ring_buffer_event *event)
5870 {
5871         if (!data->count)
5872                 return;
5873
5874         if (data->count != -1)
5875                 (data->count)--;
5876
5877         hist_enable_trigger(data, rec, event);
5878 }
5879
5880 static struct event_trigger_ops hist_enable_trigger_ops = {
5881         .func                   = hist_enable_trigger,
5882         .print                  = event_enable_trigger_print,
5883         .init                   = event_trigger_init,
5884         .free                   = event_enable_trigger_free,
5885 };
5886
5887 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5888         .func                   = hist_enable_count_trigger,
5889         .print                  = event_enable_trigger_print,
5890         .init                   = event_trigger_init,
5891         .free                   = event_enable_trigger_free,
5892 };
5893
5894 static struct event_trigger_ops hist_disable_trigger_ops = {
5895         .func                   = hist_enable_trigger,
5896         .print                  = event_enable_trigger_print,
5897         .init                   = event_trigger_init,
5898         .free                   = event_enable_trigger_free,
5899 };
5900
5901 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5902         .func                   = hist_enable_count_trigger,
5903         .print                  = event_enable_trigger_print,
5904         .init                   = event_trigger_init,
5905         .free                   = event_enable_trigger_free,
5906 };
5907
5908 static struct event_trigger_ops *
5909 hist_enable_get_trigger_ops(char *cmd, char *param)
5910 {
5911         struct event_trigger_ops *ops;
5912         bool enable;
5913
5914         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5915
5916         if (enable)
5917                 ops = param ? &hist_enable_count_trigger_ops :
5918                         &hist_enable_trigger_ops;
5919         else
5920                 ops = param ? &hist_disable_count_trigger_ops :
5921                         &hist_disable_trigger_ops;
5922
5923         return ops;
5924 }
5925
5926 static void hist_enable_unreg_all(struct trace_event_file *file)
5927 {
5928         struct event_trigger_data *test, *n;
5929
5930         list_for_each_entry_safe(test, n, &file->triggers, list) {
5931                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5932                         list_del_rcu(&test->list);
5933                         update_cond_flag(file);
5934                         trace_event_trigger_enable_disable(file, 0);
5935                         if (test->ops->free)
5936                                 test->ops->free(test->ops, test);
5937                 }
5938         }
5939 }
5940
5941 static struct event_command trigger_hist_enable_cmd = {
5942         .name                   = ENABLE_HIST_STR,
5943         .trigger_type           = ETT_HIST_ENABLE,
5944         .func                   = event_enable_trigger_func,
5945         .reg                    = event_enable_register_trigger,
5946         .unreg                  = event_enable_unregister_trigger,
5947         .unreg_all              = hist_enable_unreg_all,
5948         .get_trigger_ops        = hist_enable_get_trigger_ops,
5949         .set_filter             = set_trigger_filter,
5950 };
5951
5952 static struct event_command trigger_hist_disable_cmd = {
5953         .name                   = DISABLE_HIST_STR,
5954         .trigger_type           = ETT_HIST_ENABLE,
5955         .func                   = event_enable_trigger_func,
5956         .reg                    = event_enable_register_trigger,
5957         .unreg                  = event_enable_unregister_trigger,
5958         .unreg_all              = hist_enable_unreg_all,
5959         .get_trigger_ops        = hist_enable_get_trigger_ops,
5960         .set_filter             = set_trigger_filter,
5961 };
5962
5963 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5964 {
5965         unregister_event_command(&trigger_hist_enable_cmd);
5966         unregister_event_command(&trigger_hist_disable_cmd);
5967 }
5968
5969 __init int register_trigger_hist_enable_disable_cmds(void)
5970 {
5971         int ret;
5972
5973         ret = register_event_command(&trigger_hist_enable_cmd);
5974         if (WARN_ON(ret < 0))
5975                 return ret;
5976         ret = register_event_command(&trigger_hist_disable_cmd);
5977         if (WARN_ON(ret < 0))
5978                 unregister_trigger_hist_enable_disable_cmds();
5979
5980         return ret;
5981 }
5982
5983 static __init int trace_events_hist_init(void)
5984 {
5985         struct dentry *entry = NULL;
5986         struct dentry *d_tracer;
5987         int err = 0;
5988
5989         d_tracer = tracing_init_dentry();
5990         if (IS_ERR(d_tracer)) {
5991                 err = PTR_ERR(d_tracer);
5992                 goto err;
5993         }
5994
5995         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5996                                     NULL, &synth_events_fops);
5997         if (!entry) {
5998                 err = -ENODEV;
5999                 goto err;
6000         }
6001
6002         return err;
6003  err:
6004         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6005
6006         return err;
6007 }
6008
6009 fs_initcall(trace_events_hist_init);