tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42 #include "config.h"
  43
  44 #include "intel-pt-decoder/intel-pt-log.h"
  45 #include "intel-pt-decoder/intel-pt-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  47 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  48
  49 #define MAX_TIMESTAMP (~0ULL)
  50
  51 struct intel_pt {
  52         struct auxtrace auxtrace;
  53         struct auxtrace_queues queues;
  54         struct auxtrace_heap heap;
  55         u32 auxtrace_type;
  56         struct perf_session *session;
  57         struct machine *machine;
  58         struct perf_evsel *switch_evsel;
  59         struct thread *unknown_thread;
  60         bool timeless_decoding;
  61         bool sampling_mode;
  62         bool snapshot_mode;
  63         bool per_cpu_mmaps;
  64         bool have_tsc;
  65         bool data_queued;
  66         bool est_tsc;
  67         bool sync_switch;
  68         bool mispred_all;
  69         int have_sched_switch;
  70         u32 pmu_type;
  71         u64 kernel_start;
  72         u64 switch_ip;
  73         u64 ptss_ip;
  74
  75         struct perf_tsc_conversion tc;
  76         bool cap_user_time_zero;
  77
  78         struct itrace_synth_opts synth_opts;
  79
  80         bool sample_instructions;
  81         u64 instructions_sample_type;
  82         u64 instructions_sample_period;
  83         u64 instructions_id;
  84
  85         bool sample_branches;
  86         u32 branches_filter;
  87         u64 branches_sample_type;
  88         u64 branches_id;
  89
  90         bool sample_transactions;
  91         u64 transactions_sample_type;
  92         u64 transactions_id;
  93
  94         bool synth_needs_swap;
  95
  96         u64 tsc_bit;
  97         u64 mtc_bit;
  98         u64 mtc_freq_bits;
  99         u32 tsc_ctc_ratio_n;
 100         u32 tsc_ctc_ratio_d;
 101         u64 cyc_bit;
 102         u64 noretcomp_bit;
 103         unsigned max_non_turbo_ratio;
 104
 105         unsigned long num_events;
 106
 107         char *filter;
 108         struct addr_filters filts;
 109 };
 110
 111 enum switch_state {
 112         INTEL_PT_SS_NOT_TRACING,
 113         INTEL_PT_SS_UNKNOWN,
 114         INTEL_PT_SS_TRACING,
 115         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 116         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 117 };
 118
 119 struct intel_pt_queue {
 120         struct intel_pt *pt;
 121         unsigned int queue_nr;
 122         struct auxtrace_buffer *buffer;
 123         void *decoder;
 124         const struct intel_pt_state *state;
 125         struct ip_callchain *chain;
 126         struct branch_stack *last_branch;
 127         struct branch_stack *last_branch_rb;
 128         size_t last_branch_pos;
 129         union perf_event *event_buf;
 130         bool on_heap;
 131         bool stop;
 132         bool step_through_buffers;
 133         bool use_buffer_pid_tid;
 134         bool sync_switch;
 135         pid_t pid, tid;
 136         int cpu;
 137         int switch_state;
 138         pid_t next_tid;
 139         struct thread *thread;
 140         bool exclude_kernel;
 141         bool have_sample;
 142         u64 time;
 143         u64 timestamp;
 144         u32 flags;
 145         u16 insn_len;
 146         u64 last_insn_cnt;
 147 };
 148
 149 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 150                           unsigned char *buf, size_t len)
 151 {
 152         struct intel_pt_pkt packet;
 153         size_t pos = 0;
 154         int ret, pkt_len, i;
 155         char desc[INTEL_PT_PKT_DESC_MAX];
 156         const char *color = PERF_COLOR_BLUE;
 157
 158         color_fprintf(stdout, color,
 159                       ". ... Intel Processor Trace data: size %zu bytes\n",
 160                       len);
 161
 162         while (len) {
 163                 ret = intel_pt_get_packet(buf, len, &packet);
 164                 if (ret > 0)
 165                         pkt_len = ret;
 166                 else
 167                         pkt_len = 1;
 168                 printf(".");
 169                 color_fprintf(stdout, color, "  %08x: ", pos);
 170                 for (i = 0; i < pkt_len; i++)
 171                         color_fprintf(stdout, color, " %02x", buf[i]);
 172                 for (; i < 16; i++)
 173                         color_fprintf(stdout, color, "   ");
 174                 if (ret > 0) {
 175                         ret = intel_pt_pkt_desc(&packet, desc,
 176                                                 INTEL_PT_PKT_DESC_MAX);
 177                         if (ret > 0)
 178                                 color_fprintf(stdout, color, " %s\n", desc);
 179                 } else {
 180                         color_fprintf(stdout, color, " Bad packet!\n");
 181                 }
 182                 pos += pkt_len;
 183                 buf += pkt_len;
 184                 len -= pkt_len;
 185         }
 186 }
 187
 188 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 189                                 size_t len)
 190 {
 191         printf(".\n");
 192         intel_pt_dump(pt, buf, len);
 193 }
 194
 195 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 196                                    struct auxtrace_buffer *b)
 197 {
 198         bool consecutive = false;
 199         void *start;
 200
 201         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 202                                       pt->have_tsc, &consecutive);
 203         if (!start)
 204                 return -EINVAL;
 205         b->use_size = b->data + b->size - start;
 206         b->use_data = start;
 207         if (b->use_size && consecutive)
 208                 b->consecutive = true;
 209         return 0;
 210 }
 211
 212 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 213                                         struct auxtrace_queue *queue,
 214                                         struct auxtrace_buffer *buffer)
 215 {
 216         if (queue->cpu == -1 && buffer->cpu != -1)
 217                 ptq->cpu = buffer->cpu;
 218
 219         ptq->pid = buffer->pid;
 220         ptq->tid = buffer->tid;
 221
 222         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 223                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 224
 225         thread__zput(ptq->thread);
 226
 227         if (ptq->tid != -1) {
 228                 if (ptq->pid != -1)
 229                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 230                                                               ptq->pid,
 231                                                               ptq->tid);
 232                 else
 233                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 234                                                            ptq->tid);
 235         }
 236 }
 237
 238 /* This function assumes data is processed sequentially only */
 239 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 240 {
 241         struct intel_pt_queue *ptq = data;
 242         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 243         struct auxtrace_queue *queue;
 244
 245         if (ptq->stop) {
 246                 b->len = 0;
 247                 return 0;
 248         }
 249
 250         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 251 next:
 252         buffer = auxtrace_buffer__next(queue, buffer);
 253         if (!buffer) {
 254                 if (old_buffer)
 255                         auxtrace_buffer__drop_data(old_buffer);
 256                 b->len = 0;
 257                 return 0;
 258         }
 259
 260         ptq->buffer = buffer;
 261
 262         if (!buffer->data) {
 263                 int fd = perf_data_file__fd(ptq->pt->session->file);
 264
 265                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 266                 if (!buffer->data)
 267                         return -ENOMEM;
 268         }
 269
 270         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 271             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 272                 return -ENOMEM;
 273
 274         if (buffer->use_data) {
 275                 b->len = buffer->use_size;
 276                 b->buf = buffer->use_data;
 277         } else {
 278                 b->len = buffer->size;
 279                 b->buf = buffer->data;
 280         }
 281         b->ref_timestamp = buffer->reference;
 282
 283         /*
 284          * If in snapshot mode and the buffer has no usable data, get next
 285          * buffer and again check overlap against old_buffer.
 286          */
 287         if (ptq->pt->snapshot_mode && !b->len)
 288                 goto next;
 289
 290         if (old_buffer)
 291                 auxtrace_buffer__drop_data(old_buffer);
 292
 293         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 294                                                       !buffer->consecutive)) {
 295                 b->consecutive = false;
 296                 b->trace_nr = buffer->buffer_nr + 1;
 297         } else {
 298                 b->consecutive = true;
 299         }
 300
 301         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 302                                         ptq->tid != buffer->tid))
 303                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 304
 305         if (ptq->step_through_buffers)
 306                 ptq->stop = true;
 307
 308         if (!b->len)
 309                 return intel_pt_get_trace(b, data);
 310
 311         return 0;
 312 }
 313
 314 struct intel_pt_cache_entry {
 315         struct auxtrace_cache_entry     entry;
 316         u64                             insn_cnt;
 317         u64                             byte_cnt;
 318         enum intel_pt_insn_op           op;
 319         enum intel_pt_insn_branch       branch;
 320         int                             length;
 321         int32_t                         rel;
 322 };
 323
 324 static int intel_pt_config_div(const char *var, const char *value, void *data)
 325 {
 326         int *d = data;
 327         long val;
 328
 329         if (!strcmp(var, "intel-pt.cache-divisor")) {
 330                 val = strtol(value, NULL, 0);
 331                 if (val > 0 && val <= INT_MAX)
 332                         *d = val;
 333         }
 334
 335         return 0;
 336 }
 337
 338 static int intel_pt_cache_divisor(void)
 339 {
 340         static int d;
 341
 342         if (d)
 343                 return d;
 344
 345         perf_config(intel_pt_config_div, &d);
 346
 347         if (!d)
 348                 d = 64;
 349
 350         return d;
 351 }
 352
 353 static unsigned int intel_pt_cache_size(struct dso *dso,
 354                                         struct machine *machine)
 355 {
 356         off_t size;
 357
 358         size = dso__data_size(dso, machine);
 359         size /= intel_pt_cache_divisor();
 360         if (size < 1000)
 361                 return 10;
 362         if (size > (1 << 21))
 363                 return 21;
 364         return 32 - __builtin_clz(size);
 365 }
 366
 367 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 368                                              struct machine *machine)
 369 {
 370         struct auxtrace_cache *c;
 371         unsigned int bits;
 372
 373         if (dso->auxtrace_cache)
 374                 return dso->auxtrace_cache;
 375
 376         bits = intel_pt_cache_size(dso, machine);
 377
 378         /* Ignoring cache creation failure */
 379         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 380
 381         dso->auxtrace_cache = c;
 382
 383         return c;
 384 }
 385
 386 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 387                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 388                               struct intel_pt_insn *intel_pt_insn)
 389 {
 390         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 391         struct intel_pt_cache_entry *e;
 392         int err;
 393
 394         if (!c)
 395                 return -ENOMEM;
 396
 397         e = auxtrace_cache__alloc_entry(c);
 398         if (!e)
 399                 return -ENOMEM;
 400
 401         e->insn_cnt = insn_cnt;
 402         e->byte_cnt = byte_cnt;
 403         e->op = intel_pt_insn->op;
 404         e->branch = intel_pt_insn->branch;
 405         e->length = intel_pt_insn->length;
 406         e->rel = intel_pt_insn->rel;
 407
 408         err = auxtrace_cache__add(c, offset, &e->entry);
 409         if (err)
 410                 auxtrace_cache__free_entry(c, e);
 411
 412         return err;
 413 }
 414
 415 static struct intel_pt_cache_entry *
 416 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 417 {
 418         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 419
 420         if (!c)
 421                 return NULL;
 422
 423         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 424 }
 425
 426 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 427                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 428                                    uint64_t to_ip, uint64_t max_insn_cnt,
 429                                    void *data)
 430 {
 431         struct intel_pt_queue *ptq = data;
 432         struct machine *machine = ptq->pt->machine;
 433         struct thread *thread;
 434         struct addr_location al;
 435         unsigned char buf[1024];
 436         size_t bufsz;
 437         ssize_t len;
 438         int x86_64;
 439         u8 cpumode;
 440         u64 offset, start_offset, start_ip;
 441         u64 insn_cnt = 0;
 442         bool one_map = true;
 443
 444         if (to_ip && *ip == to_ip)
 445                 goto out_no_cache;
 446
 447         bufsz = intel_pt_insn_max_size();
 448
 449         if (*ip >= ptq->pt->kernel_start)
 450                 cpumode = PERF_RECORD_MISC_KERNEL;
 451         else
 452                 cpumode = PERF_RECORD_MISC_USER;
 453
 454         thread = ptq->thread;
 455         if (!thread) {
 456                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 457                         return -EINVAL;
 458                 thread = ptq->pt->unknown_thread;
 459         }
 460
 461         while (1) {
 462                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 463                 if (!al.map || !al.map->dso)
 464                         return -EINVAL;
 465
 466                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 467                     dso__data_status_seen(al.map->dso,
 468                                           DSO_DATA_STATUS_SEEN_ITRACE))
 469                         return -ENOENT;
 470
 471                 offset = al.map->map_ip(al.map, *ip);
 472
 473                 if (!to_ip && one_map) {
 474                         struct intel_pt_cache_entry *e;
 475
 476                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 477                         if (e &&
 478                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 479                                 *insn_cnt_ptr = e->insn_cnt;
 480                                 *ip += e->byte_cnt;
 481                                 intel_pt_insn->op = e->op;
 482                                 intel_pt_insn->branch = e->branch;
 483                                 intel_pt_insn->length = e->length;
 484                                 intel_pt_insn->rel = e->rel;
 485                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 486                                 return 0;
 487                         }
 488                 }
 489
 490                 start_offset = offset;
 491                 start_ip = *ip;
 492
 493                 /* Load maps to ensure dso->is_64_bit has been updated */
 494                 map__load(al.map);
 495
 496                 x86_64 = al.map->dso->is_64_bit;
 497
 498                 while (1) {
 499                         len = dso__data_read_offset(al.map->dso, machine,
 500                                                     offset, buf, bufsz);
 501                         if (len <= 0)
 502                                 return -EINVAL;
 503
 504                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 505                                 return -EINVAL;
 506
 507                         intel_pt_log_insn(intel_pt_insn, *ip);
 508
 509                         insn_cnt += 1;
 510
 511                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 512                                 goto out;
 513
 514                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 515                                 goto out_no_cache;
 516
 517                         *ip += intel_pt_insn->length;
 518
 519                         if (to_ip && *ip == to_ip)
 520                                 goto out_no_cache;
 521
 522                         if (*ip >= al.map->end)
 523                                 break;
 524
 525                         offset += intel_pt_insn->length;
 526                 }
 527                 one_map = false;
 528         }
 529 out:
 530         *insn_cnt_ptr = insn_cnt;
 531
 532         if (!one_map)
 533                 goto out_no_cache;
 534
 535         /*
 536          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 537          * entries.
 538          */
 539         if (to_ip) {
 540                 struct intel_pt_cache_entry *e;
 541
 542                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 543                 if (e)
 544                         return 0;
 545         }
 546
 547         /* Ignore cache errors */
 548         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 549                            *ip - start_ip, intel_pt_insn);
 550
 551         return 0;
 552
 553 out_no_cache:
 554         *insn_cnt_ptr = insn_cnt;
 555         return 0;
 556 }
 557
 558 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 559                                   uint64_t offset, const char *filename)
 560 {
 561         struct addr_filter *filt;
 562         bool have_filter   = false;
 563         bool hit_tracestop = false;
 564         bool hit_filter    = false;
 565
 566         list_for_each_entry(filt, &pt->filts.head, list) {
 567                 if (filt->start)
 568                         have_filter = true;
 569
 570                 if ((filename && !filt->filename) ||
 571                     (!filename && filt->filename) ||
 572                     (filename && strcmp(filename, filt->filename)))
 573                         continue;
 574
 575                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 576                         continue;
 577
 578                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 579                              ip, offset, filename ? filename : "[kernel]",
 580                              filt->start ? "filter" : "stop",
 581                              filt->addr, filt->size);
 582
 583                 if (filt->start)
 584                         hit_filter = true;
 585                 else
 586                         hit_tracestop = true;
 587         }
 588
 589         if (!hit_tracestop && !hit_filter)
 590                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 591                              ip, offset, filename ? filename : "[kernel]");
 592
 593         return hit_tracestop || (have_filter && !hit_filter);
 594 }
 595
 596 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 597 {
 598         struct intel_pt_queue *ptq = data;
 599         struct thread *thread;
 600         struct addr_location al;
 601         u8 cpumode;
 602         u64 offset;
 603
 604         if (ip >= ptq->pt->kernel_start)
 605                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 606
 607         cpumode = PERF_RECORD_MISC_USER;
 608
 609         thread = ptq->thread;
 610         if (!thread)
 611                 return -EINVAL;
 612
 613         thread__find_addr_map(thread, cpumode, MAP__FUNCTION, ip, &al);
 614         if (!al.map || !al.map->dso)
 615                 return -EINVAL;
 616
 617         offset = al.map->map_ip(al.map, ip);
 618
 619         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 620                                      al.map->dso->long_name);
 621 }
 622
 623 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 624 {
 625         return __intel_pt_pgd_ip(ip, data) > 0;
 626 }
 627
 628 static bool intel_pt_get_config(struct intel_pt *pt,
 629                                 struct perf_event_attr *attr, u64 *config)
 630 {
 631         if (attr->type == pt->pmu_type) {
 632                 if (config)
 633                         *config = attr->config;
 634                 return true;
 635         }
 636
 637         return false;
 638 }
 639
 640 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 641 {
 642         struct perf_evsel *evsel;
 643
 644         evlist__for_each_entry(pt->session->evlist, evsel) {
 645                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 646                     !evsel->attr.exclude_kernel)
 647                         return false;
 648         }
 649         return true;
 650 }
 651
 652 static bool intel_pt_return_compression(struct intel_pt *pt)
 653 {
 654         struct perf_evsel *evsel;
 655         u64 config;
 656
 657         if (!pt->noretcomp_bit)
 658                 return true;
 659
 660         evlist__for_each_entry(pt->session->evlist, evsel) {
 661                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 662                     (config & pt->noretcomp_bit))
 663                         return false;
 664         }
 665         return true;
 666 }
 667
 668 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 669 {
 670         struct perf_evsel *evsel;
 671         unsigned int shift;
 672         u64 config;
 673
 674         if (!pt->mtc_freq_bits)
 675                 return 0;
 676
 677         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 678                 config >>= 1;
 679
 680         evlist__for_each_entry(pt->session->evlist, evsel) {
 681                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 682                         return (config & pt->mtc_freq_bits) >> shift;
 683         }
 684         return 0;
 685 }
 686
 687 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 688 {
 689         struct perf_evsel *evsel;
 690         bool timeless_decoding = true;
 691         u64 config;
 692
 693         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 694                 return true;
 695
 696         evlist__for_each_entry(pt->session->evlist, evsel) {
 697                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 698                         return true;
 699                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 700                         if (config & pt->tsc_bit)
 701                                 timeless_decoding = false;
 702                         else
 703                                 return true;
 704                 }
 705         }
 706         return timeless_decoding;
 707 }
 708
 709 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 710 {
 711         struct perf_evsel *evsel;
 712
 713         evlist__for_each_entry(pt->session->evlist, evsel) {
 714                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 715                     !evsel->attr.exclude_kernel)
 716                         return true;
 717         }
 718         return false;
 719 }
 720
 721 static bool intel_pt_have_tsc(struct intel_pt *pt)
 722 {
 723         struct perf_evsel *evsel;
 724         bool have_tsc = false;
 725         u64 config;
 726
 727         if (!pt->tsc_bit)
 728                 return false;
 729
 730         evlist__for_each_entry(pt->session->evlist, evsel) {
 731                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 732                         if (config & pt->tsc_bit)
 733                                 have_tsc = true;
 734                         else
 735                                 return false;
 736                 }
 737         }
 738         return have_tsc;
 739 }
 740
 741 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 742 {
 743         u64 quot, rem;
 744
 745         quot = ns / pt->tc.time_mult;
 746         rem  = ns % pt->tc.time_mult;
 747         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 748                 pt->tc.time_mult;
 749 }
 750
 751 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 752                                                    unsigned int queue_nr)
 753 {
 754         struct intel_pt_params params = { .get_trace = 0, };
 755         struct perf_env *env = pt->machine->env;
 756         struct intel_pt_queue *ptq;
 757
 758         ptq = zalloc(sizeof(struct intel_pt_queue));
 759         if (!ptq)
 760                 return NULL;
 761
 762         if (pt->synth_opts.callchain) {
 763                 size_t sz = sizeof(struct ip_callchain);
 764
 765                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 766                 ptq->chain = zalloc(sz);
 767                 if (!ptq->chain)
 768                         goto out_free;
 769         }
 770
 771         if (pt->synth_opts.last_branch) {
 772                 size_t sz = sizeof(struct branch_stack);
 773
 774                 sz += pt->synth_opts.last_branch_sz *
 775                       sizeof(struct branch_entry);
 776                 ptq->last_branch = zalloc(sz);
 777                 if (!ptq->last_branch)
 778                         goto out_free;
 779                 ptq->last_branch_rb = zalloc(sz);
 780                 if (!ptq->last_branch_rb)
 781                         goto out_free;
 782         }
 783
 784         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 785         if (!ptq->event_buf)
 786                 goto out_free;
 787
 788         ptq->pt = pt;
 789         ptq->queue_nr = queue_nr;
 790         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 791         ptq->pid = -1;
 792         ptq->tid = -1;
 793         ptq->cpu = -1;
 794         ptq->next_tid = -1;
 795
 796         params.get_trace = intel_pt_get_trace;
 797         params.walk_insn = intel_pt_walk_next_insn;
 798         params.data = ptq;
 799         params.return_compression = intel_pt_return_compression(pt);
 800         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 801         params.mtc_period = intel_pt_mtc_period(pt);
 802         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 803         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 804
 805         if (pt->filts.cnt > 0)
 806                 params.pgd_ip = intel_pt_pgd_ip;
 807
 808         if (pt->synth_opts.instructions) {
 809                 if (pt->synth_opts.period) {
 810                         switch (pt->synth_opts.period_type) {
 811                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 812                                 params.period_type =
 813                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 814                                 params.period = pt->synth_opts.period;
 815                                 break;
 816                         case PERF_ITRACE_PERIOD_TICKS:
 817                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 818                                 params.period = pt->synth_opts.period;
 819                                 break;
 820                         case PERF_ITRACE_PERIOD_NANOSECS:
 821                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 822                                 params.period = intel_pt_ns_to_ticks(pt,
 823                                                         pt->synth_opts.period);
 824                                 break;
 825                         default:
 826                                 break;
 827                         }
 828                 }
 829
 830                 if (!params.period) {
 831                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 832                         params.period = 1;
 833                 }
 834         }
 835
 836         if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
 837                 params.flags |= INTEL_PT_FUP_WITH_NLIP;
 838
 839         ptq->decoder = intel_pt_decoder_new(&params);
 840         if (!ptq->decoder)
 841                 goto out_free;
 842
 843         return ptq;
 844
 845 out_free:
 846         zfree(&ptq->event_buf);
 847         zfree(&ptq->last_branch);
 848         zfree(&ptq->last_branch_rb);
 849         zfree(&ptq->chain);
 850         free(ptq);
 851         return NULL;
 852 }
 853
 854 static void intel_pt_free_queue(void *priv)
 855 {
 856         struct intel_pt_queue *ptq = priv;
 857
 858         if (!ptq)
 859                 return;
 860         thread__zput(ptq->thread);
 861         intel_pt_decoder_free(ptq->decoder);
 862         zfree(&ptq->event_buf);
 863         zfree(&ptq->last_branch);
 864         zfree(&ptq->last_branch_rb);
 865         zfree(&ptq->chain);
 866         free(ptq);
 867 }
 868
 869 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 870                                      struct auxtrace_queue *queue)
 871 {
 872         struct intel_pt_queue *ptq = queue->priv;
 873
 874         if (queue->tid == -1 || pt->have_sched_switch) {
 875                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 876                 if (ptq->tid == -1)
 877                         ptq->pid = -1;
 878                 thread__zput(ptq->thread);
 879         }
 880
 881         if (!ptq->thread && ptq->tid != -1)
 882                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 883
 884         if (ptq->thread) {
 885                 ptq->pid = ptq->thread->pid_;
 886                 if (queue->cpu == -1)
 887                         ptq->cpu = ptq->thread->cpu;
 888         }
 889 }
 890
 891 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 892 {
 893         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 894                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 895         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 896                 if (ptq->state->to_ip)
 897                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 898                                      PERF_IP_FLAG_ASYNC |
 899                                      PERF_IP_FLAG_INTERRUPT;
 900                 else
 901                         ptq->flags = PERF_IP_FLAG_BRANCH |
 902                                      PERF_IP_FLAG_TRACE_END;
 903                 ptq->insn_len = 0;
 904         } else {
 905                 if (ptq->state->from_ip)
 906                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 907                 else
 908                         ptq->flags = PERF_IP_FLAG_BRANCH |
 909                                      PERF_IP_FLAG_TRACE_BEGIN;
 910                 if (ptq->state->flags & INTEL_PT_IN_TX)
 911                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 912                 ptq->insn_len = ptq->state->insn_len;
 913         }
 914 }
 915
 916 static int intel_pt_setup_queue(struct intel_pt *pt,
 917                                 struct auxtrace_queue *queue,
 918                                 unsigned int queue_nr)
 919 {
 920         struct intel_pt_queue *ptq = queue->priv;
 921
 922         if (list_empty(&queue->head))
 923                 return 0;
 924
 925         if (!ptq) {
 926                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 927                 if (!ptq)
 928                         return -ENOMEM;
 929                 queue->priv = ptq;
 930
 931                 if (queue->cpu != -1)
 932                         ptq->cpu = queue->cpu;
 933                 ptq->tid = queue->tid;
 934
 935                 if (pt->sampling_mode) {
 936                         if (pt->timeless_decoding)
 937                                 ptq->step_through_buffers = true;
 938                         if (pt->timeless_decoding || !pt->have_sched_switch)
 939                                 ptq->use_buffer_pid_tid = true;
 940                 }
 941
 942                 ptq->sync_switch = pt->sync_switch;
 943         }
 944
 945         if (!ptq->on_heap &&
 946             (!ptq->sync_switch ||
 947              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 948                 const struct intel_pt_state *state;
 949                 int ret;
 950
 951                 if (pt->timeless_decoding)
 952                         return 0;
 953
 954                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 955                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 956                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 957                 while (1) {
 958                         state = intel_pt_decode(ptq->decoder);
 959                         if (state->err) {
 960                                 if (state->err == INTEL_PT_ERR_NODATA) {
 961                                         intel_pt_log("queue %u has no timestamp\n",
 962                                                      queue_nr);
 963                                         return 0;
 964                                 }
 965                                 continue;
 966                         }
 967                         if (state->timestamp)
 968                                 break;
 969                 }
 970
 971                 ptq->timestamp = state->timestamp;
 972                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 973                              queue_nr, ptq->timestamp);
 974                 ptq->state = state;
 975                 ptq->have_sample = true;
 976                 intel_pt_sample_flags(ptq);
 977                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 978                 if (ret)
 979                         return ret;
 980                 ptq->on_heap = true;
 981         }
 982
 983         return 0;
 984 }
 985
 986 static int intel_pt_setup_queues(struct intel_pt *pt)
 987 {
 988         unsigned int i;
 989         int ret;
 990
 991         for (i = 0; i < pt->queues.nr_queues; i++) {
 992                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 993                 if (ret)
 994                         return ret;
 995         }
 996         return 0;
 997 }
 998
 999 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1000 {
1001         struct branch_stack *bs_src = ptq->last_branch_rb;
1002         struct branch_stack *bs_dst = ptq->last_branch;
1003         size_t nr = 0;
1004
1005         bs_dst->nr = bs_src->nr;
1006
1007         if (!bs_src->nr)
1008                 return;
1009
1010         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1011         memcpy(&bs_dst->entries[0],
1012                &bs_src->entries[ptq->last_branch_pos],
1013                sizeof(struct branch_entry) * nr);
1014
1015         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1016                 memcpy(&bs_dst->entries[nr],
1017                        &bs_src->entries[0],
1018                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1019         }
1020 }
1021
1022 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1023 {
1024         ptq->last_branch_pos = 0;
1025         ptq->last_branch_rb->nr = 0;
1026 }
1027
1028 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1029 {
1030         const struct intel_pt_state *state = ptq->state;
1031         struct branch_stack *bs = ptq->last_branch_rb;
1032         struct branch_entry *be;
1033
1034         if (!ptq->last_branch_pos)
1035                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1036
1037         ptq->last_branch_pos -= 1;
1038
1039         be              = &bs->entries[ptq->last_branch_pos];
1040         be->from        = state->from_ip;
1041         be->to          = state->to_ip;
1042         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1043         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1044         /* No support for mispredict */
1045         be->flags.mispred = ptq->pt->mispred_all;
1046
1047         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1048                 bs->nr += 1;
1049 }
1050
1051 static int intel_pt_inject_event(union perf_event *event,
1052                                  struct perf_sample *sample, u64 type,
1053                                  bool swapped)
1054 {
1055         event->header.size = perf_event__sample_event_size(sample, type, 0);
1056         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
1057 }
1058
1059 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1060 {
1061         int ret;
1062         struct intel_pt *pt = ptq->pt;
1063         union perf_event *event = ptq->event_buf;
1064         struct perf_sample sample = { .ip = 0, };
1065         struct dummy_branch_stack {
1066                 u64                     nr;
1067                 struct branch_entry     entries;
1068         } dummy_bs;
1069
1070         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1071                 return 0;
1072
1073         if (pt->synth_opts.initial_skip &&
1074             pt->num_events++ < pt->synth_opts.initial_skip)
1075                 return 0;
1076
1077         event->sample.header.type = PERF_RECORD_SAMPLE;
1078         event->sample.header.misc = PERF_RECORD_MISC_USER;
1079         event->sample.header.size = sizeof(struct perf_event_header);
1080
1081         if (!pt->timeless_decoding)
1082                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1083
1084         sample.cpumode = PERF_RECORD_MISC_USER;
1085         sample.ip = ptq->state->from_ip;
1086         sample.pid = ptq->pid;
1087         sample.tid = ptq->tid;
1088         sample.addr = ptq->state->to_ip;
1089         sample.id = ptq->pt->branches_id;
1090         sample.stream_id = ptq->pt->branches_id;
1091         sample.period = 1;
1092         sample.cpu = ptq->cpu;
1093         sample.flags = ptq->flags;
1094         sample.insn_len = ptq->insn_len;
1095
1096         /*
1097          * perf report cannot handle events without a branch stack when using
1098          * SORT_MODE__BRANCH so make a dummy one.
1099          */
1100         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1101                 dummy_bs = (struct dummy_branch_stack){
1102                         .nr = 1,
1103                         .entries = {
1104                                 .from = sample.ip,
1105                                 .to = sample.addr,
1106                         },
1107                 };
1108                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1109         }
1110
1111         if (pt->synth_opts.inject) {
1112                 ret = intel_pt_inject_event(event, &sample,
1113                                             pt->branches_sample_type,
1114                                             pt->synth_needs_swap);
1115                 if (ret)
1116                         return ret;
1117         }
1118
1119         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1120         if (ret)
1121                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1122                        ret);
1123
1124         return ret;
1125 }
1126
1127 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1128 {
1129         int ret;
1130         struct intel_pt *pt = ptq->pt;
1131         union perf_event *event = ptq->event_buf;
1132         struct perf_sample sample = { .ip = 0, };
1133
1134         if (pt->synth_opts.initial_skip &&
1135             pt->num_events++ < pt->synth_opts.initial_skip)
1136                 return 0;
1137
1138         event->sample.header.type = PERF_RECORD_SAMPLE;
1139         event->sample.header.misc = PERF_RECORD_MISC_USER;
1140         event->sample.header.size = sizeof(struct perf_event_header);
1141
1142         if (!pt->timeless_decoding)
1143                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1144
1145         sample.cpumode = PERF_RECORD_MISC_USER;
1146         sample.ip = ptq->state->from_ip;
1147         sample.pid = ptq->pid;
1148         sample.tid = ptq->tid;
1149         sample.addr = ptq->state->to_ip;
1150         sample.id = ptq->pt->instructions_id;
1151         sample.stream_id = ptq->pt->instructions_id;
1152         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1153         sample.cpu = ptq->cpu;
1154         sample.flags = ptq->flags;
1155         sample.insn_len = ptq->insn_len;
1156
1157         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1158
1159         if (pt->synth_opts.callchain) {
1160                 thread_stack__sample(ptq->thread, ptq->chain,
1161                                      pt->synth_opts.callchain_sz, sample.ip);
1162                 sample.callchain = ptq->chain;
1163         }
1164
1165         if (pt->synth_opts.last_branch) {
1166                 intel_pt_copy_last_branch_rb(ptq);
1167                 sample.branch_stack = ptq->last_branch;
1168         }
1169
1170         if (pt->synth_opts.inject) {
1171                 ret = intel_pt_inject_event(event, &sample,
1172                                             pt->instructions_sample_type,
1173                                             pt->synth_needs_swap);
1174                 if (ret)
1175                         return ret;
1176         }
1177
1178         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1179         if (ret)
1180                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1181                        ret);
1182
1183         if (pt->synth_opts.last_branch)
1184                 intel_pt_reset_last_branch_rb(ptq);
1185
1186         return ret;
1187 }
1188
1189 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1190 {
1191         int ret;
1192         struct intel_pt *pt = ptq->pt;
1193         union perf_event *event = ptq->event_buf;
1194         struct perf_sample sample = { .ip = 0, };
1195
1196         if (pt->synth_opts.initial_skip &&
1197             pt->num_events++ < pt->synth_opts.initial_skip)
1198                 return 0;
1199
1200         event->sample.header.type = PERF_RECORD_SAMPLE;
1201         event->sample.header.misc = PERF_RECORD_MISC_USER;
1202         event->sample.header.size = sizeof(struct perf_event_header);
1203
1204         if (!pt->timeless_decoding)
1205                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1206
1207         sample.cpumode = PERF_RECORD_MISC_USER;
1208         sample.ip = ptq->state->from_ip;
1209         sample.pid = ptq->pid;
1210         sample.tid = ptq->tid;
1211         sample.addr = ptq->state->to_ip;
1212         sample.id = ptq->pt->transactions_id;
1213         sample.stream_id = ptq->pt->transactions_id;
1214         sample.period = 1;
1215         sample.cpu = ptq->cpu;
1216         sample.flags = ptq->flags;
1217         sample.insn_len = ptq->insn_len;
1218
1219         if (pt->synth_opts.callchain) {
1220                 thread_stack__sample(ptq->thread, ptq->chain,
1221                                      pt->synth_opts.callchain_sz, sample.ip);
1222                 sample.callchain = ptq->chain;
1223         }
1224
1225         if (pt->synth_opts.last_branch) {
1226                 intel_pt_copy_last_branch_rb(ptq);
1227                 sample.branch_stack = ptq->last_branch;
1228         }
1229
1230         if (pt->synth_opts.inject) {
1231                 ret = intel_pt_inject_event(event, &sample,
1232                                             pt->transactions_sample_type,
1233                                             pt->synth_needs_swap);
1234                 if (ret)
1235                         return ret;
1236         }
1237
1238         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1239         if (ret)
1240                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1241                        ret);
1242
1243         if (pt->synth_opts.last_branch)
1244                 intel_pt_reset_last_branch_rb(ptq);
1245
1246         return ret;
1247 }
1248
1249 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1250                                 pid_t pid, pid_t tid, u64 ip)
1251 {
1252         union perf_event event;
1253         char msg[MAX_AUXTRACE_ERROR_MSG];
1254         int err;
1255
1256         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1257
1258         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1259                              code, cpu, pid, tid, ip, msg);
1260
1261         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1262         if (err)
1263                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1264                        err);
1265
1266         return err;
1267 }
1268
1269 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1270 {
1271         struct auxtrace_queue *queue;
1272         pid_t tid = ptq->next_tid;
1273         int err;
1274
1275         if (tid == -1)
1276                 return 0;
1277
1278         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1279
1280         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1281
1282         queue = &pt->queues.queue_array[ptq->queue_nr];
1283         intel_pt_set_pid_tid_cpu(pt, queue);
1284
1285         ptq->next_tid = -1;
1286
1287         return err;
1288 }
1289
1290 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1291 {
1292         struct intel_pt *pt = ptq->pt;
1293
1294         return ip == pt->switch_ip &&
1295                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1296                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1297                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1298 }
1299
1300 static int intel_pt_sample(struct intel_pt_queue *ptq)
1301 {
1302         const struct intel_pt_state *state = ptq->state;
1303         struct intel_pt *pt = ptq->pt;
1304         int err;
1305
1306         if (!ptq->have_sample)
1307                 return 0;
1308
1309         ptq->have_sample = false;
1310
1311         if (pt->sample_instructions &&
1312             (state->type & INTEL_PT_INSTRUCTION) &&
1313             (!pt->synth_opts.initial_skip ||
1314              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1315                 err = intel_pt_synth_instruction_sample(ptq);
1316                 if (err)
1317                         return err;
1318         }
1319
1320         if (pt->sample_transactions &&
1321             (state->type & INTEL_PT_TRANSACTION) &&
1322             (!pt->synth_opts.initial_skip ||
1323              pt->num_events++ >= pt->synth_opts.initial_skip)) {
1324                 err = intel_pt_synth_transaction_sample(ptq);
1325                 if (err)
1326                         return err;
1327         }
1328
1329         if (!(state->type & INTEL_PT_BRANCH))
1330                 return 0;
1331
1332         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1333                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1334                                     state->to_ip, ptq->insn_len,
1335                                     state->trace_nr);
1336         else
1337                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1338
1339         if (pt->sample_branches) {
1340                 err = intel_pt_synth_branch_sample(ptq);
1341                 if (err)
1342                         return err;
1343         }
1344
1345         if (pt->synth_opts.last_branch)
1346                 intel_pt_update_last_branch_rb(ptq);
1347
1348         if (!ptq->sync_switch)
1349                 return 0;
1350
1351         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1352                 switch (ptq->switch_state) {
1353                 case INTEL_PT_SS_NOT_TRACING:
1354                 case INTEL_PT_SS_UNKNOWN:
1355                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1356                         err = intel_pt_next_tid(pt, ptq);
1357                         if (err)
1358                                 return err;
1359                         ptq->switch_state = INTEL_PT_SS_TRACING;
1360                         break;
1361                 default:
1362                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1363                         return 1;
1364                 }
1365         } else if (!state->to_ip) {
1366                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1367         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1368                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1369         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1370                    state->to_ip == pt->ptss_ip &&
1371                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1372                 ptq->switch_state = INTEL_PT_SS_TRACING;
1373         }
1374
1375         return 0;
1376 }
1377
1378 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1379 {
1380         struct machine *machine = pt->machine;
1381         struct map *map;
1382         struct symbol *sym, *start;
1383         u64 ip, switch_ip = 0;
1384         const char *ptss;
1385
1386         if (ptss_ip)
1387                 *ptss_ip = 0;
1388
1389         map = machine__kernel_map(machine);
1390         if (!map)
1391                 return 0;
1392
1393         if (map__load(map))
1394                 return 0;
1395
1396         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1397
1398         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1399                 if (sym->binding == STB_GLOBAL &&
1400                     !strcmp(sym->name, "__switch_to")) {
1401                         ip = map->unmap_ip(map, sym->start);
1402                         if (ip >= map->start && ip < map->end) {
1403                                 switch_ip = ip;
1404                                 break;
1405                         }
1406                 }
1407         }
1408
1409         if (!switch_ip || !ptss_ip)
1410                 return 0;
1411
1412         if (pt->have_sched_switch == 1)
1413                 ptss = "perf_trace_sched_switch";
1414         else
1415                 ptss = "__perf_event_task_sched_out";
1416
1417         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1418                 if (!strcmp(sym->name, ptss)) {
1419                         ip = map->unmap_ip(map, sym->start);
1420                         if (ip >= map->start && ip < map->end) {
1421                                 *ptss_ip = ip;
1422                                 break;
1423                         }
1424                 }
1425         }
1426
1427         return switch_ip;
1428 }
1429
1430 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
1431 {
1432         unsigned int i;
1433
1434         pt->sync_switch = true;
1435
1436         for (i = 0; i < pt->queues.nr_queues; i++) {
1437                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1438                 struct intel_pt_queue *ptq = queue->priv;
1439
1440                 if (ptq)
1441                         ptq->sync_switch = true;
1442         }
1443 }
1444
1445 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1446 {
1447         const struct intel_pt_state *state = ptq->state;
1448         struct intel_pt *pt = ptq->pt;
1449         int err;
1450
1451         if (!pt->kernel_start) {
1452                 pt->kernel_start = machine__kernel_start(pt->machine);
1453                 if (pt->per_cpu_mmaps &&
1454                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1455                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1456                     !pt->sampling_mode) {
1457                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1458                         if (pt->switch_ip) {
1459                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1460                                              pt->switch_ip, pt->ptss_ip);
1461                                 intel_pt_enable_sync_switch(pt);
1462                         }
1463                 }
1464         }
1465
1466         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1467                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1468         while (1) {
1469                 err = intel_pt_sample(ptq);
1470                 if (err)
1471                         return err;
1472
1473                 state = intel_pt_decode(ptq->decoder);
1474                 if (state->err) {
1475                         if (state->err == INTEL_PT_ERR_NODATA)
1476                                 return 1;
1477                         if (ptq->sync_switch &&
1478                             state->from_ip >= pt->kernel_start) {
1479                                 ptq->sync_switch = false;
1480                                 intel_pt_next_tid(pt, ptq);
1481                         }
1482                         if (pt->synth_opts.errors) {
1483                                 err = intel_pt_synth_error(pt, state->err,
1484                                                            ptq->cpu, ptq->pid,
1485                                                            ptq->tid,
1486                                                            state->from_ip);
1487                                 if (err)
1488                                         return err;
1489                         }
1490                         continue;
1491                 }
1492
1493                 ptq->state = state;
1494                 ptq->have_sample = true;
1495                 intel_pt_sample_flags(ptq);
1496
1497                 /* Use estimated TSC upon return to user space */
1498                 if (pt->est_tsc &&
1499                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1500                     state->to_ip && state->to_ip < pt->kernel_start) {
1501                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1502                                      state->timestamp, state->est_timestamp);
1503                         ptq->timestamp = state->est_timestamp;
1504                 /* Use estimated TSC in unknown switch state */
1505                 } else if (ptq->sync_switch &&
1506                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1507                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1508                            ptq->next_tid == -1) {
1509                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1510                                      state->timestamp, state->est_timestamp);
1511                         ptq->timestamp = state->est_timestamp;
1512                 } else if (state->timestamp > ptq->timestamp) {
1513                         ptq->timestamp = state->timestamp;
1514                 }
1515
1516                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1517                         *timestamp = ptq->timestamp;
1518                         return 0;
1519                 }
1520         }
1521         return 0;
1522 }
1523
1524 static inline int intel_pt_update_queues(struct intel_pt *pt)
1525 {
1526         if (pt->queues.new_data) {
1527                 pt->queues.new_data = false;
1528                 return intel_pt_setup_queues(pt);
1529         }
1530         return 0;
1531 }
1532
1533 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1534 {
1535         unsigned int queue_nr;
1536         u64 ts;
1537         int ret;
1538
1539         while (1) {
1540                 struct auxtrace_queue *queue;
1541                 struct intel_pt_queue *ptq;
1542
1543                 if (!pt->heap.heap_cnt)
1544                         return 0;
1545
1546                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1547                         return 0;
1548
1549                 queue_nr = pt->heap.heap_array[0].queue_nr;
1550                 queue = &pt->queues.queue_array[queue_nr];
1551                 ptq = queue->priv;
1552
1553                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1554                              queue_nr, pt->heap.heap_array[0].ordinal,
1555                              timestamp);
1556
1557                 auxtrace_heap__pop(&pt->heap);
1558
1559                 if (pt->heap.heap_cnt) {
1560                         ts = pt->heap.heap_array[0].ordinal + 1;
1561                         if (ts > timestamp)
1562                                 ts = timestamp;
1563                 } else {
1564                         ts = timestamp;
1565                 }
1566
1567                 intel_pt_set_pid_tid_cpu(pt, queue);
1568
1569                 ret = intel_pt_run_decoder(ptq, &ts);
1570
1571                 if (ret < 0) {
1572                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1573                         return ret;
1574                 }
1575
1576                 if (!ret) {
1577                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1578                         if (ret < 0)
1579                                 return ret;
1580                 } else {
1581                         ptq->on_heap = false;
1582                 }
1583         }
1584
1585         return 0;
1586 }
1587
1588 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1589                                             u64 time_)
1590 {
1591         struct auxtrace_queues *queues = &pt->queues;
1592         unsigned int i;
1593         u64 ts = 0;
1594
1595         for (i = 0; i < queues->nr_queues; i++) {
1596                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1597                 struct intel_pt_queue *ptq = queue->priv;
1598
1599                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1600                         ptq->time = time_;
1601                         intel_pt_set_pid_tid_cpu(pt, queue);
1602                         intel_pt_run_decoder(ptq, &ts);
1603                 }
1604         }
1605         return 0;
1606 }
1607
1608 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1609 {
1610         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1611                                     sample->pid, sample->tid, 0);
1612 }
1613
1614 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1615 {
1616         unsigned i, j;
1617
1618         if (cpu < 0 || !pt->queues.nr_queues)
1619                 return NULL;
1620
1621         if ((unsigned)cpu >= pt->queues.nr_queues)
1622                 i = pt->queues.nr_queues - 1;
1623         else
1624                 i = cpu;
1625
1626         if (pt->queues.queue_array[i].cpu == cpu)
1627                 return pt->queues.queue_array[i].priv;
1628
1629         for (j = 0; i > 0; j++) {
1630                 if (pt->queues.queue_array[--i].cpu == cpu)
1631                         return pt->queues.queue_array[i].priv;
1632         }
1633
1634         for (; j < pt->queues.nr_queues; j++) {
1635                 if (pt->queues.queue_array[j].cpu == cpu)
1636                         return pt->queues.queue_array[j].priv;
1637         }
1638
1639         return NULL;
1640 }
1641
1642 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1643                                 u64 timestamp)
1644 {
1645         struct intel_pt_queue *ptq;
1646         int err;
1647
1648         if (!pt->sync_switch)
1649                 return 1;
1650
1651         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1652         if (!ptq || !ptq->sync_switch)
1653                 return 1;
1654
1655         switch (ptq->switch_state) {
1656         case INTEL_PT_SS_NOT_TRACING:
1657                 ptq->next_tid = -1;
1658                 break;
1659         case INTEL_PT_SS_UNKNOWN:
1660         case INTEL_PT_SS_TRACING:
1661                 ptq->next_tid = tid;
1662                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1663                 return 0;
1664         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1665                 if (!ptq->on_heap) {
1666                         ptq->timestamp = perf_time_to_tsc(timestamp,
1667                                                           &pt->tc);
1668                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1669                                                  ptq->timestamp);
1670                         if (err)
1671                                 return err;
1672                         ptq->on_heap = true;
1673                 }
1674                 ptq->switch_state = INTEL_PT_SS_TRACING;
1675                 break;
1676         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1677                 ptq->next_tid = tid;
1678                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1679                 break;
1680         default:
1681                 break;
1682         }
1683
1684         return 1;
1685 }
1686
1687 static int intel_pt_process_switch(struct intel_pt *pt,
1688                                    struct perf_sample *sample)
1689 {
1690         struct perf_evsel *evsel;
1691         pid_t tid;
1692         int cpu, ret;
1693
1694         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1695         if (evsel != pt->switch_evsel)
1696                 return 0;
1697
1698         tid = perf_evsel__intval(evsel, sample, "next_pid");
1699         cpu = sample->cpu;
1700
1701         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1702                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1703                      &pt->tc));
1704
1705         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1706         if (ret <= 0)
1707                 return ret;
1708
1709         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1710 }
1711
1712 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1713                                    struct perf_sample *sample)
1714 {
1715         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1716         pid_t pid, tid;
1717         int cpu, ret;
1718
1719         cpu = sample->cpu;
1720
1721         if (pt->have_sched_switch == 3) {
1722                 if (!out)
1723                         return 0;
1724                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1725                         pr_err("Expecting CPU-wide context switch event\n");
1726                         return -EINVAL;
1727                 }
1728                 pid = event->context_switch.next_prev_pid;
1729                 tid = event->context_switch.next_prev_tid;
1730         } else {
1731                 if (out)
1732                         return 0;
1733                 pid = sample->pid;
1734                 tid = sample->tid;
1735         }
1736
1737         if (tid == -1)
1738                 intel_pt_log("context_switch event has no tid\n");
1739
1740         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1741                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1742                      &pt->tc));
1743
1744         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1745         if (ret <= 0)
1746                 return ret;
1747
1748         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1749 }
1750
1751 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1752                                          union perf_event *event,
1753                                          struct perf_sample *sample)
1754 {
1755         if (!pt->per_cpu_mmaps)
1756                 return 0;
1757
1758         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1759                      sample->cpu, event->itrace_start.pid,
1760                      event->itrace_start.tid, sample->time,
1761                      perf_time_to_tsc(sample->time, &pt->tc));
1762
1763         return machine__set_current_tid(pt->machine, sample->cpu,
1764                                         event->itrace_start.pid,
1765                                         event->itrace_start.tid);
1766 }
1767
1768 static int intel_pt_process_event(struct perf_session *session,
1769                                   union perf_event *event,
1770                                   struct perf_sample *sample,
1771                                   struct perf_tool *tool)
1772 {
1773         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1774                                            auxtrace);
1775         u64 timestamp;
1776         int err = 0;
1777
1778         if (dump_trace)
1779                 return 0;
1780
1781         if (!tool->ordered_events) {
1782                 pr_err("Intel Processor Trace requires ordered events\n");
1783                 return -EINVAL;
1784         }
1785
1786         if (sample->time && sample->time != (u64)-1)
1787                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1788         else
1789                 timestamp = 0;
1790
1791         if (timestamp || pt->timeless_decoding) {
1792                 err = intel_pt_update_queues(pt);
1793                 if (err)
1794                         return err;
1795         }
1796
1797         if (pt->timeless_decoding) {
1798                 if (event->header.type == PERF_RECORD_EXIT) {
1799                         err = intel_pt_process_timeless_queues(pt,
1800                                                                event->fork.tid,
1801                                                                sample->time);
1802                 }
1803         } else if (timestamp) {
1804                 err = intel_pt_process_queues(pt, timestamp);
1805         }
1806         if (err)
1807                 return err;
1808
1809         if (event->header.type == PERF_RECORD_AUX &&
1810             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1811             pt->synth_opts.errors) {
1812                 err = intel_pt_lost(pt, sample);
1813                 if (err)
1814                         return err;
1815         }
1816
1817         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1818                 err = intel_pt_process_switch(pt, sample);
1819         else if (event->header.type == PERF_RECORD_ITRACE_START)
1820                 err = intel_pt_process_itrace_start(pt, event, sample);
1821         else if (event->header.type == PERF_RECORD_SWITCH ||
1822                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1823                 err = intel_pt_context_switch(pt, event, sample);
1824
1825         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1826                      perf_event__name(event->header.type), event->header.type,
1827                      sample->cpu, sample->time, timestamp);
1828
1829         return err;
1830 }
1831
1832 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1833 {
1834         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1835                                            auxtrace);
1836         int ret;
1837
1838         if (dump_trace)
1839                 return 0;
1840
1841         if (!tool->ordered_events)
1842                 return -EINVAL;
1843
1844         ret = intel_pt_update_queues(pt);
1845         if (ret < 0)
1846                 return ret;
1847
1848         if (pt->timeless_decoding)
1849                 return intel_pt_process_timeless_queues(pt, -1,
1850                                                         MAX_TIMESTAMP - 1);
1851
1852         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1853 }
1854
1855 static void intel_pt_free_events(struct perf_session *session)
1856 {
1857         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1858                                            auxtrace);
1859         struct auxtrace_queues *queues = &pt->queues;
1860         unsigned int i;
1861
1862         for (i = 0; i < queues->nr_queues; i++) {
1863                 intel_pt_free_queue(queues->queue_array[i].priv);
1864                 queues->queue_array[i].priv = NULL;
1865         }
1866         intel_pt_log_disable();
1867         auxtrace_queues__free(queues);
1868 }
1869
1870 static void intel_pt_free(struct perf_session *session)
1871 {
1872         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1873                                            auxtrace);
1874
1875         auxtrace_heap__free(&pt->heap);
1876         intel_pt_free_events(session);
1877         session->auxtrace = NULL;
1878         thread__put(pt->unknown_thread);
1879         addr_filters__exit(&pt->filts);
1880         zfree(&pt->filter);
1881         free(pt);
1882 }
1883
1884 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1885                                            union perf_event *event,
1886                                            struct perf_tool *tool __maybe_unused)
1887 {
1888         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1889                                            auxtrace);
1890
1891         if (pt->sampling_mode)
1892                 return 0;
1893
1894         if (!pt->data_queued) {
1895                 struct auxtrace_buffer *buffer;
1896                 off_t data_offset;
1897                 int fd = perf_data_file__fd(session->file);
1898                 int err;
1899
1900                 if (perf_data_file__is_pipe(session->file)) {
1901                         data_offset = 0;
1902                 } else {
1903                         data_offset = lseek(fd, 0, SEEK_CUR);
1904                         if (data_offset == -1)
1905                                 return -errno;
1906                 }
1907
1908                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1909                                                  data_offset, &buffer);
1910                 if (err)
1911                         return err;
1912
1913                 /* Dump here now we have copied a piped trace out of the pipe */
1914                 if (dump_trace) {
1915                         if (auxtrace_buffer__get_data(buffer, fd)) {
1916                                 intel_pt_dump_event(pt, buffer->data,
1917                                                     buffer->size);
1918                                 auxtrace_buffer__put_data(buffer);
1919                         }
1920                 }
1921         }
1922
1923         return 0;
1924 }
1925
1926 struct intel_pt_synth {
1927         struct perf_tool dummy_tool;
1928         struct perf_session *session;
1929 };
1930
1931 static int intel_pt_event_synth(struct perf_tool *tool,
1932                                 union perf_event *event,
1933                                 struct perf_sample *sample __maybe_unused,
1934                                 struct machine *machine __maybe_unused)
1935 {
1936         struct intel_pt_synth *intel_pt_synth =
1937                         container_of(tool, struct intel_pt_synth, dummy_tool);
1938
1939         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1940                                                  NULL);
1941 }
1942
1943 static int intel_pt_synth_event(struct perf_session *session,
1944                                 struct perf_event_attr *attr, u64 id)
1945 {
1946         struct intel_pt_synth intel_pt_synth;
1947
1948         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1949         intel_pt_synth.session = session;
1950
1951         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1952                                            &id, intel_pt_event_synth);
1953 }
1954
1955 static int intel_pt_synth_events(struct intel_pt *pt,
1956                                  struct perf_session *session)
1957 {
1958         struct perf_evlist *evlist = session->evlist;
1959         struct perf_evsel *evsel;
1960         struct perf_event_attr attr;
1961         bool found = false;
1962         u64 id;
1963         int err;
1964
1965         evlist__for_each_entry(evlist, evsel) {
1966                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1967                         found = true;
1968                         break;
1969                 }
1970         }
1971
1972         if (!found) {
1973                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1974                 return 0;
1975         }
1976
1977         memset(&attr, 0, sizeof(struct perf_event_attr));
1978         attr.size = sizeof(struct perf_event_attr);
1979         attr.type = PERF_TYPE_HARDWARE;
1980         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1981         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1982                             PERF_SAMPLE_PERIOD;
1983         if (pt->timeless_decoding)
1984                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1985         else
1986                 attr.sample_type |= PERF_SAMPLE_TIME;
1987         if (!pt->per_cpu_mmaps)
1988                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1989         attr.exclude_user = evsel->attr.exclude_user;
1990         attr.exclude_kernel = evsel->attr.exclude_kernel;
1991         attr.exclude_hv = evsel->attr.exclude_hv;
1992         attr.exclude_host = evsel->attr.exclude_host;
1993         attr.exclude_guest = evsel->attr.exclude_guest;
1994         attr.sample_id_all = evsel->attr.sample_id_all;
1995         attr.read_format = evsel->attr.read_format;
1996
1997         id = evsel->id[0] + 1000000000;
1998         if (!id)
1999                 id = 1;
2000
2001         if (pt->synth_opts.instructions) {
2002                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2003                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2004                         attr.sample_period =
2005                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2006                 else
2007                         attr.sample_period = pt->synth_opts.period;
2008                 pt->instructions_sample_period = attr.sample_period;
2009                 if (pt->synth_opts.callchain)
2010                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2011                 if (pt->synth_opts.last_branch)
2012                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2013                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2014                          id, (u64)attr.sample_type);
2015                 err = intel_pt_synth_event(session, &attr, id);
2016                 if (err) {
2017                         pr_err("%s: failed to synthesize 'instructions' event type\n",
2018                                __func__);
2019                         return err;
2020                 }
2021                 pt->sample_instructions = true;
2022                 pt->instructions_sample_type = attr.sample_type;
2023                 pt->instructions_id = id;
2024                 id += 1;
2025         }
2026
2027         if (pt->synth_opts.transactions) {
2028                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2029                 attr.sample_period = 1;
2030                 if (pt->synth_opts.callchain)
2031                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2032                 if (pt->synth_opts.last_branch)
2033                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2034                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2035                          id, (u64)attr.sample_type);
2036                 err = intel_pt_synth_event(session, &attr, id);
2037                 if (err) {
2038                         pr_err("%s: failed to synthesize 'transactions' event type\n",
2039                                __func__);
2040                         return err;
2041                 }
2042                 pt->sample_transactions = true;
2043                 pt->transactions_id = id;
2044                 id += 1;
2045                 evlist__for_each_entry(evlist, evsel) {
2046                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
2047                                 if (evsel->name)
2048                                         zfree(&evsel->name);
2049                                 evsel->name = strdup("transactions");
2050                                 break;
2051                         }
2052                 }
2053         }
2054
2055         if (pt->synth_opts.branches) {
2056                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2057                 attr.sample_period = 1;
2058                 attr.sample_type |= PERF_SAMPLE_ADDR;
2059                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
2060                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
2061                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2062                          id, (u64)attr.sample_type);
2063                 err = intel_pt_synth_event(session, &attr, id);
2064                 if (err) {
2065                         pr_err("%s: failed to synthesize 'branches' event type\n",
2066                                __func__);
2067                         return err;
2068                 }
2069                 pt->sample_branches = true;
2070                 pt->branches_sample_type = attr.sample_type;
2071                 pt->branches_id = id;
2072         }
2073
2074         pt->synth_needs_swap = evsel->needs_swap;
2075
2076         return 0;
2077 }
2078
2079 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2080 {
2081         struct perf_evsel *evsel;
2082
2083         evlist__for_each_entry_reverse(evlist, evsel) {
2084                 const char *name = perf_evsel__name(evsel);
2085
2086                 if (!strcmp(name, "sched:sched_switch"))
2087                         return evsel;
2088         }
2089
2090         return NULL;
2091 }
2092
2093 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2094 {
2095         struct perf_evsel *evsel;
2096
2097         evlist__for_each_entry(evlist, evsel) {
2098                 if (evsel->attr.context_switch)
2099                         return true;
2100         }
2101
2102         return false;
2103 }
2104
2105 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2106 {
2107         struct intel_pt *pt = data;
2108
2109         if (!strcmp(var, "intel-pt.mispred-all"))
2110                 pt->mispred_all = perf_config_bool(var, value);
2111
2112         return 0;
2113 }
2114
2115 static const char * const intel_pt_info_fmts[] = {
2116         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2117         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2118         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2119         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2120         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2121         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2122         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2123         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2124         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2125         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2126         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2127         [INTEL_PT_MTC_FREQ_BITS]        = "  MTC freq bits       %#"PRIx64"\n",
2128         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2129         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2130         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2131         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2132         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2133 };
2134
2135 static void intel_pt_print_info(u64 *arr, int start, int finish)
2136 {
2137         int i;
2138
2139         if (!dump_trace)
2140                 return;
2141
2142         for (i = start; i <= finish; i++) {
2143                 const char *fmt = intel_pt_info_fmts[i];
2144
2145                 if (fmt)
2146                         fprintf(stdout, fmt, arr[i]);
2147         }
2148 }
2149
2150 static void intel_pt_print_info_str(const char *name, const char *str)
2151 {
2152         if (!dump_trace)
2153                 return;
2154
2155         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2156 }
2157
2158 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2159 {
2160         return auxtrace_info->header.size >=
2161                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2162 }
2163
2164 int intel_pt_process_auxtrace_info(union perf_event *event,
2165                                    struct perf_session *session)
2166 {
2167         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2168         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2169         struct intel_pt *pt;
2170         void *info_end;
2171         u64 *info;
2172         int err;
2173
2174         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2175                                         min_sz)
2176                 return -EINVAL;
2177
2178         pt = zalloc(sizeof(struct intel_pt));
2179         if (!pt)
2180                 return -ENOMEM;
2181
2182         addr_filters__init(&pt->filts);
2183
2184         perf_config(intel_pt_perf_config, pt);
2185
2186         err = auxtrace_queues__init(&pt->queues);
2187         if (err)
2188                 goto err_free;
2189
2190         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2191
2192         pt->session = session;
2193         pt->machine = &session->machines.host; /* No kvm support */
2194         pt->auxtrace_type = auxtrace_info->type;
2195         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2196         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2197         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2198         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2199         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2200         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2201         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2202         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2203         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2204         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2205         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2206                             INTEL_PT_PER_CPU_MMAPS);
2207
2208         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2209                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2210                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2211                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2212                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2213                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2214                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2215                                     INTEL_PT_CYC_BIT);
2216         }
2217
2218         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2219                 pt->max_non_turbo_ratio =
2220                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2221                 intel_pt_print_info(&auxtrace_info->priv[0],
2222                                     INTEL_PT_MAX_NONTURBO_RATIO,
2223                                     INTEL_PT_MAX_NONTURBO_RATIO);
2224         }
2225
2226         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2227         info_end = (void *)info + auxtrace_info->header.size;
2228
2229         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2230                 size_t len;
2231
2232                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2233                 intel_pt_print_info(&auxtrace_info->priv[0],
2234                                     INTEL_PT_FILTER_STR_LEN,
2235                                     INTEL_PT_FILTER_STR_LEN);
2236                 if (len) {
2237                         const char *filter = (const char *)info;
2238
2239                         len = roundup(len + 1, 8);
2240                         info += len >> 3;
2241                         if ((void *)info > info_end) {
2242                                 pr_err("%s: bad filter string length\n", __func__);
2243                                 err = -EINVAL;
2244                                 goto err_free_queues;
2245                         }
2246                         pt->filter = memdup(filter, len);
2247                         if (!pt->filter) {
2248                                 err = -ENOMEM;
2249                                 goto err_free_queues;
2250                         }
2251                         if (session->header.needs_swap)
2252                                 mem_bswap_64(pt->filter, len);
2253                         if (pt->filter[len - 1]) {
2254                                 pr_err("%s: filter string not null terminated\n", __func__);
2255                                 err = -EINVAL;
2256                                 goto err_free_queues;
2257                         }
2258                         err = addr_filters__parse_bare_filter(&pt->filts,
2259                                                               filter);
2260                         if (err)
2261                                 goto err_free_queues;
2262                 }
2263                 intel_pt_print_info_str("Filter string", pt->filter);
2264         }
2265
2266         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2267         if (pt->timeless_decoding && !pt->tc.time_mult)
2268                 pt->tc.time_mult = 1;
2269         pt->have_tsc = intel_pt_have_tsc(pt);
2270         pt->sampling_mode = false;
2271         pt->est_tsc = !pt->timeless_decoding;
2272
2273         pt->unknown_thread = thread__new(999999999, 999999999);
2274         if (!pt->unknown_thread) {
2275                 err = -ENOMEM;
2276                 goto err_free_queues;
2277         }
2278
2279         /*
2280          * Since this thread will not be kept in any rbtree not in a
2281          * list, initialize its list node so that at thread__put() the
2282          * current thread lifetime assuption is kept and we don't segfault
2283          * at list_del_init().
2284          */
2285         INIT_LIST_HEAD(&pt->unknown_thread->node);
2286
2287         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2288         if (err)
2289                 goto err_delete_thread;
2290         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2291                 err = -ENOMEM;
2292                 goto err_delete_thread;
2293         }
2294
2295         pt->auxtrace.process_event = intel_pt_process_event;
2296         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2297         pt->auxtrace.flush_events = intel_pt_flush;
2298         pt->auxtrace.free_events = intel_pt_free_events;
2299         pt->auxtrace.free = intel_pt_free;
2300         session->auxtrace = &pt->auxtrace;
2301
2302         if (dump_trace)
2303                 return 0;
2304
2305         if (pt->have_sched_switch == 1) {
2306                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2307                 if (!pt->switch_evsel) {
2308                         pr_err("%s: missing sched_switch event\n", __func__);
2309                         err = -EINVAL;
2310                         goto err_delete_thread;
2311                 }
2312         } else if (pt->have_sched_switch == 2 &&
2313                    !intel_pt_find_switch(session->evlist)) {
2314                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2315                 err = -EINVAL;
2316                 goto err_delete_thread;
2317         }
2318
2319         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2320                 pt->synth_opts = *session->itrace_synth_opts;
2321         } else {
2322                 itrace_synth_opts__set_default(&pt->synth_opts);
2323                 if (use_browser != -1) {
2324                         pt->synth_opts.branches = false;
2325                         pt->synth_opts.callchain = true;
2326                 }
2327                 if (session->itrace_synth_opts)
2328                         pt->synth_opts.thread_stack =
2329                                 session->itrace_synth_opts->thread_stack;
2330         }
2331
2332         if (pt->synth_opts.log)
2333                 intel_pt_log_enable();
2334
2335         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2336         if (pt->tc.time_mult) {
2337                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2338
2339                 if (!pt->max_non_turbo_ratio)
2340                         pt->max_non_turbo_ratio =
2341                                         (tsc_freq + 50000000) / 100000000;
2342                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2343                 intel_pt_log("Maximum non-turbo ratio %u\n",
2344                              pt->max_non_turbo_ratio);
2345         }
2346
2347         if (pt->synth_opts.calls)
2348                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2349                                        PERF_IP_FLAG_TRACE_END;
2350         if (pt->synth_opts.returns)
2351                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2352                                        PERF_IP_FLAG_TRACE_BEGIN;
2353
2354         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2355                 symbol_conf.use_callchain = true;
2356                 if (callchain_register_param(&callchain_param) < 0) {
2357                         symbol_conf.use_callchain = false;
2358                         pt->synth_opts.callchain = false;
2359                 }
2360         }
2361
2362         err = intel_pt_synth_events(pt, session);
2363         if (err)
2364                 goto err_delete_thread;
2365
2366         err = auxtrace_queues__process_index(&pt->queues, session);
2367         if (err)
2368                 goto err_delete_thread;
2369
2370         if (pt->queues.populated)
2371                 pt->data_queued = true;
2372
2373         if (pt->timeless_decoding)
2374                 pr_debug2("Intel PT decoding without timestamps\n");
2375
2376         return 0;
2377
2378 err_delete_thread:
2379         thread__zput(pt->unknown_thread);
2380 err_free_queues:
2381         intel_pt_log_disable();
2382         auxtrace_queues__free(&pt->queues);
2383         session->auxtrace = NULL;
2384 err_free:
2385         addr_filters__exit(&pt->filts);
2386         zfree(&pt->filter);
2387         free(pt);
2388         return err;
2389 }