tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         void *data;
 107         struct intel_pt_state state;
 108         const unsigned char *buf;
 109         size_t len;
 110         bool return_compression;
 111         bool mtc_insn;
 112         bool pge;
 113         bool have_tma;
 114         bool have_cyc;
 115         bool fixup_last_mtc;
 116         bool have_last_ip;
 117         enum intel_pt_param_flags flags;
 118         uint64_t pos;
 119         uint64_t last_ip;
 120         uint64_t ip;
 121         uint64_t cr3;
 122         uint64_t timestamp;
 123         uint64_t tsc_timestamp;
 124         uint64_t ref_timestamp;
 125         uint64_t sample_timestamp;
 126         uint64_t ret_addr;
 127         uint64_t ctc_timestamp;
 128         uint64_t ctc_delta;
 129         uint64_t cycle_cnt;
 130         uint64_t cyc_ref_timestamp;
 131         uint32_t last_mtc;
 132         uint32_t tsc_ctc_ratio_n;
 133         uint32_t tsc_ctc_ratio_d;
 134         uint32_t tsc_ctc_mult;
 135         uint32_t tsc_slip;
 136         uint32_t ctc_rem_mask;
 137         int mtc_shift;
 138         struct intel_pt_stack stack;
 139         enum intel_pt_pkt_state pkt_state;
 140         struct intel_pt_pkt packet;
 141         struct intel_pt_pkt tnt;
 142         int pkt_step;
 143         int pkt_len;
 144         int last_packet_type;
 145         unsigned int cbr;
 146         unsigned int max_non_turbo_ratio;
 147         double max_non_turbo_ratio_fp;
 148         double cbr_cyc_to_tsc;
 149         double calc_cyc_to_tsc;
 150         bool have_calc_cyc_to_tsc;
 151         int exec_mode;
 152         unsigned int insn_bytes;
 153         uint64_t period;
 154         enum intel_pt_period_type period_type;
 155         uint64_t tot_insn_cnt;
 156         uint64_t period_insn_cnt;
 157         uint64_t period_mask;
 158         uint64_t period_ticks;
 159         uint64_t last_masked_timestamp;
 160         bool continuous_period;
 161         bool overflow;
 162         bool set_fup_tx_flags;
 163         unsigned int fup_tx_flags;
 164         unsigned int tx_flags;
 165         uint64_t timestamp_insn_cnt;
 166         uint64_t sample_insn_cnt;
 167         uint64_t stuck_ip;
 168         int no_progress;
 169         int stuck_ip_prd;
 170         int stuck_ip_cnt;
 171         const unsigned char *next_buf;
 172         size_t next_len;
 173         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 174 };
 175
 176 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 177 {
 178         int i;
 179
 180         for (i = 0; x != 1; i++)
 181                 x >>= 1;
 182
 183         return x << i;
 184 }
 185
 186 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 187 {
 188         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 189                 uint64_t period;
 190
 191                 period = intel_pt_lower_power_of_2(decoder->period);
 192                 decoder->period_mask  = ~(period - 1);
 193                 decoder->period_ticks = period;
 194         }
 195 }
 196
 197 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 198 {
 199         if (!d)
 200                 return 0;
 201         return (t / d) * n + ((t % d) * n) / d;
 202 }
 203
 204 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 205 {
 206         struct intel_pt_decoder *decoder;
 207
 208         if (!params->get_trace || !params->walk_insn)
 209                 return NULL;
 210
 211         decoder = zalloc(sizeof(struct intel_pt_decoder));
 212         if (!decoder)
 213                 return NULL;
 214
 215         decoder->get_trace          = params->get_trace;
 216         decoder->walk_insn          = params->walk_insn;
 217         decoder->data               = params->data;
 218         decoder->return_compression = params->return_compression;
 219
 220         decoder->flags              = params->flags;
 221
 222         decoder->period             = params->period;
 223         decoder->period_type        = params->period_type;
 224
 225         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 226         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 227
 228         intel_pt_setup_period(decoder);
 229
 230         decoder->mtc_shift = params->mtc_period;
 231         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 232
 233         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 234         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 235
 236         if (!decoder->tsc_ctc_ratio_n)
 237                 decoder->tsc_ctc_ratio_d = 0;
 238
 239         if (decoder->tsc_ctc_ratio_d) {
 240                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 241                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 242                                                 decoder->tsc_ctc_ratio_d;
 243         }
 244
 245         /*
 246          * A TSC packet can slip past MTC packets so that the timestamp appears
 247          * to go backwards. One estimate is that can be up to about 40 CPU
 248          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 249          * slippage an order of magnitude more to be on the safe side.
 250          */
 251         decoder->tsc_slip = 0x10000;
 252
 253         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 254         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 255         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 256         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 257         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 258
 259         return decoder;
 260 }
 261
 262 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 263 {
 264         struct intel_pt_blk *blk = stack->blk;
 265
 266         stack->blk = blk->prev;
 267         if (!stack->spare)
 268                 stack->spare = blk;
 269         else
 270                 free(blk);
 271 }
 272
 273 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 274 {
 275         if (!stack->pos) {
 276                 if (!stack->blk)
 277                         return 0;
 278                 intel_pt_pop_blk(stack);
 279                 if (!stack->blk)
 280                         return 0;
 281                 stack->pos = INTEL_PT_BLK_SIZE;
 282         }
 283         return stack->blk->ip[--stack->pos];
 284 }
 285
 286 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 287 {
 288         struct intel_pt_blk *blk;
 289
 290         if (stack->spare) {
 291                 blk = stack->spare;
 292                 stack->spare = NULL;
 293         } else {
 294                 blk = malloc(sizeof(struct intel_pt_blk));
 295                 if (!blk)
 296                         return -ENOMEM;
 297         }
 298
 299         blk->prev = stack->blk;
 300         stack->blk = blk;
 301         stack->pos = 0;
 302         return 0;
 303 }
 304
 305 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 306 {
 307         int err;
 308
 309         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 310                 err = intel_pt_alloc_blk(stack);
 311                 if (err)
 312                         return err;
 313         }
 314
 315         stack->blk->ip[stack->pos++] = ip;
 316         return 0;
 317 }
 318
 319 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 320 {
 321         while (stack->blk)
 322                 intel_pt_pop_blk(stack);
 323         stack->pos = 0;
 324 }
 325
 326 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 327 {
 328         intel_pt_clear_stack(stack);
 329         zfree(&stack->blk);
 330         zfree(&stack->spare);
 331 }
 332
 333 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 334 {
 335         intel_pt_free_stack(&decoder->stack);
 336         free(decoder);
 337 }
 338
 339 static int intel_pt_ext_err(int code)
 340 {
 341         switch (code) {
 342         case -ENOMEM:
 343                 return INTEL_PT_ERR_NOMEM;
 344         case -ENOSYS:
 345                 return INTEL_PT_ERR_INTERN;
 346         case -EBADMSG:
 347                 return INTEL_PT_ERR_BADPKT;
 348         case -ENODATA:
 349                 return INTEL_PT_ERR_NODATA;
 350         case -EILSEQ:
 351                 return INTEL_PT_ERR_NOINSN;
 352         case -ENOENT:
 353                 return INTEL_PT_ERR_MISMAT;
 354         case -EOVERFLOW:
 355                 return INTEL_PT_ERR_OVR;
 356         case -ENOSPC:
 357                 return INTEL_PT_ERR_LOST;
 358         case -ELOOP:
 359                 return INTEL_PT_ERR_NELOOP;
 360         default:
 361                 return INTEL_PT_ERR_UNK;
 362         }
 363 }
 364
 365 static const char *intel_pt_err_msgs[] = {
 366         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 367         [INTEL_PT_ERR_INTERN] = "Internal error",
 368         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 369         [INTEL_PT_ERR_NODATA] = "No more data",
 370         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 371         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 372         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 373         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 374         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 375         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 376 };
 377
 378 int intel_pt__strerror(int code, char *buf, size_t buflen)
 379 {
 380         if (code < 1 || code > INTEL_PT_ERR_MAX)
 381                 code = INTEL_PT_ERR_UNK;
 382         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 383         return 0;
 384 }
 385
 386 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 387                                  uint64_t last_ip)
 388 {
 389         uint64_t ip;
 390
 391         switch (packet->count) {
 392         case 1:
 393                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 394                      packet->payload;
 395                 break;
 396         case 2:
 397                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 398                      packet->payload;
 399                 break;
 400         case 3:
 401                 ip = packet->payload;
 402                 /* Sign-extend 6-byte ip */
 403                 if (ip & (uint64_t)0x800000000000ULL)
 404                         ip |= (uint64_t)0xffff000000000000ULL;
 405                 break;
 406         case 4:
 407                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 408                      packet->payload;
 409                 break;
 410         case 6:
 411                 ip = packet->payload;
 412                 break;
 413         default:
 414                 return 0;
 415         }
 416
 417         return ip;
 418 }
 419
 420 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 421 {
 422         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 423         decoder->have_last_ip = true;
 424 }
 425
 426 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 427 {
 428         intel_pt_set_last_ip(decoder);
 429         decoder->ip = decoder->last_ip;
 430 }
 431
 432 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 433 {
 434         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 435                             decoder->buf);
 436 }
 437
 438 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 439 {
 440         intel_pt_log("ERROR: Internal error\n");
 441         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 442         return -ENOSYS;
 443 }
 444
 445 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 446 {
 447         decoder->tx_flags = 0;
 448 }
 449
 450 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 451 {
 452         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 453 }
 454
 455 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 456 {
 457         intel_pt_clear_tx_flags(decoder);
 458         decoder->have_tma = false;
 459         decoder->pkt_len = 1;
 460         decoder->pkt_step = 1;
 461         intel_pt_decoder_log_packet(decoder);
 462         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 463                 intel_pt_log("ERROR: Bad packet\n");
 464                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 465         }
 466         return -EBADMSG;
 467 }
 468
 469 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 470 {
 471         struct intel_pt_buffer buffer = { .buf = 0, };
 472         int ret;
 473
 474         decoder->pkt_step = 0;
 475
 476         intel_pt_log("Getting more data\n");
 477         ret = decoder->get_trace(&buffer, decoder->data);
 478         if (ret)
 479                 return ret;
 480         decoder->buf = buffer.buf;
 481         decoder->len = buffer.len;
 482         if (!decoder->len) {
 483                 intel_pt_log("No more data\n");
 484                 return -ENODATA;
 485         }
 486         if (!buffer.consecutive) {
 487                 decoder->ip = 0;
 488                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 489                 decoder->ref_timestamp = buffer.ref_timestamp;
 490                 decoder->timestamp = 0;
 491                 decoder->have_tma = false;
 492                 decoder->state.trace_nr = buffer.trace_nr;
 493                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 494                              decoder->ref_timestamp);
 495                 return -ENOLINK;
 496         }
 497
 498         return 0;
 499 }
 500
 501 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 502 {
 503         if (!decoder->next_buf)
 504                 return intel_pt_get_data(decoder);
 505
 506         decoder->buf = decoder->next_buf;
 507         decoder->len = decoder->next_len;
 508         decoder->next_buf = 0;
 509         decoder->next_len = 0;
 510         return 0;
 511 }
 512
 513 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 514 {
 515         unsigned char *buf = decoder->temp_buf;
 516         size_t old_len, len, n;
 517         int ret;
 518
 519         old_len = decoder->len;
 520         len = decoder->len;
 521         memcpy(buf, decoder->buf, len);
 522
 523         ret = intel_pt_get_data(decoder);
 524         if (ret) {
 525                 decoder->pos += old_len;
 526                 return ret < 0 ? ret : -EINVAL;
 527         }
 528
 529         n = INTEL_PT_PKT_MAX_SZ - len;
 530         if (n > decoder->len)
 531                 n = decoder->len;
 532         memcpy(buf + len, decoder->buf, n);
 533         len += n;
 534
 535         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 536         if (ret < (int)old_len) {
 537                 decoder->next_buf = decoder->buf;
 538                 decoder->next_len = decoder->len;
 539                 decoder->buf = buf;
 540                 decoder->len = old_len;
 541                 return intel_pt_bad_packet(decoder);
 542         }
 543
 544         decoder->next_buf = decoder->buf + (ret - old_len);
 545         decoder->next_len = decoder->len - (ret - old_len);
 546
 547         decoder->buf = buf;
 548         decoder->len = ret;
 549
 550         return ret;
 551 }
 552
 553 struct intel_pt_pkt_info {
 554         struct intel_pt_decoder   *decoder;
 555         struct intel_pt_pkt       packet;
 556         uint64_t                  pos;
 557         int                       pkt_len;
 558         int                       last_packet_type;
 559         void                      *data;
 560 };
 561
 562 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 563
 564 /* Lookahead packets in current buffer */
 565 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 566                                   intel_pt_pkt_cb_t cb, void *data)
 567 {
 568         struct intel_pt_pkt_info pkt_info;
 569         const unsigned char *buf = decoder->buf;
 570         size_t len = decoder->len;
 571         int ret;
 572
 573         pkt_info.decoder          = decoder;
 574         pkt_info.pos              = decoder->pos;
 575         pkt_info.pkt_len          = decoder->pkt_step;
 576         pkt_info.last_packet_type = decoder->last_packet_type;
 577         pkt_info.data             = data;
 578
 579         while (1) {
 580                 do {
 581                         pkt_info.pos += pkt_info.pkt_len;
 582                         buf          += pkt_info.pkt_len;
 583                         len          -= pkt_info.pkt_len;
 584
 585                         if (!len)
 586                                 return INTEL_PT_NEED_MORE_BYTES;
 587
 588                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 589                         if (!ret)
 590                                 return INTEL_PT_NEED_MORE_BYTES;
 591                         if (ret < 0)
 592                                 return ret;
 593
 594                         pkt_info.pkt_len = ret;
 595                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 596
 597                 ret = cb(&pkt_info);
 598                 if (ret)
 599                         return 0;
 600
 601                 pkt_info.last_packet_type = pkt_info.packet.type;
 602         }
 603 }
 604
 605 struct intel_pt_calc_cyc_to_tsc_info {
 606         uint64_t        cycle_cnt;
 607         unsigned int    cbr;
 608         uint32_t        last_mtc;
 609         uint64_t        ctc_timestamp;
 610         uint64_t        ctc_delta;
 611         uint64_t        tsc_timestamp;
 612         uint64_t        timestamp;
 613         bool            have_tma;
 614         bool            fixup_last_mtc;
 615         bool            from_mtc;
 616         double          cbr_cyc_to_tsc;
 617 };
 618
 619 /*
 620  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 621  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 622  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 623  * packet by copying the missing bits from the current MTC assuming the least
 624  * difference between the two, and that the current MTC comes after last_mtc.
 625  */
 626 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 627                                     uint32_t *last_mtc)
 628 {
 629         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 630         uint32_t mask = ~(first_missing_bit - 1);
 631
 632         *last_mtc |= mtc & mask;
 633         if (*last_mtc >= mtc) {
 634                 *last_mtc -= first_missing_bit;
 635                 *last_mtc &= 0xff;
 636         }
 637 }
 638
 639 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 640 {
 641         struct intel_pt_decoder *decoder = pkt_info->decoder;
 642         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 643         uint64_t timestamp;
 644         double cyc_to_tsc;
 645         unsigned int cbr;
 646         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 647
 648         switch (pkt_info->packet.type) {
 649         case INTEL_PT_TNT:
 650         case INTEL_PT_TIP_PGE:
 651         case INTEL_PT_TIP:
 652         case INTEL_PT_FUP:
 653         case INTEL_PT_PSB:
 654         case INTEL_PT_PIP:
 655         case INTEL_PT_MODE_EXEC:
 656         case INTEL_PT_MODE_TSX:
 657         case INTEL_PT_PSBEND:
 658         case INTEL_PT_PAD:
 659         case INTEL_PT_VMCS:
 660         case INTEL_PT_MNT:
 661                 return 0;
 662
 663         case INTEL_PT_MTC:
 664                 if (!data->have_tma)
 665                         return 0;
 666
 667                 mtc = pkt_info->packet.payload;
 668                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 669                         data->fixup_last_mtc = false;
 670                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 671                                                 &data->last_mtc);
 672                 }
 673                 if (mtc > data->last_mtc)
 674                         mtc_delta = mtc - data->last_mtc;
 675                 else
 676                         mtc_delta = mtc + 256 - data->last_mtc;
 677                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 678                 data->last_mtc = mtc;
 679
 680                 if (decoder->tsc_ctc_mult) {
 681                         timestamp = data->ctc_timestamp +
 682                                 data->ctc_delta * decoder->tsc_ctc_mult;
 683                 } else {
 684                         timestamp = data->ctc_timestamp +
 685                                 multdiv(data->ctc_delta,
 686                                         decoder->tsc_ctc_ratio_n,
 687                                         decoder->tsc_ctc_ratio_d);
 688                 }
 689
 690                 if (timestamp < data->timestamp)
 691                         return 1;
 692
 693                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 694                         data->timestamp = timestamp;
 695                         return 0;
 696                 }
 697
 698                 break;
 699
 700         case INTEL_PT_TSC:
 701                 timestamp = pkt_info->packet.payload |
 702                             (data->timestamp & (0xffULL << 56));
 703                 if (data->from_mtc && timestamp < data->timestamp &&
 704                     data->timestamp - timestamp < decoder->tsc_slip)
 705                         return 1;
 706                 if (timestamp < data->timestamp)
 707                         timestamp += (1ULL << 56);
 708                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 709                         if (data->from_mtc)
 710                                 return 1;
 711                         data->tsc_timestamp = timestamp;
 712                         data->timestamp = timestamp;
 713                         return 0;
 714                 }
 715                 break;
 716
 717         case INTEL_PT_TMA:
 718                 if (data->from_mtc)
 719                         return 1;
 720
 721                 if (!decoder->tsc_ctc_ratio_d)
 722                         return 0;
 723
 724                 ctc = pkt_info->packet.payload;
 725                 fc = pkt_info->packet.count;
 726                 ctc_rem = ctc & decoder->ctc_rem_mask;
 727
 728                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 729
 730                 data->ctc_timestamp = data->tsc_timestamp - fc;
 731                 if (decoder->tsc_ctc_mult) {
 732                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 733                 } else {
 734                         data->ctc_timestamp -=
 735                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 736                                         decoder->tsc_ctc_ratio_d);
 737                 }
 738
 739                 data->ctc_delta = 0;
 740                 data->have_tma = true;
 741                 data->fixup_last_mtc = true;
 742
 743                 return 0;
 744
 745         case INTEL_PT_CYC:
 746                 data->cycle_cnt += pkt_info->packet.payload;
 747                 return 0;
 748
 749         case INTEL_PT_CBR:
 750                 cbr = pkt_info->packet.payload;
 751                 if (data->cbr && data->cbr != cbr)
 752                         return 1;
 753                 data->cbr = cbr;
 754                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 755                 return 0;
 756
 757         case INTEL_PT_TIP_PGD:
 758         case INTEL_PT_TRACESTOP:
 759         case INTEL_PT_OVF:
 760         case INTEL_PT_BAD: /* Does not happen */
 761         default:
 762                 return 1;
 763         }
 764
 765         if (!data->cbr && decoder->cbr) {
 766                 data->cbr = decoder->cbr;
 767                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 768         }
 769
 770         if (!data->cycle_cnt)
 771                 return 1;
 772
 773         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 774
 775         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 776             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 777                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 778                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 779                 return 1;
 780         }
 781
 782         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 783         decoder->have_calc_cyc_to_tsc = true;
 784
 785         if (data->cbr) {
 786                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 787                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 788         } else {
 789                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 790                              cyc_to_tsc, pkt_info->pos);
 791         }
 792
 793         return 1;
 794 }
 795
 796 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 797                                      bool from_mtc)
 798 {
 799         struct intel_pt_calc_cyc_to_tsc_info data = {
 800                 .cycle_cnt      = 0,
 801                 .cbr            = 0,
 802                 .last_mtc       = decoder->last_mtc,
 803                 .ctc_timestamp  = decoder->ctc_timestamp,
 804                 .ctc_delta      = decoder->ctc_delta,
 805                 .tsc_timestamp  = decoder->tsc_timestamp,
 806                 .timestamp      = decoder->timestamp,
 807                 .have_tma       = decoder->have_tma,
 808                 .fixup_last_mtc = decoder->fixup_last_mtc,
 809                 .from_mtc       = from_mtc,
 810                 .cbr_cyc_to_tsc = 0,
 811         };
 812
 813         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 814 }
 815
 816 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 817 {
 818         int ret;
 819
 820         decoder->last_packet_type = decoder->packet.type;
 821
 822         do {
 823                 decoder->pos += decoder->pkt_step;
 824                 decoder->buf += decoder->pkt_step;
 825                 decoder->len -= decoder->pkt_step;
 826
 827                 if (!decoder->len) {
 828                         ret = intel_pt_get_next_data(decoder);
 829                         if (ret)
 830                                 return ret;
 831                 }
 832
 833                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 834                                           &decoder->packet);
 835                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 836                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 837                         ret = intel_pt_get_split_packet(decoder);
 838                         if (ret < 0)
 839                                 return ret;
 840                 }
 841                 if (ret <= 0)
 842                         return intel_pt_bad_packet(decoder);
 843
 844                 decoder->pkt_len = ret;
 845                 decoder->pkt_step = ret;
 846                 intel_pt_decoder_log_packet(decoder);
 847         } while (decoder->packet.type == INTEL_PT_PAD);
 848
 849         return 0;
 850 }
 851
 852 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 853 {
 854         uint64_t timestamp, masked_timestamp;
 855
 856         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 857         masked_timestamp = timestamp & decoder->period_mask;
 858         if (decoder->continuous_period) {
 859                 if (masked_timestamp > decoder->last_masked_timestamp)
 860                         return 1;
 861         } else {
 862                 timestamp += 1;
 863                 masked_timestamp = timestamp & decoder->period_mask;
 864                 if (masked_timestamp > decoder->last_masked_timestamp) {
 865                         decoder->last_masked_timestamp = masked_timestamp;
 866                         decoder->continuous_period = true;
 867                 }
 868         }
 869
 870         if (masked_timestamp < decoder->last_masked_timestamp)
 871                 return decoder->period_ticks;
 872
 873         return decoder->period_ticks - (timestamp - masked_timestamp);
 874 }
 875
 876 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 877 {
 878         switch (decoder->period_type) {
 879         case INTEL_PT_PERIOD_INSTRUCTIONS:
 880                 return decoder->period - decoder->period_insn_cnt;
 881         case INTEL_PT_PERIOD_TICKS:
 882                 return intel_pt_next_period(decoder);
 883         case INTEL_PT_PERIOD_NONE:
 884         case INTEL_PT_PERIOD_MTC:
 885         default:
 886                 return 0;
 887         }
 888 }
 889
 890 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 891 {
 892         uint64_t timestamp, masked_timestamp;
 893
 894         switch (decoder->period_type) {
 895         case INTEL_PT_PERIOD_INSTRUCTIONS:
 896                 decoder->period_insn_cnt = 0;
 897                 break;
 898         case INTEL_PT_PERIOD_TICKS:
 899                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 900                 masked_timestamp = timestamp & decoder->period_mask;
 901                 if (masked_timestamp > decoder->last_masked_timestamp)
 902                         decoder->last_masked_timestamp = masked_timestamp;
 903                 else
 904                         decoder->last_masked_timestamp += decoder->period_ticks;
 905                 break;
 906         case INTEL_PT_PERIOD_NONE:
 907         case INTEL_PT_PERIOD_MTC:
 908         default:
 909                 break;
 910         }
 911
 912         decoder->state.type |= INTEL_PT_INSTRUCTION;
 913 }
 914
 915 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 916                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 917 {
 918         uint64_t max_insn_cnt, insn_cnt = 0;
 919         int err;
 920
 921         if (!decoder->mtc_insn)
 922                 decoder->mtc_insn = true;
 923
 924         max_insn_cnt = intel_pt_next_sample(decoder);
 925
 926         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 927                                  max_insn_cnt, decoder->data);
 928
 929         decoder->tot_insn_cnt += insn_cnt;
 930         decoder->timestamp_insn_cnt += insn_cnt;
 931         decoder->sample_insn_cnt += insn_cnt;
 932         decoder->period_insn_cnt += insn_cnt;
 933
 934         if (err) {
 935                 decoder->no_progress = 0;
 936                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 937                 intel_pt_log_at("ERROR: Failed to get instruction",
 938                                 decoder->ip);
 939                 if (err == -ENOENT)
 940                         return -ENOLINK;
 941                 return -EILSEQ;
 942         }
 943
 944         if (ip && decoder->ip == ip) {
 945                 err = -EAGAIN;
 946                 goto out;
 947         }
 948
 949         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 950                 intel_pt_sample_insn(decoder);
 951
 952         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 953                 decoder->state.type = INTEL_PT_INSTRUCTION;
 954                 decoder->state.from_ip = decoder->ip;
 955                 decoder->state.to_ip = 0;
 956                 decoder->ip += intel_pt_insn->length;
 957                 err = INTEL_PT_RETURN;
 958                 goto out;
 959         }
 960
 961         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 962                 /* Zero-length calls are excluded */
 963                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 964                     intel_pt_insn->rel) {
 965                         err = intel_pt_push(&decoder->stack, decoder->ip +
 966                                             intel_pt_insn->length);
 967                         if (err)
 968                                 goto out;
 969                 }
 970         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 971                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 972         }
 973
 974         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 975                 int cnt = decoder->no_progress++;
 976
 977                 decoder->state.from_ip = decoder->ip;
 978                 decoder->ip += intel_pt_insn->length +
 979                                 intel_pt_insn->rel;
 980                 decoder->state.to_ip = decoder->ip;
 981                 err = INTEL_PT_RETURN;
 982
 983                 /*
 984                  * Check for being stuck in a loop.  This can happen if a
 985                  * decoder error results in the decoder erroneously setting the
 986                  * ip to an address that is itself in an infinite loop that
 987                  * consumes no packets.  When that happens, there must be an
 988                  * unconditional branch.
 989                  */
 990                 if (cnt) {
 991                         if (cnt == 1) {
 992                                 decoder->stuck_ip = decoder->state.to_ip;
 993                                 decoder->stuck_ip_prd = 1;
 994                                 decoder->stuck_ip_cnt = 1;
 995                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 996                                    decoder->state.to_ip == decoder->stuck_ip) {
 997                                 intel_pt_log_at("ERROR: Never-ending loop",
 998                                                 decoder->state.to_ip);
 999                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1000                                 err = -ELOOP;
1001                                 goto out;
1002                         } else if (!--decoder->stuck_ip_cnt) {
1003                                 decoder->stuck_ip_prd += 1;
1004                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1005                                 decoder->stuck_ip = decoder->state.to_ip;
1006                         }
1007                 }
1008                 goto out_no_progress;
1009         }
1010 out:
1011         decoder->no_progress = 0;
1012 out_no_progress:
1013         decoder->state.insn_op = intel_pt_insn->op;
1014         decoder->state.insn_len = intel_pt_insn->length;
1015
1016         if (decoder->tx_flags & INTEL_PT_IN_TX)
1017                 decoder->state.flags |= INTEL_PT_IN_TX;
1018
1019         return err;
1020 }
1021
1022 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1023                                           struct intel_pt_insn *intel_pt_insn,
1024                                           uint64_t ip, int err)
1025 {
1026         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1027                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1028                ip == decoder->ip + intel_pt_insn->length;
1029 }
1030
1031 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1032 {
1033         struct intel_pt_insn intel_pt_insn;
1034         uint64_t ip;
1035         int err;
1036
1037         ip = decoder->last_ip;
1038
1039         while (1) {
1040                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1041                 if (err == INTEL_PT_RETURN)
1042                         return 0;
1043                 if (err == -EAGAIN ||
1044                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1045                         if (decoder->set_fup_tx_flags) {
1046                                 decoder->set_fup_tx_flags = false;
1047                                 decoder->tx_flags = decoder->fup_tx_flags;
1048                                 decoder->state.type = INTEL_PT_TRANSACTION;
1049                                 decoder->state.from_ip = decoder->ip;
1050                                 decoder->state.to_ip = 0;
1051                                 decoder->state.flags = decoder->fup_tx_flags;
1052                                 return 0;
1053                         }
1054                         return -EAGAIN;
1055                 }
1056                 decoder->set_fup_tx_flags = false;
1057                 if (err)
1058                         return err;
1059
1060                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1061                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1062                                         decoder->ip);
1063                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1064                         return -ENOENT;
1065                 }
1066
1067                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1068                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1069                                         decoder->ip);
1070                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1071                         return -ENOENT;
1072                 }
1073
1074                 intel_pt_bug(decoder);
1075         }
1076 }
1077
1078 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1079 {
1080         struct intel_pt_insn intel_pt_insn;
1081         int err;
1082
1083         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1084         if (err == INTEL_PT_RETURN)
1085                 return 0;
1086         if (err)
1087                 return err;
1088
1089         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1090                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1091                         decoder->pge = false;
1092                         decoder->continuous_period = false;
1093                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1094                         decoder->state.from_ip = decoder->ip;
1095                         decoder->state.to_ip = 0;
1096                         if (decoder->packet.count != 0)
1097                                 decoder->ip = decoder->last_ip;
1098                 } else {
1099                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1100                         decoder->state.from_ip = decoder->ip;
1101                         if (decoder->packet.count == 0) {
1102                                 decoder->state.to_ip = 0;
1103                         } else {
1104                                 decoder->state.to_ip = decoder->last_ip;
1105                                 decoder->ip = decoder->last_ip;
1106                         }
1107                 }
1108                 return 0;
1109         }
1110
1111         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1112                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1113                                 decoder->ip);
1114                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1115                 return -ENOENT;
1116         }
1117
1118         return intel_pt_bug(decoder);
1119 }
1120
1121 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1122 {
1123         struct intel_pt_insn intel_pt_insn;
1124         int err;
1125
1126         while (1) {
1127                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1128                 if (err == INTEL_PT_RETURN)
1129                         return 0;
1130                 if (err)
1131                         return err;
1132
1133                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1134                         if (!decoder->return_compression) {
1135                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1136                                                 decoder->ip);
1137                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1138                                 return -ENOENT;
1139                         }
1140                         if (!decoder->ret_addr) {
1141                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1142                                                 decoder->ip);
1143                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1144                                 return -ENOENT;
1145                         }
1146                         if (!(decoder->tnt.payload & BIT63)) {
1147                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1148                                                 decoder->ip);
1149                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1150                                 return -ENOENT;
1151                         }
1152                         decoder->tnt.count -= 1;
1153                         if (decoder->tnt.count)
1154                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1155                         else
1156                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1157                         decoder->tnt.payload <<= 1;
1158                         decoder->state.from_ip = decoder->ip;
1159                         decoder->ip = decoder->ret_addr;
1160                         decoder->state.to_ip = decoder->ip;
1161                         return 0;
1162                 }
1163
1164                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1165                         /* Handle deferred TIPs */
1166                         err = intel_pt_get_next_packet(decoder);
1167                         if (err)
1168                                 return err;
1169                         if (decoder->packet.type != INTEL_PT_TIP ||
1170                             decoder->packet.count == 0) {
1171                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1172                                                 decoder->ip);
1173                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1174                                 decoder->pkt_step = 0;
1175                                 return -ENOENT;
1176                         }
1177                         intel_pt_set_last_ip(decoder);
1178                         decoder->state.from_ip = decoder->ip;
1179                         decoder->state.to_ip = decoder->last_ip;
1180                         decoder->ip = decoder->last_ip;
1181                         return 0;
1182                 }
1183
1184                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1185                         decoder->tnt.count -= 1;
1186                         if (decoder->tnt.count)
1187                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1188                         else
1189                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1190                         if (decoder->tnt.payload & BIT63) {
1191                                 decoder->tnt.payload <<= 1;
1192                                 decoder->state.from_ip = decoder->ip;
1193                                 decoder->ip += intel_pt_insn.length +
1194                                                intel_pt_insn.rel;
1195                                 decoder->state.to_ip = decoder->ip;
1196                                 return 0;
1197                         }
1198                         /* Instruction sample for a non-taken branch */
1199                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1200                                 decoder->tnt.payload <<= 1;
1201                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1202                                 decoder->state.from_ip = decoder->ip;
1203                                 decoder->state.to_ip = 0;
1204                                 decoder->ip += intel_pt_insn.length;
1205                                 return 0;
1206                         }
1207                         decoder->ip += intel_pt_insn.length;
1208                         if (!decoder->tnt.count) {
1209                                 decoder->sample_timestamp = decoder->timestamp;
1210                                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
1211                                 return -EAGAIN;
1212                         }
1213                         decoder->tnt.payload <<= 1;
1214                         continue;
1215                 }
1216
1217                 return intel_pt_bug(decoder);
1218         }
1219 }
1220
1221 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1222 {
1223         unsigned int fup_tx_flags;
1224         int err;
1225
1226         fup_tx_flags = decoder->packet.payload &
1227                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1228         err = intel_pt_get_next_packet(decoder);
1229         if (err)
1230                 return err;
1231         if (decoder->packet.type == INTEL_PT_FUP) {
1232                 decoder->fup_tx_flags = fup_tx_flags;
1233                 decoder->set_fup_tx_flags = true;
1234                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1235                         *no_tip = true;
1236         } else {
1237                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1238                                 decoder->pos);
1239                 intel_pt_update_in_tx(decoder);
1240         }
1241         return 0;
1242 }
1243
1244 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1245 {
1246         uint64_t timestamp;
1247
1248         decoder->have_tma = false;
1249
1250         if (decoder->ref_timestamp) {
1251                 timestamp = decoder->packet.payload |
1252                             (decoder->ref_timestamp & (0xffULL << 56));
1253                 if (timestamp < decoder->ref_timestamp) {
1254                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1255                                 timestamp += (1ULL << 56);
1256                 } else {
1257                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1258                                 timestamp -= (1ULL << 56);
1259                 }
1260                 decoder->tsc_timestamp = timestamp;
1261                 decoder->timestamp = timestamp;
1262                 decoder->ref_timestamp = 0;
1263                 decoder->timestamp_insn_cnt = 0;
1264         } else if (decoder->timestamp) {
1265                 timestamp = decoder->packet.payload |
1266                             (decoder->timestamp & (0xffULL << 56));
1267                 decoder->tsc_timestamp = timestamp;
1268                 if (timestamp < decoder->timestamp &&
1269                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1270                         intel_pt_log_to("Suppressing backwards timestamp",
1271                                         timestamp);
1272                         timestamp = decoder->timestamp;
1273                 }
1274                 if (timestamp < decoder->timestamp) {
1275                         intel_pt_log_to("Wraparound timestamp", timestamp);
1276                         timestamp += (1ULL << 56);
1277                         decoder->tsc_timestamp = timestamp;
1278                 }
1279                 decoder->timestamp = timestamp;
1280                 decoder->timestamp_insn_cnt = 0;
1281         }
1282
1283         if (decoder->last_packet_type == INTEL_PT_CYC) {
1284                 decoder->cyc_ref_timestamp = decoder->timestamp;
1285                 decoder->cycle_cnt = 0;
1286                 decoder->have_calc_cyc_to_tsc = false;
1287                 intel_pt_calc_cyc_to_tsc(decoder, false);
1288         }
1289
1290         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1291 }
1292
1293 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1294 {
1295         intel_pt_log("ERROR: Buffer overflow\n");
1296         intel_pt_clear_tx_flags(decoder);
1297         decoder->timestamp_insn_cnt = 0;
1298         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1299         decoder->overflow = true;
1300         return -EOVERFLOW;
1301 }
1302
1303 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1304 {
1305         uint32_t ctc = decoder->packet.payload;
1306         uint32_t fc = decoder->packet.count;
1307         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1308
1309         if (!decoder->tsc_ctc_ratio_d)
1310                 return;
1311
1312         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1313         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1314         if (decoder->tsc_ctc_mult) {
1315                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1316         } else {
1317                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1318                                                   decoder->tsc_ctc_ratio_n,
1319                                                   decoder->tsc_ctc_ratio_d);
1320         }
1321         decoder->ctc_delta = 0;
1322         decoder->have_tma = true;
1323         decoder->fixup_last_mtc = true;
1324         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1325                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1326 }
1327
1328 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1329 {
1330         uint64_t timestamp;
1331         uint32_t mtc, mtc_delta;
1332
1333         if (!decoder->have_tma)
1334                 return;
1335
1336         mtc = decoder->packet.payload;
1337
1338         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1339                 decoder->fixup_last_mtc = false;
1340                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1341                                         &decoder->last_mtc);
1342         }
1343
1344         if (mtc > decoder->last_mtc)
1345                 mtc_delta = mtc - decoder->last_mtc;
1346         else
1347                 mtc_delta = mtc + 256 - decoder->last_mtc;
1348
1349         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1350
1351         if (decoder->tsc_ctc_mult) {
1352                 timestamp = decoder->ctc_timestamp +
1353                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1354         } else {
1355                 timestamp = decoder->ctc_timestamp +
1356                             multdiv(decoder->ctc_delta,
1357                                     decoder->tsc_ctc_ratio_n,
1358                                     decoder->tsc_ctc_ratio_d);
1359         }
1360
1361         if (timestamp < decoder->timestamp)
1362                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1363                              timestamp, decoder->timestamp);
1364         else
1365                 decoder->timestamp = timestamp;
1366
1367         decoder->timestamp_insn_cnt = 0;
1368         decoder->last_mtc = mtc;
1369
1370         if (decoder->last_packet_type == INTEL_PT_CYC) {
1371                 decoder->cyc_ref_timestamp = decoder->timestamp;
1372                 decoder->cycle_cnt = 0;
1373                 decoder->have_calc_cyc_to_tsc = false;
1374                 intel_pt_calc_cyc_to_tsc(decoder, true);
1375         }
1376 }
1377
1378 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1379 {
1380         unsigned int cbr = decoder->packet.payload;
1381
1382         if (decoder->cbr == cbr)
1383                 return;
1384
1385         decoder->cbr = cbr;
1386         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1387 }
1388
1389 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1390 {
1391         uint64_t timestamp = decoder->cyc_ref_timestamp;
1392
1393         decoder->have_cyc = true;
1394
1395         decoder->cycle_cnt += decoder->packet.payload;
1396
1397         if (!decoder->cyc_ref_timestamp)
1398                 return;
1399
1400         if (decoder->have_calc_cyc_to_tsc)
1401                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1402         else if (decoder->cbr)
1403                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1404         else
1405                 return;
1406
1407         if (timestamp < decoder->timestamp)
1408                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1409                              timestamp, decoder->timestamp);
1410         else
1411                 decoder->timestamp = timestamp;
1412
1413         decoder->timestamp_insn_cnt = 0;
1414 }
1415
1416 /* Walk PSB+ packets when already in sync. */
1417 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1418 {
1419         int err;
1420
1421         while (1) {
1422                 err = intel_pt_get_next_packet(decoder);
1423                 if (err)
1424                         return err;
1425
1426                 switch (decoder->packet.type) {
1427                 case INTEL_PT_PSBEND:
1428                         return 0;
1429
1430                 case INTEL_PT_TIP_PGD:
1431                 case INTEL_PT_TIP_PGE:
1432                 case INTEL_PT_TIP:
1433                 case INTEL_PT_TNT:
1434                 case INTEL_PT_TRACESTOP:
1435                 case INTEL_PT_BAD:
1436                 case INTEL_PT_PSB:
1437                         decoder->have_tma = false;
1438                         intel_pt_log("ERROR: Unexpected packet\n");
1439                         return -EAGAIN;
1440
1441                 case INTEL_PT_OVF:
1442                         return intel_pt_overflow(decoder);
1443
1444                 case INTEL_PT_TSC:
1445                         intel_pt_calc_tsc_timestamp(decoder);
1446                         break;
1447
1448                 case INTEL_PT_TMA:
1449                         intel_pt_calc_tma(decoder);
1450                         break;
1451
1452                 case INTEL_PT_CBR:
1453                         intel_pt_calc_cbr(decoder);
1454                         break;
1455
1456                 case INTEL_PT_MODE_EXEC:
1457                         decoder->exec_mode = decoder->packet.payload;
1458                         break;
1459
1460                 case INTEL_PT_PIP:
1461                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1462                         break;
1463
1464                 case INTEL_PT_FUP:
1465                         decoder->pge = true;
1466                         if (decoder->packet.count)
1467                                 intel_pt_set_last_ip(decoder);
1468                         break;
1469
1470                 case INTEL_PT_MODE_TSX:
1471                         intel_pt_update_in_tx(decoder);
1472                         break;
1473
1474                 case INTEL_PT_MTC:
1475                         intel_pt_calc_mtc_timestamp(decoder);
1476                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1477                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1478                         break;
1479
1480                 case INTEL_PT_CYC:
1481                         intel_pt_calc_cyc_timestamp(decoder);
1482                         break;
1483
1484                 case INTEL_PT_VMCS:
1485                 case INTEL_PT_MNT:
1486                 case INTEL_PT_PAD:
1487                 default:
1488                         break;
1489                 }
1490         }
1491 }
1492
1493 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1494 {
1495         int err;
1496
1497         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1498                 decoder->tx_flags = 0;
1499                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1500                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1501         } else {
1502                 decoder->state.flags |= INTEL_PT_ASYNC;
1503         }
1504
1505         while (1) {
1506                 err = intel_pt_get_next_packet(decoder);
1507                 if (err)
1508                         return err;
1509
1510                 switch (decoder->packet.type) {
1511                 case INTEL_PT_TNT:
1512                 case INTEL_PT_FUP:
1513                 case INTEL_PT_TRACESTOP:
1514                 case INTEL_PT_PSB:
1515                 case INTEL_PT_TSC:
1516                 case INTEL_PT_TMA:
1517                 case INTEL_PT_MODE_TSX:
1518                 case INTEL_PT_BAD:
1519                 case INTEL_PT_PSBEND:
1520                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1521                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1522                         decoder->pkt_step = 0;
1523                         return -ENOENT;
1524
1525                 case INTEL_PT_CBR:
1526                         intel_pt_calc_cbr(decoder);
1527                         break;
1528
1529                 case INTEL_PT_OVF:
1530                         return intel_pt_overflow(decoder);
1531
1532                 case INTEL_PT_TIP_PGD:
1533                         decoder->state.from_ip = decoder->ip;
1534                         decoder->state.to_ip = 0;
1535                         if (decoder->packet.count != 0) {
1536                                 intel_pt_set_ip(decoder);
1537                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1538                                              decoder->ip);
1539                         }
1540                         decoder->pge = false;
1541                         decoder->continuous_period = false;
1542                         return 0;
1543
1544                 case INTEL_PT_TIP_PGE:
1545                         decoder->pge = true;
1546                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1547                                      decoder->ip);
1548                         decoder->state.from_ip = 0;
1549                         if (decoder->packet.count == 0) {
1550                                 decoder->state.to_ip = 0;
1551                         } else {
1552                                 intel_pt_set_ip(decoder);
1553                                 decoder->state.to_ip = decoder->ip;
1554                         }
1555                         return 0;
1556
1557                 case INTEL_PT_TIP:
1558                         decoder->state.from_ip = decoder->ip;
1559                         if (decoder->packet.count == 0) {
1560                                 decoder->state.to_ip = 0;
1561                         } else {
1562                                 intel_pt_set_ip(decoder);
1563                                 decoder->state.to_ip = decoder->ip;
1564                         }
1565                         return 0;
1566
1567                 case INTEL_PT_PIP:
1568                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1569                         break;
1570
1571                 case INTEL_PT_MTC:
1572                         intel_pt_calc_mtc_timestamp(decoder);
1573                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1574                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1575                         break;
1576
1577                 case INTEL_PT_CYC:
1578                         intel_pt_calc_cyc_timestamp(decoder);
1579                         break;
1580
1581                 case INTEL_PT_MODE_EXEC:
1582                         decoder->exec_mode = decoder->packet.payload;
1583                         break;
1584
1585                 case INTEL_PT_VMCS:
1586                 case INTEL_PT_MNT:
1587                 case INTEL_PT_PAD:
1588                         break;
1589
1590                 default:
1591                         return intel_pt_bug(decoder);
1592                 }
1593         }
1594 }
1595
1596 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1597 {
1598         bool no_tip = false;
1599         int err;
1600
1601         while (1) {
1602                 err = intel_pt_get_next_packet(decoder);
1603                 if (err)
1604                         return err;
1605 next:
1606                 switch (decoder->packet.type) {
1607                 case INTEL_PT_TNT:
1608                         if (!decoder->packet.count)
1609                                 break;
1610                         decoder->tnt = decoder->packet;
1611                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1612                         err = intel_pt_walk_tnt(decoder);
1613                         if (err == -EAGAIN)
1614                                 break;
1615                         return err;
1616
1617                 case INTEL_PT_TIP_PGD:
1618                         if (decoder->packet.count != 0)
1619                                 intel_pt_set_last_ip(decoder);
1620                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1621                         return intel_pt_walk_tip(decoder);
1622
1623                 case INTEL_PT_TIP_PGE: {
1624                         decoder->pge = true;
1625                         if (decoder->packet.count == 0) {
1626                                 intel_pt_log_at("Skipping zero TIP.PGE",
1627                                                 decoder->pos);
1628                                 break;
1629                         }
1630                         intel_pt_set_ip(decoder);
1631                         decoder->state.from_ip = 0;
1632                         decoder->state.to_ip = decoder->ip;
1633                         return 0;
1634                 }
1635
1636                 case INTEL_PT_OVF:
1637                         return intel_pt_overflow(decoder);
1638
1639                 case INTEL_PT_TIP:
1640                         if (decoder->packet.count != 0)
1641                                 intel_pt_set_last_ip(decoder);
1642                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1643                         return intel_pt_walk_tip(decoder);
1644
1645                 case INTEL_PT_FUP:
1646                         if (decoder->packet.count == 0) {
1647                                 intel_pt_log_at("Skipping zero FUP",
1648                                                 decoder->pos);
1649                                 no_tip = false;
1650                                 break;
1651                         }
1652                         intel_pt_set_last_ip(decoder);
1653                         err = intel_pt_walk_fup(decoder);
1654                         if (err != -EAGAIN) {
1655                                 if (err)
1656                                         return err;
1657                                 if (no_tip)
1658                                         decoder->pkt_state =
1659                                                 INTEL_PT_STATE_FUP_NO_TIP;
1660                                 else
1661                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1662                                 return 0;
1663                         }
1664                         if (no_tip) {
1665                                 no_tip = false;
1666                                 break;
1667                         }
1668                         return intel_pt_walk_fup_tip(decoder);
1669
1670                 case INTEL_PT_TRACESTOP:
1671                         decoder->pge = false;
1672                         decoder->continuous_period = false;
1673                         intel_pt_clear_tx_flags(decoder);
1674                         decoder->have_tma = false;
1675                         break;
1676
1677                 case INTEL_PT_PSB:
1678                         decoder->last_ip = 0;
1679                         decoder->have_last_ip = true;
1680                         intel_pt_clear_stack(&decoder->stack);
1681                         err = intel_pt_walk_psbend(decoder);
1682                         if (err == -EAGAIN)
1683                                 goto next;
1684                         if (err)
1685                                 return err;
1686                         break;
1687
1688                 case INTEL_PT_PIP:
1689                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1690                         break;
1691
1692                 case INTEL_PT_MTC:
1693                         intel_pt_calc_mtc_timestamp(decoder);
1694                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1695                                 break;
1696                         /*
1697                          * Ensure that there has been an instruction since the
1698                          * last MTC.
1699                          */
1700                         if (!decoder->mtc_insn)
1701                                 break;
1702                         decoder->mtc_insn = false;
1703                         /* Ensure that there is a timestamp */
1704                         if (!decoder->timestamp)
1705                                 break;
1706                         decoder->state.type = INTEL_PT_INSTRUCTION;
1707                         decoder->state.from_ip = decoder->ip;
1708                         decoder->state.to_ip = 0;
1709                         decoder->mtc_insn = false;
1710                         return 0;
1711
1712                 case INTEL_PT_TSC:
1713                         intel_pt_calc_tsc_timestamp(decoder);
1714                         break;
1715
1716                 case INTEL_PT_TMA:
1717                         intel_pt_calc_tma(decoder);
1718                         break;
1719
1720                 case INTEL_PT_CYC:
1721                         intel_pt_calc_cyc_timestamp(decoder);
1722                         break;
1723
1724                 case INTEL_PT_CBR:
1725                         intel_pt_calc_cbr(decoder);
1726                         break;
1727
1728                 case INTEL_PT_MODE_EXEC:
1729                         decoder->exec_mode = decoder->packet.payload;
1730                         break;
1731
1732                 case INTEL_PT_MODE_TSX:
1733                         /* MODE_TSX need not be followed by FUP */
1734                         if (!decoder->pge) {
1735                                 intel_pt_update_in_tx(decoder);
1736                                 break;
1737                         }
1738                         err = intel_pt_mode_tsx(decoder, &no_tip);
1739                         if (err)
1740                                 return err;
1741                         goto next;
1742
1743                 case INTEL_PT_BAD: /* Does not happen */
1744                         return intel_pt_bug(decoder);
1745
1746                 case INTEL_PT_PSBEND:
1747                 case INTEL_PT_VMCS:
1748                 case INTEL_PT_MNT:
1749                 case INTEL_PT_PAD:
1750                         break;
1751
1752                 default:
1753                         return intel_pt_bug(decoder);
1754                 }
1755         }
1756 }
1757
1758 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1759 {
1760         return decoder->packet.count &&
1761                (decoder->have_last_ip || decoder->packet.count == 3 ||
1762                 decoder->packet.count == 6);
1763 }
1764
1765 /* Walk PSB+ packets to get in sync. */
1766 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1767 {
1768         int err;
1769
1770         while (1) {
1771                 err = intel_pt_get_next_packet(decoder);
1772                 if (err)
1773                         return err;
1774
1775                 switch (decoder->packet.type) {
1776                 case INTEL_PT_TIP_PGD:
1777                         decoder->continuous_period = false;
1778                         __fallthrough;
1779                 case INTEL_PT_TIP_PGE:
1780                 case INTEL_PT_TIP:
1781                         intel_pt_log("ERROR: Unexpected packet\n");
1782                         return -ENOENT;
1783
1784                 case INTEL_PT_FUP:
1785                         decoder->pge = true;
1786                         if (intel_pt_have_ip(decoder)) {
1787                                 uint64_t current_ip = decoder->ip;
1788
1789                                 intel_pt_set_ip(decoder);
1790                                 if (current_ip)
1791                                         intel_pt_log_to("Setting IP",
1792                                                         decoder->ip);
1793                         }
1794                         break;
1795
1796                 case INTEL_PT_MTC:
1797                         intel_pt_calc_mtc_timestamp(decoder);
1798                         break;
1799
1800                 case INTEL_PT_TSC:
1801                         intel_pt_calc_tsc_timestamp(decoder);
1802                         break;
1803
1804                 case INTEL_PT_TMA:
1805                         intel_pt_calc_tma(decoder);
1806                         break;
1807
1808                 case INTEL_PT_CYC:
1809                         intel_pt_calc_cyc_timestamp(decoder);
1810                         break;
1811
1812                 case INTEL_PT_CBR:
1813                         intel_pt_calc_cbr(decoder);
1814                         break;
1815
1816                 case INTEL_PT_PIP:
1817                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1818                         break;
1819
1820                 case INTEL_PT_MODE_EXEC:
1821                         decoder->exec_mode = decoder->packet.payload;
1822                         break;
1823
1824                 case INTEL_PT_MODE_TSX:
1825                         intel_pt_update_in_tx(decoder);
1826                         break;
1827
1828                 case INTEL_PT_TRACESTOP:
1829                         decoder->pge = false;
1830                         decoder->continuous_period = false;
1831                         intel_pt_clear_tx_flags(decoder);
1832                         __fallthrough;
1833
1834                 case INTEL_PT_TNT:
1835                         decoder->have_tma = false;
1836                         intel_pt_log("ERROR: Unexpected packet\n");
1837                         if (decoder->ip)
1838                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1839                         else
1840                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1841                         return -ENOENT;
1842
1843                 case INTEL_PT_BAD: /* Does not happen */
1844                         return intel_pt_bug(decoder);
1845
1846                 case INTEL_PT_OVF:
1847                         return intel_pt_overflow(decoder);
1848
1849                 case INTEL_PT_PSBEND:
1850                         return 0;
1851
1852                 case INTEL_PT_PSB:
1853                 case INTEL_PT_VMCS:
1854                 case INTEL_PT_MNT:
1855                 case INTEL_PT_PAD:
1856                 default:
1857                         break;
1858                 }
1859         }
1860 }
1861
1862 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1863 {
1864         int err;
1865
1866         while (1) {
1867                 err = intel_pt_get_next_packet(decoder);
1868                 if (err)
1869                         return err;
1870
1871                 switch (decoder->packet.type) {
1872                 case INTEL_PT_TIP_PGD:
1873                         decoder->continuous_period = false;
1874                         __fallthrough;
1875                 case INTEL_PT_TIP_PGE:
1876                 case INTEL_PT_TIP:
1877                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1878                         if (intel_pt_have_ip(decoder))
1879                                 intel_pt_set_ip(decoder);
1880                         if (decoder->ip)
1881                                 return 0;
1882                         break;
1883
1884                 case INTEL_PT_FUP:
1885                         if (intel_pt_have_ip(decoder))
1886                                 intel_pt_set_ip(decoder);
1887                         if (decoder->ip)
1888                                 return 0;
1889                         break;
1890
1891                 case INTEL_PT_MTC:
1892                         intel_pt_calc_mtc_timestamp(decoder);
1893                         break;
1894
1895                 case INTEL_PT_TSC:
1896                         intel_pt_calc_tsc_timestamp(decoder);
1897                         break;
1898
1899                 case INTEL_PT_TMA:
1900                         intel_pt_calc_tma(decoder);
1901                         break;
1902
1903                 case INTEL_PT_CYC:
1904                         intel_pt_calc_cyc_timestamp(decoder);
1905                         break;
1906
1907                 case INTEL_PT_CBR:
1908                         intel_pt_calc_cbr(decoder);
1909                         break;
1910
1911                 case INTEL_PT_PIP:
1912                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1913                         break;
1914
1915                 case INTEL_PT_MODE_EXEC:
1916                         decoder->exec_mode = decoder->packet.payload;
1917                         break;
1918
1919                 case INTEL_PT_MODE_TSX:
1920                         intel_pt_update_in_tx(decoder);
1921                         break;
1922
1923                 case INTEL_PT_OVF:
1924                         return intel_pt_overflow(decoder);
1925
1926                 case INTEL_PT_BAD: /* Does not happen */
1927                         return intel_pt_bug(decoder);
1928
1929                 case INTEL_PT_TRACESTOP:
1930                         decoder->pge = false;
1931                         decoder->continuous_period = false;
1932                         intel_pt_clear_tx_flags(decoder);
1933                         decoder->have_tma = false;
1934                         break;
1935
1936                 case INTEL_PT_PSB:
1937                         decoder->last_ip = 0;
1938                         decoder->have_last_ip = true;
1939                         intel_pt_clear_stack(&decoder->stack);
1940                         err = intel_pt_walk_psb(decoder);
1941                         if (err)
1942                                 return err;
1943                         if (decoder->ip) {
1944                                 /* Do not have a sample */
1945                                 decoder->state.type = 0;
1946                                 return 0;
1947                         }
1948                         break;
1949
1950                 case INTEL_PT_TNT:
1951                 case INTEL_PT_PSBEND:
1952                 case INTEL_PT_VMCS:
1953                 case INTEL_PT_MNT:
1954                 case INTEL_PT_PAD:
1955                 default:
1956                         break;
1957                 }
1958         }
1959 }
1960
1961 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1962 {
1963         int err;
1964
1965         decoder->set_fup_tx_flags = false;
1966
1967         intel_pt_log("Scanning for full IP\n");
1968         err = intel_pt_walk_to_ip(decoder);
1969         if (err)
1970                 return err;
1971
1972         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1973         decoder->overflow = false;
1974
1975         decoder->state.from_ip = 0;
1976         decoder->state.to_ip = decoder->ip;
1977         intel_pt_log_to("Setting IP", decoder->ip);
1978
1979         return 0;
1980 }
1981
1982 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1983 {
1984         const unsigned char *end = decoder->buf + decoder->len;
1985         size_t i;
1986
1987         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1988                 if (i > decoder->len)
1989                         continue;
1990                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1991                         return i;
1992         }
1993         return 0;
1994 }
1995
1996 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1997 {
1998         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1999         const char *psb = INTEL_PT_PSB_STR;
2000
2001         if (rest_psb > decoder->len ||
2002             memcmp(decoder->buf, psb + part_psb, rest_psb))
2003                 return 0;
2004
2005         return rest_psb;
2006 }
2007
2008 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2009                                   int part_psb)
2010 {
2011         int rest_psb, ret;
2012
2013         decoder->pos += decoder->len;
2014         decoder->len = 0;
2015
2016         ret = intel_pt_get_next_data(decoder);
2017         if (ret)
2018                 return ret;
2019
2020         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2021         if (!rest_psb)
2022                 return 0;
2023
2024         decoder->pos -= part_psb;
2025         decoder->next_buf = decoder->buf + rest_psb;
2026         decoder->next_len = decoder->len - rest_psb;
2027         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2028         decoder->buf = decoder->temp_buf;
2029         decoder->len = INTEL_PT_PSB_LEN;
2030
2031         return 0;
2032 }
2033
2034 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2035 {
2036         unsigned char *next;
2037         int ret;
2038
2039         intel_pt_log("Scanning for PSB\n");
2040         while (1) {
2041                 if (!decoder->len) {
2042                         ret = intel_pt_get_next_data(decoder);
2043                         if (ret)
2044                                 return ret;
2045                 }
2046
2047                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2048                               INTEL_PT_PSB_LEN);
2049                 if (!next) {
2050                         int part_psb;
2051
2052                         part_psb = intel_pt_part_psb(decoder);
2053                         if (part_psb) {
2054                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2055                                 if (ret)
2056                                         return ret;
2057                         } else {
2058                                 decoder->pos += decoder->len;
2059                                 decoder->len = 0;
2060                         }
2061                         continue;
2062                 }
2063
2064                 decoder->pkt_step = next - decoder->buf;
2065                 return intel_pt_get_next_packet(decoder);
2066         }
2067 }
2068
2069 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2070 {
2071         int err;
2072
2073         decoder->pge = false;
2074         decoder->continuous_period = false;
2075         decoder->have_last_ip = false;
2076         decoder->last_ip = 0;
2077         decoder->ip = 0;
2078         intel_pt_clear_stack(&decoder->stack);
2079
2080         err = intel_pt_scan_for_psb(decoder);
2081         if (err)
2082                 return err;
2083
2084         decoder->have_last_ip = true;
2085         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2086
2087         err = intel_pt_walk_psb(decoder);
2088         if (err)
2089                 return err;
2090
2091         if (decoder->ip) {
2092                 decoder->state.type = 0; /* Do not have a sample */
2093                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2094         } else {
2095                 return intel_pt_sync_ip(decoder);
2096         }
2097
2098         return 0;
2099 }
2100
2101 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2102 {
2103         uint64_t est = decoder->sample_insn_cnt << 1;
2104
2105         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2106                 goto out;
2107
2108         est *= decoder->max_non_turbo_ratio;
2109         est /= decoder->cbr;
2110 out:
2111         return decoder->sample_timestamp + est;
2112 }
2113
2114 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2115 {
2116         int err;
2117
2118         do {
2119                 decoder->state.type = INTEL_PT_BRANCH;
2120                 decoder->state.flags = 0;
2121
2122                 switch (decoder->pkt_state) {
2123                 case INTEL_PT_STATE_NO_PSB:
2124                         err = intel_pt_sync(decoder);
2125                         break;
2126                 case INTEL_PT_STATE_NO_IP:
2127                         decoder->have_last_ip = false;
2128                         decoder->last_ip = 0;
2129                         decoder->ip = 0;
2130                         /* Fall through */
2131                 case INTEL_PT_STATE_ERR_RESYNC:
2132                         err = intel_pt_sync_ip(decoder);
2133                         break;
2134                 case INTEL_PT_STATE_IN_SYNC:
2135                         err = intel_pt_walk_trace(decoder);
2136                         break;
2137                 case INTEL_PT_STATE_TNT:
2138                 case INTEL_PT_STATE_TNT_CONT:
2139                         err = intel_pt_walk_tnt(decoder);
2140                         if (err == -EAGAIN)
2141                                 err = intel_pt_walk_trace(decoder);
2142                         break;
2143                 case INTEL_PT_STATE_TIP:
2144                 case INTEL_PT_STATE_TIP_PGD:
2145                         err = intel_pt_walk_tip(decoder);
2146                         break;
2147                 case INTEL_PT_STATE_FUP:
2148                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2149                         err = intel_pt_walk_fup(decoder);
2150                         if (err == -EAGAIN)
2151                                 err = intel_pt_walk_fup_tip(decoder);
2152                         else if (!err)
2153                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2154                         break;
2155                 case INTEL_PT_STATE_FUP_NO_TIP:
2156                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2157                         err = intel_pt_walk_fup(decoder);
2158                         if (err == -EAGAIN)
2159                                 err = intel_pt_walk_trace(decoder);
2160                         break;
2161                 default:
2162                         err = intel_pt_bug(decoder);
2163                         break;
2164                 }
2165         } while (err == -ENOLINK);
2166
2167         if (err) {
2168                 decoder->state.err = intel_pt_ext_err(err);
2169                 decoder->state.from_ip = decoder->ip;
2170                 decoder->sample_timestamp = decoder->timestamp;
2171                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2172         } else {
2173                 decoder->state.err = 0;
2174                 if (intel_pt_sample_time(decoder->pkt_state)) {
2175                         decoder->sample_timestamp = decoder->timestamp;
2176                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2177                 }
2178         }
2179
2180         decoder->state.timestamp = decoder->sample_timestamp;
2181         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2182         decoder->state.cr3 = decoder->cr3;
2183         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2184
2185         return &decoder->state;
2186 }
2187
2188 /**
2189  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2190  * @buf: pointer to buffer pointer
2191  * @len: size of buffer
2192  *
2193  * Updates the buffer pointer to point to the start of the next PSB packet if
2194  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2195  * @len is adjusted accordingly.
2196  *
2197  * Return: %true if a PSB packet is found, %false otherwise.
2198  */
2199 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2200 {
2201         unsigned char *next;
2202
2203         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2204         if (next) {
2205                 *len -= next - *buf;
2206                 *buf = next;
2207                 return true;
2208         }
2209         return false;
2210 }
2211
2212 /**
2213  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2214  *                     packet.
2215  * @buf: pointer to buffer pointer
2216  * @len: size of buffer
2217  *
2218  * Updates the buffer pointer to point to the start of the following PSB packet
2219  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2220  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2221  *
2222  * Return: %true if a PSB packet is found, %false otherwise.
2223  */
2224 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2225 {
2226         unsigned char *next;
2227
2228         if (!*len)
2229                 return false;
2230
2231         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2232         if (next) {
2233                 *len -= next - *buf;
2234                 *buf = next;
2235                 return true;
2236         }
2237         return false;
2238 }
2239
2240 /**
2241  * intel_pt_last_psb - find the last PSB packet in a buffer.
2242  * @buf: buffer
2243  * @len: size of buffer
2244  *
2245  * This function finds the last PSB in a buffer.
2246  *
2247  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2248  */
2249 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2250 {
2251         const char *n = INTEL_PT_PSB_STR;
2252         unsigned char *p;
2253         size_t k;
2254
2255         if (len < INTEL_PT_PSB_LEN)
2256                 return NULL;
2257
2258         k = len - INTEL_PT_PSB_LEN + 1;
2259         while (1) {
2260                 p = memrchr(buf, n[0], k);
2261                 if (!p)
2262                         return NULL;
2263                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2264                         return p;
2265                 k = p - buf;
2266                 if (!k)
2267                         return NULL;
2268         }
2269 }
2270
2271 /**
2272  * intel_pt_next_tsc - find and return next TSC.
2273  * @buf: buffer
2274  * @len: size of buffer
2275  * @tsc: TSC value returned
2276  * @rem: returns remaining size when TSC is found
2277  *
2278  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2279  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2280  * PSBEND packet is found.
2281  *
2282  * Return: %true if TSC is found, false otherwise.
2283  */
2284 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2285                               size_t *rem)
2286 {
2287         struct intel_pt_pkt packet;
2288         int ret;
2289
2290         while (len) {
2291                 ret = intel_pt_get_packet(buf, len, &packet);
2292                 if (ret <= 0)
2293                         return false;
2294                 if (packet.type == INTEL_PT_TSC) {
2295                         *tsc = packet.payload;
2296                         *rem = len;
2297                         return true;
2298                 }
2299                 if (packet.type == INTEL_PT_PSBEND)
2300                         return false;
2301                 buf += ret;
2302                 len -= ret;
2303         }
2304         return false;
2305 }
2306
2307 /**
2308  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2309  * @tsc1: first TSC to compare
2310  * @tsc2: second TSC to compare
2311  *
2312  * This function compares 7-byte TSC values allowing for the possibility that
2313  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2314  * around so for that purpose this function assumes the absolute difference is
2315  * less than half the maximum difference.
2316  *
2317  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2318  * after @tsc2.
2319  */
2320 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2321 {
2322         const uint64_t halfway = (1ULL << 55);
2323
2324         if (tsc1 == tsc2)
2325                 return 0;
2326
2327         if (tsc1 < tsc2) {
2328                 if (tsc2 - tsc1 < halfway)
2329                         return -1;
2330                 else
2331                         return 1;
2332         } else {
2333                 if (tsc1 - tsc2 < halfway)
2334                         return 1;
2335                 else
2336                         return -1;
2337         }
2338 }
2339
2340 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2341
2342 /**
2343  * adj_for_padding - adjust overlap to account for padding.
2344  * @buf_b: second buffer
2345  * @buf_a: first buffer
2346  * @len_a: size of first buffer
2347  *
2348  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2349  * accordingly.
2350  *
2351  * Return: A pointer into @buf_b from where non-overlapped data starts
2352  */
2353 static unsigned char *adj_for_padding(unsigned char *buf_b,
2354                                       unsigned char *buf_a, size_t len_a)
2355 {
2356         unsigned char *p = buf_b - MAX_PADDING;
2357         unsigned char *q = buf_a + len_a - MAX_PADDING;
2358         int i;
2359
2360         for (i = MAX_PADDING; i; i--, p++, q++) {
2361                 if (*p != *q)
2362                         break;
2363         }
2364
2365         return p;
2366 }
2367
2368 /**
2369  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2370  *                             using TSC.
2371  * @buf_a: first buffer
2372  * @len_a: size of first buffer
2373  * @buf_b: second buffer
2374  * @len_b: size of second buffer
2375  * @consecutive: returns true if there is data in buf_b that is consecutive
2376  *               to buf_a
2377  *
2378  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2379  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2380  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2381  * @buf_a and @buf_b are positioned at a PSB.
2382  *
2383  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2384  * @buf_b + @len_b if there is no non-overlapped data.
2385  */
2386 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2387                                                 size_t len_a,
2388                                                 unsigned char *buf_b,
2389                                                 size_t len_b, bool *consecutive)
2390 {
2391         uint64_t tsc_a, tsc_b;
2392         unsigned char *p;
2393         size_t len, rem_a, rem_b;
2394
2395         p = intel_pt_last_psb(buf_a, len_a);
2396         if (!p)
2397                 return buf_b; /* No PSB in buf_a => no overlap */
2398
2399         len = len_a - (p - buf_a);
2400         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2401                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2402                 len_a -= len;
2403                 p = intel_pt_last_psb(buf_a, len_a);
2404                 if (!p)
2405                         return buf_b; /* No full PSB+ => assume no overlap */
2406                 len = len_a - (p - buf_a);
2407                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2408                         return buf_b; /* No TSC in buf_a => assume no overlap */
2409         }
2410
2411         while (1) {
2412                 /* Ignore PSB+ with no TSC */
2413                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2414                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2415
2416                         /* Same TSC, so buffers are consecutive */
2417                         if (!cmp && rem_b >= rem_a) {
2418                                 unsigned char *start;
2419
2420                                 *consecutive = true;
2421                                 start = buf_b + len_b - (rem_b - rem_a);
2422                                 return adj_for_padding(start, buf_a, len_a);
2423                         }
2424                         if (cmp < 0)
2425                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2426                 }
2427
2428                 if (!intel_pt_step_psb(&buf_b, &len_b))
2429                         return buf_b + len_b; /* No PSB in buf_b => no data */
2430         }
2431 }
2432
2433 /**
2434  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2435  * @buf_a: first buffer
2436  * @len_a: size of first buffer
2437  * @buf_b: second buffer
2438  * @len_b: size of second buffer
2439  * @have_tsc: can use TSC packets to detect overlap
2440  * @consecutive: returns true if there is data in buf_b that is consecutive
2441  *               to buf_a
2442  *
2443  * When trace samples or snapshots are recorded there is the possibility that
2444  * the data overlaps.  Note that, for the purposes of decoding, data is only
2445  * useful if it begins with a PSB packet.
2446  *
2447  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2448  * @buf_b + @len_b if there is no non-overlapped data.
2449  */
2450 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2451                                      unsigned char *buf_b, size_t len_b,
2452                                      bool have_tsc, bool *consecutive)
2453 {
2454         unsigned char *found;
2455
2456         /* Buffer 'b' must start at PSB so throw away everything before that */
2457         if (!intel_pt_next_psb(&buf_b, &len_b))
2458                 return buf_b + len_b; /* No PSB */
2459
2460         if (!intel_pt_next_psb(&buf_a, &len_a))
2461                 return buf_b; /* No overlap */
2462
2463         if (have_tsc) {
2464                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2465                                                   consecutive);
2466                 if (found)
2467                         return found;
2468         }
2469
2470         /*
2471          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2472          * we can ignore the first part of buffer 'a'.
2473          */
2474         while (len_b < len_a) {
2475                 if (!intel_pt_step_psb(&buf_a, &len_a))
2476                         return buf_b; /* No overlap */
2477         }
2478
2479         /* Now len_b >= len_a */
2480         while (1) {
2481                 /* Potential overlap so check the bytes */
2482                 found = memmem(buf_a, len_a, buf_b, len_a);
2483                 if (found) {
2484                         *consecutive = true;
2485                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2486                 }
2487
2488                 /* Try again at next PSB in buffer 'a' */
2489                 if (!intel_pt_step_psb(&buf_a, &len_a))
2490                         return buf_b; /* No overlap */
2491         }
2492 }