tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT_CONT,
  62         INTEL_PT_STATE_TNT,
  63         INTEL_PT_STATE_TIP,
  64         INTEL_PT_STATE_TIP_PGD,
  65         INTEL_PT_STATE_FUP,
  66         INTEL_PT_STATE_FUP_NO_TIP,
  67 };
  68
  69 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  70 {
  71         switch (pkt_state) {
  72         case INTEL_PT_STATE_NO_PSB:
  73         case INTEL_PT_STATE_NO_IP:
  74         case INTEL_PT_STATE_ERR_RESYNC:
  75         case INTEL_PT_STATE_IN_SYNC:
  76         case INTEL_PT_STATE_TNT_CONT:
  77                 return true;
  78         case INTEL_PT_STATE_TNT:
  79         case INTEL_PT_STATE_TIP:
  80         case INTEL_PT_STATE_TIP_PGD:
  81         case INTEL_PT_STATE_FUP:
  82         case INTEL_PT_STATE_FUP_NO_TIP:
  83                 return false;
  84         default:
  85                 return true;
  86         };
  87 }
  88
  89 #ifdef INTEL_PT_STRICT
  90 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  92 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  93 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  94 #else
  95 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  96 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  97 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  98 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  99 #endif
 100
 101 struct intel_pt_decoder {
 102         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 103         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 104                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 105                          uint64_t max_insn_cnt, void *data);
 106         bool (*pgd_ip)(uint64_t ip, void *data);
 107         void *data;
 108         struct intel_pt_state state;
 109         const unsigned char *buf;
 110         size_t len;
 111         bool return_compression;
 112         bool mtc_insn;
 113         bool pge;
 114         bool have_tma;
 115         bool have_cyc;
 116         bool fixup_last_mtc;
 117         bool have_last_ip;
 118         enum intel_pt_param_flags flags;
 119         uint64_t pos;
 120         uint64_t last_ip;
 121         uint64_t ip;
 122         uint64_t cr3;
 123         uint64_t timestamp;
 124         uint64_t tsc_timestamp;
 125         uint64_t ref_timestamp;
 126         uint64_t sample_timestamp;
 127         uint64_t ret_addr;
 128         uint64_t ctc_timestamp;
 129         uint64_t ctc_delta;
 130         uint64_t cycle_cnt;
 131         uint64_t cyc_ref_timestamp;
 132         uint32_t last_mtc;
 133         uint32_t tsc_ctc_ratio_n;
 134         uint32_t tsc_ctc_ratio_d;
 135         uint32_t tsc_ctc_mult;
 136         uint32_t tsc_slip;
 137         uint32_t ctc_rem_mask;
 138         int mtc_shift;
 139         struct intel_pt_stack stack;
 140         enum intel_pt_pkt_state pkt_state;
 141         struct intel_pt_pkt packet;
 142         struct intel_pt_pkt tnt;
 143         int pkt_step;
 144         int pkt_len;
 145         int last_packet_type;
 146         unsigned int cbr;
 147         unsigned int max_non_turbo_ratio;
 148         double max_non_turbo_ratio_fp;
 149         double cbr_cyc_to_tsc;
 150         double calc_cyc_to_tsc;
 151         bool have_calc_cyc_to_tsc;
 152         int exec_mode;
 153         unsigned int insn_bytes;
 154         uint64_t period;
 155         enum intel_pt_period_type period_type;
 156         uint64_t tot_insn_cnt;
 157         uint64_t period_insn_cnt;
 158         uint64_t period_mask;
 159         uint64_t period_ticks;
 160         uint64_t last_masked_timestamp;
 161         bool continuous_period;
 162         bool overflow;
 163         bool set_fup_tx_flags;
 164         unsigned int fup_tx_flags;
 165         unsigned int tx_flags;
 166         uint64_t timestamp_insn_cnt;
 167         uint64_t sample_insn_cnt;
 168         uint64_t stuck_ip;
 169         int no_progress;
 170         int stuck_ip_prd;
 171         int stuck_ip_cnt;
 172         const unsigned char *next_buf;
 173         size_t next_len;
 174         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 175 };
 176
 177 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 178 {
 179         int i;
 180
 181         for (i = 0; x != 1; i++)
 182                 x >>= 1;
 183
 184         return x << i;
 185 }
 186
 187 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 188 {
 189         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 190                 uint64_t period;
 191
 192                 period = intel_pt_lower_power_of_2(decoder->period);
 193                 decoder->period_mask  = ~(period - 1);
 194                 decoder->period_ticks = period;
 195         }
 196 }
 197
 198 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 199 {
 200         if (!d)
 201                 return 0;
 202         return (t / d) * n + ((t % d) * n) / d;
 203 }
 204
 205 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 206 {
 207         struct intel_pt_decoder *decoder;
 208
 209         if (!params->get_trace || !params->walk_insn)
 210                 return NULL;
 211
 212         decoder = zalloc(sizeof(struct intel_pt_decoder));
 213         if (!decoder)
 214                 return NULL;
 215
 216         decoder->get_trace          = params->get_trace;
 217         decoder->walk_insn          = params->walk_insn;
 218         decoder->pgd_ip             = params->pgd_ip;
 219         decoder->data               = params->data;
 220         decoder->return_compression = params->return_compression;
 221
 222         decoder->flags              = params->flags;
 223
 224         decoder->period             = params->period;
 225         decoder->period_type        = params->period_type;
 226
 227         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 228         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 229
 230         intel_pt_setup_period(decoder);
 231
 232         decoder->mtc_shift = params->mtc_period;
 233         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 234
 235         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 236         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 237
 238         if (!decoder->tsc_ctc_ratio_n)
 239                 decoder->tsc_ctc_ratio_d = 0;
 240
 241         if (decoder->tsc_ctc_ratio_d) {
 242                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 243                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 244                                                 decoder->tsc_ctc_ratio_d;
 245         }
 246
 247         /*
 248          * A TSC packet can slip past MTC packets so that the timestamp appears
 249          * to go backwards. One estimate is that can be up to about 40 CPU
 250          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 251          * slippage an order of magnitude more to be on the safe side.
 252          */
 253         decoder->tsc_slip = 0x10000;
 254
 255         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 256         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 257         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 258         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 259         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 260
 261         return decoder;
 262 }
 263
 264 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 265 {
 266         struct intel_pt_blk *blk = stack->blk;
 267
 268         stack->blk = blk->prev;
 269         if (!stack->spare)
 270                 stack->spare = blk;
 271         else
 272                 free(blk);
 273 }
 274
 275 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 276 {
 277         if (!stack->pos) {
 278                 if (!stack->blk)
 279                         return 0;
 280                 intel_pt_pop_blk(stack);
 281                 if (!stack->blk)
 282                         return 0;
 283                 stack->pos = INTEL_PT_BLK_SIZE;
 284         }
 285         return stack->blk->ip[--stack->pos];
 286 }
 287
 288 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 289 {
 290         struct intel_pt_blk *blk;
 291
 292         if (stack->spare) {
 293                 blk = stack->spare;
 294                 stack->spare = NULL;
 295         } else {
 296                 blk = malloc(sizeof(struct intel_pt_blk));
 297                 if (!blk)
 298                         return -ENOMEM;
 299         }
 300
 301         blk->prev = stack->blk;
 302         stack->blk = blk;
 303         stack->pos = 0;
 304         return 0;
 305 }
 306
 307 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 308 {
 309         int err;
 310
 311         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 312                 err = intel_pt_alloc_blk(stack);
 313                 if (err)
 314                         return err;
 315         }
 316
 317         stack->blk->ip[stack->pos++] = ip;
 318         return 0;
 319 }
 320
 321 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 322 {
 323         while (stack->blk)
 324                 intel_pt_pop_blk(stack);
 325         stack->pos = 0;
 326 }
 327
 328 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 329 {
 330         intel_pt_clear_stack(stack);
 331         zfree(&stack->blk);
 332         zfree(&stack->spare);
 333 }
 334
 335 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 336 {
 337         intel_pt_free_stack(&decoder->stack);
 338         free(decoder);
 339 }
 340
 341 static int intel_pt_ext_err(int code)
 342 {
 343         switch (code) {
 344         case -ENOMEM:
 345                 return INTEL_PT_ERR_NOMEM;
 346         case -ENOSYS:
 347                 return INTEL_PT_ERR_INTERN;
 348         case -EBADMSG:
 349                 return INTEL_PT_ERR_BADPKT;
 350         case -ENODATA:
 351                 return INTEL_PT_ERR_NODATA;
 352         case -EILSEQ:
 353                 return INTEL_PT_ERR_NOINSN;
 354         case -ENOENT:
 355                 return INTEL_PT_ERR_MISMAT;
 356         case -EOVERFLOW:
 357                 return INTEL_PT_ERR_OVR;
 358         case -ENOSPC:
 359                 return INTEL_PT_ERR_LOST;
 360         case -ELOOP:
 361                 return INTEL_PT_ERR_NELOOP;
 362         default:
 363                 return INTEL_PT_ERR_UNK;
 364         }
 365 }
 366
 367 static const char *intel_pt_err_msgs[] = {
 368         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 369         [INTEL_PT_ERR_INTERN] = "Internal error",
 370         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 371         [INTEL_PT_ERR_NODATA] = "No more data",
 372         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 373         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 374         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 375         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 376         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 377         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 378 };
 379
 380 int intel_pt__strerror(int code, char *buf, size_t buflen)
 381 {
 382         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 383                 code = INTEL_PT_ERR_UNK;
 384         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 385         return 0;
 386 }
 387
 388 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 389                                  uint64_t last_ip)
 390 {
 391         uint64_t ip;
 392
 393         switch (packet->count) {
 394         case 1:
 395                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 396                      packet->payload;
 397                 break;
 398         case 2:
 399                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 400                      packet->payload;
 401                 break;
 402         case 3:
 403                 ip = packet->payload;
 404                 /* Sign-extend 6-byte ip */
 405                 if (ip & (uint64_t)0x800000000000ULL)
 406                         ip |= (uint64_t)0xffff000000000000ULL;
 407                 break;
 408         case 4:
 409                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 410                      packet->payload;
 411                 break;
 412         case 6:
 413                 ip = packet->payload;
 414                 break;
 415         default:
 416                 return 0;
 417         }
 418
 419         return ip;
 420 }
 421
 422 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 423 {
 424         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 425         decoder->have_last_ip = true;
 426 }
 427
 428 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 429 {
 430         intel_pt_set_last_ip(decoder);
 431         decoder->ip = decoder->last_ip;
 432 }
 433
 434 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 435 {
 436         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 437                             decoder->buf);
 438 }
 439
 440 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 441 {
 442         intel_pt_log("ERROR: Internal error\n");
 443         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 444         return -ENOSYS;
 445 }
 446
 447 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 448 {
 449         decoder->tx_flags = 0;
 450 }
 451
 452 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 453 {
 454         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 455 }
 456
 457 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 458 {
 459         intel_pt_clear_tx_flags(decoder);
 460         decoder->have_tma = false;
 461         decoder->pkt_len = 1;
 462         decoder->pkt_step = 1;
 463         intel_pt_decoder_log_packet(decoder);
 464         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 465                 intel_pt_log("ERROR: Bad packet\n");
 466                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 467         }
 468         return -EBADMSG;
 469 }
 470
 471 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 472 {
 473         struct intel_pt_buffer buffer = { .buf = 0, };
 474         int ret;
 475
 476         decoder->pkt_step = 0;
 477
 478         intel_pt_log("Getting more data\n");
 479         ret = decoder->get_trace(&buffer, decoder->data);
 480         if (ret)
 481                 return ret;
 482         decoder->buf = buffer.buf;
 483         decoder->len = buffer.len;
 484         if (!decoder->len) {
 485                 intel_pt_log("No more data\n");
 486                 return -ENODATA;
 487         }
 488         if (!buffer.consecutive) {
 489                 decoder->ip = 0;
 490                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 491                 decoder->ref_timestamp = buffer.ref_timestamp;
 492                 decoder->timestamp = 0;
 493                 decoder->have_tma = false;
 494                 decoder->state.trace_nr = buffer.trace_nr;
 495                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 496                              decoder->ref_timestamp);
 497                 return -ENOLINK;
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 504 {
 505         if (!decoder->next_buf)
 506                 return intel_pt_get_data(decoder);
 507
 508         decoder->buf = decoder->next_buf;
 509         decoder->len = decoder->next_len;
 510         decoder->next_buf = 0;
 511         decoder->next_len = 0;
 512         return 0;
 513 }
 514
 515 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 516 {
 517         unsigned char *buf = decoder->temp_buf;
 518         size_t old_len, len, n;
 519         int ret;
 520
 521         old_len = decoder->len;
 522         len = decoder->len;
 523         memcpy(buf, decoder->buf, len);
 524
 525         ret = intel_pt_get_data(decoder);
 526         if (ret) {
 527                 decoder->pos += old_len;
 528                 return ret < 0 ? ret : -EINVAL;
 529         }
 530
 531         n = INTEL_PT_PKT_MAX_SZ - len;
 532         if (n > decoder->len)
 533                 n = decoder->len;
 534         memcpy(buf + len, decoder->buf, n);
 535         len += n;
 536
 537         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 538         if (ret < (int)old_len) {
 539                 decoder->next_buf = decoder->buf;
 540                 decoder->next_len = decoder->len;
 541                 decoder->buf = buf;
 542                 decoder->len = old_len;
 543                 return intel_pt_bad_packet(decoder);
 544         }
 545
 546         decoder->next_buf = decoder->buf + (ret - old_len);
 547         decoder->next_len = decoder->len - (ret - old_len);
 548
 549         decoder->buf = buf;
 550         decoder->len = ret;
 551
 552         return ret;
 553 }
 554
 555 struct intel_pt_pkt_info {
 556         struct intel_pt_decoder   *decoder;
 557         struct intel_pt_pkt       packet;
 558         uint64_t                  pos;
 559         int                       pkt_len;
 560         int                       last_packet_type;
 561         void                      *data;
 562 };
 563
 564 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 565
 566 /* Lookahead packets in current buffer */
 567 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 568                                   intel_pt_pkt_cb_t cb, void *data)
 569 {
 570         struct intel_pt_pkt_info pkt_info;
 571         const unsigned char *buf = decoder->buf;
 572         size_t len = decoder->len;
 573         int ret;
 574
 575         pkt_info.decoder          = decoder;
 576         pkt_info.pos              = decoder->pos;
 577         pkt_info.pkt_len          = decoder->pkt_step;
 578         pkt_info.last_packet_type = decoder->last_packet_type;
 579         pkt_info.data             = data;
 580
 581         while (1) {
 582                 do {
 583                         pkt_info.pos += pkt_info.pkt_len;
 584                         buf          += pkt_info.pkt_len;
 585                         len          -= pkt_info.pkt_len;
 586
 587                         if (!len)
 588                                 return INTEL_PT_NEED_MORE_BYTES;
 589
 590                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 591                         if (!ret)
 592                                 return INTEL_PT_NEED_MORE_BYTES;
 593                         if (ret < 0)
 594                                 return ret;
 595
 596                         pkt_info.pkt_len = ret;
 597                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 598
 599                 ret = cb(&pkt_info);
 600                 if (ret)
 601                         return 0;
 602
 603                 pkt_info.last_packet_type = pkt_info.packet.type;
 604         }
 605 }
 606
 607 struct intel_pt_calc_cyc_to_tsc_info {
 608         uint64_t        cycle_cnt;
 609         unsigned int    cbr;
 610         uint32_t        last_mtc;
 611         uint64_t        ctc_timestamp;
 612         uint64_t        ctc_delta;
 613         uint64_t        tsc_timestamp;
 614         uint64_t        timestamp;
 615         bool            have_tma;
 616         bool            fixup_last_mtc;
 617         bool            from_mtc;
 618         double          cbr_cyc_to_tsc;
 619 };
 620
 621 /*
 622  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 623  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 624  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 625  * packet by copying the missing bits from the current MTC assuming the least
 626  * difference between the two, and that the current MTC comes after last_mtc.
 627  */
 628 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 629                                     uint32_t *last_mtc)
 630 {
 631         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 632         uint32_t mask = ~(first_missing_bit - 1);
 633
 634         *last_mtc |= mtc & mask;
 635         if (*last_mtc >= mtc) {
 636                 *last_mtc -= first_missing_bit;
 637                 *last_mtc &= 0xff;
 638         }
 639 }
 640
 641 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 642 {
 643         struct intel_pt_decoder *decoder = pkt_info->decoder;
 644         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 645         uint64_t timestamp;
 646         double cyc_to_tsc;
 647         unsigned int cbr;
 648         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 649
 650         switch (pkt_info->packet.type) {
 651         case INTEL_PT_TNT:
 652         case INTEL_PT_TIP_PGE:
 653         case INTEL_PT_TIP:
 654         case INTEL_PT_FUP:
 655         case INTEL_PT_PSB:
 656         case INTEL_PT_PIP:
 657         case INTEL_PT_MODE_EXEC:
 658         case INTEL_PT_MODE_TSX:
 659         case INTEL_PT_PSBEND:
 660         case INTEL_PT_PAD:
 661         case INTEL_PT_VMCS:
 662         case INTEL_PT_MNT:
 663                 return 0;
 664
 665         case INTEL_PT_MTC:
 666                 if (!data->have_tma)
 667                         return 0;
 668
 669                 mtc = pkt_info->packet.payload;
 670                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 671                         data->fixup_last_mtc = false;
 672                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 673                                                 &data->last_mtc);
 674                 }
 675                 if (mtc > data->last_mtc)
 676                         mtc_delta = mtc - data->last_mtc;
 677                 else
 678                         mtc_delta = mtc + 256 - data->last_mtc;
 679                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 680                 data->last_mtc = mtc;
 681
 682                 if (decoder->tsc_ctc_mult) {
 683                         timestamp = data->ctc_timestamp +
 684                                 data->ctc_delta * decoder->tsc_ctc_mult;
 685                 } else {
 686                         timestamp = data->ctc_timestamp +
 687                                 multdiv(data->ctc_delta,
 688                                         decoder->tsc_ctc_ratio_n,
 689                                         decoder->tsc_ctc_ratio_d);
 690                 }
 691
 692                 if (timestamp < data->timestamp)
 693                         return 1;
 694
 695                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 696                         data->timestamp = timestamp;
 697                         return 0;
 698                 }
 699
 700                 break;
 701
 702         case INTEL_PT_TSC:
 703                 timestamp = pkt_info->packet.payload |
 704                             (data->timestamp & (0xffULL << 56));
 705                 if (data->from_mtc && timestamp < data->timestamp &&
 706                     data->timestamp - timestamp < decoder->tsc_slip)
 707                         return 1;
 708                 if (timestamp < data->timestamp)
 709                         timestamp += (1ULL << 56);
 710                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 711                         if (data->from_mtc)
 712                                 return 1;
 713                         data->tsc_timestamp = timestamp;
 714                         data->timestamp = timestamp;
 715                         return 0;
 716                 }
 717                 break;
 718
 719         case INTEL_PT_TMA:
 720                 if (data->from_mtc)
 721                         return 1;
 722
 723                 if (!decoder->tsc_ctc_ratio_d)
 724                         return 0;
 725
 726                 ctc = pkt_info->packet.payload;
 727                 fc = pkt_info->packet.count;
 728                 ctc_rem = ctc & decoder->ctc_rem_mask;
 729
 730                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 731
 732                 data->ctc_timestamp = data->tsc_timestamp - fc;
 733                 if (decoder->tsc_ctc_mult) {
 734                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 735                 } else {
 736                         data->ctc_timestamp -=
 737                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 738                                         decoder->tsc_ctc_ratio_d);
 739                 }
 740
 741                 data->ctc_delta = 0;
 742                 data->have_tma = true;
 743                 data->fixup_last_mtc = true;
 744
 745                 return 0;
 746
 747         case INTEL_PT_CYC:
 748                 data->cycle_cnt += pkt_info->packet.payload;
 749                 return 0;
 750
 751         case INTEL_PT_CBR:
 752                 cbr = pkt_info->packet.payload;
 753                 if (data->cbr && data->cbr != cbr)
 754                         return 1;
 755                 data->cbr = cbr;
 756                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 757                 return 0;
 758
 759         case INTEL_PT_TIP_PGD:
 760         case INTEL_PT_TRACESTOP:
 761         case INTEL_PT_OVF:
 762         case INTEL_PT_BAD: /* Does not happen */
 763         default:
 764                 return 1;
 765         }
 766
 767         if (!data->cbr && decoder->cbr) {
 768                 data->cbr = decoder->cbr;
 769                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 770         }
 771
 772         if (!data->cycle_cnt)
 773                 return 1;
 774
 775         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 776
 777         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 778             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 779                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 780                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 781                 return 1;
 782         }
 783
 784         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 785         decoder->have_calc_cyc_to_tsc = true;
 786
 787         if (data->cbr) {
 788                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 789                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 790         } else {
 791                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 792                              cyc_to_tsc, pkt_info->pos);
 793         }
 794
 795         return 1;
 796 }
 797
 798 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 799                                      bool from_mtc)
 800 {
 801         struct intel_pt_calc_cyc_to_tsc_info data = {
 802                 .cycle_cnt      = 0,
 803                 .cbr            = 0,
 804                 .last_mtc       = decoder->last_mtc,
 805                 .ctc_timestamp  = decoder->ctc_timestamp,
 806                 .ctc_delta      = decoder->ctc_delta,
 807                 .tsc_timestamp  = decoder->tsc_timestamp,
 808                 .timestamp      = decoder->timestamp,
 809                 .have_tma       = decoder->have_tma,
 810                 .fixup_last_mtc = decoder->fixup_last_mtc,
 811                 .from_mtc       = from_mtc,
 812                 .cbr_cyc_to_tsc = 0,
 813         };
 814
 815         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 816 }
 817
 818 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 819 {
 820         int ret;
 821
 822         decoder->last_packet_type = decoder->packet.type;
 823
 824         do {
 825                 decoder->pos += decoder->pkt_step;
 826                 decoder->buf += decoder->pkt_step;
 827                 decoder->len -= decoder->pkt_step;
 828
 829                 if (!decoder->len) {
 830                         ret = intel_pt_get_next_data(decoder);
 831                         if (ret)
 832                                 return ret;
 833                 }
 834
 835                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 836                                           &decoder->packet);
 837                 if (ret == INTEL_PT_NEED_MORE_BYTES &&
 838                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 839                         ret = intel_pt_get_split_packet(decoder);
 840                         if (ret < 0)
 841                                 return ret;
 842                 }
 843                 if (ret <= 0)
 844                         return intel_pt_bad_packet(decoder);
 845
 846                 decoder->pkt_len = ret;
 847                 decoder->pkt_step = ret;
 848                 intel_pt_decoder_log_packet(decoder);
 849         } while (decoder->packet.type == INTEL_PT_PAD);
 850
 851         return 0;
 852 }
 853
 854 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 855 {
 856         uint64_t timestamp, masked_timestamp;
 857
 858         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 859         masked_timestamp = timestamp & decoder->period_mask;
 860         if (decoder->continuous_period) {
 861                 if (masked_timestamp > decoder->last_masked_timestamp)
 862                         return 1;
 863         } else {
 864                 timestamp += 1;
 865                 masked_timestamp = timestamp & decoder->period_mask;
 866                 if (masked_timestamp > decoder->last_masked_timestamp) {
 867                         decoder->last_masked_timestamp = masked_timestamp;
 868                         decoder->continuous_period = true;
 869                 }
 870         }
 871
 872         if (masked_timestamp < decoder->last_masked_timestamp)
 873                 return decoder->period_ticks;
 874
 875         return decoder->period_ticks - (timestamp - masked_timestamp);
 876 }
 877
 878 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 879 {
 880         switch (decoder->period_type) {
 881         case INTEL_PT_PERIOD_INSTRUCTIONS:
 882                 return decoder->period - decoder->period_insn_cnt;
 883         case INTEL_PT_PERIOD_TICKS:
 884                 return intel_pt_next_period(decoder);
 885         case INTEL_PT_PERIOD_NONE:
 886         case INTEL_PT_PERIOD_MTC:
 887         default:
 888                 return 0;
 889         }
 890 }
 891
 892 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 893 {
 894         uint64_t timestamp, masked_timestamp;
 895
 896         switch (decoder->period_type) {
 897         case INTEL_PT_PERIOD_INSTRUCTIONS:
 898                 decoder->period_insn_cnt = 0;
 899                 break;
 900         case INTEL_PT_PERIOD_TICKS:
 901                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 902                 masked_timestamp = timestamp & decoder->period_mask;
 903                 if (masked_timestamp > decoder->last_masked_timestamp)
 904                         decoder->last_masked_timestamp = masked_timestamp;
 905                 else
 906                         decoder->last_masked_timestamp += decoder->period_ticks;
 907                 break;
 908         case INTEL_PT_PERIOD_NONE:
 909         case INTEL_PT_PERIOD_MTC:
 910         default:
 911                 break;
 912         }
 913
 914         decoder->state.type |= INTEL_PT_INSTRUCTION;
 915 }
 916
 917 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 918                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 919 {
 920         uint64_t max_insn_cnt, insn_cnt = 0;
 921         int err;
 922
 923         if (!decoder->mtc_insn)
 924                 decoder->mtc_insn = true;
 925
 926         max_insn_cnt = intel_pt_next_sample(decoder);
 927
 928         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 929                                  max_insn_cnt, decoder->data);
 930
 931         decoder->tot_insn_cnt += insn_cnt;
 932         decoder->timestamp_insn_cnt += insn_cnt;
 933         decoder->sample_insn_cnt += insn_cnt;
 934         decoder->period_insn_cnt += insn_cnt;
 935
 936         if (err) {
 937                 decoder->no_progress = 0;
 938                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 939                 intel_pt_log_at("ERROR: Failed to get instruction",
 940                                 decoder->ip);
 941                 if (err == -ENOENT)
 942                         return -ENOLINK;
 943                 return -EILSEQ;
 944         }
 945
 946         if (ip && decoder->ip == ip) {
 947                 err = -EAGAIN;
 948                 goto out;
 949         }
 950
 951         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 952                 intel_pt_sample_insn(decoder);
 953
 954         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 955                 decoder->state.type = INTEL_PT_INSTRUCTION;
 956                 decoder->state.from_ip = decoder->ip;
 957                 decoder->state.to_ip = 0;
 958                 decoder->ip += intel_pt_insn->length;
 959                 err = INTEL_PT_RETURN;
 960                 goto out;
 961         }
 962
 963         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 964                 /* Zero-length calls are excluded */
 965                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 966                     intel_pt_insn->rel) {
 967                         err = intel_pt_push(&decoder->stack, decoder->ip +
 968                                             intel_pt_insn->length);
 969                         if (err)
 970                                 goto out;
 971                 }
 972         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
 973                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
 974         }
 975
 976         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
 977                 int cnt = decoder->no_progress++;
 978
 979                 decoder->state.from_ip = decoder->ip;
 980                 decoder->ip += intel_pt_insn->length +
 981                                 intel_pt_insn->rel;
 982                 decoder->state.to_ip = decoder->ip;
 983                 err = INTEL_PT_RETURN;
 984
 985                 /*
 986                  * Check for being stuck in a loop.  This can happen if a
 987                  * decoder error results in the decoder erroneously setting the
 988                  * ip to an address that is itself in an infinite loop that
 989                  * consumes no packets.  When that happens, there must be an
 990                  * unconditional branch.
 991                  */
 992                 if (cnt) {
 993                         if (cnt == 1) {
 994                                 decoder->stuck_ip = decoder->state.to_ip;
 995                                 decoder->stuck_ip_prd = 1;
 996                                 decoder->stuck_ip_cnt = 1;
 997                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
 998                                    decoder->state.to_ip == decoder->stuck_ip) {
 999                                 intel_pt_log_at("ERROR: Never-ending loop",
1000                                                 decoder->state.to_ip);
1001                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1002                                 err = -ELOOP;
1003                                 goto out;
1004                         } else if (!--decoder->stuck_ip_cnt) {
1005                                 decoder->stuck_ip_prd += 1;
1006                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1007                                 decoder->stuck_ip = decoder->state.to_ip;
1008                         }
1009                 }
1010                 goto out_no_progress;
1011         }
1012 out:
1013         decoder->no_progress = 0;
1014 out_no_progress:
1015         decoder->state.insn_op = intel_pt_insn->op;
1016         decoder->state.insn_len = intel_pt_insn->length;
1017
1018         if (decoder->tx_flags & INTEL_PT_IN_TX)
1019                 decoder->state.flags |= INTEL_PT_IN_TX;
1020
1021         return err;
1022 }
1023
1024 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1025                                           struct intel_pt_insn *intel_pt_insn,
1026                                           uint64_t ip, int err)
1027 {
1028         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1029                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1030                ip == decoder->ip + intel_pt_insn->length;
1031 }
1032
1033 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1034 {
1035         struct intel_pt_insn intel_pt_insn;
1036         uint64_t ip;
1037         int err;
1038
1039         ip = decoder->last_ip;
1040
1041         while (1) {
1042                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1043                 if (err == INTEL_PT_RETURN)
1044                         return 0;
1045                 if (err == -EAGAIN ||
1046                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1047                         if (decoder->set_fup_tx_flags) {
1048                                 decoder->set_fup_tx_flags = false;
1049                                 decoder->tx_flags = decoder->fup_tx_flags;
1050                                 decoder->state.type = INTEL_PT_TRANSACTION;
1051                                 decoder->state.from_ip = decoder->ip;
1052                                 decoder->state.to_ip = 0;
1053                                 decoder->state.flags = decoder->fup_tx_flags;
1054                                 return 0;
1055                         }
1056                         return -EAGAIN;
1057                 }
1058                 decoder->set_fup_tx_flags = false;
1059                 if (err)
1060                         return err;
1061
1062                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1063                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1064                                         decoder->ip);
1065                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1066                         return -ENOENT;
1067                 }
1068
1069                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1070                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1071                                         decoder->ip);
1072                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1073                         return -ENOENT;
1074                 }
1075
1076                 intel_pt_bug(decoder);
1077         }
1078 }
1079
1080 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1081 {
1082         struct intel_pt_insn intel_pt_insn;
1083         int err;
1084
1085         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1086         if (err == INTEL_PT_RETURN &&
1087             decoder->pgd_ip &&
1088             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1089             (decoder->state.type & INTEL_PT_BRANCH) &&
1090             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1091                 /* Unconditional branch leaving filter region */
1092                 decoder->no_progress = 0;
1093                 decoder->pge = false;
1094                 decoder->continuous_period = false;
1095                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1096                 decoder->state.to_ip = 0;
1097                 return 0;
1098         }
1099         if (err == INTEL_PT_RETURN)
1100                 return 0;
1101         if (err)
1102                 return err;
1103
1104         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1105                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1106                         decoder->pge = false;
1107                         decoder->continuous_period = false;
1108                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1109                         decoder->state.from_ip = decoder->ip;
1110                         decoder->state.to_ip = 0;
1111                         if (decoder->packet.count != 0)
1112                                 decoder->ip = decoder->last_ip;
1113                 } else {
1114                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1115                         decoder->state.from_ip = decoder->ip;
1116                         if (decoder->packet.count == 0) {
1117                                 decoder->state.to_ip = 0;
1118                         } else {
1119                                 decoder->state.to_ip = decoder->last_ip;
1120                                 decoder->ip = decoder->last_ip;
1121                         }
1122                 }
1123                 return 0;
1124         }
1125
1126         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1127                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1128                                  intel_pt_insn.rel;
1129
1130                 if (decoder->pgd_ip &&
1131                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1132                     decoder->pgd_ip(to_ip, decoder->data)) {
1133                         /* Conditional branch leaving filter region */
1134                         decoder->pge = false;
1135                         decoder->continuous_period = false;
1136                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1137                         decoder->ip = to_ip;
1138                         decoder->state.from_ip = decoder->ip;
1139                         decoder->state.to_ip = 0;
1140                         return 0;
1141                 }
1142                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1143                                 decoder->ip);
1144                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1145                 return -ENOENT;
1146         }
1147
1148         return intel_pt_bug(decoder);
1149 }
1150
1151 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1152 {
1153         struct intel_pt_insn intel_pt_insn;
1154         int err;
1155
1156         while (1) {
1157                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1158                 if (err == INTEL_PT_RETURN)
1159                         return 0;
1160                 if (err)
1161                         return err;
1162
1163                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1164                         if (!decoder->return_compression) {
1165                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1166                                                 decoder->ip);
1167                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1168                                 return -ENOENT;
1169                         }
1170                         if (!decoder->ret_addr) {
1171                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1172                                                 decoder->ip);
1173                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1174                                 return -ENOENT;
1175                         }
1176                         if (!(decoder->tnt.payload & BIT63)) {
1177                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1178                                                 decoder->ip);
1179                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1180                                 return -ENOENT;
1181                         }
1182                         decoder->tnt.count -= 1;
1183                         if (decoder->tnt.count)
1184                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1185                         else
1186                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1187                         decoder->tnt.payload <<= 1;
1188                         decoder->state.from_ip = decoder->ip;
1189                         decoder->ip = decoder->ret_addr;
1190                         decoder->state.to_ip = decoder->ip;
1191                         return 0;
1192                 }
1193
1194                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1195                         /* Handle deferred TIPs */
1196                         err = intel_pt_get_next_packet(decoder);
1197                         if (err)
1198                                 return err;
1199                         if (decoder->packet.type != INTEL_PT_TIP ||
1200                             decoder->packet.count == 0) {
1201                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1202                                                 decoder->ip);
1203                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1204                                 decoder->pkt_step = 0;
1205                                 return -ENOENT;
1206                         }
1207                         intel_pt_set_last_ip(decoder);
1208                         decoder->state.from_ip = decoder->ip;
1209                         decoder->state.to_ip = decoder->last_ip;
1210                         decoder->ip = decoder->last_ip;
1211                         return 0;
1212                 }
1213
1214                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1215                         decoder->tnt.count -= 1;
1216                         if (decoder->tnt.count)
1217                                 decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;
1218                         else
1219                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1220                         if (decoder->tnt.payload & BIT63) {
1221                                 decoder->tnt.payload <<= 1;
1222                                 decoder->state.from_ip = decoder->ip;
1223                                 decoder->ip += intel_pt_insn.length +
1224                                                intel_pt_insn.rel;
1225                                 decoder->state.to_ip = decoder->ip;
1226                                 return 0;
1227                         }
1228                         /* Instruction sample for a non-taken branch */
1229                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1230                                 decoder->tnt.payload <<= 1;
1231                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1232                                 decoder->state.from_ip = decoder->ip;
1233                                 decoder->state.to_ip = 0;
1234                                 decoder->ip += intel_pt_insn.length;
1235                                 return 0;
1236                         }
1237                         decoder->ip += intel_pt_insn.length;
1238                         if (!decoder->tnt.count) {
1239                                 decoder->sample_timestamp = decoder->timestamp;
1240                                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
1241                                 return -EAGAIN;
1242                         }
1243                         decoder->tnt.payload <<= 1;
1244                         continue;
1245                 }
1246
1247                 return intel_pt_bug(decoder);
1248         }
1249 }
1250
1251 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1252 {
1253         unsigned int fup_tx_flags;
1254         int err;
1255
1256         fup_tx_flags = decoder->packet.payload &
1257                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1258         err = intel_pt_get_next_packet(decoder);
1259         if (err)
1260                 return err;
1261         if (decoder->packet.type == INTEL_PT_FUP) {
1262                 decoder->fup_tx_flags = fup_tx_flags;
1263                 decoder->set_fup_tx_flags = true;
1264                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1265                         *no_tip = true;
1266         } else {
1267                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1268                                 decoder->pos);
1269                 intel_pt_update_in_tx(decoder);
1270         }
1271         return 0;
1272 }
1273
1274 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1275 {
1276         uint64_t timestamp;
1277
1278         decoder->have_tma = false;
1279
1280         if (decoder->ref_timestamp) {
1281                 timestamp = decoder->packet.payload |
1282                             (decoder->ref_timestamp & (0xffULL << 56));
1283                 if (timestamp < decoder->ref_timestamp) {
1284                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1285                                 timestamp += (1ULL << 56);
1286                 } else {
1287                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1288                                 timestamp -= (1ULL << 56);
1289                 }
1290                 decoder->tsc_timestamp = timestamp;
1291                 decoder->timestamp = timestamp;
1292                 decoder->ref_timestamp = 0;
1293                 decoder->timestamp_insn_cnt = 0;
1294         } else if (decoder->timestamp) {
1295                 timestamp = decoder->packet.payload |
1296                             (decoder->timestamp & (0xffULL << 56));
1297                 decoder->tsc_timestamp = timestamp;
1298                 if (timestamp < decoder->timestamp &&
1299                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1300                         intel_pt_log_to("Suppressing backwards timestamp",
1301                                         timestamp);
1302                         timestamp = decoder->timestamp;
1303                 }
1304                 if (timestamp < decoder->timestamp) {
1305                         intel_pt_log_to("Wraparound timestamp", timestamp);
1306                         timestamp += (1ULL << 56);
1307                         decoder->tsc_timestamp = timestamp;
1308                 }
1309                 decoder->timestamp = timestamp;
1310                 decoder->timestamp_insn_cnt = 0;
1311         }
1312
1313         if (decoder->last_packet_type == INTEL_PT_CYC) {
1314                 decoder->cyc_ref_timestamp = decoder->timestamp;
1315                 decoder->cycle_cnt = 0;
1316                 decoder->have_calc_cyc_to_tsc = false;
1317                 intel_pt_calc_cyc_to_tsc(decoder, false);
1318         }
1319
1320         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1321 }
1322
1323 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1324 {
1325         intel_pt_log("ERROR: Buffer overflow\n");
1326         intel_pt_clear_tx_flags(decoder);
1327         decoder->timestamp_insn_cnt = 0;
1328         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1329         decoder->overflow = true;
1330         return -EOVERFLOW;
1331 }
1332
1333 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1334 {
1335         uint32_t ctc = decoder->packet.payload;
1336         uint32_t fc = decoder->packet.count;
1337         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1338
1339         if (!decoder->tsc_ctc_ratio_d)
1340                 return;
1341
1342         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1343         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1344         if (decoder->tsc_ctc_mult) {
1345                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1346         } else {
1347                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1348                                                   decoder->tsc_ctc_ratio_n,
1349                                                   decoder->tsc_ctc_ratio_d);
1350         }
1351         decoder->ctc_delta = 0;
1352         decoder->have_tma = true;
1353         decoder->fixup_last_mtc = true;
1354         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1355                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1356 }
1357
1358 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1359 {
1360         uint64_t timestamp;
1361         uint32_t mtc, mtc_delta;
1362
1363         if (!decoder->have_tma)
1364                 return;
1365
1366         mtc = decoder->packet.payload;
1367
1368         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1369                 decoder->fixup_last_mtc = false;
1370                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1371                                         &decoder->last_mtc);
1372         }
1373
1374         if (mtc > decoder->last_mtc)
1375                 mtc_delta = mtc - decoder->last_mtc;
1376         else
1377                 mtc_delta = mtc + 256 - decoder->last_mtc;
1378
1379         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1380
1381         if (decoder->tsc_ctc_mult) {
1382                 timestamp = decoder->ctc_timestamp +
1383                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1384         } else {
1385                 timestamp = decoder->ctc_timestamp +
1386                             multdiv(decoder->ctc_delta,
1387                                     decoder->tsc_ctc_ratio_n,
1388                                     decoder->tsc_ctc_ratio_d);
1389         }
1390
1391         if (timestamp < decoder->timestamp)
1392                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1393                              timestamp, decoder->timestamp);
1394         else
1395                 decoder->timestamp = timestamp;
1396
1397         decoder->timestamp_insn_cnt = 0;
1398         decoder->last_mtc = mtc;
1399
1400         if (decoder->last_packet_type == INTEL_PT_CYC) {
1401                 decoder->cyc_ref_timestamp = decoder->timestamp;
1402                 decoder->cycle_cnt = 0;
1403                 decoder->have_calc_cyc_to_tsc = false;
1404                 intel_pt_calc_cyc_to_tsc(decoder, true);
1405         }
1406 }
1407
1408 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1409 {
1410         unsigned int cbr = decoder->packet.payload;
1411
1412         if (decoder->cbr == cbr)
1413                 return;
1414
1415         decoder->cbr = cbr;
1416         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1417 }
1418
1419 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1420 {
1421         uint64_t timestamp = decoder->cyc_ref_timestamp;
1422
1423         decoder->have_cyc = true;
1424
1425         decoder->cycle_cnt += decoder->packet.payload;
1426
1427         if (!decoder->cyc_ref_timestamp)
1428                 return;
1429
1430         if (decoder->have_calc_cyc_to_tsc)
1431                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1432         else if (decoder->cbr)
1433                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1434         else
1435                 return;
1436
1437         if (timestamp < decoder->timestamp)
1438                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1439                              timestamp, decoder->timestamp);
1440         else
1441                 decoder->timestamp = timestamp;
1442
1443         decoder->timestamp_insn_cnt = 0;
1444 }
1445
1446 /* Walk PSB+ packets when already in sync. */
1447 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1448 {
1449         int err;
1450
1451         while (1) {
1452                 err = intel_pt_get_next_packet(decoder);
1453                 if (err)
1454                         return err;
1455
1456                 switch (decoder->packet.type) {
1457                 case INTEL_PT_PSBEND:
1458                         return 0;
1459
1460                 case INTEL_PT_TIP_PGD:
1461                 case INTEL_PT_TIP_PGE:
1462                 case INTEL_PT_TIP:
1463                 case INTEL_PT_TNT:
1464                 case INTEL_PT_TRACESTOP:
1465                 case INTEL_PT_BAD:
1466                 case INTEL_PT_PSB:
1467                         decoder->have_tma = false;
1468                         intel_pt_log("ERROR: Unexpected packet\n");
1469                         return -EAGAIN;
1470
1471                 case INTEL_PT_OVF:
1472                         return intel_pt_overflow(decoder);
1473
1474                 case INTEL_PT_TSC:
1475                         intel_pt_calc_tsc_timestamp(decoder);
1476                         break;
1477
1478                 case INTEL_PT_TMA:
1479                         intel_pt_calc_tma(decoder);
1480                         break;
1481
1482                 case INTEL_PT_CBR:
1483                         intel_pt_calc_cbr(decoder);
1484                         break;
1485
1486                 case INTEL_PT_MODE_EXEC:
1487                         decoder->exec_mode = decoder->packet.payload;
1488                         break;
1489
1490                 case INTEL_PT_PIP:
1491                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1492                         break;
1493
1494                 case INTEL_PT_FUP:
1495                         decoder->pge = true;
1496                         if (decoder->packet.count)
1497                                 intel_pt_set_last_ip(decoder);
1498                         break;
1499
1500                 case INTEL_PT_MODE_TSX:
1501                         intel_pt_update_in_tx(decoder);
1502                         break;
1503
1504                 case INTEL_PT_MTC:
1505                         intel_pt_calc_mtc_timestamp(decoder);
1506                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1507                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1508                         break;
1509
1510                 case INTEL_PT_CYC:
1511                         intel_pt_calc_cyc_timestamp(decoder);
1512                         break;
1513
1514                 case INTEL_PT_VMCS:
1515                 case INTEL_PT_MNT:
1516                 case INTEL_PT_PAD:
1517                 default:
1518                         break;
1519                 }
1520         }
1521 }
1522
1523 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1524 {
1525         int err;
1526
1527         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1528                 decoder->tx_flags = 0;
1529                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1530                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1531         } else {
1532                 decoder->state.flags |= INTEL_PT_ASYNC;
1533         }
1534
1535         while (1) {
1536                 err = intel_pt_get_next_packet(decoder);
1537                 if (err)
1538                         return err;
1539
1540                 switch (decoder->packet.type) {
1541                 case INTEL_PT_TNT:
1542                 case INTEL_PT_FUP:
1543                 case INTEL_PT_TRACESTOP:
1544                 case INTEL_PT_PSB:
1545                 case INTEL_PT_TSC:
1546                 case INTEL_PT_TMA:
1547                 case INTEL_PT_MODE_TSX:
1548                 case INTEL_PT_BAD:
1549                 case INTEL_PT_PSBEND:
1550                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1551                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1552                         decoder->pkt_step = 0;
1553                         return -ENOENT;
1554
1555                 case INTEL_PT_CBR:
1556                         intel_pt_calc_cbr(decoder);
1557                         break;
1558
1559                 case INTEL_PT_OVF:
1560                         return intel_pt_overflow(decoder);
1561
1562                 case INTEL_PT_TIP_PGD:
1563                         decoder->state.from_ip = decoder->ip;
1564                         decoder->state.to_ip = 0;
1565                         if (decoder->packet.count != 0) {
1566                                 intel_pt_set_ip(decoder);
1567                                 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1568                                              decoder->ip);
1569                         }
1570                         decoder->pge = false;
1571                         decoder->continuous_period = false;
1572                         return 0;
1573
1574                 case INTEL_PT_TIP_PGE:
1575                         decoder->pge = true;
1576                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1577                                      decoder->ip);
1578                         decoder->state.from_ip = 0;
1579                         if (decoder->packet.count == 0) {
1580                                 decoder->state.to_ip = 0;
1581                         } else {
1582                                 intel_pt_set_ip(decoder);
1583                                 decoder->state.to_ip = decoder->ip;
1584                         }
1585                         return 0;
1586
1587                 case INTEL_PT_TIP:
1588                         decoder->state.from_ip = decoder->ip;
1589                         if (decoder->packet.count == 0) {
1590                                 decoder->state.to_ip = 0;
1591                         } else {
1592                                 intel_pt_set_ip(decoder);
1593                                 decoder->state.to_ip = decoder->ip;
1594                         }
1595                         return 0;
1596
1597                 case INTEL_PT_PIP:
1598                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1599                         break;
1600
1601                 case INTEL_PT_MTC:
1602                         intel_pt_calc_mtc_timestamp(decoder);
1603                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1604                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1605                         break;
1606
1607                 case INTEL_PT_CYC:
1608                         intel_pt_calc_cyc_timestamp(decoder);
1609                         break;
1610
1611                 case INTEL_PT_MODE_EXEC:
1612                         decoder->exec_mode = decoder->packet.payload;
1613                         break;
1614
1615                 case INTEL_PT_VMCS:
1616                 case INTEL_PT_MNT:
1617                 case INTEL_PT_PAD:
1618                         break;
1619
1620                 default:
1621                         return intel_pt_bug(decoder);
1622                 }
1623         }
1624 }
1625
1626 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1627 {
1628         bool no_tip = false;
1629         int err;
1630
1631         while (1) {
1632                 err = intel_pt_get_next_packet(decoder);
1633                 if (err)
1634                         return err;
1635 next:
1636                 switch (decoder->packet.type) {
1637                 case INTEL_PT_TNT:
1638                         if (!decoder->packet.count)
1639                                 break;
1640                         decoder->tnt = decoder->packet;
1641                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1642                         err = intel_pt_walk_tnt(decoder);
1643                         if (err == -EAGAIN)
1644                                 break;
1645                         return err;
1646
1647                 case INTEL_PT_TIP_PGD:
1648                         if (decoder->packet.count != 0)
1649                                 intel_pt_set_last_ip(decoder);
1650                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1651                         return intel_pt_walk_tip(decoder);
1652
1653                 case INTEL_PT_TIP_PGE: {
1654                         decoder->pge = true;
1655                         if (decoder->packet.count == 0) {
1656                                 intel_pt_log_at("Skipping zero TIP.PGE",
1657                                                 decoder->pos);
1658                                 break;
1659                         }
1660                         intel_pt_set_ip(decoder);
1661                         decoder->state.from_ip = 0;
1662                         decoder->state.to_ip = decoder->ip;
1663                         return 0;
1664                 }
1665
1666                 case INTEL_PT_OVF:
1667                         return intel_pt_overflow(decoder);
1668
1669                 case INTEL_PT_TIP:
1670                         if (decoder->packet.count != 0)
1671                                 intel_pt_set_last_ip(decoder);
1672                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1673                         return intel_pt_walk_tip(decoder);
1674
1675                 case INTEL_PT_FUP:
1676                         if (decoder->packet.count == 0) {
1677                                 intel_pt_log_at("Skipping zero FUP",
1678                                                 decoder->pos);
1679                                 no_tip = false;
1680                                 break;
1681                         }
1682                         intel_pt_set_last_ip(decoder);
1683                         err = intel_pt_walk_fup(decoder);
1684                         if (err != -EAGAIN) {
1685                                 if (err)
1686                                         return err;
1687                                 if (no_tip)
1688                                         decoder->pkt_state =
1689                                                 INTEL_PT_STATE_FUP_NO_TIP;
1690                                 else
1691                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1692                                 return 0;
1693                         }
1694                         if (no_tip) {
1695                                 no_tip = false;
1696                                 break;
1697                         }
1698                         return intel_pt_walk_fup_tip(decoder);
1699
1700                 case INTEL_PT_TRACESTOP:
1701                         decoder->pge = false;
1702                         decoder->continuous_period = false;
1703                         intel_pt_clear_tx_flags(decoder);
1704                         decoder->have_tma = false;
1705                         break;
1706
1707                 case INTEL_PT_PSB:
1708                         decoder->last_ip = 0;
1709                         decoder->have_last_ip = true;
1710                         intel_pt_clear_stack(&decoder->stack);
1711                         err = intel_pt_walk_psbend(decoder);
1712                         if (err == -EAGAIN)
1713                                 goto next;
1714                         if (err)
1715                                 return err;
1716                         break;
1717
1718                 case INTEL_PT_PIP:
1719                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1720                         break;
1721
1722                 case INTEL_PT_MTC:
1723                         intel_pt_calc_mtc_timestamp(decoder);
1724                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1725                                 break;
1726                         /*
1727                          * Ensure that there has been an instruction since the
1728                          * last MTC.
1729                          */
1730                         if (!decoder->mtc_insn)
1731                                 break;
1732                         decoder->mtc_insn = false;
1733                         /* Ensure that there is a timestamp */
1734                         if (!decoder->timestamp)
1735                                 break;
1736                         decoder->state.type = INTEL_PT_INSTRUCTION;
1737                         decoder->state.from_ip = decoder->ip;
1738                         decoder->state.to_ip = 0;
1739                         decoder->mtc_insn = false;
1740                         return 0;
1741
1742                 case INTEL_PT_TSC:
1743                         intel_pt_calc_tsc_timestamp(decoder);
1744                         break;
1745
1746                 case INTEL_PT_TMA:
1747                         intel_pt_calc_tma(decoder);
1748                         break;
1749
1750                 case INTEL_PT_CYC:
1751                         intel_pt_calc_cyc_timestamp(decoder);
1752                         break;
1753
1754                 case INTEL_PT_CBR:
1755                         intel_pt_calc_cbr(decoder);
1756                         break;
1757
1758                 case INTEL_PT_MODE_EXEC:
1759                         decoder->exec_mode = decoder->packet.payload;
1760                         break;
1761
1762                 case INTEL_PT_MODE_TSX:
1763                         /* MODE_TSX need not be followed by FUP */
1764                         if (!decoder->pge) {
1765                                 intel_pt_update_in_tx(decoder);
1766                                 break;
1767                         }
1768                         err = intel_pt_mode_tsx(decoder, &no_tip);
1769                         if (err)
1770                                 return err;
1771                         goto next;
1772
1773                 case INTEL_PT_BAD: /* Does not happen */
1774                         return intel_pt_bug(decoder);
1775
1776                 case INTEL_PT_PSBEND:
1777                 case INTEL_PT_VMCS:
1778                 case INTEL_PT_MNT:
1779                 case INTEL_PT_PAD:
1780                         break;
1781
1782                 default:
1783                         return intel_pt_bug(decoder);
1784                 }
1785         }
1786 }
1787
1788 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1789 {
1790         return decoder->packet.count &&
1791                (decoder->have_last_ip || decoder->packet.count == 3 ||
1792                 decoder->packet.count == 6);
1793 }
1794
1795 /* Walk PSB+ packets to get in sync. */
1796 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1797 {
1798         int err;
1799
1800         while (1) {
1801                 err = intel_pt_get_next_packet(decoder);
1802                 if (err)
1803                         return err;
1804
1805                 switch (decoder->packet.type) {
1806                 case INTEL_PT_TIP_PGD:
1807                         decoder->continuous_period = false;
1808                         __fallthrough;
1809                 case INTEL_PT_TIP_PGE:
1810                 case INTEL_PT_TIP:
1811                         intel_pt_log("ERROR: Unexpected packet\n");
1812                         return -ENOENT;
1813
1814                 case INTEL_PT_FUP:
1815                         decoder->pge = true;
1816                         if (intel_pt_have_ip(decoder)) {
1817                                 uint64_t current_ip = decoder->ip;
1818
1819                                 intel_pt_set_ip(decoder);
1820                                 if (current_ip)
1821                                         intel_pt_log_to("Setting IP",
1822                                                         decoder->ip);
1823                         }
1824                         break;
1825
1826                 case INTEL_PT_MTC:
1827                         intel_pt_calc_mtc_timestamp(decoder);
1828                         break;
1829
1830                 case INTEL_PT_TSC:
1831                         intel_pt_calc_tsc_timestamp(decoder);
1832                         break;
1833
1834                 case INTEL_PT_TMA:
1835                         intel_pt_calc_tma(decoder);
1836                         break;
1837
1838                 case INTEL_PT_CYC:
1839                         intel_pt_calc_cyc_timestamp(decoder);
1840                         break;
1841
1842                 case INTEL_PT_CBR:
1843                         intel_pt_calc_cbr(decoder);
1844                         break;
1845
1846                 case INTEL_PT_PIP:
1847                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1848                         break;
1849
1850                 case INTEL_PT_MODE_EXEC:
1851                         decoder->exec_mode = decoder->packet.payload;
1852                         break;
1853
1854                 case INTEL_PT_MODE_TSX:
1855                         intel_pt_update_in_tx(decoder);
1856                         break;
1857
1858                 case INTEL_PT_TRACESTOP:
1859                         decoder->pge = false;
1860                         decoder->continuous_period = false;
1861                         intel_pt_clear_tx_flags(decoder);
1862                         __fallthrough;
1863
1864                 case INTEL_PT_TNT:
1865                         decoder->have_tma = false;
1866                         intel_pt_log("ERROR: Unexpected packet\n");
1867                         if (decoder->ip)
1868                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1869                         else
1870                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1871                         return -ENOENT;
1872
1873                 case INTEL_PT_BAD: /* Does not happen */
1874                         return intel_pt_bug(decoder);
1875
1876                 case INTEL_PT_OVF:
1877                         return intel_pt_overflow(decoder);
1878
1879                 case INTEL_PT_PSBEND:
1880                         return 0;
1881
1882                 case INTEL_PT_PSB:
1883                 case INTEL_PT_VMCS:
1884                 case INTEL_PT_MNT:
1885                 case INTEL_PT_PAD:
1886                 default:
1887                         break;
1888                 }
1889         }
1890 }
1891
1892 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1893 {
1894         int err;
1895
1896         while (1) {
1897                 err = intel_pt_get_next_packet(decoder);
1898                 if (err)
1899                         return err;
1900
1901                 switch (decoder->packet.type) {
1902                 case INTEL_PT_TIP_PGD:
1903                         decoder->continuous_period = false;
1904                         __fallthrough;
1905                 case INTEL_PT_TIP_PGE:
1906                 case INTEL_PT_TIP:
1907                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1908                         if (intel_pt_have_ip(decoder))
1909                                 intel_pt_set_ip(decoder);
1910                         if (decoder->ip)
1911                                 return 0;
1912                         break;
1913
1914                 case INTEL_PT_FUP:
1915                         if (intel_pt_have_ip(decoder))
1916                                 intel_pt_set_ip(decoder);
1917                         if (decoder->ip)
1918                                 return 0;
1919                         break;
1920
1921                 case INTEL_PT_MTC:
1922                         intel_pt_calc_mtc_timestamp(decoder);
1923                         break;
1924
1925                 case INTEL_PT_TSC:
1926                         intel_pt_calc_tsc_timestamp(decoder);
1927                         break;
1928
1929                 case INTEL_PT_TMA:
1930                         intel_pt_calc_tma(decoder);
1931                         break;
1932
1933                 case INTEL_PT_CYC:
1934                         intel_pt_calc_cyc_timestamp(decoder);
1935                         break;
1936
1937                 case INTEL_PT_CBR:
1938                         intel_pt_calc_cbr(decoder);
1939                         break;
1940
1941                 case INTEL_PT_PIP:
1942                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1943                         break;
1944
1945                 case INTEL_PT_MODE_EXEC:
1946                         decoder->exec_mode = decoder->packet.payload;
1947                         break;
1948
1949                 case INTEL_PT_MODE_TSX:
1950                         intel_pt_update_in_tx(decoder);
1951                         break;
1952
1953                 case INTEL_PT_OVF:
1954                         return intel_pt_overflow(decoder);
1955
1956                 case INTEL_PT_BAD: /* Does not happen */
1957                         return intel_pt_bug(decoder);
1958
1959                 case INTEL_PT_TRACESTOP:
1960                         decoder->pge = false;
1961                         decoder->continuous_period = false;
1962                         intel_pt_clear_tx_flags(decoder);
1963                         decoder->have_tma = false;
1964                         break;
1965
1966                 case INTEL_PT_PSB:
1967                         decoder->last_ip = 0;
1968                         decoder->have_last_ip = true;
1969                         intel_pt_clear_stack(&decoder->stack);
1970                         err = intel_pt_walk_psb(decoder);
1971                         if (err)
1972                                 return err;
1973                         if (decoder->ip) {
1974                                 /* Do not have a sample */
1975                                 decoder->state.type = 0;
1976                                 return 0;
1977                         }
1978                         break;
1979
1980                 case INTEL_PT_TNT:
1981                 case INTEL_PT_PSBEND:
1982                 case INTEL_PT_VMCS:
1983                 case INTEL_PT_MNT:
1984                 case INTEL_PT_PAD:
1985                 default:
1986                         break;
1987                 }
1988         }
1989 }
1990
1991 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1992 {
1993         int err;
1994
1995         decoder->set_fup_tx_flags = false;
1996
1997         intel_pt_log("Scanning for full IP\n");
1998         err = intel_pt_walk_to_ip(decoder);
1999         if (err)
2000                 return err;
2001
2002         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2003         decoder->overflow = false;
2004
2005         decoder->state.from_ip = 0;
2006         decoder->state.to_ip = decoder->ip;
2007         intel_pt_log_to("Setting IP", decoder->ip);
2008
2009         return 0;
2010 }
2011
2012 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2013 {
2014         const unsigned char *end = decoder->buf + decoder->len;
2015         size_t i;
2016
2017         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2018                 if (i > decoder->len)
2019                         continue;
2020                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2021                         return i;
2022         }
2023         return 0;
2024 }
2025
2026 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2027 {
2028         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2029         const char *psb = INTEL_PT_PSB_STR;
2030
2031         if (rest_psb > decoder->len ||
2032             memcmp(decoder->buf, psb + part_psb, rest_psb))
2033                 return 0;
2034
2035         return rest_psb;
2036 }
2037
2038 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2039                                   int part_psb)
2040 {
2041         int rest_psb, ret;
2042
2043         decoder->pos += decoder->len;
2044         decoder->len = 0;
2045
2046         ret = intel_pt_get_next_data(decoder);
2047         if (ret)
2048                 return ret;
2049
2050         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2051         if (!rest_psb)
2052                 return 0;
2053
2054         decoder->pos -= part_psb;
2055         decoder->next_buf = decoder->buf + rest_psb;
2056         decoder->next_len = decoder->len - rest_psb;
2057         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2058         decoder->buf = decoder->temp_buf;
2059         decoder->len = INTEL_PT_PSB_LEN;
2060
2061         return 0;
2062 }
2063
2064 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2065 {
2066         unsigned char *next;
2067         int ret;
2068
2069         intel_pt_log("Scanning for PSB\n");
2070         while (1) {
2071                 if (!decoder->len) {
2072                         ret = intel_pt_get_next_data(decoder);
2073                         if (ret)
2074                                 return ret;
2075                 }
2076
2077                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2078                               INTEL_PT_PSB_LEN);
2079                 if (!next) {
2080                         int part_psb;
2081
2082                         part_psb = intel_pt_part_psb(decoder);
2083                         if (part_psb) {
2084                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2085                                 if (ret)
2086                                         return ret;
2087                         } else {
2088                                 decoder->pos += decoder->len;
2089                                 decoder->len = 0;
2090                         }
2091                         continue;
2092                 }
2093
2094                 decoder->pkt_step = next - decoder->buf;
2095                 return intel_pt_get_next_packet(decoder);
2096         }
2097 }
2098
2099 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2100 {
2101         int err;
2102
2103         decoder->pge = false;
2104         decoder->continuous_period = false;
2105         decoder->have_last_ip = false;
2106         decoder->last_ip = 0;
2107         decoder->ip = 0;
2108         intel_pt_clear_stack(&decoder->stack);
2109
2110         err = intel_pt_scan_for_psb(decoder);
2111         if (err)
2112                 return err;
2113
2114         decoder->have_last_ip = true;
2115         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2116
2117         err = intel_pt_walk_psb(decoder);
2118         if (err)
2119                 return err;
2120
2121         if (decoder->ip) {
2122                 decoder->state.type = 0; /* Do not have a sample */
2123                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2124         } else {
2125                 return intel_pt_sync_ip(decoder);
2126         }
2127
2128         return 0;
2129 }
2130
2131 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2132 {
2133         uint64_t est = decoder->sample_insn_cnt << 1;
2134
2135         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2136                 goto out;
2137
2138         est *= decoder->max_non_turbo_ratio;
2139         est /= decoder->cbr;
2140 out:
2141         return decoder->sample_timestamp + est;
2142 }
2143
2144 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2145 {
2146         int err;
2147
2148         do {
2149                 decoder->state.type = INTEL_PT_BRANCH;
2150                 decoder->state.flags = 0;
2151
2152                 switch (decoder->pkt_state) {
2153                 case INTEL_PT_STATE_NO_PSB:
2154                         err = intel_pt_sync(decoder);
2155                         break;
2156                 case INTEL_PT_STATE_NO_IP:
2157                         decoder->have_last_ip = false;
2158                         decoder->last_ip = 0;
2159                         decoder->ip = 0;
2160                         /* Fall through */
2161                 case INTEL_PT_STATE_ERR_RESYNC:
2162                         err = intel_pt_sync_ip(decoder);
2163                         break;
2164                 case INTEL_PT_STATE_IN_SYNC:
2165                         err = intel_pt_walk_trace(decoder);
2166                         break;
2167                 case INTEL_PT_STATE_TNT:
2168                 case INTEL_PT_STATE_TNT_CONT:
2169                         err = intel_pt_walk_tnt(decoder);
2170                         if (err == -EAGAIN)
2171                                 err = intel_pt_walk_trace(decoder);
2172                         break;
2173                 case INTEL_PT_STATE_TIP:
2174                 case INTEL_PT_STATE_TIP_PGD:
2175                         err = intel_pt_walk_tip(decoder);
2176                         break;
2177                 case INTEL_PT_STATE_FUP:
2178                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2179                         err = intel_pt_walk_fup(decoder);
2180                         if (err == -EAGAIN)
2181                                 err = intel_pt_walk_fup_tip(decoder);
2182                         else if (!err)
2183                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2184                         break;
2185                 case INTEL_PT_STATE_FUP_NO_TIP:
2186                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2187                         err = intel_pt_walk_fup(decoder);
2188                         if (err == -EAGAIN)
2189                                 err = intel_pt_walk_trace(decoder);
2190                         break;
2191                 default:
2192                         err = intel_pt_bug(decoder);
2193                         break;
2194                 }
2195         } while (err == -ENOLINK);
2196
2197         if (err) {
2198                 decoder->state.err = intel_pt_ext_err(err);
2199                 decoder->state.from_ip = decoder->ip;
2200                 decoder->sample_timestamp = decoder->timestamp;
2201                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2202         } else {
2203                 decoder->state.err = 0;
2204                 if (intel_pt_sample_time(decoder->pkt_state)) {
2205                         decoder->sample_timestamp = decoder->timestamp;
2206                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2207                 }
2208         }
2209
2210         decoder->state.timestamp = decoder->sample_timestamp;
2211         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2212         decoder->state.cr3 = decoder->cr3;
2213         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2214
2215         return &decoder->state;
2216 }
2217
2218 /**
2219  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2220  * @buf: pointer to buffer pointer
2221  * @len: size of buffer
2222  *
2223  * Updates the buffer pointer to point to the start of the next PSB packet if
2224  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2225  * @len is adjusted accordingly.
2226  *
2227  * Return: %true if a PSB packet is found, %false otherwise.
2228  */
2229 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2230 {
2231         unsigned char *next;
2232
2233         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2234         if (next) {
2235                 *len -= next - *buf;
2236                 *buf = next;
2237                 return true;
2238         }
2239         return false;
2240 }
2241
2242 /**
2243  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2244  *                     packet.
2245  * @buf: pointer to buffer pointer
2246  * @len: size of buffer
2247  *
2248  * Updates the buffer pointer to point to the start of the following PSB packet
2249  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2250  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2251  *
2252  * Return: %true if a PSB packet is found, %false otherwise.
2253  */
2254 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2255 {
2256         unsigned char *next;
2257
2258         if (!*len)
2259                 return false;
2260
2261         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2262         if (next) {
2263                 *len -= next - *buf;
2264                 *buf = next;
2265                 return true;
2266         }
2267         return false;
2268 }
2269
2270 /**
2271  * intel_pt_last_psb - find the last PSB packet in a buffer.
2272  * @buf: buffer
2273  * @len: size of buffer
2274  *
2275  * This function finds the last PSB in a buffer.
2276  *
2277  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2278  */
2279 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2280 {
2281         const char *n = INTEL_PT_PSB_STR;
2282         unsigned char *p;
2283         size_t k;
2284
2285         if (len < INTEL_PT_PSB_LEN)
2286                 return NULL;
2287
2288         k = len - INTEL_PT_PSB_LEN + 1;
2289         while (1) {
2290                 p = memrchr(buf, n[0], k);
2291                 if (!p)
2292                         return NULL;
2293                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2294                         return p;
2295                 k = p - buf;
2296                 if (!k)
2297                         return NULL;
2298         }
2299 }
2300
2301 /**
2302  * intel_pt_next_tsc - find and return next TSC.
2303  * @buf: buffer
2304  * @len: size of buffer
2305  * @tsc: TSC value returned
2306  * @rem: returns remaining size when TSC is found
2307  *
2308  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2309  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2310  * PSBEND packet is found.
2311  *
2312  * Return: %true if TSC is found, false otherwise.
2313  */
2314 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2315                               size_t *rem)
2316 {
2317         struct intel_pt_pkt packet;
2318         int ret;
2319
2320         while (len) {
2321                 ret = intel_pt_get_packet(buf, len, &packet);
2322                 if (ret <= 0)
2323                         return false;
2324                 if (packet.type == INTEL_PT_TSC) {
2325                         *tsc = packet.payload;
2326                         *rem = len;
2327                         return true;
2328                 }
2329                 if (packet.type == INTEL_PT_PSBEND)
2330                         return false;
2331                 buf += ret;
2332                 len -= ret;
2333         }
2334         return false;
2335 }
2336
2337 /**
2338  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2339  * @tsc1: first TSC to compare
2340  * @tsc2: second TSC to compare
2341  *
2342  * This function compares 7-byte TSC values allowing for the possibility that
2343  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2344  * around so for that purpose this function assumes the absolute difference is
2345  * less than half the maximum difference.
2346  *
2347  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2348  * after @tsc2.
2349  */
2350 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2351 {
2352         const uint64_t halfway = (1ULL << 55);
2353
2354         if (tsc1 == tsc2)
2355                 return 0;
2356
2357         if (tsc1 < tsc2) {
2358                 if (tsc2 - tsc1 < halfway)
2359                         return -1;
2360                 else
2361                         return 1;
2362         } else {
2363                 if (tsc1 - tsc2 < halfway)
2364                         return 1;
2365                 else
2366                         return -1;
2367         }
2368 }
2369
2370 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2371
2372 /**
2373  * adj_for_padding - adjust overlap to account for padding.
2374  * @buf_b: second buffer
2375  * @buf_a: first buffer
2376  * @len_a: size of first buffer
2377  *
2378  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2379  * accordingly.
2380  *
2381  * Return: A pointer into @buf_b from where non-overlapped data starts
2382  */
2383 static unsigned char *adj_for_padding(unsigned char *buf_b,
2384                                       unsigned char *buf_a, size_t len_a)
2385 {
2386         unsigned char *p = buf_b - MAX_PADDING;
2387         unsigned char *q = buf_a + len_a - MAX_PADDING;
2388         int i;
2389
2390         for (i = MAX_PADDING; i; i--, p++, q++) {
2391                 if (*p != *q)
2392                         break;
2393         }
2394
2395         return p;
2396 }
2397
2398 /**
2399  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2400  *                             using TSC.
2401  * @buf_a: first buffer
2402  * @len_a: size of first buffer
2403  * @buf_b: second buffer
2404  * @len_b: size of second buffer
2405  * @consecutive: returns true if there is data in buf_b that is consecutive
2406  *               to buf_a
2407  *
2408  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2409  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2410  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2411  * @buf_a and @buf_b are positioned at a PSB.
2412  *
2413  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2414  * @buf_b + @len_b if there is no non-overlapped data.
2415  */
2416 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2417                                                 size_t len_a,
2418                                                 unsigned char *buf_b,
2419                                                 size_t len_b, bool *consecutive)
2420 {
2421         uint64_t tsc_a, tsc_b;
2422         unsigned char *p;
2423         size_t len, rem_a, rem_b;
2424
2425         p = intel_pt_last_psb(buf_a, len_a);
2426         if (!p)
2427                 return buf_b; /* No PSB in buf_a => no overlap */
2428
2429         len = len_a - (p - buf_a);
2430         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2431                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2432                 len_a -= len;
2433                 p = intel_pt_last_psb(buf_a, len_a);
2434                 if (!p)
2435                         return buf_b; /* No full PSB+ => assume no overlap */
2436                 len = len_a - (p - buf_a);
2437                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2438                         return buf_b; /* No TSC in buf_a => assume no overlap */
2439         }
2440
2441         while (1) {
2442                 /* Ignore PSB+ with no TSC */
2443                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2444                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2445
2446                         /* Same TSC, so buffers are consecutive */
2447                         if (!cmp && rem_b >= rem_a) {
2448                                 unsigned char *start;
2449
2450                                 *consecutive = true;
2451                                 start = buf_b + len_b - (rem_b - rem_a);
2452                                 return adj_for_padding(start, buf_a, len_a);
2453                         }
2454                         if (cmp < 0)
2455                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2456                 }
2457
2458                 if (!intel_pt_step_psb(&buf_b, &len_b))
2459                         return buf_b + len_b; /* No PSB in buf_b => no data */
2460         }
2461 }
2462
2463 /**
2464  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2465  * @buf_a: first buffer
2466  * @len_a: size of first buffer
2467  * @buf_b: second buffer
2468  * @len_b: size of second buffer
2469  * @have_tsc: can use TSC packets to detect overlap
2470  * @consecutive: returns true if there is data in buf_b that is consecutive
2471  *               to buf_a
2472  *
2473  * When trace samples or snapshots are recorded there is the possibility that
2474  * the data overlaps.  Note that, for the purposes of decoding, data is only
2475  * useful if it begins with a PSB packet.
2476  *
2477  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2478  * @buf_b + @len_b if there is no non-overlapped data.
2479  */
2480 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2481                                      unsigned char *buf_b, size_t len_b,
2482                                      bool have_tsc, bool *consecutive)
2483 {
2484         unsigned char *found;
2485
2486         /* Buffer 'b' must start at PSB so throw away everything before that */
2487         if (!intel_pt_next_psb(&buf_b, &len_b))
2488                 return buf_b + len_b; /* No PSB */
2489
2490         if (!intel_pt_next_psb(&buf_a, &len_a))
2491                 return buf_b; /* No overlap */
2492
2493         if (have_tsc) {
2494                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2495                                                   consecutive);
2496                 if (found)
2497                         return found;
2498         }
2499
2500         /*
2501          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2502          * we can ignore the first part of buffer 'a'.
2503          */
2504         while (len_b < len_a) {
2505                 if (!intel_pt_step_psb(&buf_a, &len_a))
2506                         return buf_b; /* No overlap */
2507         }
2508
2509         /* Now len_b >= len_a */
2510         while (1) {
2511                 /* Potential overlap so check the bytes */
2512                 found = memmem(buf_a, len_a, buf_b, len_a);
2513                 if (found) {
2514                         *consecutive = true;
2515                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2516                 }
2517
2518                 /* Try again at next PSB in buffer 'a' */
2519                 if (!intel_pt_step_psb(&buf_a, &len_a))
2520                         return buf_b; /* No overlap */
2521         }
2522 }