drivers/hwtracing/coresight/coresight-tmc-etf.c

   1 /*
   2  * Copyright(C) 2016 Linaro Limited. All rights reserved.
   3  * Author: Mathieu Poirier <mathieu.poirier@linaro.org>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published by
   7  * the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 #include <linux/circ_buf.h>
  19 #include <linux/coresight.h>
  20 #include <linux/perf_event.h>
  21 #include <linux/slab.h>
  22 #include "coresight-priv.h"
  23 #include "coresight-tmc.h"
  24
  25 static void tmc_etb_enable_hw(struct tmc_drvdata *drvdata)
  26 {
  27         CS_UNLOCK(drvdata->base);
  28
  29         /* Wait for TMCSReady bit to be set */
  30         tmc_wait_for_tmcready(drvdata);
  31
  32         writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
  33         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
  34                        TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
  35                        TMC_FFCR_TRIGON_TRIGIN,
  36                        drvdata->base + TMC_FFCR);
  37
  38         writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
  39         tmc_enable_hw(drvdata);
  40
  41         CS_LOCK(drvdata->base);
  42 }
  43
  44 static void tmc_etb_dump_hw(struct tmc_drvdata *drvdata)
  45 {
  46         bool lost = false;
  47         char *bufp;
  48         const u32 *barrier;
  49         u32 read_data, status;
  50         int i;
  51
  52         /*
  53          * Get a hold of the status register and see if a wrap around
  54          * has occurred.
  55          */
  56         status = readl_relaxed(drvdata->base + TMC_STS);
  57         if (status & TMC_STS_FULL)
  58                 lost = true;
  59
  60         bufp = drvdata->buf;
  61         drvdata->len = 0;
  62         barrier = barrier_pkt;
  63         while (1) {
  64                 for (i = 0; i < drvdata->memwidth; i++) {
  65                         read_data = readl_relaxed(drvdata->base + TMC_RRD);
  66                         if (read_data == 0xFFFFFFFF)
  67                                 return;
  68
  69                         if (lost && *barrier) {
  70                                 read_data = *barrier;
  71                                 barrier++;
  72                         }
  73
  74                         memcpy(bufp, &read_data, 4);
  75                         bufp += 4;
  76                         drvdata->len += 4;
  77                 }
  78         }
  79 }
  80
  81 static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata)
  82 {
  83         CS_UNLOCK(drvdata->base);
  84
  85         tmc_flush_and_stop(drvdata);
  86         /*
  87          * When operating in sysFS mode the content of the buffer needs to be
  88          * read before the TMC is disabled.
  89          */
  90         if (drvdata->mode == CS_MODE_SYSFS)
  91                 tmc_etb_dump_hw(drvdata);
  92         tmc_disable_hw(drvdata);
  93
  94         CS_LOCK(drvdata->base);
  95 }
  96
  97 static void tmc_etf_enable_hw(struct tmc_drvdata *drvdata)
  98 {
  99         CS_UNLOCK(drvdata->base);
 100
 101         /* Wait for TMCSReady bit to be set */
 102         tmc_wait_for_tmcready(drvdata);
 103
 104         writel_relaxed(TMC_MODE_HARDWARE_FIFO, drvdata->base + TMC_MODE);
 105         writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI,
 106                        drvdata->base + TMC_FFCR);
 107         writel_relaxed(0x0, drvdata->base + TMC_BUFWM);
 108         tmc_enable_hw(drvdata);
 109
 110         CS_LOCK(drvdata->base);
 111 }
 112
 113 static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata)
 114 {
 115         CS_UNLOCK(drvdata->base);
 116
 117         tmc_flush_and_stop(drvdata);
 118         tmc_disable_hw(drvdata);
 119
 120         CS_LOCK(drvdata->base);
 121 }
 122
 123 static int tmc_enable_etf_sink_sysfs(struct coresight_device *csdev)
 124 {
 125         int ret = 0;
 126         bool used = false;
 127         char *buf = NULL;
 128         unsigned long flags;
 129         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 130
 131         /*
 132          * If we don't have a buffer release the lock and allocate memory.
 133          * Otherwise keep the lock and move along.
 134          */
 135         spin_lock_irqsave(&drvdata->spinlock, flags);
 136         if (!drvdata->buf) {
 137                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 138
 139                 /* Allocating the memory here while outside of the spinlock */
 140                 buf = kzalloc(drvdata->size, GFP_KERNEL);
 141                 if (!buf)
 142                         return -ENOMEM;
 143
 144                 /* Let's try again */
 145                 spin_lock_irqsave(&drvdata->spinlock, flags);
 146         }
 147
 148         if (drvdata->reading) {
 149                 ret = -EBUSY;
 150                 goto out;
 151         }
 152
 153         /*
 154          * In sysFS mode we can have multiple writers per sink.  Since this
 155          * sink is already enabled no memory is needed and the HW need not be
 156          * touched.
 157          */
 158         if (drvdata->mode == CS_MODE_SYSFS)
 159                 goto out;
 160
 161         /*
 162          * If drvdata::buf isn't NULL, memory was allocated for a previous
 163          * trace run but wasn't read.  If so simply zero-out the memory.
 164          * Otherwise use the memory allocated above.
 165          *
 166          * The memory is freed when users read the buffer using the
 167          * /dev/xyz.{etf|etb} interface.  See tmc_read_unprepare_etf() for
 168          * details.
 169          */
 170         if (drvdata->buf) {
 171                 memset(drvdata->buf, 0, drvdata->size);
 172         } else {
 173                 used = true;
 174                 drvdata->buf = buf;
 175         }
 176
 177         drvdata->mode = CS_MODE_SYSFS;
 178         tmc_etb_enable_hw(drvdata);
 179 out:
 180         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 181
 182         /* Free memory outside the spinlock if need be */
 183         if (!used)
 184                 kfree(buf);
 185
 186         return ret;
 187 }
 188
 189 static int tmc_enable_etf_sink_perf(struct coresight_device *csdev)
 190 {
 191         int ret = 0;
 192         unsigned long flags;
 193         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 194
 195         spin_lock_irqsave(&drvdata->spinlock, flags);
 196         if (drvdata->reading) {
 197                 ret = -EINVAL;
 198                 goto out;
 199         }
 200
 201         /*
 202          * In Perf mode there can be only one writer per sink.  There
 203          * is also no need to continue if the ETB/ETR is already operated
 204          * from sysFS.
 205          */
 206         if (drvdata->mode != CS_MODE_DISABLED) {
 207                 ret = -EINVAL;
 208                 goto out;
 209         }
 210
 211         drvdata->mode = CS_MODE_PERF;
 212         tmc_etb_enable_hw(drvdata);
 213 out:
 214         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 215
 216         return ret;
 217 }
 218
 219 static int tmc_enable_etf_sink(struct coresight_device *csdev, u32 mode)
 220 {
 221         int ret;
 222         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 223
 224         switch (mode) {
 225         case CS_MODE_SYSFS:
 226                 ret = tmc_enable_etf_sink_sysfs(csdev);
 227                 break;
 228         case CS_MODE_PERF:
 229                 ret = tmc_enable_etf_sink_perf(csdev);
 230                 break;
 231         /* We shouldn't be here */
 232         default:
 233                 ret = -EINVAL;
 234                 break;
 235         }
 236
 237         if (ret)
 238                 return ret;
 239
 240         dev_info(drvdata->dev, "TMC-ETB/ETF enabled\n");
 241         return 0;
 242 }
 243
 244 static void tmc_disable_etf_sink(struct coresight_device *csdev)
 245 {
 246         unsigned long flags;
 247         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 248
 249         spin_lock_irqsave(&drvdata->spinlock, flags);
 250         if (drvdata->reading) {
 251                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 252                 return;
 253         }
 254
 255         /* Disable the TMC only if it needs to */
 256         if (drvdata->mode != CS_MODE_DISABLED) {
 257                 tmc_etb_disable_hw(drvdata);
 258                 drvdata->mode = CS_MODE_DISABLED;
 259         }
 260
 261         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 262
 263         dev_info(drvdata->dev, "TMC-ETB/ETF disabled\n");
 264 }
 265
 266 static int tmc_enable_etf_link(struct coresight_device *csdev,
 267                                int inport, int outport)
 268 {
 269         unsigned long flags;
 270         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 271
 272         spin_lock_irqsave(&drvdata->spinlock, flags);
 273         if (drvdata->reading) {
 274                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 275                 return -EBUSY;
 276         }
 277
 278         tmc_etf_enable_hw(drvdata);
 279         drvdata->mode = CS_MODE_SYSFS;
 280         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 281
 282         dev_info(drvdata->dev, "TMC-ETF enabled\n");
 283         return 0;
 284 }
 285
 286 static void tmc_disable_etf_link(struct coresight_device *csdev,
 287                                  int inport, int outport)
 288 {
 289         unsigned long flags;
 290         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 291
 292         spin_lock_irqsave(&drvdata->spinlock, flags);
 293         if (drvdata->reading) {
 294                 spin_unlock_irqrestore(&drvdata->spinlock, flags);
 295                 return;
 296         }
 297
 298         tmc_etf_disable_hw(drvdata);
 299         drvdata->mode = CS_MODE_DISABLED;
 300         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 301
 302         dev_info(drvdata->dev, "TMC-ETF disabled\n");
 303 }
 304
 305 static void *tmc_alloc_etf_buffer(struct coresight_device *csdev, int cpu,
 306                                   void **pages, int nr_pages, bool overwrite)
 307 {
 308         int node;
 309         struct cs_buffers *buf;
 310
 311         node = (cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu);
 312
 313         /* Allocate memory structure for interaction with Perf */
 314         buf = kzalloc_node(sizeof(struct cs_buffers), GFP_KERNEL, node);
 315         if (!buf)
 316                 return NULL;
 317
 318         buf->snapshot = overwrite;
 319         buf->nr_pages = nr_pages;
 320         buf->data_pages = pages;
 321
 322         return buf;
 323 }
 324
 325 static void tmc_free_etf_buffer(void *config)
 326 {
 327         struct cs_buffers *buf = config;
 328
 329         kfree(buf);
 330 }
 331
 332 static int tmc_set_etf_buffer(struct coresight_device *csdev,
 333                               struct perf_output_handle *handle,
 334                               void *sink_config)
 335 {
 336         int ret = 0;
 337         unsigned long head;
 338         struct cs_buffers *buf = sink_config;
 339
 340         /* wrap head around to the amount of space we have */
 341         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 342
 343         /* find the page to write to */
 344         buf->cur = head / PAGE_SIZE;
 345
 346         /* and offset within that page */
 347         buf->offset = head % PAGE_SIZE;
 348
 349         local_set(&buf->data_size, 0);
 350
 351         return ret;
 352 }
 353
 354 static unsigned long tmc_reset_etf_buffer(struct coresight_device *csdev,
 355                                           struct perf_output_handle *handle,
 356                                           void *sink_config)
 357 {
 358         long size = 0;
 359         struct cs_buffers *buf = sink_config;
 360
 361         if (buf) {
 362                 /*
 363                  * In snapshot mode ->data_size holds the new address of the
 364                  * ring buffer's head.  The size itself is the whole address
 365                  * range since we want the latest information.
 366                  */
 367                 if (buf->snapshot)
 368                         handle->head = local_xchg(&buf->data_size,
 369                                                   buf->nr_pages << PAGE_SHIFT);
 370                 /*
 371                  * Tell the tracer PMU how much we got in this run and if
 372                  * something went wrong along the way.  Nobody else can use
 373                  * this cs_buffers instance until we are done.  As such
 374                  * resetting parameters here and squaring off with the ring
 375                  * buffer API in the tracer PMU is fine.
 376                  */
 377                 size = local_xchg(&buf->data_size, 0);
 378         }
 379
 380         return size;
 381 }
 382
 383 static void tmc_update_etf_buffer(struct coresight_device *csdev,
 384                                   struct perf_output_handle *handle,
 385                                   void *sink_config)
 386 {
 387         bool lost = false;
 388         int i, cur;
 389         const u32 *barrier;
 390         u32 *buf_ptr;
 391         u64 read_ptr, write_ptr;
 392         u32 status, to_read;
 393         unsigned long offset;
 394         struct cs_buffers *buf = sink_config;
 395         struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 396
 397         if (!buf)
 398                 return;
 399
 400         /* This shouldn't happen */
 401         if (WARN_ON_ONCE(drvdata->mode != CS_MODE_PERF))
 402                 return;
 403
 404         CS_UNLOCK(drvdata->base);
 405
 406         tmc_flush_and_stop(drvdata);
 407
 408         read_ptr = tmc_read_rrp(drvdata);
 409         write_ptr = tmc_read_rwp(drvdata);
 410
 411         /*
 412          * Get a hold of the status register and see if a wrap around
 413          * has occurred.  If so adjust things accordingly.
 414          */
 415         status = readl_relaxed(drvdata->base + TMC_STS);
 416         if (status & TMC_STS_FULL) {
 417                 lost = true;
 418                 to_read = drvdata->size;
 419         } else {
 420                 to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size);
 421         }
 422
 423         /*
 424          * The TMC RAM buffer may be bigger than the space available in the
 425          * perf ring buffer (handle->size).  If so advance the RRP so that we
 426          * get the latest trace data.
 427          */
 428         if (to_read > handle->size) {
 429                 u32 mask = 0;
 430
 431                 /*
 432                  * The value written to RRP must be byte-address aligned to
 433                  * the width of the trace memory databus _and_ to a frame
 434                  * boundary (16 byte), whichever is the biggest. For example,
 435                  * for 32-bit, 64-bit and 128-bit wide trace memory, the four
 436                  * LSBs must be 0s. For 256-bit wide trace memory, the five
 437                  * LSBs must be 0s.
 438                  */
 439                 switch (drvdata->memwidth) {
 440                 case TMC_MEM_INTF_WIDTH_32BITS:
 441                 case TMC_MEM_INTF_WIDTH_64BITS:
 442                 case TMC_MEM_INTF_WIDTH_128BITS:
 443                         mask = GENMASK(31, 4);
 444                         break;
 445                 case TMC_MEM_INTF_WIDTH_256BITS:
 446                         mask = GENMASK(31, 5);
 447                         break;
 448                 }
 449
 450                 /*
 451                  * Make sure the new size is aligned in accordance with the
 452                  * requirement explained above.
 453                  */
 454                 to_read = handle->size & mask;
 455                 /* Move the RAM read pointer up */
 456                 read_ptr = (write_ptr + drvdata->size) - to_read;
 457                 /* Make sure we are still within our limits */
 458                 if (read_ptr > (drvdata->size - 1))
 459                         read_ptr -= drvdata->size;
 460                 /* Tell the HW */
 461                 tmc_write_rrp(drvdata, read_ptr);
 462                 lost = true;
 463         }
 464
 465         if (lost)
 466                 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 467
 468         cur = buf->cur;
 469         offset = buf->offset;
 470         barrier = barrier_pkt;
 471
 472         /* for every byte to read */
 473         for (i = 0; i < to_read; i += 4) {
 474                 buf_ptr = buf->data_pages[cur] + offset;
 475                 *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD);
 476
 477                 if (lost && *barrier) {
 478                         *buf_ptr = *barrier;
 479                         barrier++;
 480                 }
 481
 482                 offset += 4;
 483                 if (offset >= PAGE_SIZE) {
 484                         offset = 0;
 485                         cur++;
 486                         /* wrap around at the end of the buffer */
 487                         cur &= buf->nr_pages - 1;
 488                 }
 489         }
 490
 491         /*
 492          * In snapshot mode all we have to do is communicate to
 493          * perf_aux_output_end() the address of the current head.  In full
 494          * trace mode the same function expects a size to move rb->aux_head
 495          * forward.
 496          */
 497         if (buf->snapshot)
 498                 local_set(&buf->data_size, (cur * PAGE_SIZE) + offset);
 499         else
 500                 local_add(to_read, &buf->data_size);
 501
 502         CS_LOCK(drvdata->base);
 503 }
 504
 505 static const struct coresight_ops_sink tmc_etf_sink_ops = {
 506         .enable         = tmc_enable_etf_sink,
 507         .disable        = tmc_disable_etf_sink,
 508         .alloc_buffer   = tmc_alloc_etf_buffer,
 509         .free_buffer    = tmc_free_etf_buffer,
 510         .set_buffer     = tmc_set_etf_buffer,
 511         .reset_buffer   = tmc_reset_etf_buffer,
 512         .update_buffer  = tmc_update_etf_buffer,
 513 };
 514
 515 static const struct coresight_ops_link tmc_etf_link_ops = {
 516         .enable         = tmc_enable_etf_link,
 517         .disable        = tmc_disable_etf_link,
 518 };
 519
 520 const struct coresight_ops tmc_etb_cs_ops = {
 521         .sink_ops       = &tmc_etf_sink_ops,
 522 };
 523
 524 const struct coresight_ops tmc_etf_cs_ops = {
 525         .sink_ops       = &tmc_etf_sink_ops,
 526         .link_ops       = &tmc_etf_link_ops,
 527 };
 528
 529 int tmc_read_prepare_etb(struct tmc_drvdata *drvdata)
 530 {
 531         enum tmc_mode mode;
 532         int ret = 0;
 533         unsigned long flags;
 534
 535         /* config types are set a boot time and never change */
 536         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 537                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 538                 return -EINVAL;
 539
 540         spin_lock_irqsave(&drvdata->spinlock, flags);
 541
 542         if (drvdata->reading) {
 543                 ret = -EBUSY;
 544                 goto out;
 545         }
 546
 547         /* There is no point in reading a TMC in HW FIFO mode */
 548         mode = readl_relaxed(drvdata->base + TMC_MODE);
 549         if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 550                 ret = -EINVAL;
 551                 goto out;
 552         }
 553
 554         /* Don't interfere if operated from Perf */
 555         if (drvdata->mode == CS_MODE_PERF) {
 556                 ret = -EINVAL;
 557                 goto out;
 558         }
 559
 560         /* If drvdata::buf is NULL the trace data has been read already */
 561         if (drvdata->buf == NULL) {
 562                 ret = -EINVAL;
 563                 goto out;
 564         }
 565
 566         /* Disable the TMC if need be */
 567         if (drvdata->mode == CS_MODE_SYSFS)
 568                 tmc_etb_disable_hw(drvdata);
 569
 570         drvdata->reading = true;
 571 out:
 572         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 573
 574         return ret;
 575 }
 576
 577 int tmc_read_unprepare_etb(struct tmc_drvdata *drvdata)
 578 {
 579         char *buf = NULL;
 580         enum tmc_mode mode;
 581         unsigned long flags;
 582
 583         /* config types are set a boot time and never change */
 584         if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETB &&
 585                          drvdata->config_type != TMC_CONFIG_TYPE_ETF))
 586                 return -EINVAL;
 587
 588         spin_lock_irqsave(&drvdata->spinlock, flags);
 589
 590         /* Re-enable the TMC if need be */
 591         if (drvdata->mode == CS_MODE_SYSFS) {
 592                 /* There is no point in reading a TMC in HW FIFO mode */
 593                 mode = readl_relaxed(drvdata->base + TMC_MODE);
 594                 if (mode != TMC_MODE_CIRCULAR_BUFFER) {
 595                         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 596                         return -EINVAL;
 597                 }
 598                 /*
 599                  * The trace run will continue with the same allocated trace
 600                  * buffer. As such zero-out the buffer so that we don't end
 601                  * up with stale data.
 602                  *
 603                  * Since the tracer is still enabled drvdata::buf
 604                  * can't be NULL.
 605                  */
 606                 memset(drvdata->buf, 0, drvdata->size);
 607                 tmc_etb_enable_hw(drvdata);
 608         } else {
 609                 /*
 610                  * The ETB/ETF is not tracing and the buffer was just read.
 611                  * As such prepare to free the trace buffer.
 612                  */
 613                 buf = drvdata->buf;
 614                 drvdata->buf = NULL;
 615         }
 616
 617         drvdata->reading = false;
 618         spin_unlock_irqrestore(&drvdata->spinlock, flags);
 619
 620         /*
 621          * Free allocated memory outside of the spinlock.  There is no need
 622          * to assert the validity of 'buf' since calling kfree(NULL) is safe.
 623          */
 624         kfree(buf);
 625
 626         return 0;
 627 }