drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_address.h>
  22 #include <linux/io.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched_clock.h>
  25 #include <linux/acpi.h>
  26
  27 #include <asm/arch_timer.h>
  28 #include <asm/virt.h>
  29
  30 #include <clocksource/arm_arch_timer.h>
  31
  32 #define CNTTIDR         0x08
  33 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  34
  35 #define CNTVCT_LO       0x08
  36 #define CNTVCT_HI       0x0c
  37 #define CNTFRQ          0x10
  38 #define CNTP_TVAL       0x28
  39 #define CNTP_CTL        0x2c
  40 #define CNTV_TVAL       0x38
  41 #define CNTV_CTL        0x3c
  42
  43 #define ARCH_CP15_TIMER BIT(0)
  44 #define ARCH_MEM_TIMER  BIT(1)
  45 static unsigned arch_timers_present __initdata;
  46
  47 static void __iomem *arch_counter_base;
  48
  49 struct arch_timer {
  50         void __iomem *base;
  51         struct clock_event_device evt;
  52 };
  53
  54 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  55
  56 static u32 arch_timer_rate;
  57
  58 enum ppi_nr {
  59         PHYS_SECURE_PPI,
  60         PHYS_NONSECURE_PPI,
  61         VIRT_PPI,
  62         HYP_PPI,
  63         MAX_TIMER_PPI
  64 };
  65
  66 static int arch_timer_ppi[MAX_TIMER_PPI];
  67
  68 static struct clock_event_device __percpu *arch_timer_evt;
  69
  70 static bool arch_timer_use_virtual = true;
  71 static bool arch_timer_c3stop;
  72 static bool arch_timer_mem_use_virtual;
  73
  74 /*
  75  * Architected system timer support.
  76  */
  77
  78 static __always_inline
  79 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  80                           struct clock_event_device *clk)
  81 {
  82         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  83                 struct arch_timer *timer = to_arch_timer(clk);
  84                 switch (reg) {
  85                 case ARCH_TIMER_REG_CTRL:
  86                         writel_relaxed(val, timer->base + CNTP_CTL);
  87                         break;
  88                 case ARCH_TIMER_REG_TVAL:
  89                         writel_relaxed(val, timer->base + CNTP_TVAL);
  90                         break;
  91                 }
  92         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
  93                 struct arch_timer *timer = to_arch_timer(clk);
  94                 switch (reg) {
  95                 case ARCH_TIMER_REG_CTRL:
  96                         writel_relaxed(val, timer->base + CNTV_CTL);
  97                         break;
  98                 case ARCH_TIMER_REG_TVAL:
  99                         writel_relaxed(val, timer->base + CNTV_TVAL);
 100                         break;
 101                 }
 102         } else {
 103                 arch_timer_reg_write_cp15(access, reg, val);
 104         }
 105 }
 106
 107 static __always_inline
 108 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 109                         struct clock_event_device *clk)
 110 {
 111         u32 val;
 112
 113         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 114                 struct arch_timer *timer = to_arch_timer(clk);
 115                 switch (reg) {
 116                 case ARCH_TIMER_REG_CTRL:
 117                         val = readl_relaxed(timer->base + CNTP_CTL);
 118                         break;
 119                 case ARCH_TIMER_REG_TVAL:
 120                         val = readl_relaxed(timer->base + CNTP_TVAL);
 121                         break;
 122                 }
 123         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 124                 struct arch_timer *timer = to_arch_timer(clk);
 125                 switch (reg) {
 126                 case ARCH_TIMER_REG_CTRL:
 127                         val = readl_relaxed(timer->base + CNTV_CTL);
 128                         break;
 129                 case ARCH_TIMER_REG_TVAL:
 130                         val = readl_relaxed(timer->base + CNTV_TVAL);
 131                         break;
 132                 }
 133         } else {
 134                 val = arch_timer_reg_read_cp15(access, reg);
 135         }
 136
 137         return val;
 138 }
 139
 140 static __always_inline irqreturn_t timer_handler(const int access,
 141                                         struct clock_event_device *evt)
 142 {
 143         unsigned long ctrl;
 144
 145         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 146         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 147                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 148                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 149                 evt->event_handler(evt);
 150                 return IRQ_HANDLED;
 151         }
 152
 153         return IRQ_NONE;
 154 }
 155
 156 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 157 {
 158         struct clock_event_device *evt = dev_id;
 159
 160         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 161 }
 162
 163 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 164 {
 165         struct clock_event_device *evt = dev_id;
 166
 167         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 168 }
 169
 170 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 171 {
 172         struct clock_event_device *evt = dev_id;
 173
 174         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 175 }
 176
 177 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 178 {
 179         struct clock_event_device *evt = dev_id;
 180
 181         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 182 }
 183
 184 static __always_inline int timer_shutdown(const int access,
 185                                           struct clock_event_device *clk)
 186 {
 187         unsigned long ctrl;
 188
 189         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 190         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 191         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 192
 193         return 0;
 194 }
 195
 196 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 197 {
 198         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 199 }
 200
 201 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 202 {
 203         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 204 }
 205
 206 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 207 {
 208         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 209 }
 210
 211 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 212 {
 213         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 214 }
 215
 216 static __always_inline void set_next_event(const int access, unsigned long evt,
 217                                            struct clock_event_device *clk)
 218 {
 219         unsigned long ctrl;
 220         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 221         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 222         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 223         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 224         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 225 }
 226
 227 static int arch_timer_set_next_event_virt(unsigned long evt,
 228                                           struct clock_event_device *clk)
 229 {
 230         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 231         return 0;
 232 }
 233
 234 static int arch_timer_set_next_event_phys(unsigned long evt,
 235                                           struct clock_event_device *clk)
 236 {
 237         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 238         return 0;
 239 }
 240
 241 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 242                                               struct clock_event_device *clk)
 243 {
 244         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 245         return 0;
 246 }
 247
 248 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 249                                               struct clock_event_device *clk)
 250 {
 251         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 252         return 0;
 253 }
 254
 255 static void __arch_timer_setup(unsigned type,
 256                                struct clock_event_device *clk)
 257 {
 258         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 259
 260         if (type == ARCH_CP15_TIMER) {
 261                 if (arch_timer_c3stop)
 262                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 263                 clk->name = "arch_sys_timer";
 264                 clk->rating = 450;
 265                 clk->cpumask = cpumask_of(smp_processor_id());
 266                 if (arch_timer_use_virtual) {
 267                         clk->irq = arch_timer_ppi[VIRT_PPI];
 268                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 269                         clk->set_next_event = arch_timer_set_next_event_virt;
 270                 } else {
 271                         clk->irq = arch_timer_ppi[PHYS_SECURE_PPI];
 272                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 273                         clk->set_next_event = arch_timer_set_next_event_phys;
 274                 }
 275         } else {
 276                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 277                 clk->name = "arch_mem_timer";
 278                 clk->rating = 400;
 279                 clk->cpumask = cpu_all_mask;
 280                 if (arch_timer_mem_use_virtual) {
 281                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 282                         clk->set_next_event =
 283                                 arch_timer_set_next_event_virt_mem;
 284                 } else {
 285                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 286                         clk->set_next_event =
 287                                 arch_timer_set_next_event_phys_mem;
 288                 }
 289         }
 290
 291         clk->set_state_shutdown(clk);
 292
 293         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 294 }
 295
 296 static void arch_timer_evtstrm_enable(int divider)
 297 {
 298         u32 cntkctl = arch_timer_get_cntkctl();
 299
 300         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 301         /* Set the divider and enable virtual event stream */
 302         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 303                         | ARCH_TIMER_VIRT_EVT_EN;
 304         arch_timer_set_cntkctl(cntkctl);
 305         elf_hwcap |= HWCAP_EVTSTRM;
 306 #ifdef CONFIG_COMPAT
 307         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 308 #endif
 309 }
 310
 311 static void arch_timer_configure_evtstream(void)
 312 {
 313         int evt_stream_div, lsb;
 314
 315         /*
 316          * As the event stream can at most be generated at half the frequency
 317          * of the counter, use half the frequency when computing the divider.
 318          */
 319         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 320
 321         /*
 322          * Find the closest power of two to the divisor. If the adjacent bit
 323          * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 324          */
 325         lsb = fls(evt_stream_div) - 1;
 326         if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 327                 lsb++;
 328
 329         /* enable event stream */
 330         arch_timer_evtstrm_enable(max(0, min(lsb, 15)));
 331 }
 332
 333 static void arch_counter_set_user_access(void)
 334 {
 335         u32 cntkctl = arch_timer_get_cntkctl();
 336
 337         /* Disable user access to the timers and the physical counter */
 338         /* Also disable virtual event stream */
 339         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 340                         | ARCH_TIMER_USR_VT_ACCESS_EN
 341                         | ARCH_TIMER_VIRT_EVT_EN
 342                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 343
 344         /* Enable user access to the virtual counter */
 345         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 346
 347         arch_timer_set_cntkctl(cntkctl);
 348 }
 349
 350 static int arch_timer_setup(struct clock_event_device *clk)
 351 {
 352         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 353
 354         if (arch_timer_use_virtual)
 355                 enable_percpu_irq(arch_timer_ppi[VIRT_PPI], 0);
 356         else {
 357                 enable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI], 0);
 358                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 359                         enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], 0);
 360         }
 361
 362         arch_counter_set_user_access();
 363         if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM))
 364                 arch_timer_configure_evtstream();
 365
 366         return 0;
 367 }
 368
 369 static void
 370 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 371 {
 372         /* Who has more than one independent system counter? */
 373         if (arch_timer_rate)
 374                 return;
 375
 376         /*
 377          * Try to determine the frequency from the device tree or CNTFRQ,
 378          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 379          */
 380         if (!acpi_disabled ||
 381             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 382                 if (cntbase)
 383                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 384                 else
 385                         arch_timer_rate = arch_timer_get_cntfrq();
 386         }
 387
 388         /* Check the timer frequency. */
 389         if (arch_timer_rate == 0)
 390                 pr_warn("Architected timer frequency not available\n");
 391 }
 392
 393 static void arch_timer_banner(unsigned type)
 394 {
 395         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 396                      type & ARCH_CP15_TIMER ? "cp15" : "",
 397                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 398                      type & ARCH_MEM_TIMER ? "mmio" : "",
 399                      (unsigned long)arch_timer_rate / 1000000,
 400                      (unsigned long)(arch_timer_rate / 10000) % 100,
 401                      type & ARCH_CP15_TIMER ?
 402                         arch_timer_use_virtual ? "virt" : "phys" :
 403                         "",
 404                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 405                      type & ARCH_MEM_TIMER ?
 406                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 407                         "");
 408 }
 409
 410 u32 arch_timer_get_rate(void)
 411 {
 412         return arch_timer_rate;
 413 }
 414
 415 static u64 arch_counter_get_cntvct_mem(void)
 416 {
 417         u32 vct_lo, vct_hi, tmp_hi;
 418
 419         do {
 420                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 421                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 422                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 423         } while (vct_hi != tmp_hi);
 424
 425         return ((u64) vct_hi << 32) | vct_lo;
 426 }
 427
 428 /*
 429  * Default to cp15 based access because arm64 uses this function for
 430  * sched_clock() before DT is probed and the cp15 method is guaranteed
 431  * to exist on arm64. arm doesn't use this before DT is probed so even
 432  * if we don't have the cp15 accessors we won't have a problem.
 433  */
 434 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 435
 436 static cycle_t arch_counter_read(struct clocksource *cs)
 437 {
 438         return arch_timer_read_counter();
 439 }
 440
 441 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 442 {
 443         return arch_timer_read_counter();
 444 }
 445
 446 static struct clocksource clocksource_counter = {
 447         .name   = "arch_sys_counter",
 448         .rating = 400,
 449         .read   = arch_counter_read,
 450         .mask   = CLOCKSOURCE_MASK(56),
 451         .flags  = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP,
 452 };
 453
 454 static struct cyclecounter cyclecounter = {
 455         .read   = arch_counter_read_cc,
 456         .mask   = CLOCKSOURCE_MASK(56),
 457 };
 458
 459 static struct timecounter timecounter;
 460
 461 struct timecounter *arch_timer_get_timecounter(void)
 462 {
 463         return &timecounter;
 464 }
 465
 466 static void __init arch_counter_register(unsigned type)
 467 {
 468         u64 start_count;
 469
 470         /* Register the CP15 based counter if we have one */
 471         if (type & ARCH_CP15_TIMER) {
 472                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_use_virtual)
 473                         arch_timer_read_counter = arch_counter_get_cntvct;
 474                 else
 475                         arch_timer_read_counter = arch_counter_get_cntpct;
 476         } else {
 477                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 478
 479                 /* If the clocksource name is "arch_sys_counter" the
 480                  * VDSO will attempt to read the CP15-based counter.
 481                  * Ensure this does not happen when CP15-based
 482                  * counter is not available.
 483                  */
 484                 clocksource_counter.name = "arch_mem_counter";
 485         }
 486
 487         start_count = arch_timer_read_counter();
 488         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 489         cyclecounter.mult = clocksource_counter.mult;
 490         cyclecounter.shift = clocksource_counter.shift;
 491         timecounter_init(&timecounter, &cyclecounter, start_count);
 492
 493         /* 56 bits minimum, so we assume worst case rollover */
 494         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 495 }
 496
 497 static void arch_timer_stop(struct clock_event_device *clk)
 498 {
 499         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 500                  clk->irq, smp_processor_id());
 501
 502         if (arch_timer_use_virtual)
 503                 disable_percpu_irq(arch_timer_ppi[VIRT_PPI]);
 504         else {
 505                 disable_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI]);
 506                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 507                         disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 508         }
 509
 510         clk->set_state_shutdown(clk);
 511 }
 512
 513 static int arch_timer_cpu_notify(struct notifier_block *self,
 514                                            unsigned long action, void *hcpu)
 515 {
 516         /*
 517          * Grab cpu pointer in each case to avoid spurious
 518          * preemptible warnings
 519          */
 520         switch (action & ~CPU_TASKS_FROZEN) {
 521         case CPU_STARTING:
 522                 arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 523                 break;
 524         case CPU_DYING:
 525                 arch_timer_stop(this_cpu_ptr(arch_timer_evt));
 526                 break;
 527         }
 528
 529         return NOTIFY_OK;
 530 }
 531
 532 static struct notifier_block arch_timer_cpu_nb = {
 533         .notifier_call = arch_timer_cpu_notify,
 534 };
 535
 536 #ifdef CONFIG_CPU_PM
 537 static unsigned int saved_cntkctl;
 538 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 539                                     unsigned long action, void *hcpu)
 540 {
 541         if (action == CPU_PM_ENTER)
 542                 saved_cntkctl = arch_timer_get_cntkctl();
 543         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 544                 arch_timer_set_cntkctl(saved_cntkctl);
 545         return NOTIFY_OK;
 546 }
 547
 548 static struct notifier_block arch_timer_cpu_pm_notifier = {
 549         .notifier_call = arch_timer_cpu_pm_notify,
 550 };
 551
 552 static int __init arch_timer_cpu_pm_init(void)
 553 {
 554         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 555 }
 556 #else
 557 static int __init arch_timer_cpu_pm_init(void)
 558 {
 559         return 0;
 560 }
 561 #endif
 562
 563 static int __init arch_timer_register(void)
 564 {
 565         int err;
 566         int ppi;
 567
 568         arch_timer_evt = alloc_percpu(struct clock_event_device);
 569         if (!arch_timer_evt) {
 570                 err = -ENOMEM;
 571                 goto out;
 572         }
 573
 574         if (arch_timer_use_virtual) {
 575                 ppi = arch_timer_ppi[VIRT_PPI];
 576                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 577                                          "arch_timer", arch_timer_evt);
 578         } else {
 579                 ppi = arch_timer_ppi[PHYS_SECURE_PPI];
 580                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 581                                          "arch_timer", arch_timer_evt);
 582                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 583                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 584                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 585                                                  "arch_timer", arch_timer_evt);
 586                         if (err)
 587                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 588                                                 arch_timer_evt);
 589                 }
 590         }
 591
 592         if (err) {
 593                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 594                        ppi, err);
 595                 goto out_free;
 596         }
 597
 598         err = register_cpu_notifier(&arch_timer_cpu_nb);
 599         if (err)
 600                 goto out_free_irq;
 601
 602         err = arch_timer_cpu_pm_init();
 603         if (err)
 604                 goto out_unreg_notify;
 605
 606         /* Immediately configure the timer on the boot CPU */
 607         arch_timer_setup(this_cpu_ptr(arch_timer_evt));
 608
 609         return 0;
 610
 611 out_unreg_notify:
 612         unregister_cpu_notifier(&arch_timer_cpu_nb);
 613 out_free_irq:
 614         if (arch_timer_use_virtual)
 615                 free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt);
 616         else {
 617                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 618                                 arch_timer_evt);
 619                 if (arch_timer_ppi[PHYS_NONSECURE_PPI])
 620                         free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 621                                         arch_timer_evt);
 622         }
 623
 624 out_free:
 625         free_percpu(arch_timer_evt);
 626 out:
 627         return err;
 628 }
 629
 630 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 631 {
 632         int ret;
 633         irq_handler_t func;
 634         struct arch_timer *t;
 635
 636         t = kzalloc(sizeof(*t), GFP_KERNEL);
 637         if (!t)
 638                 return -ENOMEM;
 639
 640         t->base = base;
 641         t->evt.irq = irq;
 642         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 643
 644         if (arch_timer_mem_use_virtual)
 645                 func = arch_timer_handler_virt_mem;
 646         else
 647                 func = arch_timer_handler_phys_mem;
 648
 649         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 650         if (ret) {
 651                 pr_err("arch_timer: Failed to request mem timer irq\n");
 652                 kfree(t);
 653         }
 654
 655         return ret;
 656 }
 657
 658 static const struct of_device_id arch_timer_of_match[] __initconst = {
 659         { .compatible   = "arm,armv7-timer",    },
 660         { .compatible   = "arm,armv8-timer",    },
 661         {},
 662 };
 663
 664 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 665         { .compatible   = "arm,armv7-timer-mem", },
 666         {},
 667 };
 668
 669 static bool __init
 670 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 671 {
 672         struct device_node *dn;
 673         bool needs_probing = false;
 674
 675         dn = of_find_matching_node(NULL, matches);
 676         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 677                 needs_probing = true;
 678         of_node_put(dn);
 679
 680         return needs_probing;
 681 }
 682
 683 static void __init arch_timer_common_init(void)
 684 {
 685         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 686
 687         /* Wait until both nodes are probed if we have two timers */
 688         if ((arch_timers_present & mask) != mask) {
 689                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 690                         return;
 691                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 692                         return;
 693         }
 694
 695         arch_timer_banner(arch_timers_present);
 696         arch_counter_register(arch_timers_present);
 697         arch_timer_arch_init();
 698 }
 699
 700 static void __init arch_timer_init(void)
 701 {
 702         /*
 703          * If HYP mode is available, we know that the physical timer
 704          * has been configured to be accessible from PL1. Use it, so
 705          * that a guest can use the virtual timer instead.
 706          *
 707          * If no interrupt provided for virtual timer, we'll have to
 708          * stick to the physical timer. It'd better be accessible...
 709          */
 710         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 711                 arch_timer_use_virtual = false;
 712
 713                 if (!arch_timer_ppi[PHYS_SECURE_PPI] ||
 714                     !arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 715                         pr_warn("arch_timer: No interrupt available, giving up\n");
 716                         return;
 717                 }
 718         }
 719
 720         arch_timer_register();
 721         arch_timer_common_init();
 722 }
 723
 724 static void __init arch_timer_of_init(struct device_node *np)
 725 {
 726         int i;
 727
 728         if (arch_timers_present & ARCH_CP15_TIMER) {
 729                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 730                 return;
 731         }
 732
 733         arch_timers_present |= ARCH_CP15_TIMER;
 734         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 735                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 736
 737         arch_timer_detect_rate(NULL, np);
 738
 739         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 740
 741         /*
 742          * If we cannot rely on firmware initializing the timer registers then
 743          * we should use the physical timers instead.
 744          */
 745         if (IS_ENABLED(CONFIG_ARM) &&
 746             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 747                         arch_timer_use_virtual = false;
 748
 749         arch_timer_init();
 750 }
 751 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 752 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 753
 754 static void __init arch_timer_mem_init(struct device_node *np)
 755 {
 756         struct device_node *frame, *best_frame = NULL;
 757         void __iomem *cntctlbase, *base;
 758         unsigned int irq;
 759         u32 cnttidr;
 760
 761         arch_timers_present |= ARCH_MEM_TIMER;
 762         cntctlbase = of_iomap(np, 0);
 763         if (!cntctlbase) {
 764                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 765                 return;
 766         }
 767
 768         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 769         iounmap(cntctlbase);
 770
 771         /*
 772          * Try to find a virtual capable frame. Otherwise fall back to a
 773          * physical capable frame.
 774          */
 775         for_each_available_child_of_node(np, frame) {
 776                 int n;
 777
 778                 if (of_property_read_u32(frame, "frame-number", &n)) {
 779                         pr_err("arch_timer: Missing frame-number\n");
 780                         of_node_put(best_frame);
 781                         of_node_put(frame);
 782                         return;
 783                 }
 784
 785                 if (cnttidr & CNTTIDR_VIRT(n)) {
 786                         of_node_put(best_frame);
 787                         best_frame = frame;
 788                         arch_timer_mem_use_virtual = true;
 789                         break;
 790                 }
 791                 of_node_put(best_frame);
 792                 best_frame = of_node_get(frame);
 793         }
 794
 795         base = arch_counter_base = of_iomap(best_frame, 0);
 796         if (!base) {
 797                 pr_err("arch_timer: Can't map frame's registers\n");
 798                 of_node_put(best_frame);
 799                 return;
 800         }
 801
 802         if (arch_timer_mem_use_virtual)
 803                 irq = irq_of_parse_and_map(best_frame, 1);
 804         else
 805                 irq = irq_of_parse_and_map(best_frame, 0);
 806         of_node_put(best_frame);
 807         if (!irq) {
 808                 pr_err("arch_timer: Frame missing %s irq",
 809                        arch_timer_mem_use_virtual ? "virt" : "phys");
 810                 return;
 811         }
 812
 813         arch_timer_detect_rate(base, np);
 814         arch_timer_mem_register(base, irq);
 815         arch_timer_common_init();
 816 }
 817 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
 818                        arch_timer_mem_init);
 819
 820 #ifdef CONFIG_ACPI
 821 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
 822 {
 823         int trigger, polarity;
 824
 825         if (!interrupt)
 826                 return 0;
 827
 828         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
 829                         : ACPI_LEVEL_SENSITIVE;
 830
 831         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
 832                         : ACPI_ACTIVE_HIGH;
 833
 834         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
 835 }
 836
 837 /* Initialize per-processor generic timer */
 838 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
 839 {
 840         struct acpi_table_gtdt *gtdt;
 841
 842         if (arch_timers_present & ARCH_CP15_TIMER) {
 843                 pr_warn("arch_timer: already initialized, skipping\n");
 844                 return -EINVAL;
 845         }
 846
 847         gtdt = container_of(table, struct acpi_table_gtdt, header);
 848
 849         arch_timers_present |= ARCH_CP15_TIMER;
 850
 851         arch_timer_ppi[PHYS_SECURE_PPI] =
 852                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
 853                 gtdt->secure_el1_flags);
 854
 855         arch_timer_ppi[PHYS_NONSECURE_PPI] =
 856                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
 857                 gtdt->non_secure_el1_flags);
 858
 859         arch_timer_ppi[VIRT_PPI] =
 860                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
 861                 gtdt->virtual_timer_flags);
 862
 863         arch_timer_ppi[HYP_PPI] =
 864                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
 865                 gtdt->non_secure_el2_flags);
 866
 867         /* Get the frequency from CNTFRQ */
 868         arch_timer_detect_rate(NULL, NULL);
 869
 870         /* Always-on capability */
 871         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
 872
 873         arch_timer_init();
 874         return 0;
 875 }
 876 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
 877 #endif