virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/kvm.h>
  22 #include <linux/kvm_host.h>
  23 #include <linux/interrupt.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26 #include <asm/arch_timer.h>
  27
  28 #include <kvm/arm_vgic.h>
  29 #include <kvm/arm_arch_timer.h>
  30
  31 #include "trace.h"
  32
  33 static struct timecounter *timecounter;
  34 static struct workqueue_struct *wqueue;
  35 static unsigned int host_vtimer_irq;
  36
  37 static cycle_t kvm_phys_timer_read(void)
  38 {
  39         return timecounter->cc->read(timecounter->cc);
  40 }
  41
  42 static bool timer_is_armed(struct arch_timer_cpu *timer)
  43 {
  44         return timer->armed;
  45 }
  46
  47 /* timer_arm: as in "arm the timer", not as in ARM the company */
  48 static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  49 {
  50         timer->armed = true;
  51         hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  52                       HRTIMER_MODE_ABS);
  53 }
  54
  55 static void timer_disarm(struct arch_timer_cpu *timer)
  56 {
  57         if (timer_is_armed(timer)) {
  58                 hrtimer_cancel(&timer->timer);
  59                 cancel_work_sync(&timer->expired);
  60                 timer->armed = false;
  61         }
  62 }
  63
  64 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  65 {
  66         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  67
  68         /*
  69          * We disable the timer in the world switch and let it be
  70          * handled by kvm_timer_sync_hwstate(). Getting a timer
  71          * interrupt at this point is a sure sign of some major
  72          * breakage.
  73          */
  74         pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
  75         return IRQ_HANDLED;
  76 }
  77
  78 /*
  79  * Work function for handling the backup timer that we schedule when a vcpu is
  80  * no longer running, but had a timer programmed to fire in the future.
  81  */
  82 static void kvm_timer_inject_irq_work(struct work_struct *work)
  83 {
  84         struct kvm_vcpu *vcpu;
  85
  86         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
  87
  88         /*
  89          * If the vcpu is blocked we want to wake it up so that it will see
  90          * the timer has expired when entering the guest.
  91          */
  92         kvm_vcpu_kick(vcpu);
  93 }
  94
  95 static u64 kvm_timer_compute_delta(struct kvm_vcpu *vcpu)
  96 {
  97         cycle_t cval, now;
  98
  99         cval = vcpu->arch.timer_cpu.cntv_cval;
 100         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 101
 102         if (now < cval) {
 103                 u64 ns;
 104
 105                 ns = cyclecounter_cyc2ns(timecounter->cc,
 106                                          cval - now,
 107                                          timecounter->mask,
 108                                          &timecounter->frac);
 109                 return ns;
 110         }
 111
 112         return 0;
 113 }
 114
 115 static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
 116 {
 117         struct arch_timer_cpu *timer;
 118         struct kvm_vcpu *vcpu;
 119         u64 ns;
 120
 121         timer = container_of(hrt, struct arch_timer_cpu, timer);
 122         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 123
 124         /*
 125          * Check that the timer has really expired from the guest's
 126          * PoV (NTP on the host may have forced it to expire
 127          * early). If we should have slept longer, restart it.
 128          */
 129         ns = kvm_timer_compute_delta(vcpu);
 130         if (unlikely(ns)) {
 131                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 132                 return HRTIMER_RESTART;
 133         }
 134
 135         queue_work(wqueue, &timer->expired);
 136         return HRTIMER_NORESTART;
 137 }
 138
 139 static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
 140 {
 141         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 142
 143         return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 144                 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
 145 }
 146
 147 bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
 148 {
 149         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 150         cycle_t cval, now;
 151
 152         if (!kvm_timer_irq_can_fire(vcpu))
 153                 return false;
 154
 155         cval = timer->cntv_cval;
 156         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 157
 158         return cval <= now;
 159 }
 160
 161 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
 162 {
 163         int ret;
 164         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 165
 166         BUG_ON(!vgic_initialized(vcpu->kvm));
 167
 168         timer->irq.level = new_level;
 169         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->map->virt_irq,
 170                                    timer->irq.level);
 171         ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
 172                                          timer->map,
 173                                          timer->irq.level);
 174         WARN_ON(ret);
 175 }
 176
 177 /*
 178  * Check if there was a change in the timer state (should we raise or lower
 179  * the line level to the GIC).
 180  */
 181 static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
 182 {
 183         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 184
 185         /*
 186          * If userspace modified the timer registers via SET_ONE_REG before
 187          * the vgic was initialized, we mustn't set the timer->irq.level value
 188          * because the guest would never see the interrupt.  Instead wait
 189          * until we call this function from kvm_timer_flush_hwstate.
 190          */
 191         if (!vgic_initialized(vcpu->kvm))
 192                 return -ENODEV;
 193
 194         if (kvm_timer_should_fire(vcpu) != timer->irq.level)
 195                 kvm_timer_update_irq(vcpu, !timer->irq.level);
 196
 197         return 0;
 198 }
 199
 200 /*
 201  * Schedule the background timer before calling kvm_vcpu_block, so that this
 202  * thread is removed from its waitqueue and made runnable when there's a timer
 203  * interrupt to handle.
 204  */
 205 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 206 {
 207         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 208
 209         BUG_ON(timer_is_armed(timer));
 210
 211         /*
 212          * No need to schedule a background timer if the guest timer has
 213          * already expired, because kvm_vcpu_block will return before putting
 214          * the thread to sleep.
 215          */
 216         if (kvm_timer_should_fire(vcpu))
 217                 return;
 218
 219         /*
 220          * If the timer is not capable of raising interrupts (disabled or
 221          * masked), then there's no more work for us to do.
 222          */
 223         if (!kvm_timer_irq_can_fire(vcpu))
 224                 return;
 225
 226         /*  The timer has not yet expired, schedule a background timer */
 227         timer_arm(timer, kvm_timer_compute_delta(vcpu));
 228 }
 229
 230 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 231 {
 232         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 233         timer_disarm(timer);
 234 }
 235
 236 /**
 237  * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
 238  * @vcpu: The vcpu pointer
 239  *
 240  * Check if the virtual timer has expired while we were running in the host,
 241  * and inject an interrupt if that was the case.
 242  */
 243 void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 244 {
 245         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 246         bool phys_active;
 247         int ret;
 248
 249         if (kvm_timer_update_state(vcpu))
 250                 return;
 251
 252         /*
 253         * If we enter the guest with the virtual input level to the VGIC
 254         * asserted, then we have already told the VGIC what we need to, and
 255         * we don't need to exit from the guest until the guest deactivates
 256         * the already injected interrupt, so therefore we should set the
 257         * hardware active state to prevent unnecessary exits from the guest.
 258         *
 259         * Also, if we enter the guest with the virtual timer interrupt active,
 260         * then it must be active on the physical distributor, because we set
 261         * the HW bit and the guest must be able to deactivate the virtual and
 262         * physical interrupt at the same time.
 263         *
 264         * Conversely, if the virtual input level is deasserted and the virtual
 265         * interrupt is not active, then always clear the hardware active state
 266         * to ensure that hardware interrupts from the timer triggers a guest
 267         * exit.
 268         */
 269         if (timer->irq.level || kvm_vgic_map_is_active(vcpu, timer->map))
 270                 phys_active = true;
 271         else
 272                 phys_active = false;
 273
 274         ret = irq_set_irqchip_state(timer->map->irq,
 275                                     IRQCHIP_STATE_ACTIVE,
 276                                     phys_active);
 277         WARN_ON(ret);
 278 }
 279
 280 /**
 281  * kvm_timer_sync_hwstate - sync timer state from cpu
 282  * @vcpu: The vcpu pointer
 283  *
 284  * Check if the virtual timer has expired while we were running in the guest,
 285  * and inject an interrupt if that was the case.
 286  */
 287 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 288 {
 289         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 290
 291         BUG_ON(timer_is_armed(timer));
 292
 293         /*
 294          * The guest could have modified the timer registers or the timer
 295          * could have expired, update the timer state.
 296          */
 297         kvm_timer_update_state(vcpu);
 298 }
 299
 300 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
 301                          const struct kvm_irq_level *irq)
 302 {
 303         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 304         struct irq_phys_map *map;
 305
 306         /*
 307          * The vcpu timer irq number cannot be determined in
 308          * kvm_timer_vcpu_init() because it is called much before
 309          * kvm_vcpu_set_target(). To handle this, we determine
 310          * vcpu timer irq number when the vcpu is reset.
 311          */
 312         timer->irq.irq = irq->irq;
 313
 314         /*
 315          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 316          * and to 0 for ARMv7.  We provide an implementation that always
 317          * resets the timer to be disabled and unmasked and is compliant with
 318          * the ARMv7 architecture.
 319          */
 320         timer->cntv_ctl = 0;
 321         kvm_timer_update_state(vcpu);
 322
 323         /*
 324          * Tell the VGIC that the virtual interrupt is tied to a
 325          * physical interrupt. We do that once per VCPU.
 326          */
 327         map = kvm_vgic_map_phys_irq(vcpu, irq->irq, host_vtimer_irq);
 328         if (WARN_ON(IS_ERR(map)))
 329                 return PTR_ERR(map);
 330
 331         timer->map = map;
 332         return 0;
 333 }
 334
 335 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 336 {
 337         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 338
 339         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 340         hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 341         timer->timer.function = kvm_timer_expire;
 342 }
 343
 344 static void kvm_timer_init_interrupt(void *info)
 345 {
 346         enable_percpu_irq(host_vtimer_irq, 0);
 347 }
 348
 349 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 350 {
 351         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 352
 353         switch (regid) {
 354         case KVM_REG_ARM_TIMER_CTL:
 355                 timer->cntv_ctl = value;
 356                 break;
 357         case KVM_REG_ARM_TIMER_CNT:
 358                 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
 359                 break;
 360         case KVM_REG_ARM_TIMER_CVAL:
 361                 timer->cntv_cval = value;
 362                 break;
 363         default:
 364                 return -1;
 365         }
 366
 367         kvm_timer_update_state(vcpu);
 368         return 0;
 369 }
 370
 371 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 372 {
 373         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 374
 375         switch (regid) {
 376         case KVM_REG_ARM_TIMER_CTL:
 377                 return timer->cntv_ctl;
 378         case KVM_REG_ARM_TIMER_CNT:
 379                 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 380         case KVM_REG_ARM_TIMER_CVAL:
 381                 return timer->cntv_cval;
 382         }
 383         return (u64)-1;
 384 }
 385
 386 static int kvm_timer_cpu_notify(struct notifier_block *self,
 387                                 unsigned long action, void *cpu)
 388 {
 389         switch (action) {
 390         case CPU_STARTING:
 391         case CPU_STARTING_FROZEN:
 392                 kvm_timer_init_interrupt(NULL);
 393                 break;
 394         case CPU_DYING:
 395         case CPU_DYING_FROZEN:
 396                 disable_percpu_irq(host_vtimer_irq);
 397                 break;
 398         }
 399
 400         return NOTIFY_OK;
 401 }
 402
 403 static struct notifier_block kvm_timer_cpu_nb = {
 404         .notifier_call = kvm_timer_cpu_notify,
 405 };
 406
 407 static const struct of_device_id arch_timer_of_match[] = {
 408         { .compatible   = "arm,armv7-timer",    },
 409         { .compatible   = "arm,armv8-timer",    },
 410         {},
 411 };
 412
 413 int kvm_timer_hyp_init(void)
 414 {
 415         struct device_node *np;
 416         unsigned int ppi;
 417         int err;
 418
 419         timecounter = arch_timer_get_timecounter();
 420         if (!timecounter)
 421                 return -ENODEV;
 422
 423         np = of_find_matching_node(NULL, arch_timer_of_match);
 424         if (!np) {
 425                 kvm_err("kvm_arch_timer: can't find DT node\n");
 426                 return -ENODEV;
 427         }
 428
 429         ppi = irq_of_parse_and_map(np, 2);
 430         if (!ppi) {
 431                 kvm_err("kvm_arch_timer: no virtual timer interrupt\n");
 432                 err = -EINVAL;
 433                 goto out;
 434         }
 435
 436         err = request_percpu_irq(ppi, kvm_arch_timer_handler,
 437                                  "kvm guest timer", kvm_get_running_vcpus());
 438         if (err) {
 439                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 440                         ppi, err);
 441                 goto out;
 442         }
 443
 444         host_vtimer_irq = ppi;
 445
 446         err = __register_cpu_notifier(&kvm_timer_cpu_nb);
 447         if (err) {
 448                 kvm_err("Cannot register timer CPU notifier\n");
 449                 goto out_free;
 450         }
 451
 452         wqueue = create_singlethread_workqueue("kvm_arch_timer");
 453         if (!wqueue) {
 454                 err = -ENOMEM;
 455                 goto out_free;
 456         }
 457
 458         kvm_info("%s IRQ%d\n", np->name, ppi);
 459         on_each_cpu(kvm_timer_init_interrupt, NULL, 1);
 460
 461         goto out;
 462 out_free:
 463         free_percpu_irq(ppi, kvm_get_running_vcpus());
 464 out:
 465         of_node_put(np);
 466         return err;
 467 }
 468
 469 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 470 {
 471         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 472
 473         timer_disarm(timer);
 474         if (timer->map)
 475                 kvm_vgic_unmap_phys_irq(vcpu, timer->map);
 476 }
 477
 478 void kvm_timer_enable(struct kvm *kvm)
 479 {
 480         if (kvm->arch.timer.enabled)
 481                 return;
 482
 483         /*
 484          * There is a potential race here between VCPUs starting for the first
 485          * time, which may be enabling the timer multiple times.  That doesn't
 486          * hurt though, because we're just setting a variable to the same
 487          * variable that it already was.  The important thing is that all
 488          * VCPUs have the enabled variable set, before entering the guest, if
 489          * the arch timers are enabled.
 490          */
 491         if (timecounter && wqueue)
 492                 kvm->arch.timer.enabled = 1;
 493 }
 494
 495 void kvm_timer_init(struct kvm *kvm)
 496 {
 497         kvm->arch.timer.cntvoff = kvm_phys_timer_read();
 498 }