virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/irq.h>
  24
  25 #include <clocksource/arm_arch_timer.h>
  26 #include <asm/arch_timer.h>
  27
  28 #include <kvm/arm_vgic.h>
  29 #include <kvm/arm_arch_timer.h>
  30
  31 #include "trace.h"
  32
  33 static struct timecounter *timecounter;
  34 static unsigned int host_vtimer_irq;
  35 static u32 host_vtimer_irq_flags;
  36
  37 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
  38 {
  39         vcpu->arch.timer_cpu.active_cleared_last = false;
  40 }
  41
  42 static cycle_t kvm_phys_timer_read(void)
  43 {
  44         return timecounter->cc->read(timecounter->cc);
  45 }
  46
  47 static bool timer_is_armed(struct arch_timer_cpu *timer)
  48 {
  49         return timer->armed;
  50 }
  51
  52 /* timer_arm: as in "arm the timer", not as in ARM the company */
  53 static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  54 {
  55         timer->armed = true;
  56         hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  57                       HRTIMER_MODE_ABS);
  58 }
  59
  60 static void timer_disarm(struct arch_timer_cpu *timer)
  61 {
  62         if (timer_is_armed(timer)) {
  63                 hrtimer_cancel(&timer->timer);
  64                 cancel_work_sync(&timer->expired);
  65                 timer->armed = false;
  66         }
  67 }
  68
  69 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  70 {
  71         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  72
  73         /*
  74          * We disable the timer in the world switch and let it be
  75          * handled by kvm_timer_sync_hwstate(). Getting a timer
  76          * interrupt at this point is a sure sign of some major
  77          * breakage.
  78          */
  79         pr_warn("Unexpected interrupt %d on vcpu %p\n", irq, vcpu);
  80         return IRQ_HANDLED;
  81 }
  82
  83 /*
  84  * Work function for handling the backup timer that we schedule when a vcpu is
  85  * no longer running, but had a timer programmed to fire in the future.
  86  */
  87 static void kvm_timer_inject_irq_work(struct work_struct *work)
  88 {
  89         struct kvm_vcpu *vcpu;
  90
  91         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
  92
  93         /*
  94          * If the vcpu is blocked we want to wake it up so that it will see
  95          * the timer has expired when entering the guest.
  96          */
  97         kvm_vcpu_kick(vcpu);
  98 }
  99
 100 static u64 kvm_timer_compute_delta(struct kvm_vcpu *vcpu)
 101 {
 102         cycle_t cval, now;
 103
 104         cval = vcpu->arch.timer_cpu.cntv_cval;
 105         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 106
 107         if (now < cval) {
 108                 u64 ns;
 109
 110                 ns = cyclecounter_cyc2ns(timecounter->cc,
 111                                          cval - now,
 112                                          timecounter->mask,
 113                                          &timecounter->frac);
 114                 return ns;
 115         }
 116
 117         return 0;
 118 }
 119
 120 static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
 121 {
 122         struct arch_timer_cpu *timer;
 123         struct kvm_vcpu *vcpu;
 124         u64 ns;
 125
 126         timer = container_of(hrt, struct arch_timer_cpu, timer);
 127         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 128
 129         /*
 130          * Check that the timer has really expired from the guest's
 131          * PoV (NTP on the host may have forced it to expire
 132          * early). If we should have slept longer, restart it.
 133          */
 134         ns = kvm_timer_compute_delta(vcpu);
 135         if (unlikely(ns)) {
 136                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 137                 return HRTIMER_RESTART;
 138         }
 139
 140         schedule_work(&timer->expired);
 141         return HRTIMER_NORESTART;
 142 }
 143
 144 static bool kvm_timer_irq_can_fire(struct kvm_vcpu *vcpu)
 145 {
 146         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 147
 148         return !(timer->cntv_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 149                 (timer->cntv_ctl & ARCH_TIMER_CTRL_ENABLE);
 150 }
 151
 152 bool kvm_timer_should_fire(struct kvm_vcpu *vcpu)
 153 {
 154         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 155         cycle_t cval, now;
 156
 157         if (!kvm_timer_irq_can_fire(vcpu))
 158                 return false;
 159
 160         cval = timer->cntv_cval;
 161         now = kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 162
 163         return cval <= now;
 164 }
 165
 166 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level)
 167 {
 168         int ret;
 169         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 170
 171         BUG_ON(!vgic_initialized(vcpu->kvm));
 172
 173         timer->active_cleared_last = false;
 174         timer->irq.level = new_level;
 175         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer->irq.irq,
 176                                    timer->irq.level);
 177         ret = kvm_vgic_inject_mapped_irq(vcpu->kvm, vcpu->vcpu_id,
 178                                          timer->irq.irq,
 179                                          timer->irq.level);
 180         WARN_ON(ret);
 181 }
 182
 183 /*
 184  * Check if there was a change in the timer state (should we raise or lower
 185  * the line level to the GIC).
 186  */
 187 static int kvm_timer_update_state(struct kvm_vcpu *vcpu)
 188 {
 189         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 190
 191         /*
 192          * If userspace modified the timer registers via SET_ONE_REG before
 193          * the vgic was initialized, we mustn't set the timer->irq.level value
 194          * because the guest would never see the interrupt.  Instead wait
 195          * until we call this function from kvm_timer_flush_hwstate.
 196          */
 197         if (!vgic_initialized(vcpu->kvm) || !timer->enabled)
 198                 return -ENODEV;
 199
 200         if (kvm_timer_should_fire(vcpu) != timer->irq.level)
 201                 kvm_timer_update_irq(vcpu, !timer->irq.level);
 202
 203         return 0;
 204 }
 205
 206 /*
 207  * Schedule the background timer before calling kvm_vcpu_block, so that this
 208  * thread is removed from its waitqueue and made runnable when there's a timer
 209  * interrupt to handle.
 210  */
 211 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 212 {
 213         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 214
 215         BUG_ON(timer_is_armed(timer));
 216
 217         /*
 218          * No need to schedule a background timer if the guest timer has
 219          * already expired, because kvm_vcpu_block will return before putting
 220          * the thread to sleep.
 221          */
 222         if (kvm_timer_should_fire(vcpu))
 223                 return;
 224
 225         /*
 226          * If the timer is not capable of raising interrupts (disabled or
 227          * masked), then there's no more work for us to do.
 228          */
 229         if (!kvm_timer_irq_can_fire(vcpu))
 230                 return;
 231
 232         /*  The timer has not yet expired, schedule a background timer */
 233         timer_arm(timer, kvm_timer_compute_delta(vcpu));
 234 }
 235
 236 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 237 {
 238         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 239         timer_disarm(timer);
 240 }
 241
 242 /**
 243  * kvm_timer_flush_hwstate - prepare to move the virt timer to the cpu
 244  * @vcpu: The vcpu pointer
 245  *
 246  * Check if the virtual timer has expired while we were running in the host,
 247  * and inject an interrupt if that was the case.
 248  */
 249 void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
 250 {
 251         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 252         bool phys_active;
 253         int ret;
 254
 255         if (kvm_timer_update_state(vcpu))
 256                 return;
 257
 258         /*
 259         * If we enter the guest with the virtual input level to the VGIC
 260         * asserted, then we have already told the VGIC what we need to, and
 261         * we don't need to exit from the guest until the guest deactivates
 262         * the already injected interrupt, so therefore we should set the
 263         * hardware active state to prevent unnecessary exits from the guest.
 264         *
 265         * Also, if we enter the guest with the virtual timer interrupt active,
 266         * then it must be active on the physical distributor, because we set
 267         * the HW bit and the guest must be able to deactivate the virtual and
 268         * physical interrupt at the same time.
 269         *
 270         * Conversely, if the virtual input level is deasserted and the virtual
 271         * interrupt is not active, then always clear the hardware active state
 272         * to ensure that hardware interrupts from the timer triggers a guest
 273         * exit.
 274         */
 275         phys_active = timer->irq.level ||
 276                         kvm_vgic_map_is_active(vcpu, timer->irq.irq);
 277
 278         /*
 279          * We want to avoid hitting the (re)distributor as much as
 280          * possible, as this is a potentially expensive MMIO access
 281          * (not to mention locks in the irq layer), and a solution for
 282          * this is to cache the "active" state in memory.
 283          *
 284          * Things to consider: we cannot cache an "active set" state,
 285          * because the HW can change this behind our back (it becomes
 286          * "clear" in the HW). We must then restrict the caching to
 287          * the "clear" state.
 288          *
 289          * The cache is invalidated on:
 290          * - vcpu put, indicating that the HW cannot be trusted to be
 291          *   in a sane state on the next vcpu load,
 292          * - any change in the interrupt state
 293          *
 294          * Usage conditions:
 295          * - cached value is "active clear"
 296          * - value to be programmed is "active clear"
 297          */
 298         if (timer->active_cleared_last && !phys_active)
 299                 return;
 300
 301         ret = irq_set_irqchip_state(host_vtimer_irq,
 302                                     IRQCHIP_STATE_ACTIVE,
 303                                     phys_active);
 304         WARN_ON(ret);
 305
 306         timer->active_cleared_last = !phys_active;
 307 }
 308
 309 /**
 310  * kvm_timer_sync_hwstate - sync timer state from cpu
 311  * @vcpu: The vcpu pointer
 312  *
 313  * Check if the virtual timer has expired while we were running in the guest,
 314  * and inject an interrupt if that was the case.
 315  */
 316 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 317 {
 318         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 319
 320         BUG_ON(timer_is_armed(timer));
 321
 322         /*
 323          * The guest could have modified the timer registers or the timer
 324          * could have expired, update the timer state.
 325          */
 326         kvm_timer_update_state(vcpu);
 327 }
 328
 329 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu,
 330                          const struct kvm_irq_level *irq)
 331 {
 332         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 333
 334         /*
 335          * The vcpu timer irq number cannot be determined in
 336          * kvm_timer_vcpu_init() because it is called much before
 337          * kvm_vcpu_set_target(). To handle this, we determine
 338          * vcpu timer irq number when the vcpu is reset.
 339          */
 340         timer->irq.irq = irq->irq;
 341
 342         /*
 343          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 344          * and to 0 for ARMv7.  We provide an implementation that always
 345          * resets the timer to be disabled and unmasked and is compliant with
 346          * the ARMv7 architecture.
 347          */
 348         timer->cntv_ctl = 0;
 349         kvm_timer_update_state(vcpu);
 350
 351         return 0;
 352 }
 353
 354 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 355 {
 356         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 357
 358         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 359         hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 360         timer->timer.function = kvm_timer_expire;
 361 }
 362
 363 static void kvm_timer_init_interrupt(void *info)
 364 {
 365         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 366 }
 367
 368 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 369 {
 370         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 371
 372         switch (regid) {
 373         case KVM_REG_ARM_TIMER_CTL:
 374                 timer->cntv_ctl = value;
 375                 break;
 376         case KVM_REG_ARM_TIMER_CNT:
 377                 vcpu->kvm->arch.timer.cntvoff = kvm_phys_timer_read() - value;
 378                 break;
 379         case KVM_REG_ARM_TIMER_CVAL:
 380                 timer->cntv_cval = value;
 381                 break;
 382         default:
 383                 return -1;
 384         }
 385
 386         kvm_timer_update_state(vcpu);
 387         return 0;
 388 }
 389
 390 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 391 {
 392         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 393
 394         switch (regid) {
 395         case KVM_REG_ARM_TIMER_CTL:
 396                 return timer->cntv_ctl;
 397         case KVM_REG_ARM_TIMER_CNT:
 398                 return kvm_phys_timer_read() - vcpu->kvm->arch.timer.cntvoff;
 399         case KVM_REG_ARM_TIMER_CVAL:
 400                 return timer->cntv_cval;
 401         }
 402         return (u64)-1;
 403 }
 404
 405 static int kvm_timer_starting_cpu(unsigned int cpu)
 406 {
 407         kvm_timer_init_interrupt(NULL);
 408         return 0;
 409 }
 410
 411 static int kvm_timer_dying_cpu(unsigned int cpu)
 412 {
 413         disable_percpu_irq(host_vtimer_irq);
 414         return 0;
 415 }
 416
 417 int kvm_timer_hyp_init(void)
 418 {
 419         struct arch_timer_kvm_info *info;
 420         int err;
 421
 422         info = arch_timer_get_kvm_info();
 423         timecounter = &info->timecounter;
 424
 425         if (info->virtual_irq <= 0) {
 426                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 427                         info->virtual_irq);
 428                 return -ENODEV;
 429         }
 430         host_vtimer_irq = info->virtual_irq;
 431
 432         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 433         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 434             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 435                 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
 436                         host_vtimer_irq);
 437                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 438         }
 439
 440         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 441                                  "kvm guest timer", kvm_get_running_vcpus());
 442         if (err) {
 443                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 444                         host_vtimer_irq, err);
 445                 return err;
 446         }
 447
 448         kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
 449
 450         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 451                           "AP_KVM_ARM_TIMER_STARTING", kvm_timer_starting_cpu,
 452                           kvm_timer_dying_cpu);
 453         return err;
 454 }
 455
 456 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 457 {
 458         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 459
 460         timer_disarm(timer);
 461         kvm_vgic_unmap_phys_irq(vcpu, timer->irq.irq);
 462 }
 463
 464 int kvm_timer_enable(struct kvm_vcpu *vcpu)
 465 {
 466         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 467         struct irq_desc *desc;
 468         struct irq_data *data;
 469         int phys_irq;
 470         int ret;
 471
 472         if (timer->enabled)
 473                 return 0;
 474
 475         /*
 476          * Find the physical IRQ number corresponding to the host_vtimer_irq
 477          */
 478         desc = irq_to_desc(host_vtimer_irq);
 479         if (!desc) {
 480                 kvm_err("%s: no interrupt descriptor\n", __func__);
 481                 return -EINVAL;
 482         }
 483
 484         data = irq_desc_get_irq_data(desc);
 485         while (data->parent_data)
 486                 data = data->parent_data;
 487
 488         phys_irq = data->hwirq;
 489
 490         /*
 491          * Tell the VGIC that the virtual interrupt is tied to a
 492          * physical interrupt. We do that once per VCPU.
 493          */
 494         ret = kvm_vgic_map_phys_irq(vcpu, timer->irq.irq, phys_irq);
 495         if (ret)
 496                 return ret;
 497
 498
 499         /*
 500          * There is a potential race here between VCPUs starting for the first
 501          * time, which may be enabling the timer multiple times.  That doesn't
 502          * hurt though, because we're just setting a variable to the same
 503          * variable that it already was.  The important thing is that all
 504          * VCPUs have the enabled variable set, before entering the guest, if
 505          * the arch timers are enabled.
 506          */
 507         if (timecounter)
 508                 timer->enabled = 1;
 509
 510         return 0;
 511 }
 512
 513 void kvm_timer_init(struct kvm *kvm)
 514 {
 515         kvm->arch.timer.cntvoff = kvm_phys_timer_read();
 516 }