2 * Copyright (C) 2012-2015 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 #include <linux/compiler.h>
19 #include <linux/irqchip/arm-gic-v3.h>
20 #include <linux/kvm_host.h>
22 #include <asm/kvm_emulate.h>
23 #include <asm/kvm_hyp.h>
25 #define vtr_to_max_lr_idx(v) ((v) & 0xf)
26 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
27 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
29 static u64 __hyp_text __gic_v3_get_lr(unsigned int lr)
33 return read_gicreg(ICH_LR0_EL2);
35 return read_gicreg(ICH_LR1_EL2);
37 return read_gicreg(ICH_LR2_EL2);
39 return read_gicreg(ICH_LR3_EL2);
41 return read_gicreg(ICH_LR4_EL2);
43 return read_gicreg(ICH_LR5_EL2);
45 return read_gicreg(ICH_LR6_EL2);
47 return read_gicreg(ICH_LR7_EL2);
49 return read_gicreg(ICH_LR8_EL2);
51 return read_gicreg(ICH_LR9_EL2);
53 return read_gicreg(ICH_LR10_EL2);
55 return read_gicreg(ICH_LR11_EL2);
57 return read_gicreg(ICH_LR12_EL2);
59 return read_gicreg(ICH_LR13_EL2);
61 return read_gicreg(ICH_LR14_EL2);
63 return read_gicreg(ICH_LR15_EL2);
69 static void __hyp_text __gic_v3_set_lr(u64 val, int lr)
73 write_gicreg(val, ICH_LR0_EL2);
76 write_gicreg(val, ICH_LR1_EL2);
79 write_gicreg(val, ICH_LR2_EL2);
82 write_gicreg(val, ICH_LR3_EL2);
85 write_gicreg(val, ICH_LR4_EL2);
88 write_gicreg(val, ICH_LR5_EL2);
91 write_gicreg(val, ICH_LR6_EL2);
94 write_gicreg(val, ICH_LR7_EL2);
97 write_gicreg(val, ICH_LR8_EL2);
100 write_gicreg(val, ICH_LR9_EL2);
103 write_gicreg(val, ICH_LR10_EL2);
106 write_gicreg(val, ICH_LR11_EL2);
109 write_gicreg(val, ICH_LR12_EL2);
112 write_gicreg(val, ICH_LR13_EL2);
115 write_gicreg(val, ICH_LR14_EL2);
118 write_gicreg(val, ICH_LR15_EL2);
123 static void __hyp_text __vgic_v3_write_ap0rn(u32 val, int n)
127 write_gicreg(val, ICH_AP0R0_EL2);
130 write_gicreg(val, ICH_AP0R1_EL2);
133 write_gicreg(val, ICH_AP0R2_EL2);
136 write_gicreg(val, ICH_AP0R3_EL2);
141 static void __hyp_text __vgic_v3_write_ap1rn(u32 val, int n)
145 write_gicreg(val, ICH_AP1R0_EL2);
148 write_gicreg(val, ICH_AP1R1_EL2);
151 write_gicreg(val, ICH_AP1R2_EL2);
154 write_gicreg(val, ICH_AP1R3_EL2);
159 static u32 __hyp_text __vgic_v3_read_ap0rn(int n)
165 val = read_gicreg(ICH_AP0R0_EL2);
168 val = read_gicreg(ICH_AP0R1_EL2);
171 val = read_gicreg(ICH_AP0R2_EL2);
174 val = read_gicreg(ICH_AP0R3_EL2);
183 static u32 __hyp_text __vgic_v3_read_ap1rn(int n)
189 val = read_gicreg(ICH_AP1R0_EL2);
192 val = read_gicreg(ICH_AP1R1_EL2);
195 val = read_gicreg(ICH_AP1R2_EL2);
198 val = read_gicreg(ICH_AP1R3_EL2);
207 void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu)
209 struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
210 u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
214 * Make sure stores to the GIC via the memory mapped interface
215 * are now visible to the system register interface.
217 if (!cpu_if->vgic_sre) {
220 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
227 cpu_if->vgic_elrsr = read_gicreg(ICH_ELSR_EL2);
229 write_gicreg(0, ICH_HCR_EL2);
230 val = read_gicreg(ICH_VTR_EL2);
231 nr_pre_bits = vtr_to_nr_pre_bits(val);
233 for (i = 0; i < used_lrs; i++) {
234 if (cpu_if->vgic_elrsr & (1 << i))
235 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
237 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
239 __gic_v3_set_lr(0, i);
242 switch (nr_pre_bits) {
244 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
245 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
247 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
249 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
252 switch (nr_pre_bits) {
254 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
255 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
257 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
259 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
262 if (static_branch_unlikely(&vgic_v3_cpuif_trap))
263 write_gicreg(0, ICH_HCR_EL2);
265 cpu_if->vgic_elrsr = 0xffff;
266 cpu_if->vgic_ap0r[0] = 0;
267 cpu_if->vgic_ap0r[1] = 0;
268 cpu_if->vgic_ap0r[2] = 0;
269 cpu_if->vgic_ap0r[3] = 0;
270 cpu_if->vgic_ap1r[0] = 0;
271 cpu_if->vgic_ap1r[1] = 0;
272 cpu_if->vgic_ap1r[2] = 0;
273 cpu_if->vgic_ap1r[3] = 0;
276 val = read_gicreg(ICC_SRE_EL2);
277 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
279 if (!cpu_if->vgic_sre) {
280 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
282 write_gicreg(1, ICC_SRE_EL1);
286 void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu)
288 struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3;
289 u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs;
295 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
296 * Group0 interrupt (as generated in GICv2 mode) to be
297 * delivered as a FIQ to the guest, with potentially fatal
298 * consequences. So we must make sure that ICC_SRE_EL1 has
299 * been actually programmed with the value we want before
300 * starting to mess with the rest of the GIC, and VMCR_EL2 in
303 if (!cpu_if->vgic_sre) {
304 write_gicreg(0, ICC_SRE_EL1);
306 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
309 val = read_gicreg(ICH_VTR_EL2);
310 nr_pre_bits = vtr_to_nr_pre_bits(val);
313 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
315 switch (nr_pre_bits) {
317 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
318 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
320 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
322 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
325 switch (nr_pre_bits) {
327 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
328 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
330 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
332 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
335 for (i = 0; i < used_lrs; i++)
336 __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
339 * If we need to trap system registers, we must write
340 * ICH_HCR_EL2 anyway, even if no interrupts are being
343 if (static_branch_unlikely(&vgic_v3_cpuif_trap))
344 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
348 * Ensures that the above will have reached the
349 * (re)distributors. This ensure the guest will read the
350 * correct values from the memory-mapped interface.
352 if (!cpu_if->vgic_sre) {
358 * Prevent the guest from touching the GIC system registers if
359 * SRE isn't enabled for GICv3 emulation.
361 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
365 void __hyp_text __vgic_v3_init_lrs(void)
367 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
370 for (i = 0; i <= max_lr_idx; i++)
371 __gic_v3_set_lr(0, i);
374 u64 __hyp_text __vgic_v3_get_ich_vtr_el2(void)
376 return read_gicreg(ICH_VTR_EL2);
379 u64 __hyp_text __vgic_v3_read_vmcr(void)
381 return read_gicreg(ICH_VMCR_EL2);
384 void __hyp_text __vgic_v3_write_vmcr(u32 vmcr)
386 write_gicreg(vmcr, ICH_VMCR_EL2);
391 static int __hyp_text __vgic_v3_bpr_min(void)
393 /* See Pseudocode for VPriorityGroup */
394 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
397 static int __hyp_text __vgic_v3_get_group(struct kvm_vcpu *vcpu)
399 u32 esr = kvm_vcpu_get_hsr(vcpu);
400 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
405 #define GICv3_IDLE_PRIORITY 0xff
407 static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu,
411 unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
412 u8 priority = GICv3_IDLE_PRIORITY;
415 for (i = 0; i < used_lrs; i++) {
416 u64 val = __gic_v3_get_lr(i);
417 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
419 /* Not pending in the state? */
420 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
423 /* Group-0 interrupt, but Group-0 disabled? */
424 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
427 /* Group-1 interrupt, but Group-1 disabled? */
428 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
431 /* Not the highest priority? */
432 if (lr_prio >= priority)
435 /* This is a candidate */
442 *lr_val = ICC_IAR1_EL1_SPURIOUS;
447 static int __hyp_text __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu,
448 int intid, u64 *lr_val)
450 unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs;
453 for (i = 0; i < used_lrs; i++) {
454 u64 val = __gic_v3_get_lr(i);
456 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
457 (val & ICH_LR_ACTIVE_BIT)) {
463 *lr_val = ICC_IAR1_EL1_SPURIOUS;
467 static int __hyp_text __vgic_v3_get_highest_active_priority(void)
469 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
473 for (i = 0; i < nr_apr_regs; i++) {
477 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
478 * contain the active priority levels for this VCPU
479 * for the maximum number of supported priority
480 * levels, and we return the full priority level only
481 * if the BPR is programmed to its minimum, otherwise
482 * we return a combination of the priority level and
483 * subpriority, as determined by the setting of the
484 * BPR, but without the full subpriority.
486 val = __vgic_v3_read_ap0rn(i);
487 val |= __vgic_v3_read_ap1rn(i);
493 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
496 return GICv3_IDLE_PRIORITY;
499 static unsigned int __hyp_text __vgic_v3_get_bpr0(u32 vmcr)
501 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
504 static unsigned int __hyp_text __vgic_v3_get_bpr1(u32 vmcr)
508 if (vmcr & ICH_VMCR_CBPR_MASK) {
509 bpr = __vgic_v3_get_bpr0(vmcr);
513 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
520 * Convert a priority to a preemption level, taking the relevant BPR
521 * into account by zeroing the sub-priority bits.
523 static u8 __hyp_text __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
528 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
530 bpr = __vgic_v3_get_bpr1(vmcr);
532 return pri & (GENMASK(7, 0) << bpr);
536 * The priority value is independent of any of the BPR values, so we
537 * normalize it using the minumal BPR value. This guarantees that no
538 * matter what the guest does with its BPR, we can always set/get the
539 * same value of a priority.
541 static void __hyp_text __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
547 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
548 ap = pre >> __vgic_v3_bpr_min();
552 val = __vgic_v3_read_ap0rn(apr);
553 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
555 val = __vgic_v3_read_ap1rn(apr);
556 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
560 static int __hyp_text __vgic_v3_clear_highest_active_priority(void)
562 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
566 for (i = 0; i < nr_apr_regs; i++) {
570 ap0 = __vgic_v3_read_ap0rn(i);
571 ap1 = __vgic_v3_read_ap1rn(i);
577 c0 = ap0 ? __ffs(ap0) : 32;
578 c1 = ap1 ? __ffs(ap1) : 32;
580 /* Always clear the LSB, which is the highest priority */
583 __vgic_v3_write_ap0rn(ap0, i);
587 __vgic_v3_write_ap1rn(ap1, i);
591 /* Rescale to 8 bits of priority */
592 return hap << __vgic_v3_bpr_min();
595 return GICv3_IDLE_PRIORITY;
598 static void __hyp_text __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
604 grp = __vgic_v3_get_group(vcpu);
606 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
610 if (grp != !!(lr_val & ICH_LR_GROUP))
613 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
614 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
618 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
621 lr_val &= ~ICH_LR_STATE;
622 /* No active state for LPIs */
623 if ((lr_val & ICH_LR_VIRTUAL_ID_MASK) <= VGIC_MAX_SPI)
624 lr_val |= ICH_LR_ACTIVE_BIT;
625 __gic_v3_set_lr(lr_val, lr);
626 __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
627 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
631 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
634 static void __hyp_text __vgic_v3_clear_active_lr(int lr, u64 lr_val)
636 lr_val &= ~ICH_LR_ACTIVE_BIT;
637 if (lr_val & ICH_LR_HW) {
640 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
644 __gic_v3_set_lr(lr_val, lr);
647 static void __hyp_text __vgic_v3_bump_eoicount(void)
651 hcr = read_gicreg(ICH_HCR_EL2);
652 hcr += 1 << ICH_HCR_EOIcount_SHIFT;
653 write_gicreg(hcr, ICH_HCR_EL2);
656 static void __hyp_text __vgic_v3_write_dir(struct kvm_vcpu *vcpu,
659 u32 vid = vcpu_get_reg(vcpu, rt);
663 /* EOImode == 0, nothing to be done here */
664 if (!(vmcr & ICH_VMCR_EOIM_MASK))
667 /* No deactivate to be performed on an LPI */
668 if (vid >= VGIC_MIN_LPI)
671 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
673 __vgic_v3_bump_eoicount();
677 __vgic_v3_clear_active_lr(lr, lr_val);
680 static void __hyp_text __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
682 u32 vid = vcpu_get_reg(vcpu, rt);
684 u8 lr_prio, act_prio;
687 grp = __vgic_v3_get_group(vcpu);
689 /* Drop priority in any case */
690 act_prio = __vgic_v3_clear_highest_active_priority();
692 /* If EOIing an LPI, no deactivate to be performed */
693 if (vid >= VGIC_MIN_LPI)
696 /* EOImode == 1, nothing to be done here */
697 if (vmcr & ICH_VMCR_EOIM_MASK)
700 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
702 __vgic_v3_bump_eoicount();
706 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
708 /* If priorities or group do not match, the guest has fscked-up. */
709 if (grp != !!(lr_val & ICH_LR_GROUP) ||
710 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
713 /* Let's now perform the deactivation */
714 __vgic_v3_clear_active_lr(lr, lr_val);
717 static void __hyp_text __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
719 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
722 static void __hyp_text __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
724 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
727 static void __hyp_text __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
729 u64 val = vcpu_get_reg(vcpu, rt);
732 vmcr |= ICH_VMCR_ENG0_MASK;
734 vmcr &= ~ICH_VMCR_ENG0_MASK;
736 __vgic_v3_write_vmcr(vmcr);
739 static void __hyp_text __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
741 u64 val = vcpu_get_reg(vcpu, rt);
744 vmcr |= ICH_VMCR_ENG1_MASK;
746 vmcr &= ~ICH_VMCR_ENG1_MASK;
748 __vgic_v3_write_vmcr(vmcr);
751 static void __hyp_text __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
753 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
756 static void __hyp_text __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
758 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
761 static void __hyp_text __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
763 u64 val = vcpu_get_reg(vcpu, rt);
764 u8 bpr_min = __vgic_v3_bpr_min() - 1;
766 /* Enforce BPR limiting */
770 val <<= ICH_VMCR_BPR0_SHIFT;
771 val &= ICH_VMCR_BPR0_MASK;
772 vmcr &= ~ICH_VMCR_BPR0_MASK;
775 __vgic_v3_write_vmcr(vmcr);
778 static void __hyp_text __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
780 u64 val = vcpu_get_reg(vcpu, rt);
781 u8 bpr_min = __vgic_v3_bpr_min();
783 if (vmcr & ICH_VMCR_CBPR_MASK)
786 /* Enforce BPR limiting */
790 val <<= ICH_VMCR_BPR1_SHIFT;
791 val &= ICH_VMCR_BPR1_MASK;
792 vmcr &= ~ICH_VMCR_BPR1_MASK;
795 __vgic_v3_write_vmcr(vmcr);
798 static void __hyp_text __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
802 if (!__vgic_v3_get_group(vcpu))
803 val = __vgic_v3_read_ap0rn(n);
805 val = __vgic_v3_read_ap1rn(n);
807 vcpu_set_reg(vcpu, rt, val);
810 static void __hyp_text __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
812 u32 val = vcpu_get_reg(vcpu, rt);
814 if (!__vgic_v3_get_group(vcpu))
815 __vgic_v3_write_ap0rn(val, n);
817 __vgic_v3_write_ap1rn(val, n);
820 static void __hyp_text __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
823 __vgic_v3_read_apxrn(vcpu, rt, 0);
826 static void __hyp_text __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
829 __vgic_v3_read_apxrn(vcpu, rt, 1);
832 static void __hyp_text __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu,
835 __vgic_v3_read_apxrn(vcpu, rt, 2);
838 static void __hyp_text __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu,
841 __vgic_v3_read_apxrn(vcpu, rt, 3);
844 static void __hyp_text __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu,
847 __vgic_v3_write_apxrn(vcpu, rt, 0);
850 static void __hyp_text __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu,
853 __vgic_v3_write_apxrn(vcpu, rt, 1);
856 static void __hyp_text __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu,
859 __vgic_v3_write_apxrn(vcpu, rt, 2);
862 static void __hyp_text __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu,
865 __vgic_v3_write_apxrn(vcpu, rt, 3);
868 static void __hyp_text __vgic_v3_read_hppir(struct kvm_vcpu *vcpu,
874 grp = __vgic_v3_get_group(vcpu);
876 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
880 lr_grp = !!(lr_val & ICH_LR_GROUP);
882 lr_val = ICC_IAR1_EL1_SPURIOUS;
885 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
888 static void __hyp_text __vgic_v3_read_pmr(struct kvm_vcpu *vcpu,
891 vmcr &= ICH_VMCR_PMR_MASK;
892 vmcr >>= ICH_VMCR_PMR_SHIFT;
893 vcpu_set_reg(vcpu, rt, vmcr);
896 static void __hyp_text __vgic_v3_write_pmr(struct kvm_vcpu *vcpu,
899 u32 val = vcpu_get_reg(vcpu, rt);
901 val <<= ICH_VMCR_PMR_SHIFT;
902 val &= ICH_VMCR_PMR_MASK;
903 vmcr &= ~ICH_VMCR_PMR_MASK;
906 write_gicreg(vmcr, ICH_VMCR_EL2);
909 static void __hyp_text __vgic_v3_read_rpr(struct kvm_vcpu *vcpu,
912 u32 val = __vgic_v3_get_highest_active_priority();
913 vcpu_set_reg(vcpu, rt, val);
916 static void __hyp_text __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu,
921 vtr = read_gicreg(ICH_VTR_EL2);
923 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
925 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
927 val |= ((vtr >> 22) & 1) << ICC_CTLR_EL1_SEIS_SHIFT;
929 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
931 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
933 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
935 vcpu_set_reg(vcpu, rt, val);
938 static void __hyp_text __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu,
941 u32 val = vcpu_get_reg(vcpu, rt);
943 if (val & ICC_CTLR_EL1_CBPR_MASK)
944 vmcr |= ICH_VMCR_CBPR_MASK;
946 vmcr &= ~ICH_VMCR_CBPR_MASK;
948 if (val & ICC_CTLR_EL1_EOImode_MASK)
949 vmcr |= ICH_VMCR_EOIM_MASK;
951 vmcr &= ~ICH_VMCR_EOIM_MASK;
953 write_gicreg(vmcr, ICH_VMCR_EL2);
956 int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
961 void (*fn)(struct kvm_vcpu *, u32, int);
965 esr = kvm_vcpu_get_hsr(vcpu);
966 if (vcpu_mode_is_32bit(vcpu)) {
967 if (!kvm_condition_valid(vcpu))
970 sysreg = esr_cp15_to_sysreg(esr);
972 sysreg = esr_sys64_to_sysreg(esr);
975 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
978 case SYS_ICC_IAR0_EL1:
979 case SYS_ICC_IAR1_EL1:
980 if (unlikely(!is_read))
982 fn = __vgic_v3_read_iar;
984 case SYS_ICC_EOIR0_EL1:
985 case SYS_ICC_EOIR1_EL1:
986 if (unlikely(is_read))
988 fn = __vgic_v3_write_eoir;
990 case SYS_ICC_IGRPEN1_EL1:
992 fn = __vgic_v3_read_igrpen1;
994 fn = __vgic_v3_write_igrpen1;
996 case SYS_ICC_BPR1_EL1:
998 fn = __vgic_v3_read_bpr1;
1000 fn = __vgic_v3_write_bpr1;
1002 case SYS_ICC_AP0Rn_EL1(0):
1003 case SYS_ICC_AP1Rn_EL1(0):
1005 fn = __vgic_v3_read_apxr0;
1007 fn = __vgic_v3_write_apxr0;
1009 case SYS_ICC_AP0Rn_EL1(1):
1010 case SYS_ICC_AP1Rn_EL1(1):
1012 fn = __vgic_v3_read_apxr1;
1014 fn = __vgic_v3_write_apxr1;
1016 case SYS_ICC_AP0Rn_EL1(2):
1017 case SYS_ICC_AP1Rn_EL1(2):
1019 fn = __vgic_v3_read_apxr2;
1021 fn = __vgic_v3_write_apxr2;
1023 case SYS_ICC_AP0Rn_EL1(3):
1024 case SYS_ICC_AP1Rn_EL1(3):
1026 fn = __vgic_v3_read_apxr3;
1028 fn = __vgic_v3_write_apxr3;
1030 case SYS_ICC_HPPIR0_EL1:
1031 case SYS_ICC_HPPIR1_EL1:
1032 if (unlikely(!is_read))
1034 fn = __vgic_v3_read_hppir;
1036 case SYS_ICC_IGRPEN0_EL1:
1038 fn = __vgic_v3_read_igrpen0;
1040 fn = __vgic_v3_write_igrpen0;
1042 case SYS_ICC_BPR0_EL1:
1044 fn = __vgic_v3_read_bpr0;
1046 fn = __vgic_v3_write_bpr0;
1048 case SYS_ICC_DIR_EL1:
1049 if (unlikely(is_read))
1051 fn = __vgic_v3_write_dir;
1053 case SYS_ICC_RPR_EL1:
1054 if (unlikely(!is_read))
1056 fn = __vgic_v3_read_rpr;
1058 case SYS_ICC_CTLR_EL1:
1060 fn = __vgic_v3_read_ctlr;
1062 fn = __vgic_v3_write_ctlr;
1064 case SYS_ICC_PMR_EL1:
1066 fn = __vgic_v3_read_pmr;
1068 fn = __vgic_v3_write_pmr;
1074 vmcr = __vgic_v3_read_vmcr();
1075 rt = kvm_vcpu_sys_get_rt(vcpu);