GNU Linux-libre 4.14.266-gnu1
[releases.git] / arch / x86 / xen / pmu.c
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/types.h>
3 #include <linux/interrupt.h>
4
5 #include <asm/xen/hypercall.h>
6 #include <xen/page.h>
7 #include <xen/interface/xen.h>
8 #include <xen/interface/vcpu.h>
9 #include <xen/interface/xenpmu.h>
10
11 #include "xen-ops.h"
12 #include "pmu.h"
13
14 /* x86_pmu.handle_irq definition */
15 #include "../events/perf_event.h"
16
17 #define XENPMU_IRQ_PROCESSING    1
18 struct xenpmu {
19         /* Shared page between hypervisor and domain */
20         struct xen_pmu_data *xenpmu_data;
21
22         uint8_t flags;
23 };
24 static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
25 #define get_xenpmu_data()    (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
26 #define get_xenpmu_flags()   (this_cpu_ptr(&xenpmu_shared)->flags)
27
28 /* Macro for computing address of a PMU MSR bank */
29 #define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
30                                             (uintptr_t)ctxt->field))
31
32 /* AMD PMU */
33 #define F15H_NUM_COUNTERS   6
34 #define F10H_NUM_COUNTERS   4
35
36 static __read_mostly uint32_t amd_counters_base;
37 static __read_mostly uint32_t amd_ctrls_base;
38 static __read_mostly int amd_msr_step;
39 static __read_mostly int k7_counters_mirrored;
40 static __read_mostly int amd_num_counters;
41
42 /* Intel PMU */
43 #define MSR_TYPE_COUNTER            0
44 #define MSR_TYPE_CTRL               1
45 #define MSR_TYPE_GLOBAL             2
46 #define MSR_TYPE_ARCH_COUNTER       3
47 #define MSR_TYPE_ARCH_CTRL          4
48
49 /* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
50 #define PMU_GENERAL_NR_SHIFT        8
51 #define PMU_GENERAL_NR_BITS         8
52 #define PMU_GENERAL_NR_MASK         (((1 << PMU_GENERAL_NR_BITS) - 1) \
53                                      << PMU_GENERAL_NR_SHIFT)
54
55 /* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
56 #define PMU_FIXED_NR_SHIFT          0
57 #define PMU_FIXED_NR_BITS           5
58 #define PMU_FIXED_NR_MASK           (((1 << PMU_FIXED_NR_BITS) - 1) \
59                                      << PMU_FIXED_NR_SHIFT)
60
61 /* Alias registers (0x4c1) for full-width writes to PMCs */
62 #define MSR_PMC_ALIAS_MASK          (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
63
64 #define INTEL_PMC_TYPE_SHIFT        30
65
66 static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
67
68
69 static void xen_pmu_arch_init(void)
70 {
71         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
72
73                 switch (boot_cpu_data.x86) {
74                 case 0x15:
75                         amd_num_counters = F15H_NUM_COUNTERS;
76                         amd_counters_base = MSR_F15H_PERF_CTR;
77                         amd_ctrls_base = MSR_F15H_PERF_CTL;
78                         amd_msr_step = 2;
79                         k7_counters_mirrored = 1;
80                         break;
81                 case 0x10:
82                 case 0x12:
83                 case 0x14:
84                 case 0x16:
85                 default:
86                         amd_num_counters = F10H_NUM_COUNTERS;
87                         amd_counters_base = MSR_K7_PERFCTR0;
88                         amd_ctrls_base = MSR_K7_EVNTSEL0;
89                         amd_msr_step = 1;
90                         k7_counters_mirrored = 0;
91                         break;
92                 }
93         } else {
94                 uint32_t eax, ebx, ecx, edx;
95
96                 cpuid(0xa, &eax, &ebx, &ecx, &edx);
97
98                 intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
99                         PMU_GENERAL_NR_SHIFT;
100                 intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
101                         PMU_FIXED_NR_SHIFT;
102         }
103 }
104
105 static inline uint32_t get_fam15h_addr(u32 addr)
106 {
107         switch (addr) {
108         case MSR_K7_PERFCTR0:
109         case MSR_K7_PERFCTR1:
110         case MSR_K7_PERFCTR2:
111         case MSR_K7_PERFCTR3:
112                 return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
113         case MSR_K7_EVNTSEL0:
114         case MSR_K7_EVNTSEL1:
115         case MSR_K7_EVNTSEL2:
116         case MSR_K7_EVNTSEL3:
117                 return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
118         default:
119                 break;
120         }
121
122         return addr;
123 }
124
125 static inline bool is_amd_pmu_msr(unsigned int msr)
126 {
127         if ((msr >= MSR_F15H_PERF_CTL &&
128              msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
129             (msr >= MSR_K7_EVNTSEL0 &&
130              msr < MSR_K7_PERFCTR0 + amd_num_counters))
131                 return true;
132
133         return false;
134 }
135
136 static int is_intel_pmu_msr(u32 msr_index, int *type, int *index)
137 {
138         u32 msr_index_pmc;
139
140         switch (msr_index) {
141         case MSR_CORE_PERF_FIXED_CTR_CTRL:
142         case MSR_IA32_DS_AREA:
143         case MSR_IA32_PEBS_ENABLE:
144                 *type = MSR_TYPE_CTRL;
145                 return true;
146
147         case MSR_CORE_PERF_GLOBAL_CTRL:
148         case MSR_CORE_PERF_GLOBAL_STATUS:
149         case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
150                 *type = MSR_TYPE_GLOBAL;
151                 return true;
152
153         default:
154
155                 if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
156                     (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
157                                  intel_num_fixed_counters)) {
158                         *index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
159                         *type = MSR_TYPE_COUNTER;
160                         return true;
161                 }
162
163                 if ((msr_index >= MSR_P6_EVNTSEL0) &&
164                     (msr_index < MSR_P6_EVNTSEL0 +  intel_num_arch_counters)) {
165                         *index = msr_index - MSR_P6_EVNTSEL0;
166                         *type = MSR_TYPE_ARCH_CTRL;
167                         return true;
168                 }
169
170                 msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
171                 if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
172                     (msr_index_pmc < MSR_IA32_PERFCTR0 +
173                                      intel_num_arch_counters)) {
174                         *type = MSR_TYPE_ARCH_COUNTER;
175                         *index = msr_index_pmc - MSR_IA32_PERFCTR0;
176                         return true;
177                 }
178                 return false;
179         }
180 }
181
182 static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
183                                   int index, bool is_read)
184 {
185         uint64_t *reg = NULL;
186         struct xen_pmu_intel_ctxt *ctxt;
187         uint64_t *fix_counters;
188         struct xen_pmu_cntr_pair *arch_cntr_pair;
189         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
190         uint8_t xenpmu_flags = get_xenpmu_flags();
191
192
193         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
194                 return false;
195
196         ctxt = &xenpmu_data->pmu.c.intel;
197
198         switch (msr) {
199         case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
200                 reg = &ctxt->global_ovf_ctrl;
201                 break;
202         case MSR_CORE_PERF_GLOBAL_STATUS:
203                 reg = &ctxt->global_status;
204                 break;
205         case MSR_CORE_PERF_GLOBAL_CTRL:
206                 reg = &ctxt->global_ctrl;
207                 break;
208         case MSR_CORE_PERF_FIXED_CTR_CTRL:
209                 reg = &ctxt->fixed_ctrl;
210                 break;
211         default:
212                 switch (type) {
213                 case MSR_TYPE_COUNTER:
214                         fix_counters = field_offset(ctxt, fixed_counters);
215                         reg = &fix_counters[index];
216                         break;
217                 case MSR_TYPE_ARCH_COUNTER:
218                         arch_cntr_pair = field_offset(ctxt, arch_counters);
219                         reg = &arch_cntr_pair[index].counter;
220                         break;
221                 case MSR_TYPE_ARCH_CTRL:
222                         arch_cntr_pair = field_offset(ctxt, arch_counters);
223                         reg = &arch_cntr_pair[index].control;
224                         break;
225                 default:
226                         return false;
227                 }
228         }
229
230         if (reg) {
231                 if (is_read)
232                         *val = *reg;
233                 else {
234                         *reg = *val;
235
236                         if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
237                                 ctxt->global_status &= (~(*val));
238                 }
239                 return true;
240         }
241
242         return false;
243 }
244
245 static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
246 {
247         uint64_t *reg = NULL;
248         int i, off = 0;
249         struct xen_pmu_amd_ctxt *ctxt;
250         uint64_t *counter_regs, *ctrl_regs;
251         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
252         uint8_t xenpmu_flags = get_xenpmu_flags();
253
254         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
255                 return false;
256
257         if (k7_counters_mirrored &&
258             ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
259                 msr = get_fam15h_addr(msr);
260
261         ctxt = &xenpmu_data->pmu.c.amd;
262         for (i = 0; i < amd_num_counters; i++) {
263                 if (msr == amd_ctrls_base + off) {
264                         ctrl_regs = field_offset(ctxt, ctrls);
265                         reg = &ctrl_regs[i];
266                         break;
267                 } else if (msr == amd_counters_base + off) {
268                         counter_regs = field_offset(ctxt, counters);
269                         reg = &counter_regs[i];
270                         break;
271                 }
272                 off += amd_msr_step;
273         }
274
275         if (reg) {
276                 if (is_read)
277                         *val = *reg;
278                 else
279                         *reg = *val;
280
281                 return true;
282         }
283         return false;
284 }
285
286 bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
287 {
288         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
289                 if (is_amd_pmu_msr(msr)) {
290                         if (!xen_amd_pmu_emulate(msr, val, 1))
291                                 *val = native_read_msr_safe(msr, err);
292                         return true;
293                 }
294         } else {
295                 int type, index;
296
297                 if (is_intel_pmu_msr(msr, &type, &index)) {
298                         if (!xen_intel_pmu_emulate(msr, val, type, index, 1))
299                                 *val = native_read_msr_safe(msr, err);
300                         return true;
301                 }
302         }
303
304         return false;
305 }
306
307 bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
308 {
309         uint64_t val = ((uint64_t)high << 32) | low;
310
311         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
312                 if (is_amd_pmu_msr(msr)) {
313                         if (!xen_amd_pmu_emulate(msr, &val, 0))
314                                 *err = native_write_msr_safe(msr, low, high);
315                         return true;
316                 }
317         } else {
318                 int type, index;
319
320                 if (is_intel_pmu_msr(msr, &type, &index)) {
321                         if (!xen_intel_pmu_emulate(msr, &val, type, index, 0))
322                                 *err = native_write_msr_safe(msr, low, high);
323                         return true;
324                 }
325         }
326
327         return false;
328 }
329
330 static unsigned long long xen_amd_read_pmc(int counter)
331 {
332         struct xen_pmu_amd_ctxt *ctxt;
333         uint64_t *counter_regs;
334         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
335         uint8_t xenpmu_flags = get_xenpmu_flags();
336
337         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
338                 uint32_t msr;
339                 int err;
340
341                 msr = amd_counters_base + (counter * amd_msr_step);
342                 return native_read_msr_safe(msr, &err);
343         }
344
345         ctxt = &xenpmu_data->pmu.c.amd;
346         counter_regs = field_offset(ctxt, counters);
347         return counter_regs[counter];
348 }
349
350 static unsigned long long xen_intel_read_pmc(int counter)
351 {
352         struct xen_pmu_intel_ctxt *ctxt;
353         uint64_t *fixed_counters;
354         struct xen_pmu_cntr_pair *arch_cntr_pair;
355         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
356         uint8_t xenpmu_flags = get_xenpmu_flags();
357
358         if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
359                 uint32_t msr;
360                 int err;
361
362                 if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
363                         msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
364                 else
365                         msr = MSR_IA32_PERFCTR0 + counter;
366
367                 return native_read_msr_safe(msr, &err);
368         }
369
370         ctxt = &xenpmu_data->pmu.c.intel;
371         if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
372                 fixed_counters = field_offset(ctxt, fixed_counters);
373                 return fixed_counters[counter & 0xffff];
374         }
375
376         arch_cntr_pair = field_offset(ctxt, arch_counters);
377         return arch_cntr_pair[counter].counter;
378 }
379
380 unsigned long long xen_read_pmc(int counter)
381 {
382         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
383                 return xen_amd_read_pmc(counter);
384         else
385                 return xen_intel_read_pmc(counter);
386 }
387
388 int pmu_apic_update(uint32_t val)
389 {
390         int ret;
391         struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
392
393         if (!xenpmu_data) {
394                 pr_warn_once("%s: pmudata not initialized\n", __func__);
395                 return -EINVAL;
396         }
397
398         xenpmu_data->pmu.l.lapic_lvtpc = val;
399
400         if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
401                 return 0;
402
403         ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
404
405         return ret;
406 }
407
408 /* perf callbacks */
409 static int xen_is_in_guest(void)
410 {
411         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
412
413         if (!xenpmu_data) {
414                 pr_warn_once("%s: pmudata not initialized\n", __func__);
415                 return 0;
416         }
417
418         if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
419                 return 0;
420
421         return 1;
422 }
423
424 static int xen_is_user_mode(void)
425 {
426         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
427
428         if (!xenpmu_data) {
429                 pr_warn_once("%s: pmudata not initialized\n", __func__);
430                 return 0;
431         }
432
433         if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV)
434                 return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER);
435         else
436                 return !!(xenpmu_data->pmu.r.regs.cpl & 3);
437 }
438
439 static unsigned long xen_get_guest_ip(void)
440 {
441         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
442
443         if (!xenpmu_data) {
444                 pr_warn_once("%s: pmudata not initialized\n", __func__);
445                 return 0;
446         }
447
448         return xenpmu_data->pmu.r.regs.ip;
449 }
450
451 static struct perf_guest_info_callbacks xen_guest_cbs = {
452         .is_in_guest            = xen_is_in_guest,
453         .is_user_mode           = xen_is_user_mode,
454         .get_guest_ip           = xen_get_guest_ip,
455 };
456
457 /* Convert registers from Xen's format to Linux' */
458 static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
459                              struct pt_regs *regs, uint64_t pmu_flags)
460 {
461         regs->ip = xen_regs->ip;
462         regs->cs = xen_regs->cs;
463         regs->sp = xen_regs->sp;
464
465         if (pmu_flags & PMU_SAMPLE_PV) {
466                 if (pmu_flags & PMU_SAMPLE_USER)
467                         regs->cs |= 3;
468                 else
469                         regs->cs &= ~3;
470         } else {
471                 if (xen_regs->cpl)
472                         regs->cs |= 3;
473                 else
474                         regs->cs &= ~3;
475         }
476 }
477
478 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
479 {
480         int err, ret = IRQ_NONE;
481         struct pt_regs regs = {0};
482         const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
483         uint8_t xenpmu_flags = get_xenpmu_flags();
484
485         if (!xenpmu_data) {
486                 pr_warn_once("%s: pmudata not initialized\n", __func__);
487                 return ret;
488         }
489
490         this_cpu_ptr(&xenpmu_shared)->flags =
491                 xenpmu_flags | XENPMU_IRQ_PROCESSING;
492         xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
493                          xenpmu_data->pmu.pmu_flags);
494         if (x86_pmu.handle_irq(&regs))
495                 ret = IRQ_HANDLED;
496
497         /* Write out cached context to HW */
498         err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
499         this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
500         if (err) {
501                 pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
502                 return IRQ_NONE;
503         }
504
505         return ret;
506 }
507
508 bool is_xen_pmu(int cpu)
509 {
510         return (get_xenpmu_data() != NULL);
511 }
512
513 void xen_pmu_init(int cpu)
514 {
515         int err;
516         struct xen_pmu_params xp;
517         unsigned long pfn;
518         struct xen_pmu_data *xenpmu_data;
519
520         BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
521
522         if (xen_hvm_domain())
523                 return;
524
525         xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
526         if (!xenpmu_data) {
527                 pr_err("VPMU init: No memory\n");
528                 return;
529         }
530         pfn = virt_to_pfn(xenpmu_data);
531
532         xp.val = pfn_to_mfn(pfn);
533         xp.vcpu = cpu;
534         xp.version.maj = XENPMU_VER_MAJ;
535         xp.version.min = XENPMU_VER_MIN;
536         err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
537         if (err)
538                 goto fail;
539
540         per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
541         per_cpu(xenpmu_shared, cpu).flags = 0;
542
543         if (cpu == 0) {
544                 perf_register_guest_info_callbacks(&xen_guest_cbs);
545                 xen_pmu_arch_init();
546         }
547
548         return;
549
550 fail:
551         if (err == -EOPNOTSUPP || err == -ENOSYS)
552                 pr_info_once("VPMU disabled by hypervisor.\n");
553         else
554                 pr_info_once("Could not initialize VPMU for cpu %d, error %d\n",
555                         cpu, err);
556         free_pages((unsigned long)xenpmu_data, 0);
557 }
558
559 void xen_pmu_finish(int cpu)
560 {
561         struct xen_pmu_params xp;
562
563         if (xen_hvm_domain())
564                 return;
565
566         xp.vcpu = cpu;
567         xp.version.maj = XENPMU_VER_MAJ;
568         xp.version.min = XENPMU_VER_MIN;
569
570         (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
571
572         free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
573         per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
574 }