GNU Linux-libre 4.19.286-gnu1
[releases.git] / kernel / irq / irqdesc.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
4  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
5  *
6  * This file contains the interrupt descriptor management code. Detailed
7  * information is available in Documentation/core-api/genericirq.rst
8  *
9  */
10 #include <linux/irq.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/radix-tree.h>
16 #include <linux/bitmap.h>
17 #include <linux/irqdomain.h>
18 #include <linux/sysfs.h>
19
20 #include "internals.h"
21
22 /*
23  * lockdep: we want to handle all irq_desc locks as a single lock-class:
24  */
25 static struct lock_class_key irq_desc_lock_class;
26
27 #if defined(CONFIG_SMP)
28 static int __init irq_affinity_setup(char *str)
29 {
30         alloc_bootmem_cpumask_var(&irq_default_affinity);
31         cpulist_parse(str, irq_default_affinity);
32         /*
33          * Set at least the boot cpu. We don't want to end up with
34          * bugreports caused by random comandline masks
35          */
36         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
37         return 1;
38 }
39 __setup("irqaffinity=", irq_affinity_setup);
40
41 static void __init init_irq_default_affinity(void)
42 {
43         if (!cpumask_available(irq_default_affinity))
44                 zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
45         if (cpumask_empty(irq_default_affinity))
46                 cpumask_setall(irq_default_affinity);
47 }
48 #else
49 static void __init init_irq_default_affinity(void)
50 {
51 }
52 #endif
53
54 #ifdef CONFIG_SMP
55 static int alloc_masks(struct irq_desc *desc, int node)
56 {
57         if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
58                                      GFP_KERNEL, node))
59                 return -ENOMEM;
60
61 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
62         if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,
63                                      GFP_KERNEL, node)) {
64                 free_cpumask_var(desc->irq_common_data.affinity);
65                 return -ENOMEM;
66         }
67 #endif
68
69 #ifdef CONFIG_GENERIC_PENDING_IRQ
70         if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {
71 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
72                 free_cpumask_var(desc->irq_common_data.effective_affinity);
73 #endif
74                 free_cpumask_var(desc->irq_common_data.affinity);
75                 return -ENOMEM;
76         }
77 #endif
78         return 0;
79 }
80
81 static void desc_smp_init(struct irq_desc *desc, int node,
82                           const struct cpumask *affinity)
83 {
84         if (!affinity)
85                 affinity = irq_default_affinity;
86         cpumask_copy(desc->irq_common_data.affinity, affinity);
87
88 #ifdef CONFIG_GENERIC_PENDING_IRQ
89         cpumask_clear(desc->pending_mask);
90 #endif
91 #ifdef CONFIG_NUMA
92         desc->irq_common_data.node = node;
93 #endif
94 }
95
96 #else
97 static inline int
98 alloc_masks(struct irq_desc *desc, int node) { return 0; }
99 static inline void
100 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
101 #endif
102
103 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
104                               const struct cpumask *affinity, struct module *owner)
105 {
106         int cpu;
107
108         desc->irq_common_data.handler_data = NULL;
109         desc->irq_common_data.msi_desc = NULL;
110
111         desc->irq_data.common = &desc->irq_common_data;
112         desc->irq_data.irq = irq;
113         desc->irq_data.chip = &no_irq_chip;
114         desc->irq_data.chip_data = NULL;
115         irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
116         irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
117         irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
118         desc->handle_irq = handle_bad_irq;
119         desc->depth = 1;
120         desc->irq_count = 0;
121         desc->irqs_unhandled = 0;
122         desc->tot_count = 0;
123         desc->name = NULL;
124         desc->owner = owner;
125         for_each_possible_cpu(cpu)
126                 *per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
127         desc_smp_init(desc, node, affinity);
128 }
129
130 int nr_irqs = NR_IRQS;
131 EXPORT_SYMBOL_GPL(nr_irqs);
132
133 static DEFINE_MUTEX(sparse_irq_lock);
134 static DECLARE_BITMAP(allocated_irqs, IRQ_BITMAP_BITS);
135
136 #ifdef CONFIG_SPARSE_IRQ
137
138 static void irq_kobj_release(struct kobject *kobj);
139
140 #ifdef CONFIG_SYSFS
141 static struct kobject *irq_kobj_base;
142
143 #define IRQ_ATTR_RO(_name) \
144 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
145
146 static ssize_t per_cpu_count_show(struct kobject *kobj,
147                                   struct kobj_attribute *attr, char *buf)
148 {
149         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
150         int cpu, irq = desc->irq_data.irq;
151         ssize_t ret = 0;
152         char *p = "";
153
154         for_each_possible_cpu(cpu) {
155                 unsigned int c = kstat_irqs_cpu(irq, cpu);
156
157                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c);
158                 p = ",";
159         }
160
161         ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
162         return ret;
163 }
164 IRQ_ATTR_RO(per_cpu_count);
165
166 static ssize_t chip_name_show(struct kobject *kobj,
167                               struct kobj_attribute *attr, char *buf)
168 {
169         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
170         ssize_t ret = 0;
171
172         raw_spin_lock_irq(&desc->lock);
173         if (desc->irq_data.chip && desc->irq_data.chip->name) {
174                 ret = scnprintf(buf, PAGE_SIZE, "%s\n",
175                                 desc->irq_data.chip->name);
176         }
177         raw_spin_unlock_irq(&desc->lock);
178
179         return ret;
180 }
181 IRQ_ATTR_RO(chip_name);
182
183 static ssize_t hwirq_show(struct kobject *kobj,
184                           struct kobj_attribute *attr, char *buf)
185 {
186         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
187         ssize_t ret = 0;
188
189         raw_spin_lock_irq(&desc->lock);
190         if (desc->irq_data.domain)
191                 ret = sprintf(buf, "%d\n", (int)desc->irq_data.hwirq);
192         raw_spin_unlock_irq(&desc->lock);
193
194         return ret;
195 }
196 IRQ_ATTR_RO(hwirq);
197
198 static ssize_t type_show(struct kobject *kobj,
199                          struct kobj_attribute *attr, char *buf)
200 {
201         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
202         ssize_t ret = 0;
203
204         raw_spin_lock_irq(&desc->lock);
205         ret = sprintf(buf, "%s\n",
206                       irqd_is_level_type(&desc->irq_data) ? "level" : "edge");
207         raw_spin_unlock_irq(&desc->lock);
208
209         return ret;
210
211 }
212 IRQ_ATTR_RO(type);
213
214 static ssize_t wakeup_show(struct kobject *kobj,
215                            struct kobj_attribute *attr, char *buf)
216 {
217         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
218         ssize_t ret = 0;
219
220         raw_spin_lock_irq(&desc->lock);
221         ret = sprintf(buf, "%s\n",
222                       irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled");
223         raw_spin_unlock_irq(&desc->lock);
224
225         return ret;
226
227 }
228 IRQ_ATTR_RO(wakeup);
229
230 static ssize_t name_show(struct kobject *kobj,
231                          struct kobj_attribute *attr, char *buf)
232 {
233         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
234         ssize_t ret = 0;
235
236         raw_spin_lock_irq(&desc->lock);
237         if (desc->name)
238                 ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name);
239         raw_spin_unlock_irq(&desc->lock);
240
241         return ret;
242 }
243 IRQ_ATTR_RO(name);
244
245 static ssize_t actions_show(struct kobject *kobj,
246                             struct kobj_attribute *attr, char *buf)
247 {
248         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
249         struct irqaction *action;
250         ssize_t ret = 0;
251         char *p = "";
252
253         raw_spin_lock_irq(&desc->lock);
254         for (action = desc->action; action != NULL; action = action->next) {
255                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
256                                  p, action->name);
257                 p = ",";
258         }
259         raw_spin_unlock_irq(&desc->lock);
260
261         if (ret)
262                 ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
263
264         return ret;
265 }
266 IRQ_ATTR_RO(actions);
267
268 static struct attribute *irq_attrs[] = {
269         &per_cpu_count_attr.attr,
270         &chip_name_attr.attr,
271         &hwirq_attr.attr,
272         &type_attr.attr,
273         &wakeup_attr.attr,
274         &name_attr.attr,
275         &actions_attr.attr,
276         NULL
277 };
278
279 static struct kobj_type irq_kobj_type = {
280         .release        = irq_kobj_release,
281         .sysfs_ops      = &kobj_sysfs_ops,
282         .default_attrs  = irq_attrs,
283 };
284
285 static void irq_sysfs_add(int irq, struct irq_desc *desc)
286 {
287         if (irq_kobj_base) {
288                 /*
289                  * Continue even in case of failure as this is nothing
290                  * crucial.
291                  */
292                 if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))
293                         pr_warn("Failed to add kobject for irq %d\n", irq);
294         }
295 }
296
297 static void irq_sysfs_del(struct irq_desc *desc)
298 {
299         /*
300          * If irq_sysfs_init() has not yet been invoked (early boot), then
301          * irq_kobj_base is NULL and the descriptor was never added.
302          * kobject_del() complains about a object with no parent, so make
303          * it conditional.
304          */
305         if (irq_kobj_base)
306                 kobject_del(&desc->kobj);
307 }
308
309 static int __init irq_sysfs_init(void)
310 {
311         struct irq_desc *desc;
312         int irq;
313
314         /* Prevent concurrent irq alloc/free */
315         irq_lock_sparse();
316
317         irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);
318         if (!irq_kobj_base) {
319                 irq_unlock_sparse();
320                 return -ENOMEM;
321         }
322
323         /* Add the already allocated interrupts */
324         for_each_irq_desc(irq, desc)
325                 irq_sysfs_add(irq, desc);
326         irq_unlock_sparse();
327
328         return 0;
329 }
330 postcore_initcall(irq_sysfs_init);
331
332 #else /* !CONFIG_SYSFS */
333
334 static struct kobj_type irq_kobj_type = {
335         .release        = irq_kobj_release,
336 };
337
338 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}
339 static void irq_sysfs_del(struct irq_desc *desc) {}
340
341 #endif /* CONFIG_SYSFS */
342
343 static RADIX_TREE(irq_desc_tree, GFP_KERNEL);
344
345 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
346 {
347         radix_tree_insert(&irq_desc_tree, irq, desc);
348 }
349
350 struct irq_desc *irq_to_desc(unsigned int irq)
351 {
352         return radix_tree_lookup(&irq_desc_tree, irq);
353 }
354 EXPORT_SYMBOL(irq_to_desc);
355
356 static void delete_irq_desc(unsigned int irq)
357 {
358         radix_tree_delete(&irq_desc_tree, irq);
359 }
360
361 #ifdef CONFIG_SMP
362 static void free_masks(struct irq_desc *desc)
363 {
364 #ifdef CONFIG_GENERIC_PENDING_IRQ
365         free_cpumask_var(desc->pending_mask);
366 #endif
367         free_cpumask_var(desc->irq_common_data.affinity);
368 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
369         free_cpumask_var(desc->irq_common_data.effective_affinity);
370 #endif
371 }
372 #else
373 static inline void free_masks(struct irq_desc *desc) { }
374 #endif
375
376 void irq_lock_sparse(void)
377 {
378         mutex_lock(&sparse_irq_lock);
379 }
380
381 void irq_unlock_sparse(void)
382 {
383         mutex_unlock(&sparse_irq_lock);
384 }
385
386 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
387                                    const struct cpumask *affinity,
388                                    struct module *owner)
389 {
390         struct irq_desc *desc;
391
392         desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node);
393         if (!desc)
394                 return NULL;
395         /* allocate based on nr_cpu_ids */
396         desc->kstat_irqs = alloc_percpu(unsigned int);
397         if (!desc->kstat_irqs)
398                 goto err_desc;
399
400         if (alloc_masks(desc, node))
401                 goto err_kstat;
402
403         raw_spin_lock_init(&desc->lock);
404         lockdep_set_class(&desc->lock, &irq_desc_lock_class);
405         mutex_init(&desc->request_mutex);
406         init_rcu_head(&desc->rcu);
407         init_waitqueue_head(&desc->wait_for_threads);
408
409         desc_set_defaults(irq, desc, node, affinity, owner);
410         irqd_set(&desc->irq_data, flags);
411         kobject_init(&desc->kobj, &irq_kobj_type);
412
413         return desc;
414
415 err_kstat:
416         free_percpu(desc->kstat_irqs);
417 err_desc:
418         kfree(desc);
419         return NULL;
420 }
421
422 static void irq_kobj_release(struct kobject *kobj)
423 {
424         struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
425
426         free_masks(desc);
427         free_percpu(desc->kstat_irqs);
428         kfree(desc);
429 }
430
431 static void delayed_free_desc(struct rcu_head *rhp)
432 {
433         struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);
434
435         kobject_put(&desc->kobj);
436 }
437
438 static void free_desc(unsigned int irq)
439 {
440         struct irq_desc *desc = irq_to_desc(irq);
441
442         irq_remove_debugfs_entry(desc);
443         unregister_irq_proc(irq, desc);
444
445         /*
446          * sparse_irq_lock protects also show_interrupts() and
447          * kstat_irq_usr(). Once we deleted the descriptor from the
448          * sparse tree we can free it. Access in proc will fail to
449          * lookup the descriptor.
450          *
451          * The sysfs entry must be serialized against a concurrent
452          * irq_sysfs_init() as well.
453          */
454         irq_sysfs_del(desc);
455         delete_irq_desc(irq);
456
457         /*
458          * We free the descriptor, masks and stat fields via RCU. That
459          * allows demultiplex interrupts to do rcu based management of
460          * the child interrupts.
461          * This also allows us to use rcu in kstat_irqs_usr().
462          */
463         call_rcu(&desc->rcu, delayed_free_desc);
464 }
465
466 static int alloc_descs(unsigned int start, unsigned int cnt, int node,
467                        const struct cpumask *affinity, struct module *owner)
468 {
469         const struct cpumask *mask = NULL;
470         struct irq_desc *desc;
471         unsigned int flags;
472         int i;
473
474         /* Validate affinity mask(s) */
475         if (affinity) {
476                 for (i = 0, mask = affinity; i < cnt; i++, mask++) {
477                         if (cpumask_empty(mask))
478                                 return -EINVAL;
479                 }
480         }
481
482         flags = affinity ? IRQD_AFFINITY_MANAGED | IRQD_MANAGED_SHUTDOWN : 0;
483         mask = NULL;
484
485         for (i = 0; i < cnt; i++) {
486                 if (affinity) {
487                         node = cpu_to_node(cpumask_first(affinity));
488                         mask = affinity;
489                         affinity++;
490                 }
491                 desc = alloc_desc(start + i, node, flags, mask, owner);
492                 if (!desc)
493                         goto err;
494                 irq_insert_desc(start + i, desc);
495                 irq_sysfs_add(start + i, desc);
496                 irq_add_debugfs_entry(start + i, desc);
497         }
498         bitmap_set(allocated_irqs, start, cnt);
499         return start;
500
501 err:
502         for (i--; i >= 0; i--)
503                 free_desc(start + i);
504         return -ENOMEM;
505 }
506
507 static int irq_expand_nr_irqs(unsigned int nr)
508 {
509         if (nr > IRQ_BITMAP_BITS)
510                 return -ENOMEM;
511         nr_irqs = nr;
512         return 0;
513 }
514
515 int __init early_irq_init(void)
516 {
517         int i, initcnt, node = first_online_node;
518         struct irq_desc *desc;
519
520         init_irq_default_affinity();
521
522         /* Let arch update nr_irqs and return the nr of preallocated irqs */
523         initcnt = arch_probe_nr_irqs();
524         printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n",
525                NR_IRQS, nr_irqs, initcnt);
526
527         if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
528                 nr_irqs = IRQ_BITMAP_BITS;
529
530         if (WARN_ON(initcnt > IRQ_BITMAP_BITS))
531                 initcnt = IRQ_BITMAP_BITS;
532
533         if (initcnt > nr_irqs)
534                 nr_irqs = initcnt;
535
536         for (i = 0; i < initcnt; i++) {
537                 desc = alloc_desc(i, node, 0, NULL, NULL);
538                 set_bit(i, allocated_irqs);
539                 irq_insert_desc(i, desc);
540         }
541         return arch_early_irq_init();
542 }
543
544 #else /* !CONFIG_SPARSE_IRQ */
545
546 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
547         [0 ... NR_IRQS-1] = {
548                 .handle_irq     = handle_bad_irq,
549                 .depth          = 1,
550                 .lock           = __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
551         }
552 };
553
554 int __init early_irq_init(void)
555 {
556         int count, i, node = first_online_node;
557         struct irq_desc *desc;
558
559         init_irq_default_affinity();
560
561         printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS);
562
563         desc = irq_desc;
564         count = ARRAY_SIZE(irq_desc);
565
566         for (i = 0; i < count; i++) {
567                 desc[i].kstat_irqs = alloc_percpu(unsigned int);
568                 alloc_masks(&desc[i], node);
569                 raw_spin_lock_init(&desc[i].lock);
570                 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
571                 mutex_init(&desc[i].request_mutex);
572                 init_waitqueue_head(&desc[i].wait_for_threads);
573                 desc_set_defaults(i, &desc[i], node, NULL, NULL);
574         }
575         return arch_early_irq_init();
576 }
577
578 struct irq_desc *irq_to_desc(unsigned int irq)
579 {
580         return (irq < NR_IRQS) ? irq_desc + irq : NULL;
581 }
582 EXPORT_SYMBOL(irq_to_desc);
583
584 static void free_desc(unsigned int irq)
585 {
586         struct irq_desc *desc = irq_to_desc(irq);
587         unsigned long flags;
588
589         raw_spin_lock_irqsave(&desc->lock, flags);
590         desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);
591         raw_spin_unlock_irqrestore(&desc->lock, flags);
592 }
593
594 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
595                               const struct cpumask *affinity,
596                               struct module *owner)
597 {
598         u32 i;
599
600         for (i = 0; i < cnt; i++) {
601                 struct irq_desc *desc = irq_to_desc(start + i);
602
603                 desc->owner = owner;
604         }
605         bitmap_set(allocated_irqs, start, cnt);
606         return start;
607 }
608
609 static int irq_expand_nr_irqs(unsigned int nr)
610 {
611         return -ENOMEM;
612 }
613
614 void irq_mark_irq(unsigned int irq)
615 {
616         mutex_lock(&sparse_irq_lock);
617         bitmap_set(allocated_irqs, irq, 1);
618         mutex_unlock(&sparse_irq_lock);
619 }
620
621 #ifdef CONFIG_GENERIC_IRQ_LEGACY
622 void irq_init_desc(unsigned int irq)
623 {
624         free_desc(irq);
625 }
626 #endif
627
628 #endif /* !CONFIG_SPARSE_IRQ */
629
630 /**
631  * generic_handle_irq - Invoke the handler for a particular irq
632  * @irq:        The irq number to handle
633  *
634  */
635 int generic_handle_irq(unsigned int irq)
636 {
637         struct irq_desc *desc = irq_to_desc(irq);
638
639         if (!desc)
640                 return -EINVAL;
641         generic_handle_irq_desc(desc);
642         return 0;
643 }
644 EXPORT_SYMBOL_GPL(generic_handle_irq);
645
646 #ifdef CONFIG_HANDLE_DOMAIN_IRQ
647 /**
648  * __handle_domain_irq - Invoke the handler for a HW irq belonging to a domain
649  * @domain:     The domain where to perform the lookup
650  * @hwirq:      The HW irq number to convert to a logical one
651  * @lookup:     Whether to perform the domain lookup or not
652  * @regs:       Register file coming from the low-level handling code
653  *
654  * Returns:     0 on success, or -EINVAL if conversion has failed
655  */
656 int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
657                         bool lookup, struct pt_regs *regs)
658 {
659         struct pt_regs *old_regs = set_irq_regs(regs);
660         unsigned int irq = hwirq;
661         int ret = 0;
662
663         irq_enter();
664
665 #ifdef CONFIG_IRQ_DOMAIN
666         if (lookup)
667                 irq = irq_find_mapping(domain, hwirq);
668 #endif
669
670         /*
671          * Some hardware gives randomly wrong interrupts.  Rather
672          * than crashing, do something sensible.
673          */
674         if (unlikely(!irq || irq >= nr_irqs)) {
675                 ack_bad_irq(irq);
676                 ret = -EINVAL;
677         } else {
678                 generic_handle_irq(irq);
679         }
680
681         irq_exit();
682         set_irq_regs(old_regs);
683         return ret;
684 }
685 #endif
686
687 /* Dynamic interrupt handling */
688
689 /**
690  * irq_free_descs - free irq descriptors
691  * @from:       Start of descriptor range
692  * @cnt:        Number of consecutive irqs to free
693  */
694 void irq_free_descs(unsigned int from, unsigned int cnt)
695 {
696         int i;
697
698         if (from >= nr_irqs || (from + cnt) > nr_irqs)
699                 return;
700
701         mutex_lock(&sparse_irq_lock);
702         for (i = 0; i < cnt; i++)
703                 free_desc(from + i);
704
705         bitmap_clear(allocated_irqs, from, cnt);
706         mutex_unlock(&sparse_irq_lock);
707 }
708 EXPORT_SYMBOL_GPL(irq_free_descs);
709
710 /**
711  * irq_alloc_descs - allocate and initialize a range of irq descriptors
712  * @irq:        Allocate for specific irq number if irq >= 0
713  * @from:       Start the search from this irq number
714  * @cnt:        Number of consecutive irqs to allocate.
715  * @node:       Preferred node on which the irq descriptor should be allocated
716  * @owner:      Owning module (can be NULL)
717  * @affinity:   Optional pointer to an affinity mask array of size @cnt which
718  *              hints where the irq descriptors should be allocated and which
719  *              default affinities to use
720  *
721  * Returns the first irq number or error code
722  */
723 int __ref
724 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
725                   struct module *owner, const struct cpumask *affinity)
726 {
727         int start, ret;
728
729         if (!cnt)
730                 return -EINVAL;
731
732         if (irq >= 0) {
733                 if (from > irq)
734                         return -EINVAL;
735                 from = irq;
736         } else {
737                 /*
738                  * For interrupts which are freely allocated the
739                  * architecture can force a lower bound to the @from
740                  * argument. x86 uses this to exclude the GSI space.
741                  */
742                 from = arch_dynirq_lower_bound(from);
743         }
744
745         mutex_lock(&sparse_irq_lock);
746
747         start = bitmap_find_next_zero_area(allocated_irqs, IRQ_BITMAP_BITS,
748                                            from, cnt, 0);
749         ret = -EEXIST;
750         if (irq >=0 && start != irq)
751                 goto unlock;
752
753         if (start + cnt > nr_irqs) {
754                 ret = irq_expand_nr_irqs(start + cnt);
755                 if (ret)
756                         goto unlock;
757         }
758         ret = alloc_descs(start, cnt, node, affinity, owner);
759 unlock:
760         mutex_unlock(&sparse_irq_lock);
761         return ret;
762 }
763 EXPORT_SYMBOL_GPL(__irq_alloc_descs);
764
765 #ifdef CONFIG_GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
766 /**
767  * irq_alloc_hwirqs - Allocate an irq descriptor and initialize the hardware
768  * @cnt:        number of interrupts to allocate
769  * @node:       node on which to allocate
770  *
771  * Returns an interrupt number > 0 or 0, if the allocation fails.
772  */
773 unsigned int irq_alloc_hwirqs(int cnt, int node)
774 {
775         int i, irq = __irq_alloc_descs(-1, 0, cnt, node, NULL, NULL);
776
777         if (irq < 0)
778                 return 0;
779
780         for (i = irq; cnt > 0; i++, cnt--) {
781                 if (arch_setup_hwirq(i, node))
782                         goto err;
783                 irq_clear_status_flags(i, _IRQ_NOREQUEST);
784         }
785         return irq;
786
787 err:
788         for (i--; i >= irq; i--) {
789                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
790                 arch_teardown_hwirq(i);
791         }
792         irq_free_descs(irq, cnt);
793         return 0;
794 }
795 EXPORT_SYMBOL_GPL(irq_alloc_hwirqs);
796
797 /**
798  * irq_free_hwirqs - Free irq descriptor and cleanup the hardware
799  * @from:       Free from irq number
800  * @cnt:        number of interrupts to free
801  *
802  */
803 void irq_free_hwirqs(unsigned int from, int cnt)
804 {
805         int i, j;
806
807         for (i = from, j = cnt; j > 0; i++, j--) {
808                 irq_set_status_flags(i, _IRQ_NOREQUEST | _IRQ_NOPROBE);
809                 arch_teardown_hwirq(i);
810         }
811         irq_free_descs(from, cnt);
812 }
813 EXPORT_SYMBOL_GPL(irq_free_hwirqs);
814 #endif
815
816 /**
817  * irq_get_next_irq - get next allocated irq number
818  * @offset:     where to start the search
819  *
820  * Returns next irq number after offset or nr_irqs if none is found.
821  */
822 unsigned int irq_get_next_irq(unsigned int offset)
823 {
824         return find_next_bit(allocated_irqs, nr_irqs, offset);
825 }
826
827 struct irq_desc *
828 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
829                     unsigned int check)
830 {
831         struct irq_desc *desc = irq_to_desc(irq);
832
833         if (desc) {
834                 if (check & _IRQ_DESC_CHECK) {
835                         if ((check & _IRQ_DESC_PERCPU) &&
836                             !irq_settings_is_per_cpu_devid(desc))
837                                 return NULL;
838
839                         if (!(check & _IRQ_DESC_PERCPU) &&
840                             irq_settings_is_per_cpu_devid(desc))
841                                 return NULL;
842                 }
843
844                 if (bus)
845                         chip_bus_lock(desc);
846                 raw_spin_lock_irqsave(&desc->lock, *flags);
847         }
848         return desc;
849 }
850
851 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
852 {
853         raw_spin_unlock_irqrestore(&desc->lock, flags);
854         if (bus)
855                 chip_bus_sync_unlock(desc);
856 }
857
858 int irq_set_percpu_devid_partition(unsigned int irq,
859                                    const struct cpumask *affinity)
860 {
861         struct irq_desc *desc = irq_to_desc(irq);
862
863         if (!desc)
864                 return -EINVAL;
865
866         if (desc->percpu_enabled)
867                 return -EINVAL;
868
869         desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
870
871         if (!desc->percpu_enabled)
872                 return -ENOMEM;
873
874         if (affinity)
875                 desc->percpu_affinity = affinity;
876         else
877                 desc->percpu_affinity = cpu_possible_mask;
878
879         irq_set_percpu_devid_flags(irq);
880         return 0;
881 }
882
883 int irq_set_percpu_devid(unsigned int irq)
884 {
885         return irq_set_percpu_devid_partition(irq, NULL);
886 }
887
888 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)
889 {
890         struct irq_desc *desc = irq_to_desc(irq);
891
892         if (!desc || !desc->percpu_enabled)
893                 return -EINVAL;
894
895         if (affinity)
896                 cpumask_copy(affinity, desc->percpu_affinity);
897
898         return 0;
899 }
900 EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition);
901
902 void kstat_incr_irq_this_cpu(unsigned int irq)
903 {
904         kstat_incr_irqs_this_cpu(irq_to_desc(irq));
905 }
906
907 /**
908  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
909  * @irq:        The interrupt number
910  * @cpu:        The cpu number
911  *
912  * Returns the sum of interrupt counts on @cpu since boot for
913  * @irq. The caller must ensure that the interrupt is not removed
914  * concurrently.
915  */
916 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
917 {
918         struct irq_desc *desc = irq_to_desc(irq);
919
920         return desc && desc->kstat_irqs ?
921                         *per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
922 }
923
924 /**
925  * kstat_irqs - Get the statistics for an interrupt
926  * @irq:        The interrupt number
927  *
928  * Returns the sum of interrupt counts on all cpus since boot for
929  * @irq. The caller must ensure that the interrupt is not removed
930  * concurrently.
931  */
932 unsigned int kstat_irqs(unsigned int irq)
933 {
934         struct irq_desc *desc = irq_to_desc(irq);
935         unsigned int sum = 0;
936         int cpu;
937
938         if (!desc || !desc->kstat_irqs)
939                 return 0;
940         if (!irq_settings_is_per_cpu_devid(desc) &&
941             !irq_settings_is_per_cpu(desc))
942             return desc->tot_count;
943
944         for_each_possible_cpu(cpu)
945                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
946         return sum;
947 }
948
949 /**
950  * kstat_irqs_usr - Get the statistics for an interrupt
951  * @irq:        The interrupt number
952  *
953  * Returns the sum of interrupt counts on all cpus since boot for @irq.
954  * Contrary to kstat_irqs() this can be called from any context.
955  * It uses rcu since a concurrent removal of an interrupt descriptor is
956  * observing an rcu grace period before delayed_free_desc()/irq_kobj_release().
957  */
958 unsigned int kstat_irqs_usr(unsigned int irq)
959 {
960         unsigned int sum;
961
962         rcu_read_lock();
963         sum = kstat_irqs(irq);
964         rcu_read_unlock();
965         return sum;
966 }