--- zzzz-none-000/linux-4.9.279/arch/x86/kernel/apic/vector.c 2021-08-08 06:38:54.000000000 +0000 +++ puma7-atom-6591-750/linux-4.9.279/arch/x86/kernel/apic/vector.c 2023-02-08 11:43:42.000000000 +0000 @@ -12,9 +12,14 @@ */ #include #include +#include #include #include #include +#ifdef CONFIG_INTEL_PCI_MULTI_MSI +#include /* struct pci_dev */ +#include /* struct msi_desc */ +#endif #include #include #include @@ -110,19 +115,7 @@ static int __assign_irq_vector(int irq, struct apic_chip_data *d, const struct cpumask *mask) { - /* - * NOTE! The local APIC isn't very good at handling - * multiple interrupts at the same interrupt level. - * As the interrupt level is determined by taking the - * vector number and shifting that right by 4, we - * want to spread these out a bit so that they don't - * all fall in the same interrupt level. - * - * Also, we've got to be careful not to trash gate - * 0x80, because int 0x80 is hm, kind of importantish. ;) - */ - static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START; - static int current_offset = VECTOR_OFFSET_START % 16; + static int current_vector = FIRST_EXTERNAL_VECTOR; int cpu, vector; /* @@ -138,7 +131,7 @@ cpumask_clear(searched_cpumask); cpu = cpumask_first_and(mask, cpu_online_mask); while (cpu < nr_cpu_ids) { - int new_cpu, offset; + int new_cpu; /* Get the possible target cpus for @mask/@cpu from the apic */ apic->vector_allocation_domain(cpu, vector_cpumask, mask); @@ -167,13 +160,10 @@ } vector = current_vector; - offset = current_offset; next: - vector += 16; - if (vector >= first_system_vector) { - offset = (offset + 1) % 16; - vector = FIRST_EXTERNAL_VECTOR + offset; - } + vector++; + if (vector >= first_system_vector) + vector = FIRST_EXTERNAL_VECTOR; /* If the search wrapped around, try the next cpu */ if (unlikely(current_vector == vector)) @@ -188,7 +178,6 @@ } /* Found one! */ current_vector = vector; - current_offset = offset; /* Schedule the old vector for cleanup on all cpus */ if (d->cfg.vector) cpumask_copy(d->old_domain, d->domain); @@ -328,6 +317,203 @@ } } +#ifdef CONFIG_INTEL_PCI_MULTI_MSI +static int __assign_irq_vector_block(struct irq_domain *domain, int irq, + const struct cpumask *mask, int count) +{ + static int current_vector = FIRST_EXTERNAL_VECTOR; + int cpu, i, vector, rcount; + struct irq_data *irq_data; + struct apic_chip_data *chip_data; + + rcount = roundup_pow_of_two(count); + + BUG_ON((irq + rcount) > NR_IRQS); + + /* + * If there is still a move in progress or the previous move has not + * been cleaned up completely, tell the caller to come back later. + */ + for (i = 0; i < count; i++) { + irq_data = irq_domain_get_irq_data(domain, irq + i); + chip_data = irq_data->chip_data; + + BUG_ON(!irq_data || !chip_data); + + if (irq_data && chip_data) + if (chip_data->move_in_progress || + cpumask_intersects(chip_data->old_domain, cpu_online_mask)) + return -EBUSY; + + cpumask_clear(chip_data->old_domain); + } + + /* Only try and allocate irqs on cpus that are present */ + cpumask_clear(searched_cpumask); + cpu = cpumask_first_and(mask, cpu_online_mask); + while (cpu < nr_cpu_ids) { + int new_cpu; + + /* Get the possible target cpus for @mask/@cpu from the apic */ + apic->vector_allocation_domain(cpu, vector_cpumask, mask); + + /* + * Clear the offline cpus from @vector_cpumask for searching + * and verify whether the result overlaps with @mask. If true, + * then the call to apic->cpu_mask_to_apicid_and() will + * succeed as well. If not, no point in trying to find a + * vector in this mask. + */ + cpumask_and(vector_searchmask, vector_cpumask, cpu_online_mask); + + if (!cpumask_intersects(vector_searchmask, mask)) + goto next_cpu; + + vector = current_vector & ~(rcount - 1); +next: + vector += rcount; + if (vector + rcount >= first_system_vector) { + vector = FIRST_EXTERNAL_VECTOR & ~(rcount - 1); + if (vector < FIRST_EXTERNAL_VECTOR) + vector += rcount; + } + + /* If the search wrapped around, try the next cpu */ + if ((current_vector & ~(rcount - 1)) == vector) + goto next_cpu; + + for (i = 0; i < count; i++) + if (test_bit(vector + i, used_vectors)) + goto next; + + for_each_cpu(new_cpu, vector_searchmask) { + for (i = 0; i < count; i++) { + if (!IS_ERR_OR_NULL(per_cpu(vector_irq, new_cpu)[vector + i])) + goto next; + } + } + + /* Found one! */ + current_vector = vector + count - 1; + + pr_debug("Found one! vector start since %d", vector); + + for (i = 0; i < count ; i++) { + irq_data = irq_domain_get_irq_data(domain, irq + i); + BUG_ON(!irq_data); + chip_data = irq_data->chip_data; + + /* Schedule the old vector for cleanup on all cpus */ + if (chip_data->cfg.vector) + cpumask_copy(chip_data->old_domain, chip_data->domain); + + for_each_cpu(new_cpu, vector_searchmask) { + per_cpu(vector_irq, new_cpu)[vector + i] = irq_to_desc(irq + i); + + pr_debug("%d.vector_irq[%d] = %p (%d)\n", + new_cpu, vector + i, irq_to_desc(irq + i), irq + i); + } + + /* + * Exclude offline cpus from the cleanup mask and set the + * move_in_progress flag when the result is not empty. + */ + + cpumask_and(chip_data->old_domain, chip_data->old_domain, cpu_online_mask); + chip_data->move_in_progress = !cpumask_empty(chip_data->old_domain); + chip_data->cfg.old_vector = chip_data->move_in_progress ? chip_data->cfg.vector : 0; + chip_data->cfg.vector = vector + i; + cpumask_copy(chip_data->domain, vector_cpumask); + + BUG_ON(apic->cpu_mask_to_apicid_and(mask, chip_data->domain, + &chip_data->cfg.dest_apicid)); + } + + goto success; +next_cpu: + /* + * We exclude the current @vector_cpumask from the requested + * @mask and try again with the next online cpu in the + * result. We cannot modify @mask, so we use @vector_cpumask + * as a temporary buffer here as it will be reassigned when + * calling apic->vector_allocation_domain() above. + */ + cpumask_or(searched_cpumask, searched_cpumask, vector_cpumask); + cpumask_andnot(vector_cpumask, mask, searched_cpumask); + cpu = cpumask_first_and(vector_cpumask, cpu_online_mask); + continue; + } + + return -ENOSPC; + +success: + return 0; +} + +static int assign_irq_vector_block(struct irq_domain * domain, int irq, + const struct cpumask *mask, int count) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector_block(domain, irq, mask, count); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +static int assign_irq_vector_policy_block(struct irq_domain *domain, int irq, int node, + struct irq_alloc_info *info, int count) +{ + if (info && info->mask) + return assign_irq_vector_block(domain, irq, info->mask, count); + if (node != NUMA_NO_NODE && + assign_irq_vector_block(domain, irq, cpumask_of_node(node), count) == 0) + return 0; + return assign_irq_vector_block(domain, irq, apic->target_cpus(), count); +} + +static int x86_vector_alloc_irqs_block(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct irq_alloc_info *info = arg; + struct apic_chip_data *data; + struct irq_data *irq_data; + int i, err, node; + + + for (i = 0; i < nr_irqs; i++) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + BUG_ON(!irq_data); + node = irq_data_get_node(irq_data); +#ifdef CONFIG_X86_IO_APIC + if (virq < nr_legacy_irqs() && legacy_irq_data[virq + i]) + data = legacy_irq_data[virq + i]; + else +#endif + data = alloc_apic_chip_data(node); + + if (unlikely(!data)) { + err = -ENOMEM; + goto error; + } + + irq_data->chip = &lapic_controller; + irq_data->chip_data = data; + irq_data->hwirq = virq + i; + } + + err = assign_irq_vector_policy_block(domain, virq, node, info, nr_irqs); + if (unlikely(err)) + goto error; + + return 0; +error: + x86_vector_free_irqs(domain, virq, i + (i < nr_irqs)); + return err; +} +#endif + static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { @@ -340,8 +526,16 @@ return -ENXIO; /* Currently vector allocator can't guarantee contiguous allocations */ - if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && nr_irqs > 1) + if ((info->flags & X86_IRQ_ALLOC_CONTIGUOUS_VECTORS) && (nr_irqs > 1)) { +#ifdef CONFIG_INTEL_PCI_MULTI_MSI + if (info->msi_dev->mmsi_support) { + pr_debug("allocating contiguous virtual irqs vector: %d-%d...\n", virq, virq + nr_irqs - 1); + return x86_vector_alloc_irqs_block(domain, virq, nr_irqs, arg); + } + else +#endif return -ENOSYS; + } for (i = 0; i < nr_irqs; i++) { irq_data = irq_domain_get_irq_data(domain, virq + i); @@ -376,9 +570,54 @@ return err; } +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +void x86_vector_debug_show(struct seq_file *m, struct irq_domain *d, + struct irq_data *irqd, int ind) +{ + cpumask_var_t domain, old_domain; + unsigned int vec, prev_vec; + struct apic_chip_data *apicd; + unsigned long flags; + int irq; + + if (!irqd) + return; + + irq = irqd->irq; + if (irq < nr_legacy_irqs() && !test_bit(irq, &io_apic_irqs)) { + seq_printf(m, "%*sVector: %5d\n", ind, "", ISA_IRQ_VECTOR(irq)); + seq_printf(m, "%*sTarget: Legacy PIC all CPUs\n", ind, ""); + return; + } + + apicd = irqd->chip_data; + if (!apicd) { + seq_printf(m, "%*sVector: Not assigned\n", ind, ""); + return; + } + + raw_spin_lock_irqsave(&vector_lock, flags); + cpumask_copy(domain, apicd->domain); + cpumask_copy(old_domain, apicd->old_domain); + vec = apicd->cfg.vector; + prev_vec = apicd->cfg.old_vector; + raw_spin_unlock_irqrestore(&vector_lock, flags); + seq_printf(m, "%*sVector: %5u\n", ind, "", vec); + seq_printf(m, "%*sTarget: %*pbl\n", ind, "", cpumask_pr_args(domain)); + if (prev_vec) { + seq_printf(m, "%*sPrevious vector: %5u\n", ind, "", prev_vec); + seq_printf(m, "%*sPrevious target: %*pbl\n", ind, "", + cpumask_pr_args(old_domain)); + } +} +#endif + static const struct irq_domain_ops x86_vector_domain_ops = { - .alloc = x86_vector_alloc_irqs, - .free = x86_vector_free_irqs, + .alloc = x86_vector_alloc_irqs, + .free = x86_vector_free_irqs, +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS + .debug_show = x86_vector_debug_show, +#endif }; int __init arch_probe_nr_irqs(void) @@ -533,6 +772,15 @@ if (!cpumask_intersects(dest, cpu_online_mask)) return -EINVAL; +#ifdef CONFIG_INTEL_PCI_MULTI_MSI + if (irq_data->common && + irq_data->common->msi_desc && + irq_data->common->msi_desc->msi_attrib.multiple) { + /* TODO: setting cpu affinity for multiple msi isn't supported yet */ + return -ENOSYS; + } + else +#endif err = assign_irq_vector(irq, data, dest); return err ? err : IRQ_SET_MASK_OK; }