kernel/irq/ipi.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2015 Imagination Technologies Ltd
  * Author: Qais Yousef <qais.yousef@imgtec.com>
  *
  * This file contains driver APIs to the IPI subsystem.
  */

 #define pr_fmt(fmt) "genirq/ipi: " fmt

 #include <linux/irqdomain.h>
 #include <linux/irq.h>

 /**
  * irq_reserve_ipi() - Setup an IPI to destination cpumask
  * @domain:	IPI domain
  * @dest:	cpumask of CPUs which can receive the IPI
  *
  * Allocate a virq that can be used to send IPI to any CPU in dest mask.
  *
  * Return: Linux IRQ number on success or error code on failure
  */
 int irq_reserve_ipi(struct irq_domain *domain,
 			     const struct cpumask *dest)
 {
 	unsigned int nr_irqs, offset;
 	struct irq_data *data;
 	int virq, i;

 	if (!domain ||!irq_domain_is_ipi(domain)) {
 		pr_warn("Reservation on a non IPI domain\n");
 		return -EINVAL;
 	}

 	if (!cpumask_subset(dest, cpu_possible_mask)) {
 		pr_warn("Reservation is not in possible_cpu_mask\n");
 		return -EINVAL;
 	}

 	nr_irqs = cpumask_weight(dest);
 	if (!nr_irqs) {
 		pr_warn("Reservation for empty destination mask\n");
 		return -EINVAL;
 	}

 	if (irq_domain_is_ipi_single(domain)) {
 		/*
 		 * If the underlying implementation uses a single HW irq on
 		 * all cpus then we only need a single Linux irq number for
 		 * it. We have no restrictions vs. the destination mask. The
 		 * underlying implementation can deal with holes nicely.
 		 */
 		nr_irqs = 1;
 		offset = 0;
 	} else {
 		unsigned int next;

 		/*
 		 * The IPI requires a separate HW irq on each CPU. We require
 		 * that the destination mask is consecutive. If an
 		 * implementation needs to support holes, it can reserve
 		 * several IPI ranges.
 		 */
 		offset = cpumask_first(dest);
 		/*
 		 * Find a hole and if found look for another set bit after the
 		 * hole. For now we don't support this scenario.
 		 */
 		next = cpumask_next_zero(offset, dest);
 		if (next < nr_cpu_ids)
 			next = cpumask_next(next, dest);
 		if (next < nr_cpu_ids) {
 			pr_warn("Destination mask has holes\n");
 			return -EINVAL;
 		}
 	}

 	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
 	if (virq <= 0) {
 		pr_warn("Can't reserve IPI, failed to alloc descs\n");
 		return -ENOMEM;
 	}

 	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
 				       (void *) dest, true, NULL);

 	if (virq <= 0) {
 		pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
 		goto free_descs;
 	}

 	for (i = 0; i < nr_irqs; i++) {
 		data = irq_get_irq_data(virq + i);
 		cpumask_copy(data->common->affinity, dest);
 		data->common->ipi_offset = offset;
 		irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
 	}
 	return virq;

 free_descs:
 	irq_free_descs(virq, nr_irqs);
 	return -EBUSY;
 }

 /**
  * irq_destroy_ipi() - unreserve an IPI that was previously allocated
  * @irq:	Linux IRQ number to be destroyed
  * @dest:	cpumask of CPUs which should have the IPI removed
  *
  * The IPIs allocated with irq_reserve_ipi() are returned to the system
  * destroying all virqs associated with them.
  *
  * Return: %0 on success or error code on failure.
  */
 int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
 {
 	struct irq_data *data = irq_get_irq_data(irq);
 	const struct cpumask *ipimask;
 	struct irq_domain *domain;
 	unsigned int nr_irqs;

 	if (!irq || !data)
 		return -EINVAL;

 	domain = data->domain;
 	if (WARN_ON(domain == NULL))
 		return -EINVAL;

 	if (!irq_domain_is_ipi(domain)) {
 		pr_warn("Trying to destroy a non IPI domain!\n");
 		return -EINVAL;
 	}

 	ipimask = irq_data_get_affinity_mask(data);
 	if (!ipimask || WARN_ON(!cpumask_subset(dest, ipimask)))
 		/*
 		 * Must be destroying a subset of CPUs to which this IPI
 		 * was set up to target
 		 */
 		return -EINVAL;

 	if (irq_domain_is_ipi_per_cpu(domain)) {
 		irq = irq + cpumask_first(dest) - data->common->ipi_offset;
 		nr_irqs = cpumask_weight(dest);
 	} else {
 		nr_irqs = 1;
 	}

 	irq_domain_free_irqs(irq, nr_irqs);
 	return 0;
 }

 /**
  * ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU
  * @irq:	Linux IRQ number
  * @cpu:	the target CPU
  *
  * When dealing with coprocessors IPI, we need to inform the coprocessor of
  * the hwirq it needs to use to receive and send IPIs.
  *
  * Return: hwirq value on success or INVALID_HWIRQ on failure.
  */
 irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
 {
 	struct irq_data *data = irq_get_irq_data(irq);
 	const struct cpumask *ipimask;

 	if (!data || cpu >= nr_cpu_ids)
 		return INVALID_HWIRQ;

 	ipimask = irq_data_get_affinity_mask(data);
 	if (!ipimask || !cpumask_test_cpu(cpu, ipimask))
 		return INVALID_HWIRQ;

 	/*
 	 * Get the real hardware irq number if the underlying implementation
 	 * uses a separate irq per cpu. If the underlying implementation uses
 	 * a single hardware irq for all cpus then the IPI send mechanism
 	 * needs to take care of the cpu destinations.
 	 */
 	if (irq_domain_is_ipi_per_cpu(data->domain))
 		data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);

 	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
 }
 EXPORT_SYMBOL_GPL(ipi_get_hwirq);

 static int ipi_send_verify(struct irq_chip *chip, struct irq_data *data,
 			   const struct cpumask *dest, unsigned int cpu)
 {
 	const struct cpumask *ipimask;

 	if (!chip || !data)
 		return -EINVAL;

 	if (!chip->ipi_send_single && !chip->ipi_send_mask)
 		return -EINVAL;

 	if (cpu >= nr_cpu_ids)
 		return -EINVAL;

 	ipimask = irq_data_get_affinity_mask(data);
 	if (!ipimask)
 		return -EINVAL;

 	if (dest) {
 		if (!cpumask_subset(dest, ipimask))
 			return -EINVAL;
 	} else {
 		if (!cpumask_test_cpu(cpu, ipimask))
 			return -EINVAL;
 	}
 	return 0;
 }

 /**
  * __ipi_send_single - send an IPI to a target Linux SMP CPU
  * @desc:	pointer to irq_desc of the IRQ
  * @cpu:	destination CPU, must in the destination mask passed to
  *		irq_reserve_ipi()
  *
  * This function is for architecture or core code to speed up IPI sending. Not
  * usable from driver code.
  *
  * Return: %0 on success or negative error number on failure.
  */
 int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
 {
 	struct irq_data *data = irq_desc_get_irq_data(desc);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);

 #ifdef DEBUG
 	/*
 	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
 	 * Since the callers should be arch or core code which is generally
 	 * trusted, only check for errors when debugging.
 	 */
 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
 		return -EINVAL;
 #endif
 	if (!chip->ipi_send_single) {
 		chip->ipi_send_mask(data, cpumask_of(cpu));
 		return 0;
 	}

 	/* FIXME: Store this information in irqdata flags */
 	if (irq_domain_is_ipi_per_cpu(data->domain) &&
 	    cpu != data->common->ipi_offset) {
 		/* use the correct data for that cpu */
 		unsigned irq = data->irq + cpu - data->common->ipi_offset;

 		data = irq_get_irq_data(irq);
 	}
 	chip->ipi_send_single(data, cpu);
 	return 0;
 }

 /**
  * __ipi_send_mask - send an IPI to target Linux SMP CPU(s)
  * @desc:	pointer to irq_desc of the IRQ
  * @dest:	dest CPU(s), must be a subset of the mask passed to
  *		irq_reserve_ipi()
  *
  * This function is for architecture or core code to speed up IPI sending. Not
  * usable from driver code.
  *
  * Return: %0 on success or negative error number on failure.
  */
 int __ipi_send_mask(struct irq_desc *desc, const struct cpumask *dest)
 {
 	struct irq_data *data = irq_desc_get_irq_data(desc);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	unsigned int cpu;

 #ifdef DEBUG
 	/*
 	 * Minimise the overhead by omitting the checks for Linux SMP IPIs.
 	 * Since the callers should be arch or core code which is generally
 	 * trusted, only check for errors when debugging.
 	 */
 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
 		return -EINVAL;
 #endif
 	if (chip->ipi_send_mask) {
 		chip->ipi_send_mask(data, dest);
 		return 0;
 	}

 	if (irq_domain_is_ipi_per_cpu(data->domain)) {
 		unsigned int base = data->irq;

 		for_each_cpu(cpu, dest) {
 			unsigned irq = base + cpu - data->common->ipi_offset;

 			data = irq_get_irq_data(irq);
 			chip->ipi_send_single(data, cpu);
 		}
 	} else {
 		for_each_cpu(cpu, dest)
 			chip->ipi_send_single(data, cpu);
 	}
 	return 0;
 }

 /**
  * ipi_send_single - Send an IPI to a single CPU
  * @virq:	Linux IRQ number from irq_reserve_ipi()
  * @cpu:	destination CPU, must in the destination mask passed to
  *		irq_reserve_ipi()
  *
  * Return: %0 on success or negative error number on failure.
  */
 int ipi_send_single(unsigned int virq, unsigned int cpu)
 {
 	struct irq_desc *desc = irq_to_desc(virq);
 	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
 	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;

 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
 		return -EINVAL;

 	return __ipi_send_single(desc, cpu);
 }
 EXPORT_SYMBOL_GPL(ipi_send_single);

 /**
  * ipi_send_mask - Send an IPI to target CPU(s)
  * @virq:	Linux IRQ number from irq_reserve_ipi()
  * @dest:	dest CPU(s), must be a subset of the mask passed to
  *		irq_reserve_ipi()
  *
  * Return: %0 on success or negative error number on failure.
  */
 int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
 {
 	struct irq_desc *desc = irq_to_desc(virq);
 	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
 	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;

 	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
 		return -EINVAL;

 	return __ipi_send_mask(desc, dest);
 }
 EXPORT_SYMBOL_GPL(ipi_send_mask);
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (C) 2015 Imagination Technologies Ltd
	* Author: Qais Yousef <qais.yousef@imgtec.com>
	*
	* This file contains driver APIs to the IPI subsystem.
	*/

	#define pr_fmt(fmt) "genirq/ipi: " fmt

	#include <linux/irqdomain.h>
	#include <linux/irq.h>

	/**
	* irq_reserve_ipi() - Setup an IPI to destination cpumask
	* @domain: IPI domain
	* @dest: cpumask of CPUs which can receive the IPI
	*
	* Allocate a virq that can be used to send IPI to any CPU in dest mask.
	*
	* Return: Linux IRQ number on success or error code on failure
	*/
	int irq_reserve_ipi(struct irq_domain *domain,
	const struct cpumask *dest)
	{
	unsigned int nr_irqs, offset;
	struct irq_data *data;
	int virq, i;

	if (!domain \|\|!irq_domain_is_ipi(domain)) {
	pr_warn("Reservation on a non IPI domain\n");
	return -EINVAL;
	}

	if (!cpumask_subset(dest, cpu_possible_mask)) {
	pr_warn("Reservation is not in possible_cpu_mask\n");
	return -EINVAL;
	}

	nr_irqs = cpumask_weight(dest);
	if (!nr_irqs) {
	pr_warn("Reservation for empty destination mask\n");
	return -EINVAL;
	}

	if (irq_domain_is_ipi_single(domain)) {
	/*
	* If the underlying implementation uses a single HW irq on
	* all cpus then we only need a single Linux irq number for
	* it. We have no restrictions vs. the destination mask. The
	* underlying implementation can deal with holes nicely.
	*/
	nr_irqs = 1;
	offset = 0;
	} else {
	unsigned int next;

	/*
	* The IPI requires a separate HW irq on each CPU. We require
	* that the destination mask is consecutive. If an
	* implementation needs to support holes, it can reserve
	* several IPI ranges.
	*/
	offset = cpumask_first(dest);
	/*
	* Find a hole and if found look for another set bit after the
	* hole. For now we don't support this scenario.
	*/
	next = cpumask_next_zero(offset, dest);
	if (next < nr_cpu_ids)
	next = cpumask_next(next, dest);
	if (next < nr_cpu_ids) {
	pr_warn("Destination mask has holes\n");
	return -EINVAL;
	}
	}

	virq = irq_domain_alloc_descs(-1, nr_irqs, 0, NUMA_NO_NODE, NULL);
	if (virq <= 0) {
	pr_warn("Can't reserve IPI, failed to alloc descs\n");
	return -ENOMEM;
	}

	virq = __irq_domain_alloc_irqs(domain, virq, nr_irqs, NUMA_NO_NODE,
	(void *) dest, true, NULL);

	if (virq <= 0) {
	pr_warn("Can't reserve IPI, failed to alloc hw irqs\n");
	goto free_descs;
	}

	for (i = 0; i < nr_irqs; i++) {
	data = irq_get_irq_data(virq + i);
	cpumask_copy(data->common->affinity, dest);
	data->common->ipi_offset = offset;
	irq_set_status_flags(virq + i, IRQ_NO_BALANCING);
	}
	return virq;

	free_descs:
	irq_free_descs(virq, nr_irqs);
	return -EBUSY;
	}

	/**
	* irq_destroy_ipi() - unreserve an IPI that was previously allocated
	* @irq: Linux IRQ number to be destroyed
	* @dest: cpumask of CPUs which should have the IPI removed
	*
	* The IPIs allocated with irq_reserve_ipi() are returned to the system
	* destroying all virqs associated with them.
	*
	* Return: %0 on success or error code on failure.
	*/
	int irq_destroy_ipi(unsigned int irq, const struct cpumask *dest)
	{
	struct irq_data *data = irq_get_irq_data(irq);
	const struct cpumask *ipimask;
	struct irq_domain *domain;
	unsigned int nr_irqs;

	if (!irq \|\| !data)
	return -EINVAL;

	domain = data->domain;
	if (WARN_ON(domain == NULL))
	return -EINVAL;

	if (!irq_domain_is_ipi(domain)) {
	pr_warn("Trying to destroy a non IPI domain!\n");
	return -EINVAL;
	}

	ipimask = irq_data_get_affinity_mask(data);
	if (!ipimask \|\| WARN_ON(!cpumask_subset(dest, ipimask)))
	/*
	* Must be destroying a subset of CPUs to which this IPI
	* was set up to target
	*/
	return -EINVAL;

	if (irq_domain_is_ipi_per_cpu(domain)) {
	irq = irq + cpumask_first(dest) - data->common->ipi_offset;
	nr_irqs = cpumask_weight(dest);
	} else {
	nr_irqs = 1;
	}

	irq_domain_free_irqs(irq, nr_irqs);
	return 0;
	}

	/**
	* ipi_get_hwirq - Get the hwirq associated with an IPI to a CPU
	* @irq: Linux IRQ number
	* @cpu: the target CPU
	*
	* When dealing with coprocessors IPI, we need to inform the coprocessor of
	* the hwirq it needs to use to receive and send IPIs.
	*
	* Return: hwirq value on success or INVALID_HWIRQ on failure.
	*/
	irq_hw_number_t ipi_get_hwirq(unsigned int irq, unsigned int cpu)
	{
	struct irq_data *data = irq_get_irq_data(irq);
	const struct cpumask *ipimask;

	if (!data \|\| cpu >= nr_cpu_ids)
	return INVALID_HWIRQ;

	ipimask = irq_data_get_affinity_mask(data);
	if (!ipimask \|\| !cpumask_test_cpu(cpu, ipimask))
	return INVALID_HWIRQ;

	/*
	* Get the real hardware irq number if the underlying implementation
	* uses a separate irq per cpu. If the underlying implementation uses
	* a single hardware irq for all cpus then the IPI send mechanism
	* needs to take care of the cpu destinations.
	*/
	if (irq_domain_is_ipi_per_cpu(data->domain))
	data = irq_get_irq_data(irq + cpu - data->common->ipi_offset);

	return data ? irqd_to_hwirq(data) : INVALID_HWIRQ;
	}
	EXPORT_SYMBOL_GPL(ipi_get_hwirq);

	static int ipi_send_verify(struct irq_chip chip, struct irq_data data,
	const struct cpumask *dest, unsigned int cpu)
	{
	const struct cpumask *ipimask;

	if (!chip \|\| !data)
	return -EINVAL;

	if (!chip->ipi_send_single && !chip->ipi_send_mask)
	return -EINVAL;

	if (cpu >= nr_cpu_ids)
	return -EINVAL;

	ipimask = irq_data_get_affinity_mask(data);
	if (!ipimask)
	return -EINVAL;

	if (dest) {
	if (!cpumask_subset(dest, ipimask))
	return -EINVAL;
	} else {
	if (!cpumask_test_cpu(cpu, ipimask))
	return -EINVAL;
	}
	return 0;
	}

	/**
	* __ipi_send_single - send an IPI to a target Linux SMP CPU
	* @desc: pointer to irq_desc of the IRQ
	* @cpu: destination CPU, must in the destination mask passed to
	* irq_reserve_ipi()
	*
	* This function is for architecture or core code to speed up IPI sending. Not
	* usable from driver code.
	*
	* Return: %0 on success or negative error number on failure.
	*/
	int __ipi_send_single(struct irq_desc *desc, unsigned int cpu)
	{
	struct irq_data *data = irq_desc_get_irq_data(desc);
	struct irq_chip *chip = irq_data_get_irq_chip(data);

	#ifdef DEBUG
	/*
	* Minimise the overhead by omitting the checks for Linux SMP IPIs.
	* Since the callers should be arch or core code which is generally
	* trusted, only check for errors when debugging.
	*/
	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
	return -EINVAL;
	#endif
	if (!chip->ipi_send_single) {
	chip->ipi_send_mask(data, cpumask_of(cpu));
	return 0;
	}

	/* FIXME: Store this information in irqdata flags */
	if (irq_domain_is_ipi_per_cpu(data->domain) &&
	cpu != data->common->ipi_offset) {
	/* use the correct data for that cpu */
	unsigned irq = data->irq + cpu - data->common->ipi_offset;

	data = irq_get_irq_data(irq);
	}
	chip->ipi_send_single(data, cpu);
	return 0;
	}

	/**
	* __ipi_send_mask - send an IPI to target Linux SMP CPU(s)
	* @desc: pointer to irq_desc of the IRQ
	* @dest: dest CPU(s), must be a subset of the mask passed to
	* irq_reserve_ipi()
	*
	* This function is for architecture or core code to speed up IPI sending. Not
	* usable from driver code.
	*
	* Return: %0 on success or negative error number on failure.
	*/
	int __ipi_send_mask(struct irq_desc desc, const struct cpumask dest)
	{
	struct irq_data *data = irq_desc_get_irq_data(desc);
	struct irq_chip *chip = irq_data_get_irq_chip(data);
	unsigned int cpu;

	#ifdef DEBUG
	/*
	* Minimise the overhead by omitting the checks for Linux SMP IPIs.
	* Since the callers should be arch or core code which is generally
	* trusted, only check for errors when debugging.
	*/
	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
	return -EINVAL;
	#endif
	if (chip->ipi_send_mask) {
	chip->ipi_send_mask(data, dest);
	return 0;
	}

	if (irq_domain_is_ipi_per_cpu(data->domain)) {
	unsigned int base = data->irq;

	for_each_cpu(cpu, dest) {
	unsigned irq = base + cpu - data->common->ipi_offset;

	data = irq_get_irq_data(irq);
	chip->ipi_send_single(data, cpu);
	}
	} else {
	for_each_cpu(cpu, dest)
	chip->ipi_send_single(data, cpu);
	}
	return 0;
	}

	/**
	* ipi_send_single - Send an IPI to a single CPU
	* @virq: Linux IRQ number from irq_reserve_ipi()
	* @cpu: destination CPU, must in the destination mask passed to
	* irq_reserve_ipi()
	*
	* Return: %0 on success or negative error number on failure.
	*/
	int ipi_send_single(unsigned int virq, unsigned int cpu)
	{
	struct irq_desc *desc = irq_to_desc(virq);
	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;

	if (WARN_ON_ONCE(ipi_send_verify(chip, data, NULL, cpu)))
	return -EINVAL;

	return __ipi_send_single(desc, cpu);
	}
	EXPORT_SYMBOL_GPL(ipi_send_single);

	/**
	* ipi_send_mask - Send an IPI to target CPU(s)
	* @virq: Linux IRQ number from irq_reserve_ipi()
	* @dest: dest CPU(s), must be a subset of the mask passed to
	* irq_reserve_ipi()
	*
	* Return: %0 on success or negative error number on failure.
	*/
	int ipi_send_mask(unsigned int virq, const struct cpumask *dest)
	{
	struct irq_desc *desc = irq_to_desc(virq);
	struct irq_data *data = desc ? irq_desc_get_irq_data(desc) : NULL;
	struct irq_chip *chip = data ? irq_data_get_irq_chip(data) : NULL;

	if (WARN_ON_ONCE(ipi_send_verify(chip, data, dest, 0)))
	return -EINVAL;

	return __ipi_send_mask(desc, dest);
	}
	EXPORT_SYMBOL_GPL(ipi_send_mask);