stop_machine: add ALL_CPUS option
-allow stop_mahcine_run() to call a function on all cpus. Calling
stop_machine_run() with a 'ALL_CPUS' invokes this new behavior.
stop_machine_run() proceeds as normal until the calling cpu has
invoked 'fn'. Then, we tell all the other cpus to call 'fn'.
Signed-off-by: Jason Baron <jbaron@redhat.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
CC: Adrian Bunk <bunk@stusta.de>
CC: Andi Kleen <andi@firstfloor.org>
CC: Alexey Dobriyan <adobriyan@gmail.com>
CC: Christoph Hellwig <hch@infradead.org>
CC: mingo@elte.hu
CC: akpm@osdl.org
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 738b411..a473bd0 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -22,9 +22,17 @@
STOPMACHINE_WAIT,
STOPMACHINE_PREPARE,
STOPMACHINE_DISABLE_IRQ,
+ STOPMACHINE_RUN,
STOPMACHINE_EXIT,
};
+struct stop_machine_data {
+ int (*fn)(void *);
+ void *data;
+ struct completion done;
+ int run_all;
+} smdata;
+
static enum stopmachine_state stopmachine_state;
static unsigned int stopmachine_num_threads;
static atomic_t stopmachine_thread_ack;
@@ -33,6 +41,7 @@
{
int irqs_disabled = 0;
int prepared = 0;
+ int ran = 0;
cpumask_of_cpu_ptr(cpumask, (int)(long)cpu);
set_cpus_allowed_ptr(current, cpumask);
@@ -58,6 +67,11 @@
prepared = 1;
smp_mb(); /* Must read state first. */
atomic_inc(&stopmachine_thread_ack);
+ } else if (stopmachine_state == STOPMACHINE_RUN && !ran) {
+ smdata.fn(smdata.data);
+ ran = 1;
+ smp_mb(); /* Must read state first. */
+ atomic_inc(&stopmachine_thread_ack);
}
/* Yield in first stage: migration threads need to
* help our sisters onto their CPUs. */
@@ -136,11 +150,10 @@
preempt_enable_no_resched();
}
-struct stop_machine_data {
- int (*fn)(void *);
- void *data;
- struct completion done;
-};
+static void run_other_cpus(void)
+{
+ stopmachine_set_state(STOPMACHINE_RUN);
+}
static int do_stop(void *_smdata)
{
@@ -150,6 +163,8 @@
ret = stop_machine();
if (ret == 0) {
ret = smdata->fn(smdata->data);
+ if (smdata->run_all)
+ run_other_cpus();
restart_machine();
}
@@ -173,14 +188,17 @@
struct stop_machine_data smdata;
struct task_struct *p;
- smdata.fn = fn;
- smdata.data = data;
- init_completion(&smdata.done);
-
mutex_lock(&stopmachine_mutex);
+ smdata.fn = fn;
+ smdata.data = data;
+ smdata.run_all = (cpu == ALL_CPUS) ? 1 : 0;
+ init_completion(&smdata.done);
+
+ smp_wmb(); /* make sure other cpus see smdata updates */
+
/* If they don't care which CPU fn runs on, bind to any online one. */
- if (cpu == NR_CPUS)
+ if (cpu == NR_CPUS || cpu == ALL_CPUS)
cpu = raw_smp_processor_id();
p = kthread_create(do_stop, &smdata, "kstopmachine");
