Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 1 | /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. |
Rusty Russell | e5582ca | 2006-09-29 02:01:35 -0700 | [diff] [blame] | 2 | * GPL v2 and any later version. |
| 3 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 4 | #include <linux/cpu.h> |
| 5 | #include <linux/err.h> |
Prarit Bhargava | ee527cd | 2007-05-08 00:25:08 -0700 | [diff] [blame] | 6 | #include <linux/kthread.h> |
| 7 | #include <linux/module.h> |
| 8 | #include <linux/sched.h> |
| 9 | #include <linux/stop_machine.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | #include <linux/syscalls.h> |
Benjamin Herrenschmidt | a12bb44 | 2007-05-10 22:22:47 -0700 | [diff] [blame] | 11 | #include <linux/interrupt.h> |
| 12 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 13 | #include <asm/atomic.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 14 | #include <asm/uaccess.h> |
| 15 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 16 | /* This controls the threads on each CPU. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 17 | enum stopmachine_state { |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 18 | /* Dummy starting state for thread. */ |
| 19 | STOPMACHINE_NONE, |
| 20 | /* Awaiting everyone to be scheduled. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | STOPMACHINE_PREPARE, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 22 | /* Disable interrupts. */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 23 | STOPMACHINE_DISABLE_IRQ, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 24 | /* Run the function */ |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 25 | STOPMACHINE_RUN, |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 26 | /* Exit */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 27 | STOPMACHINE_EXIT, |
| 28 | }; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 29 | static enum stopmachine_state state; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 31 | struct stop_machine_data { |
| 32 | int (*fn)(void *); |
| 33 | void *data; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 34 | int fnret; |
| 35 | }; |
Jason Baron | 5c2aed6 | 2008-02-28 11:33:03 -0500 | [diff] [blame] | 36 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 37 | /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ |
| 38 | static unsigned int num_threads; |
| 39 | static atomic_t thread_ack; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 40 | static DEFINE_MUTEX(lock); |
Heiko Carstens | 9ea09af | 2008-12-22 12:36:30 +0100 | [diff] [blame] | 41 | /* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */ |
| 42 | static DEFINE_MUTEX(setup_lock); |
| 43 | /* Users of stop_machine. */ |
| 44 | static int refcount; |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 45 | static struct workqueue_struct *stop_machine_wq; |
| 46 | static struct stop_machine_data active, idle; |
| 47 | static const cpumask_t *active_cpus; |
| 48 | static void *stop_machine_work; |
| 49 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 50 | static void set_state(enum stopmachine_state newstate) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 51 | { |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 52 | /* Reset ack counter. */ |
| 53 | atomic_set(&thread_ack, num_threads); |
| 54 | smp_wmb(); |
| 55 | state = newstate; |
| 56 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 58 | /* Last one to ack a state moves to the next state. */ |
| 59 | static void ack_state(void) |
| 60 | { |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 61 | if (atomic_dec_and_test(&thread_ack)) |
| 62 | set_state(state + 1); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 63 | } |
Andrew Morton | d8cb7c1 | 2006-07-03 17:32:22 -0700 | [diff] [blame] | 64 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 65 | /* This is the actual function which stops the CPU. It runs |
| 66 | * in the context of a dedicated stopmachine workqueue. */ |
| 67 | static void stop_cpu(struct work_struct *unused) |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 68 | { |
| 69 | enum stopmachine_state curstate = STOPMACHINE_NONE; |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 70 | struct stop_machine_data *smdata = &idle; |
| 71 | int cpu = smp_processor_id(); |
Heiko Carstens | 8163bca | 2008-10-22 10:00:26 -0500 | [diff] [blame] | 72 | int err; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 73 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 74 | if (!active_cpus) { |
Rusty Russell | 41c7bb9 | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 75 | if (cpu == cpumask_first(cpu_online_mask)) |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 76 | smdata = &active; |
| 77 | } else { |
Rusty Russell | 41c7bb9 | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 78 | if (cpumask_test_cpu(cpu, active_cpus)) |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 79 | smdata = &active; |
| 80 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 81 | /* Simple state machine */ |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 82 | do { |
| 83 | /* Chill out and ensure we re-read stopmachine_state. */ |
Christian Borntraeger | 3401a61e | 2008-05-08 15:20:38 +0200 | [diff] [blame] | 84 | cpu_relax(); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 85 | if (state != curstate) { |
| 86 | curstate = state; |
| 87 | switch (curstate) { |
| 88 | case STOPMACHINE_DISABLE_IRQ: |
| 89 | local_irq_disable(); |
| 90 | hard_irq_disable(); |
| 91 | break; |
| 92 | case STOPMACHINE_RUN: |
Heiko Carstens | 8163bca | 2008-10-22 10:00:26 -0500 | [diff] [blame] | 93 | /* On multiple CPUs only a single error code |
| 94 | * is needed to tell that something failed. */ |
| 95 | err = smdata->fn(smdata->data); |
| 96 | if (err) |
| 97 | smdata->fnret = err; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 98 | break; |
| 99 | default: |
| 100 | break; |
| 101 | } |
| 102 | ack_state(); |
| 103 | } |
| 104 | } while (curstate != STOPMACHINE_EXIT); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 105 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 106 | local_irq_enable(); |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 107 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 109 | /* Callback for CPUs which aren't supposed to do anything. */ |
| 110 | static int chill(void *unused) |
| 111 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 112 | return 0; |
| 113 | } |
| 114 | |
Heiko Carstens | 9ea09af | 2008-12-22 12:36:30 +0100 | [diff] [blame] | 115 | int stop_machine_create(void) |
| 116 | { |
| 117 | mutex_lock(&setup_lock); |
| 118 | if (refcount) |
| 119 | goto done; |
| 120 | stop_machine_wq = create_rt_workqueue("kstop"); |
| 121 | if (!stop_machine_wq) |
| 122 | goto err_out; |
| 123 | stop_machine_work = alloc_percpu(struct work_struct); |
| 124 | if (!stop_machine_work) |
| 125 | goto err_out; |
| 126 | done: |
| 127 | refcount++; |
| 128 | mutex_unlock(&setup_lock); |
| 129 | return 0; |
| 130 | |
| 131 | err_out: |
| 132 | if (stop_machine_wq) |
| 133 | destroy_workqueue(stop_machine_wq); |
| 134 | mutex_unlock(&setup_lock); |
| 135 | return -ENOMEM; |
| 136 | } |
| 137 | EXPORT_SYMBOL_GPL(stop_machine_create); |
| 138 | |
| 139 | void stop_machine_destroy(void) |
| 140 | { |
| 141 | mutex_lock(&setup_lock); |
| 142 | refcount--; |
| 143 | if (refcount) |
| 144 | goto done; |
| 145 | destroy_workqueue(stop_machine_wq); |
| 146 | free_percpu(stop_machine_work); |
| 147 | done: |
| 148 | mutex_unlock(&setup_lock); |
| 149 | } |
| 150 | EXPORT_SYMBOL_GPL(stop_machine_destroy); |
| 151 | |
Rusty Russell | 41c7bb9 | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 152 | int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | { |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 154 | struct work_struct *sm_work; |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 155 | int i, ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 156 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 157 | /* Set up initial state. */ |
| 158 | mutex_lock(&lock); |
| 159 | num_threads = num_online_cpus(); |
| 160 | active_cpus = cpus; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 161 | active.fn = fn; |
| 162 | active.data = data; |
| 163 | active.fnret = 0; |
| 164 | idle.fn = chill; |
| 165 | idle.data = NULL; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 167 | set_state(STOPMACHINE_PREPARE); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 168 | |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 169 | /* Schedule the stop_cpu work on all cpus: hold this CPU so one |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 170 | * doesn't hit this CPU until we're ready. */ |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 171 | get_cpu(); |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 172 | for_each_online_cpu(i) { |
| 173 | sm_work = percpu_ptr(stop_machine_work, i); |
| 174 | INIT_WORK(sm_work, stop_cpu); |
| 175 | queue_work_on(i, stop_machine_wq, sm_work); |
| 176 | } |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 177 | /* This will release the thread on our CPU. */ |
| 178 | put_cpu(); |
Heiko Carstens | c9583e5 | 2008-10-13 23:50:10 +0200 | [diff] [blame] | 179 | flush_workqueue(stop_machine_wq); |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 180 | ret = active.fnret; |
Rusty Russell | ffdb597 | 2008-07-28 12:16:28 -0500 | [diff] [blame] | 181 | mutex_unlock(&lock); |
Rusty Russell | e14c8bf | 2008-11-17 08:22:18 +1030 | [diff] [blame] | 182 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 183 | } |
| 184 | |
Rusty Russell | 41c7bb9 | 2009-01-01 10:12:28 +1030 | [diff] [blame] | 185 | int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 186 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 187 | int ret; |
| 188 | |
Heiko Carstens | 9ea09af | 2008-12-22 12:36:30 +0100 | [diff] [blame] | 189 | ret = stop_machine_create(); |
| 190 | if (ret) |
| 191 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 192 | /* No CPUs can come up or down during this. */ |
Gautham R Shenoy | 86ef5c9 | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 193 | get_online_cpus(); |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 194 | ret = __stop_machine(fn, data, cpus); |
Gautham R Shenoy | 86ef5c9 | 2008-01-25 21:08:02 +0100 | [diff] [blame] | 195 | put_online_cpus(); |
Heiko Carstens | 9ea09af | 2008-12-22 12:36:30 +0100 | [diff] [blame] | 196 | stop_machine_destroy(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | return ret; |
| 198 | } |
Rusty Russell | eeec4fa | 2008-07-28 12:16:30 -0500 | [diff] [blame] | 199 | EXPORT_SYMBOL_GPL(stop_machine); |