File: /usr/src/linux/arch/ia64/kernel/smp.c
1 /*
2 * SMP Support
3 *
4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
5 * Copyright (C) 1999, 2001 David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * Lots of stuff stolen from arch/alpha/kernel/smp.c
8 *
9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized
10 * the existing code (on the lines of x86 port).
11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
12 * calibration on each CPU.
13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor
15 * & cpu_online_map now gets done here (instead of setup.c)
16 * 99/10/05 davidm Update to bring it in sync with new command-line processing
17 * scheme.
18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
19 * smp_call_function_single to resend IPI on timeouts
20 */
21 #define __KERNEL_SYSCALLS__
22
23 #include <linux/config.h>
24
25 #include <linux/kernel.h>
26 #include <linux/sched.h>
27 #include <linux/init.h>
28 #include <linux/interrupt.h>
29 #include <linux/smp.h>
30 #include <linux/kernel_stat.h>
31 #include <linux/mm.h>
32 #include <linux/delay.h>
33
34 #include <asm/atomic.h>
35 #include <asm/bitops.h>
36 #include <asm/current.h>
37 #include <asm/delay.h>
38 #include <asm/efi.h>
39 #include <asm/machvec.h>
40
41 #include <asm/io.h>
42 #include <asm/irq.h>
43 #include <asm/page.h>
44 #include <asm/pgalloc.h>
45 #include <asm/pgtable.h>
46 #include <asm/processor.h>
47 #include <asm/ptrace.h>
48 #include <asm/sal.h>
49 #include <asm/system.h>
50 #include <asm/unistd.h>
51
52 /* The 'big kernel lock' */
53 spinlock_t kernel_flag = SPIN_LOCK_UNLOCKED;
54
55 /*
56 * Structure and data for smp_call_function(). This is designed to minimise static memory
57 * requirements. It also looks cleaner.
58 */
59 static spinlock_t call_lock = SPIN_LOCK_UNLOCKED;
60
61 struct call_data_struct {
62 void (*func) (void *info);
63 void *info;
64 long wait;
65 atomic_t started;
66 atomic_t finished;
67 };
68
69 static volatile struct call_data_struct *call_data;
70
71 #define IPI_CALL_FUNC 0
72 #define IPI_CPU_STOP 1
73 #ifndef CONFIG_ITANIUM_PTCG
74 # define IPI_FLUSH_TLB 2
75 #endif /*!CONFIG_ITANIUM_PTCG */
76
77 static void
78 stop_this_cpu (void)
79 {
80 /*
81 * Remove this CPU:
82 */
83 clear_bit(smp_processor_id(), &cpu_online_map);
84 max_xtp();
85 __cli();
86 for (;;);
87 }
88
89 void
90 handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
91 {
92 int this_cpu = smp_processor_id();
93 unsigned long *pending_ipis = &local_cpu_data->ipi_operation;
94 unsigned long ops;
95
96 /* Count this now; we may make a call that never returns. */
97 local_cpu_data->ipi_count++;
98
99 mb(); /* Order interrupt and bit testing. */
100 while ((ops = xchg(pending_ipis, 0)) != 0) {
101 mb(); /* Order bit clearing and data access. */
102 do {
103 unsigned long which;
104
105 which = ffz(~ops);
106 ops &= ~(1 << which);
107
108 switch (which) {
109 case IPI_CALL_FUNC:
110 {
111 struct call_data_struct *data;
112 void (*func)(void *info);
113 void *info;
114 int wait;
115
116 /* release the 'pointer lock' */
117 data = (struct call_data_struct *) call_data;
118 func = data->func;
119 info = data->info;
120 wait = data->wait;
121
122 mb();
123 atomic_inc(&data->started);
124
125 /* At this point the structure may be gone unless wait is true. */
126 (*func)(info);
127
128 /* Notify the sending CPU that the task is done. */
129 mb();
130 if (wait)
131 atomic_inc(&data->finished);
132 }
133 break;
134
135 case IPI_CPU_STOP:
136 stop_this_cpu();
137 break;
138
139 #ifndef CONFIG_ITANIUM_PTCG
140 case IPI_FLUSH_TLB:
141 {
142 extern unsigned long flush_start, flush_end, flush_nbits, flush_rid;
143 extern atomic_t flush_cpu_count;
144 unsigned long saved_rid = ia64_get_rr(flush_start);
145 unsigned long end = flush_end;
146 unsigned long start = flush_start;
147 unsigned long nbits = flush_nbits;
148
149 /*
150 * Current CPU may be running with different RID so we need to
151 * reload the RID of flushed address. Purging the translation
152 * also needs ALAT invalidation; we do not need "invala" here
153 * since it is done in ia64_leave_kernel.
154 */
155 ia64_srlz_d();
156 if (saved_rid != flush_rid) {
157 ia64_set_rr(flush_start, flush_rid);
158 ia64_srlz_d();
159 }
160
161 do {
162 /*
163 * Purge local TLB entries.
164 */
165 __asm__ __volatile__ ("ptc.l %0,%1" ::
166 "r"(start), "r"(nbits<<2) : "memory");
167 start += (1UL << nbits);
168 } while (start < end);
169
170 ia64_insn_group_barrier();
171 ia64_srlz_i(); /* srlz.i implies srlz.d */
172
173 if (saved_rid != flush_rid) {
174 ia64_set_rr(flush_start, saved_rid);
175 ia64_srlz_d();
176 }
177 atomic_dec(&flush_cpu_count);
178 break;
179 }
180 #endif /* !CONFIG_ITANIUM_PTCG */
181
182 default:
183 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", this_cpu, which);
184 break;
185 } /* Switch */
186 } while (ops);
187
188 mb(); /* Order data access and bit testing. */
189 }
190 }
191
192 static inline void
193 send_IPI_single (int dest_cpu, int op)
194 {
195 set_bit(op, &cpu_data(dest_cpu)->ipi_operation);
196 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
197 }
198
199 static inline void
200 send_IPI_allbutself (int op)
201 {
202 int i;
203
204 for (i = 0; i < smp_num_cpus; i++) {
205 if (i != smp_processor_id())
206 send_IPI_single(i, op);
207 }
208 }
209
210 static inline void
211 send_IPI_all (int op)
212 {
213 int i;
214
215 for (i = 0; i < smp_num_cpus; i++)
216 send_IPI_single(i, op);
217 }
218
219 static inline void
220 send_IPI_self (int op)
221 {
222 send_IPI_single(smp_processor_id(), op);
223 }
224
225 void
226 smp_send_reschedule (int cpu)
227 {
228 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
229 }
230
231 #ifndef CONFIG_ITANIUM_PTCG
232
233 void
234 smp_send_flush_tlb (void)
235 {
236 send_IPI_allbutself(IPI_FLUSH_TLB);
237 }
238
239 void
240 smp_resend_flush_tlb (void)
241 {
242 int i;
243
244 /*
245 * Really need a null IPI but since this rarely should happen & since this code
246 * will go away, lets not add one.
247 */
248 for (i = 0; i < smp_num_cpus; ++i)
249 if (i != smp_processor_id())
250 smp_send_reschedule(i);
251 }
252
253 #endif /* !CONFIG_ITANIUM_PTCG */
254
255 void
256 smp_flush_tlb_all (void)
257 {
258 smp_call_function ((void (*)(void *))__flush_tlb_all,0,1,1);
259 __flush_tlb_all();
260 }
261
262 /*
263 * Run a function on another CPU
264 * <func> The function to run. This must be fast and non-blocking.
265 * <info> An arbitrary pointer to pass to the function.
266 * <nonatomic> Currently unused.
267 * <wait> If true, wait until function has completed on other CPUs.
268 * [RETURNS] 0 on success, else a negative status code.
269 *
270 * Does not return until the remote CPU is nearly ready to execute <func>
271 * or is or has executed.
272 */
273
274 int
275 smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic,
276 int wait)
277 {
278 struct call_data_struct data;
279 int cpus = 1;
280 #if (defined(CONFIG_ITANIUM_B0_SPECIFIC) \
281 || defined(CONFIG_ITANIUM_B1_SPECIFIC) || defined(CONFIG_ITANIUM_B2_SPECIFIC))
282 unsigned long timeout;
283 #endif
284
285 if (cpuid == smp_processor_id()) {
286 printk(__FUNCTION__" trying to call self\n");
287 return -EBUSY;
288 }
289
290 data.func = func;
291 data.info = info;
292 atomic_set(&data.started, 0);
293 data.wait = wait;
294 if (wait)
295 atomic_set(&data.finished, 0);
296
297 spin_lock_bh(&call_lock);
298 call_data = &data;
299
300 #if (defined(CONFIG_ITANIUM_B0_SPECIFIC) \
301 || defined(CONFIG_ITANIUM_B1_SPECIFIC) || defined(CONFIG_ITANIUM_B2_SPECIFIC))
302 resend:
303 send_IPI_single(cpuid, IPI_CALL_FUNC);
304
305 /* Wait for response */
306 timeout = jiffies + HZ;
307 while ((atomic_read(&data.started) != cpus) && time_before(jiffies, timeout))
308 barrier();
309 if (atomic_read(&data.started) != cpus)
310 goto resend;
311 #else
312 send_IPI_single(cpuid, IPI_CALL_FUNC);
313
314 /* Wait for response */
315 while (atomic_read(&data.started) != cpus)
316 barrier();
317 #endif
318 if (wait)
319 while (atomic_read(&data.finished) != cpus)
320 barrier();
321 call_data = NULL;
322
323 spin_unlock_bh(&call_lock);
324 return 0;
325 }
326
327 /*
328 * this function sends a 'generic call function' IPI to all other CPUs
329 * in the system.
330 */
331
332 /*
333 * [SUMMARY] Run a function on all other CPUs.
334 * <func> The function to run. This must be fast and non-blocking.
335 * <info> An arbitrary pointer to pass to the function.
336 * <nonatomic> currently unused.
337 * <wait> If true, wait (atomically) until function has completed on other CPUs.
338 * [RETURNS] 0 on success, else a negative status code.
339 *
340 * Does not return until remote CPUs are nearly ready to execute <func> or are or have
341 * executed.
342 *
343 * You must not call this function with disabled interrupts or from a hardware interrupt
344 * handler, you may call it from a bottom half handler.
345 */
346 int
347 smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait)
348 {
349 struct call_data_struct data;
350 int cpus = smp_num_cpus-1;
351 #if (defined(CONFIG_ITANIUM_B0_SPECIFIC) \
352 || defined(CONFIG_ITANIUM_B1_SPECIFIC) || defined(CONFIG_ITANIUM_B2_SPECIFIC))
353 unsigned long timeout;
354 #endif
355
356 if (!cpus)
357 return 0;
358
359 data.func = func;
360 data.info = info;
361 atomic_set(&data.started, 0);
362 data.wait = wait;
363 if (wait)
364 atomic_set(&data.finished, 0);
365
366 spin_lock_bh(&call_lock);
367 call_data = &data;
368
369 #if (defined(CONFIG_ITANIUM_B0_SPECIFIC) \
370 || defined(CONFIG_ITANIUM_B1_SPECIFIC) || defined(CONFIG_ITANIUM_B2_SPECIFIC))
371 resend:
372 /* Send a message to all other CPUs and wait for them to respond */
373 send_IPI_allbutself(IPI_CALL_FUNC);
374
375 /* Wait for response */
376 timeout = jiffies + HZ;
377 while ((atomic_read(&data.started) != cpus) && time_before(jiffies, timeout))
378 barrier();
379 if (atomic_read(&data.started) != cpus)
380 goto resend;
381 #else
382 send_IPI_allbutself(IPI_CALL_FUNC);
383
384 /* Wait for response */
385 while (atomic_read(&data.started) != cpus)
386 barrier();
387 #endif
388
389 if (wait)
390 while (atomic_read(&data.finished) != cpus)
391 barrier();
392 call_data = NULL;
393
394 spin_unlock_bh(&call_lock);
395 return 0;
396 }
397
398 void
399 smp_do_timer (struct pt_regs *regs)
400 {
401 int user = user_mode(regs);
402
403 if (--local_cpu_data->prof_counter <= 0) {
404 local_cpu_data->prof_counter = local_cpu_data->prof_multiplier;
405 update_process_times(user);
406 }
407 }
408
409 /*
410 * this function calls the 'stop' function on all other CPUs in the system.
411 */
412 void
413 smp_send_stop (void)
414 {
415 send_IPI_allbutself(IPI_CPU_STOP);
416 smp_num_cpus = 1;
417 }
418
419 int __init
420 setup_profiling_timer (unsigned int multiplier)
421 {
422 return -EINVAL;
423 }
424