File: /usr/src/linux/kernel/softirq.c
1 /*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
6 * Fixed a disable_bh()/enable_bh() race (was causing a console lockup)
7 * due bh_mask_count not atomic handling. Copyright (C) 1998 Andrea Arcangeli
8 *
9 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
10 */
11
12 #include <linux/config.h>
13 #include <linux/mm.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/interrupt.h>
16 #include <linux/smp_lock.h>
17 #include <linux/init.h>
18 #include <linux/tqueue.h>
19
20 /*
21 - No shared variables, all the data are CPU local.
22 - If a softirq needs serialization, let it serialize itself
23 by its own spinlocks.
24 - Even if softirq is serialized, only local cpu is marked for
25 execution. Hence, we get something sort of weak cpu binding.
26 Though it is still not clear, will it result in better locality
27 or will not.
28 - These softirqs are not masked by global cli() and start_bh_atomic()
29 (by clear reasons). Hence, old parts of code still using global locks
30 MUST NOT use softirqs, but insert interfacing routines acquiring
31 global locks. F.e. look at BHs implementation.
32
33 Examples:
34 - NET RX softirq. It is multithreaded and does not require
35 any global serialization.
36 - NET TX softirq. It kicks software netdevice queues, hence
37 it is logically serialized per device, but this serialization
38 is invisible to common code.
39 - Tasklets: serialized wrt itself.
40 - Bottom halves: globally serialized, grr...
41 */
42
43 irq_cpustat_t irq_stat[NR_CPUS];
44
45 static struct softirq_action softirq_vec[32] __cacheline_aligned;
46
47 /*
48 * we cannot loop indefinitely here to avoid userspace starvation,
49 * but we also don't want to introduce a worst case 1/HZ latency
50 * to the pending events, so lets the scheduler to balance
51 * the softirq load for us.
52 */
53 static inline void wakeup_softirqd(unsigned cpu)
54 {
55 struct task_struct * tsk = ksoftirqd_task(cpu);
56
57 if (tsk && tsk->state != TASK_RUNNING)
58 wake_up_process(tsk);
59 }
60
61 asmlinkage void do_softirq()
62 {
63 int cpu = smp_processor_id();
64 __u32 pending;
65 long flags;
66 __u32 mask;
67
68 if (in_interrupt())
69 return;
70
71 local_irq_save(flags);
72
73 pending = softirq_pending(cpu);
74
75 if (pending) {
76 struct softirq_action *h;
77
78 mask = ~pending;
79 local_bh_disable();
80 restart:
81 /* Reset the pending bitmask before enabling irqs */
82 softirq_pending(cpu) = 0;
83
84 local_irq_enable();
85
86 h = softirq_vec;
87
88 do {
89 if (pending & 1)
90 h->action(h);
91 h++;
92 pending >>= 1;
93 } while (pending);
94
95 local_irq_disable();
96
97 pending = softirq_pending(cpu);
98 if (pending & mask) {
99 mask &= ~pending;
100 goto restart;
101 }
102 __local_bh_enable();
103
104 if (pending)
105 wakeup_softirqd(cpu);
106 }
107
108 local_irq_restore(flags);
109 }
110
111 /*
112 * This function must run with irq disabled!
113 */
114 inline void cpu_raise_softirq(unsigned int cpu, unsigned int nr)
115 {
116 __cpu_raise_softirq(cpu, nr);
117
118 /*
119 * If we're in an interrupt or bh, we're done
120 * (this also catches bh-disabled code). We will
121 * actually run the softirq once we return from
122 * the irq or bh.
123 *
124 * Otherwise we wake up ksoftirqd to make sure we
125 * schedule the softirq soon.
126 */
127 if (!(local_irq_count(cpu) | local_bh_count(cpu)))
128 wakeup_softirqd(cpu);
129 }
130
131 void raise_softirq(unsigned int nr)
132 {
133 long flags;
134
135 local_irq_save(flags);
136 cpu_raise_softirq(smp_processor_id(), nr);
137 local_irq_restore(flags);
138 }
139
140 void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
141 {
142 softirq_vec[nr].data = data;
143 softirq_vec[nr].action = action;
144 }
145
146
147 /* Tasklets */
148
149 struct tasklet_head tasklet_vec[NR_CPUS] __cacheline_aligned;
150 struct tasklet_head tasklet_hi_vec[NR_CPUS] __cacheline_aligned;
151
152 void __tasklet_schedule(struct tasklet_struct *t)
153 {
154 int cpu = smp_processor_id();
155 unsigned long flags;
156
157 local_irq_save(flags);
158 t->next = tasklet_vec[cpu].list;
159 tasklet_vec[cpu].list = t;
160 cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
161 local_irq_restore(flags);
162 }
163
164 void __tasklet_hi_schedule(struct tasklet_struct *t)
165 {
166 int cpu = smp_processor_id();
167 unsigned long flags;
168
169 local_irq_save(flags);
170 t->next = tasklet_hi_vec[cpu].list;
171 tasklet_hi_vec[cpu].list = t;
172 cpu_raise_softirq(cpu, HI_SOFTIRQ);
173 local_irq_restore(flags);
174 }
175
176 static void tasklet_action(struct softirq_action *a)
177 {
178 int cpu = smp_processor_id();
179 struct tasklet_struct *list;
180
181 local_irq_disable();
182 list = tasklet_vec[cpu].list;
183 tasklet_vec[cpu].list = NULL;
184 local_irq_enable();
185
186 while (list) {
187 struct tasklet_struct *t = list;
188
189 list = list->next;
190
191 if (tasklet_trylock(t)) {
192 if (!atomic_read(&t->count)) {
193 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
194 BUG();
195 t->func(t->data);
196 tasklet_unlock(t);
197 continue;
198 }
199 tasklet_unlock(t);
200 }
201
202 local_irq_disable();
203 t->next = tasklet_vec[cpu].list;
204 tasklet_vec[cpu].list = t;
205 __cpu_raise_softirq(cpu, TASKLET_SOFTIRQ);
206 local_irq_enable();
207 }
208 }
209
210 static void tasklet_hi_action(struct softirq_action *a)
211 {
212 int cpu = smp_processor_id();
213 struct tasklet_struct *list;
214
215 local_irq_disable();
216 list = tasklet_hi_vec[cpu].list;
217 tasklet_hi_vec[cpu].list = NULL;
218 local_irq_enable();
219
220 while (list) {
221 struct tasklet_struct *t = list;
222
223 list = list->next;
224
225 if (tasklet_trylock(t)) {
226 if (!atomic_read(&t->count)) {
227 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
228 BUG();
229 t->func(t->data);
230 tasklet_unlock(t);
231 continue;
232 }
233 tasklet_unlock(t);
234 }
235
236 local_irq_disable();
237 t->next = tasklet_hi_vec[cpu].list;
238 tasklet_hi_vec[cpu].list = t;
239 __cpu_raise_softirq(cpu, HI_SOFTIRQ);
240 local_irq_enable();
241 }
242 }
243
244
245 void tasklet_init(struct tasklet_struct *t,
246 void (*func)(unsigned long), unsigned long data)
247 {
248 t->next = NULL;
249 t->state = 0;
250 atomic_set(&t->count, 0);
251 t->func = func;
252 t->data = data;
253 }
254
255 void tasklet_kill(struct tasklet_struct *t)
256 {
257 if (in_interrupt())
258 printk("Attempt to kill tasklet from interrupt\n");
259
260 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
261 current->state = TASK_RUNNING;
262 do {
263 current->policy |= SCHED_YIELD;
264 schedule();
265 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
266 }
267 tasklet_unlock_wait(t);
268 clear_bit(TASKLET_STATE_SCHED, &t->state);
269 }
270
271
272
273 /* Old style BHs */
274
275 static void (*bh_base[32])(void);
276 struct tasklet_struct bh_task_vec[32];
277
278 /* BHs are serialized by spinlock global_bh_lock.
279
280 It is still possible to make synchronize_bh() as
281 spin_unlock_wait(&global_bh_lock). This operation is not used
282 by kernel now, so that this lock is not made private only
283 due to wait_on_irq().
284
285 It can be removed only after auditing all the BHs.
286 */
287 spinlock_t global_bh_lock = SPIN_LOCK_UNLOCKED;
288
289 static void bh_action(unsigned long nr)
290 {
291 int cpu = smp_processor_id();
292
293 if (!spin_trylock(&global_bh_lock))
294 goto resched;
295
296 if (!hardirq_trylock(cpu))
297 goto resched_unlock;
298
299 if (bh_base[nr])
300 bh_base[nr]();
301
302 hardirq_endlock(cpu);
303 spin_unlock(&global_bh_lock);
304 return;
305
306 resched_unlock:
307 spin_unlock(&global_bh_lock);
308 resched:
309 mark_bh(nr);
310 }
311
312 void init_bh(int nr, void (*routine)(void))
313 {
314 bh_base[nr] = routine;
315 mb();
316 }
317
318 void remove_bh(int nr)
319 {
320 tasklet_kill(bh_task_vec+nr);
321 bh_base[nr] = NULL;
322 }
323
324 void __init softirq_init()
325 {
326 int i;
327
328 for (i=0; i<32; i++)
329 tasklet_init(bh_task_vec+i, bh_action, i);
330
331 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
332 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
333 }
334
335 void __run_task_queue(task_queue *list)
336 {
337 struct list_head head, *next;
338 unsigned long flags;
339
340 spin_lock_irqsave(&tqueue_lock, flags);
341 list_add(&head, list);
342 list_del_init(list);
343 spin_unlock_irqrestore(&tqueue_lock, flags);
344
345 next = head.next;
346 while (next != &head) {
347 void (*f) (void *);
348 struct tq_struct *p;
349 void *data;
350
351 p = list_entry(next, struct tq_struct, list);
352 next = next->next;
353 f = p->routine;
354 data = p->data;
355 wmb();
356 p->sync = 0;
357 if (f)
358 f(data);
359 }
360 }
361
362 static int ksoftirqd(void * __bind_cpu)
363 {
364 int bind_cpu = *(int *) __bind_cpu;
365 int cpu = cpu_logical_map(bind_cpu);
366
367 daemonize();
368 current->nice = 19;
369 sigfillset(¤t->blocked);
370
371 /* Migrate to the right CPU */
372 current->cpus_allowed = 1UL << cpu;
373 while (smp_processor_id() != cpu)
374 schedule();
375
376 sprintf(current->comm, "ksoftirqd_CPU%d", bind_cpu);
377
378 __set_current_state(TASK_INTERRUPTIBLE);
379 mb();
380
381 ksoftirqd_task(cpu) = current;
382
383 for (;;) {
384 if (!softirq_pending(cpu))
385 schedule();
386
387 __set_current_state(TASK_RUNNING);
388
389 while (softirq_pending(cpu)) {
390 do_softirq();
391 if (current->need_resched)
392 schedule();
393 }
394
395 __set_current_state(TASK_INTERRUPTIBLE);
396 }
397 }
398
399 static __init int spawn_ksoftirqd(void)
400 {
401 int cpu;
402
403 for (cpu = 0; cpu < smp_num_cpus; cpu++) {
404 if (kernel_thread(ksoftirqd, (void *) &cpu,
405 CLONE_FS | CLONE_FILES | CLONE_SIGNAL) < 0)
406 printk("spawn_ksoftirqd() failed for cpu %d\n", cpu);
407 else {
408 while (!ksoftirqd_task(cpu_logical_map(cpu))) {
409 current->policy |= SCHED_YIELD;
410 schedule();
411 }
412 }
413 }
414
415 return 0;
416 }
417
418 __initcall(spawn_ksoftirqd);
419