File: /usr/src/linux/arch/sparc64/kernel/irq.c
1 /* $Id: irq.c,v 1.101 2001/06/04 06:50:18 ecd Exp $
2 * irq.c: UltraSparc IRQ handling/init/registry.
3 *
4 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be)
6 * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz)
7 */
8
9 #include <linux/config.h>
10 #include <linux/ptrace.h>
11 #include <linux/errno.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/signal.h>
14 #include <linux/mm.h>
15 #include <linux/interrupt.h>
16 #include <linux/slab.h>
17 #include <linux/random.h> /* XXX ADD add_foo_randomness() calls... -DaveM */
18 #include <linux/init.h>
19 #include <linux/delay.h>
20
21 #include <asm/ptrace.h>
22 #include <asm/processor.h>
23 #include <asm/atomic.h>
24 #include <asm/system.h>
25 #include <asm/irq.h>
26 #include <asm/sbus.h>
27 #include <asm/iommu.h>
28 #include <asm/upa.h>
29 #include <asm/oplib.h>
30 #include <asm/timer.h>
31 #include <asm/smp.h>
32 #include <asm/hardirq.h>
33 #include <asm/softirq.h>
34 #include <asm/starfire.h>
35
36 /* Internal flag, should not be visible elsewhere at all. */
37 #define SA_IMAP_MASKED 0x100
38 #define SA_DMA_SYNC 0x200
39
40 #ifdef CONFIG_SMP
41 static void distribute_irqs(void);
42 #endif
43
44 /* UPA nodes send interrupt packet to UltraSparc with first data reg
45 * value low 5 (7 on Starfire) bits holding the IRQ identifier being
46 * delivered. We must translate this into a non-vector IRQ so we can
47 * set the softint on this cpu.
48 *
49 * To make processing these packets efficient and race free we use
50 * an array of irq buckets below. The interrupt vector handler in
51 * entry.S feeds incoming packets into per-cpu pil-indexed lists.
52 * The IVEC handler does not need to act atomically, the PIL dispatch
53 * code uses CAS to get an atomic snapshot of the list and clear it
54 * at the same time.
55 */
56
57 struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (64)));
58
59 #ifndef CONFIG_SMP
60 unsigned int __up_workvec[16] __attribute__ ((aligned (64)));
61 #define irq_work(__cpu, __pil) &(__up_workvec[(void)(__cpu), (__pil)])
62 #else
63 #define irq_work(__cpu, __pil) &(cpu_data[(__cpu)].irq_worklists[(__pil)])
64 #endif
65
66 #ifdef CONFIG_PCI
67 /* This is a table of physical addresses used to deal with SA_DMA_SYNC.
68 * It is used for PCI only to synchronize DMA transfers with IRQ delivery
69 * for devices behind busses other than APB on Sabre systems.
70 *
71 * Currently these physical addresses are just config space accesses
72 * to the command register for that device.
73 */
74 unsigned long pci_dma_wsync;
75 unsigned long dma_sync_reg_table[256];
76 unsigned char dma_sync_reg_table_entry = 0;
77 #endif
78
79 /* This is based upon code in the 32-bit Sparc kernel written mostly by
80 * David Redman (djhr@tadpole.co.uk).
81 */
82 #define MAX_STATIC_ALLOC 4
83 static struct irqaction static_irqaction[MAX_STATIC_ALLOC];
84 static int static_irq_count = 0;
85
86 /* This is exported so that fast IRQ handlers can get at it... -DaveM */
87 struct irqaction *irq_action[NR_IRQS+1] = {
88 NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL,
89 NULL, NULL, NULL, NULL, NULL, NULL , NULL, NULL
90 };
91
92 int get_irq_list(char *buf)
93 {
94 int i, len = 0;
95 struct irqaction *action;
96 #ifdef CONFIG_SMP
97 int j;
98 #endif
99
100 for(i = 0; i < (NR_IRQS + 1); i++) {
101 if(!(action = *(i + irq_action)))
102 continue;
103 len += sprintf(buf + len, "%3d: ", i);
104 #ifndef CONFIG_SMP
105 len += sprintf(buf + len, "%10u ", kstat_irqs(i));
106 #else
107 for (j = 0; j < smp_num_cpus; j++)
108 len += sprintf(buf + len, "%10u ",
109 kstat.irqs[cpu_logical_map(j)][i]);
110 #endif
111 len += sprintf(buf + len, "%c %s",
112 (action->flags & SA_INTERRUPT) ? '+' : ' ',
113 action->name);
114 for(action = action->next; action; action = action->next) {
115 len += sprintf(buf+len, ",%s %s",
116 (action->flags & SA_INTERRUPT) ? " +" : "",
117 action->name);
118 }
119 len += sprintf(buf + len, "\n");
120 }
121 return len;
122 }
123
124 /* Now these are always passed a true fully specified sun4u INO. */
125 void enable_irq(unsigned int irq)
126 {
127 struct ino_bucket *bucket = __bucket(irq);
128 unsigned long imap;
129 unsigned long tid;
130
131 imap = bucket->imap;
132 if (imap == 0UL)
133 return;
134
135 if (tlb_type == cheetah) {
136 /* We set it to our Safari AID. */
137 __asm__ __volatile__("ldxa [%%g0] %1, %0"
138 : "=r" (tid)
139 : "i" (ASI_SAFARI_CONFIG));
140 tid = ((tid & (0x3ffUL<<17)) << 9);
141 tid &= IMAP_AID_SAFARI;
142 } else if (this_is_starfire == 0) {
143 /* We set it to our UPA MID. */
144 __asm__ __volatile__("ldxa [%%g0] %1, %0"
145 : "=r" (tid)
146 : "i" (ASI_UPA_CONFIG));
147 tid = ((tid & UPA_CONFIG_MID) << 9);
148 tid &= IMAP_TID_UPA;
149 } else {
150 tid = (starfire_translate(imap, current->processor) << 26);
151 tid &= IMAP_TID_UPA;
152 }
153
154 /* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product
155 * of this SYSIO's preconfigured IGN in the SYSIO Control
156 * Register, the hardware just mirrors that value here.
157 * However for Graphics and UPA Slave devices the full
158 * IMAP_INR field can be set by the programmer here.
159 *
160 * Things like FFB can now be handled via the new IRQ mechanism.
161 */
162 upa_writel(tid | IMAP_VALID, imap);
163 }
164
165 /* This now gets passed true ino's as well. */
166 void disable_irq(unsigned int irq)
167 {
168 struct ino_bucket *bucket = __bucket(irq);
169 unsigned long imap;
170
171 imap = bucket->imap;
172 if (imap != 0UL) {
173 u32 tmp;
174
175 /* NOTE: We do not want to futz with the IRQ clear registers
176 * and move the state to IDLE, the SCSI code does call
177 * disable_irq() to assure atomicity in the queue cmd
178 * SCSI adapter driver code. Thus we'd lose interrupts.
179 */
180 tmp = upa_readl(imap);
181 tmp &= ~IMAP_VALID;
182 upa_writel(tmp, imap);
183 }
184 }
185
186 /* The timer is the one "weird" interrupt which is generated by
187 * the CPU %tick register and not by some normal vectored interrupt
188 * source. To handle this special case, we use this dummy INO bucket.
189 */
190 static struct ino_bucket pil0_dummy_bucket = {
191 0, /* irq_chain */
192 0, /* pil */
193 0, /* pending */
194 0, /* flags */
195 0, /* __unused */
196 NULL, /* irq_info */
197 0UL, /* iclr */
198 0UL, /* imap */
199 };
200
201 unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap)
202 {
203 struct ino_bucket *bucket;
204 int ino;
205
206 if(pil == 0) {
207 if(iclr != 0UL || imap != 0UL) {
208 prom_printf("Invalid dummy bucket for PIL0 (%lx:%lx)\n",
209 iclr, imap);
210 prom_halt();
211 }
212 return __irq(&pil0_dummy_bucket);
213 }
214
215 /* RULE: Both must be specified in all other cases. */
216 if (iclr == 0UL || imap == 0UL) {
217 prom_printf("Invalid build_irq %d %d %016lx %016lx\n",
218 pil, inofixup, iclr, imap);
219 prom_halt();
220 }
221
222 ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
223 if(ino > NUM_IVECS) {
224 prom_printf("Invalid INO %04x (%d:%d:%016lx:%016lx)\n",
225 ino, pil, inofixup, iclr, imap);
226 prom_halt();
227 }
228
229 /* Ok, looks good, set it up. Don't touch the irq_chain or
230 * the pending flag.
231 */
232 bucket = &ivector_table[ino];
233 if ((bucket->flags & IBF_ACTIVE) ||
234 (bucket->irq_info != NULL)) {
235 /* This is a gross fatal error if it happens here. */
236 prom_printf("IRQ: Trying to reinit INO bucket, fatal error.\n");
237 prom_printf("IRQ: Request INO %04x (%d:%d:%016lx:%016lx)\n",
238 ino, pil, inofixup, iclr, imap);
239 prom_printf("IRQ: Existing (%d:%016lx:%016lx)\n",
240 bucket->pil, bucket->iclr, bucket->imap);
241 prom_printf("IRQ: Cannot continue, halting...\n");
242 prom_halt();
243 }
244 bucket->imap = imap;
245 bucket->iclr = iclr;
246 bucket->pil = pil;
247 bucket->flags = 0;
248
249 bucket->irq_info = NULL;
250
251 return __irq(bucket);
252 }
253
254 static void atomic_bucket_insert(struct ino_bucket *bucket)
255 {
256 unsigned long pstate;
257 unsigned int *ent;
258
259 __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
260 __asm__ __volatile__("wrpr %0, %1, %%pstate"
261 : : "r" (pstate), "i" (PSTATE_IE));
262 ent = irq_work(smp_processor_id(), bucket->pil);
263 bucket->irq_chain = *ent;
264 *ent = __irq(bucket);
265 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
266 }
267
268 int request_irq(unsigned int irq, void (*handler)(int, void *, struct pt_regs *),
269 unsigned long irqflags, const char *name, void *dev_id)
270 {
271 struct irqaction *action, *tmp = NULL;
272 struct ino_bucket *bucket = __bucket(irq);
273 unsigned long flags;
274 int pending = 0;
275
276 if ((bucket != &pil0_dummy_bucket) &&
277 (bucket < &ivector_table[0] ||
278 bucket >= &ivector_table[NUM_IVECS])) {
279 unsigned int *caller;
280
281 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
282 printk(KERN_CRIT "request_irq: Old style IRQ registry attempt "
283 "from %p, irq %08x.\n", caller, irq);
284 return -EINVAL;
285 }
286 if(!handler)
287 return -EINVAL;
288
289 if (!bucket->pil)
290 irqflags &= ~SA_IMAP_MASKED;
291 else {
292 irqflags |= SA_IMAP_MASKED;
293 if (bucket->flags & IBF_PCI) {
294 /*
295 * PCI IRQs should never use SA_INTERRUPT.
296 */
297 irqflags &= ~(SA_INTERRUPT);
298
299 /*
300 * Check wether we _should_ use DMA Write Sync
301 * (for devices behind bridges behind APB).
302 */
303 if (bucket->flags & IBF_DMA_SYNC)
304 irqflags |= SA_DMA_SYNC;
305 }
306 }
307
308 save_and_cli(flags);
309
310 action = *(bucket->pil + irq_action);
311 if(action) {
312 if((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ))
313 for (tmp = action; tmp->next; tmp = tmp->next)
314 ;
315 else {
316 restore_flags(flags);
317 return -EBUSY;
318 }
319 if((action->flags & SA_INTERRUPT) ^ (irqflags & SA_INTERRUPT)) {
320 printk("Attempt to mix fast and slow interrupts on IRQ%d "
321 "denied\n", bucket->pil);
322 restore_flags(flags);
323 return -EBUSY;
324 }
325 action = NULL; /* Or else! */
326 }
327
328 /* If this is flagged as statically allocated then we use our
329 * private struct which is never freed.
330 */
331 if(irqflags & SA_STATIC_ALLOC) {
332 if(static_irq_count < MAX_STATIC_ALLOC)
333 action = &static_irqaction[static_irq_count++];
334 else
335 printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
336 "using kmalloc\n", irq, name);
337 }
338 if(action == NULL)
339 action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
340 GFP_KERNEL);
341
342 if(!action) {
343 restore_flags(flags);
344 return -ENOMEM;
345 }
346
347 if ((irqflags & SA_IMAP_MASKED) == 0) {
348 bucket->irq_info = action;
349 bucket->flags |= IBF_ACTIVE;
350 } else {
351 if((bucket->flags & IBF_ACTIVE) != 0) {
352 void *orig = bucket->irq_info;
353 void **vector = NULL;
354
355 if((bucket->flags & IBF_PCI) == 0) {
356 printk("IRQ: Trying to share non-PCI bucket.\n");
357 goto free_and_ebusy;
358 }
359 if((bucket->flags & IBF_MULTI) == 0) {
360 vector = kmalloc(sizeof(void *) * 4, GFP_KERNEL);
361 if(vector == NULL)
362 goto free_and_enomem;
363
364 /* We might have slept. */
365 if ((bucket->flags & IBF_MULTI) != 0) {
366 int ent;
367
368 kfree(vector);
369 vector = (void **)bucket->irq_info;
370 for(ent = 0; ent < 4; ent++) {
371 if (vector[ent] == NULL) {
372 vector[ent] = action;
373 break;
374 }
375 }
376 if (ent == 4)
377 goto free_and_ebusy;
378 } else {
379 vector[0] = orig;
380 vector[1] = action;
381 vector[2] = NULL;
382 vector[3] = NULL;
383 bucket->irq_info = vector;
384 bucket->flags |= IBF_MULTI;
385 }
386 } else {
387 int ent;
388
389 vector = (void **)orig;
390 for(ent = 0; ent < 4; ent++) {
391 if(vector[ent] == NULL) {
392 vector[ent] = action;
393 break;
394 }
395 }
396 if (ent == 4)
397 goto free_and_ebusy;
398 }
399 } else {
400 bucket->irq_info = action;
401 bucket->flags |= IBF_ACTIVE;
402 }
403 pending = bucket->pending;
404 if(pending)
405 bucket->pending = 0;
406 }
407
408 action->mask = (unsigned long) bucket;
409 action->handler = handler;
410 action->flags = irqflags;
411 action->name = name;
412 action->next = NULL;
413 action->dev_id = dev_id;
414
415 if(tmp)
416 tmp->next = action;
417 else
418 *(bucket->pil + irq_action) = action;
419
420 enable_irq(irq);
421
422 /* We ate the IVEC already, this makes sure it does not get lost. */
423 if(pending) {
424 atomic_bucket_insert(bucket);
425 set_softint(1 << bucket->pil);
426 }
427 restore_flags(flags);
428
429 #ifdef CONFIG_SMP
430 distribute_irqs();
431 #endif
432 return 0;
433
434 free_and_ebusy:
435 kfree(action);
436 restore_flags(flags);
437 return -EBUSY;
438
439 free_and_enomem:
440 kfree(action);
441 restore_flags(flags);
442 return -ENOMEM;
443 }
444
445 void free_irq(unsigned int irq, void *dev_id)
446 {
447 struct irqaction *action;
448 struct irqaction *tmp = NULL;
449 unsigned long flags;
450 struct ino_bucket *bucket = __bucket(irq), *bp;
451
452 if ((bucket != &pil0_dummy_bucket) &&
453 (bucket < &ivector_table[0] ||
454 bucket >= &ivector_table[NUM_IVECS])) {
455 unsigned int *caller;
456
457 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
458 printk(KERN_CRIT "free_irq: Old style IRQ removal attempt "
459 "from %p, irq %08x.\n", caller, irq);
460 return;
461 }
462
463 action = *(bucket->pil + irq_action);
464 if(!action->handler) {
465 printk("Freeing free IRQ %d\n", bucket->pil);
466 return;
467 }
468 if(dev_id) {
469 for( ; action; action = action->next) {
470 if(action->dev_id == dev_id)
471 break;
472 tmp = action;
473 }
474 if(!action) {
475 printk("Trying to free free shared IRQ %d\n", bucket->pil);
476 return;
477 }
478 } else if(action->flags & SA_SHIRQ) {
479 printk("Trying to free shared IRQ %d with NULL device ID\n", bucket->pil);
480 return;
481 }
482
483 if(action->flags & SA_STATIC_ALLOC) {
484 printk("Attempt to free statically allocated IRQ %d (%s)\n",
485 bucket->pil, action->name);
486 return;
487 }
488
489 save_and_cli(flags);
490 if(action && tmp)
491 tmp->next = action->next;
492 else
493 *(bucket->pil + irq_action) = action->next;
494
495 if(action->flags & SA_IMAP_MASKED) {
496 unsigned long imap = bucket->imap;
497 void **vector, *orig;
498 int ent;
499
500 orig = bucket->irq_info;
501 vector = (void **)orig;
502
503 if ((bucket->flags & IBF_MULTI) != 0) {
504 int other = 0;
505 void *orphan = NULL;
506 for(ent = 0; ent < 4; ent++) {
507 if(vector[ent] == action)
508 vector[ent] = NULL;
509 else if(vector[ent] != NULL) {
510 orphan = vector[ent];
511 other++;
512 }
513 }
514
515 /* Only free when no other shared irq
516 * uses this bucket.
517 */
518 if(other) {
519 if (other == 1) {
520 /* Convert back to non-shared bucket. */
521 bucket->irq_info = orphan;
522 bucket->flags &= ~(IBF_MULTI);
523 kfree(vector);
524 }
525 goto out;
526 }
527 } else {
528 bucket->irq_info = NULL;
529 }
530
531 /* This unique interrupt source is now inactive. */
532 bucket->flags &= ~IBF_ACTIVE;
533
534 /* See if any other buckets share this bucket's IMAP
535 * and are still active.
536 */
537 for(ent = 0; ent < NUM_IVECS; ent++) {
538 bp = &ivector_table[ent];
539 if(bp != bucket &&
540 bp->imap == imap &&
541 (bp->flags & IBF_ACTIVE) != 0)
542 break;
543 }
544
545 /* Only disable when no other sub-irq levels of
546 * the same IMAP are active.
547 */
548 if (ent == NUM_IVECS)
549 disable_irq(irq);
550 }
551
552 out:
553 kfree(action);
554 restore_flags(flags);
555 }
556
557 #ifdef CONFIG_SMP
558
559 /* Who has the global irq brlock */
560 unsigned char global_irq_holder = NO_PROC_ID;
561
562 static void show(char * str)
563 {
564 int cpu = smp_processor_id();
565 int i;
566
567 printk("\n%s, CPU %d:\n", str, cpu);
568 printk("irq: %d [ ", irqs_running());
569 for (i = 0; i < smp_num_cpus; i++)
570 printk("%u ", __brlock_array[i][BR_GLOBALIRQ_LOCK]);
571 printk("]\nbh: %d [ ",
572 (spin_is_locked(&global_bh_lock) ? 1 : 0));
573 for (i = 0; i < smp_num_cpus; i++)
574 printk("%u ", local_bh_count(i));
575 printk("]\n");
576 }
577
578 #define MAXCOUNT 100000000
579
580 #if 0
581 #define SYNC_OTHER_ULTRAS(x) udelay(x+1)
582 #else
583 #define SYNC_OTHER_ULTRAS(x) membar("#Sync");
584 #endif
585
586 void synchronize_irq(void)
587 {
588 if (irqs_running()) {
589 cli();
590 sti();
591 }
592 }
593
594 static inline void get_irqlock(int cpu)
595 {
596 int count;
597
598 if ((unsigned char)cpu == global_irq_holder)
599 return;
600
601 count = MAXCOUNT;
602 again:
603 br_write_lock(BR_GLOBALIRQ_LOCK);
604 for (;;) {
605 spinlock_t *lock;
606
607 if (!irqs_running() &&
608 (local_bh_count(smp_processor_id()) || !spin_is_locked(&global_bh_lock)))
609 break;
610
611 br_write_unlock(BR_GLOBALIRQ_LOCK);
612 lock = &__br_write_locks[BR_GLOBALIRQ_LOCK].lock;
613 while (irqs_running() ||
614 spin_is_locked(lock) ||
615 (!local_bh_count(smp_processor_id()) && spin_is_locked(&global_bh_lock))) {
616 if (!--count) {
617 show("get_irqlock");
618 count = (~0 >> 1);
619 }
620 __sti();
621 SYNC_OTHER_ULTRAS(cpu);
622 __cli();
623 }
624 goto again;
625 }
626
627 global_irq_holder = cpu;
628 }
629
630 void __global_cli(void)
631 {
632 unsigned long flags;
633
634 __save_flags(flags);
635 if(flags == 0) {
636 int cpu = smp_processor_id();
637 __cli();
638 if (! local_irq_count(cpu))
639 get_irqlock(cpu);
640 }
641 }
642
643 void __global_sti(void)
644 {
645 int cpu = smp_processor_id();
646
647 if (! local_irq_count(cpu))
648 release_irqlock(cpu);
649 __sti();
650 }
651
652 unsigned long __global_save_flags(void)
653 {
654 unsigned long flags, local_enabled, retval;
655
656 __save_flags(flags);
657 local_enabled = ((flags == 0) ? 1 : 0);
658 retval = 2 + local_enabled;
659 if (! local_irq_count(smp_processor_id())) {
660 if (local_enabled)
661 retval = 1;
662 if (global_irq_holder == (unsigned char) smp_processor_id())
663 retval = 0;
664 }
665 return retval;
666 }
667
668 void __global_restore_flags(unsigned long flags)
669 {
670 switch (flags) {
671 case 0:
672 __global_cli();
673 break;
674 case 1:
675 __global_sti();
676 break;
677 case 2:
678 __cli();
679 break;
680 case 3:
681 __sti();
682 break;
683 default:
684 {
685 unsigned long pc;
686 __asm__ __volatile__("mov %%i7, %0" : "=r" (pc));
687 printk("global_restore_flags: Bogon flags(%016lx) caller %016lx\n",
688 flags, pc);
689 }
690 }
691 }
692
693 #endif /* CONFIG_SMP */
694
695 void catch_disabled_ivec(struct pt_regs *regs)
696 {
697 int cpu = smp_processor_id();
698 struct ino_bucket *bucket = __bucket(*irq_work(cpu, 0));
699
700 /* We can actually see this on Ultra/PCI PCI cards, which are bridges
701 * to other devices. Here a single IMAP enabled potentially multiple
702 * unique interrupt sources (which each do have a unique ICLR register.
703 *
704 * So what we do is just register that the IVEC arrived, when registered
705 * for real the request_irq() code will check the bit and signal
706 * a local CPU interrupt for it.
707 */
708 #if 0
709 printk("IVEC: Spurious interrupt vector (%x) received at (%016lx)\n",
710 bucket - &ivector_table[0], regs->tpc);
711 #endif
712 *irq_work(cpu, 0) = 0;
713 bucket->pending = 1;
714 }
715
716 /* Tune this... */
717 #define FORWARD_VOLUME 12
718
719 void handler_irq(int irq, struct pt_regs *regs)
720 {
721 struct ino_bucket *bp, *nbp;
722 int cpu = smp_processor_id();
723 #ifdef CONFIG_SMP
724 int should_forward = (this_is_starfire == 0 &&
725 irq < 10 &&
726 current->pid != 0);
727 unsigned int buddy = 0;
728
729 /* 'cpu' is the MID (ie. UPAID), calculate the MID
730 * of our buddy.
731 */
732 if (should_forward != 0) {
733 buddy = cpu_number_map(cpu) + 1;
734 if (buddy >= NR_CPUS ||
735 (buddy = cpu_logical_map(buddy)) == -1)
736 buddy = cpu_logical_map(0);
737
738 /* Voo-doo programming. */
739 if (cpu_data[buddy].idle_volume < FORWARD_VOLUME)
740 should_forward = 0;
741
742 /* This just so happens to be correct on Cheetah
743 * at the moment.
744 */
745 buddy <<= 26;
746 }
747 #endif
748
749 #ifndef CONFIG_SMP
750 /*
751 * Check for TICK_INT on level 14 softint.
752 */
753 {
754 unsigned long clr_mask = 1 << irq;
755 unsigned long tick_mask;
756
757 if (SPARC64_USE_STICK)
758 tick_mask = (1UL << 16);
759 else
760 tick_mask = (1UL << 0);
761 if ((irq == 14) && (get_softint() & tick_mask)) {
762 irq = 0;
763 clr_mask = tick_mask;
764 }
765 clear_softint(clr_mask);
766 }
767 #else
768 clear_softint(1 << irq);
769 #endif
770
771 irq_enter(cpu, irq);
772 kstat.irqs[cpu][irq]++;
773
774 /* Sliiiick... */
775 #ifndef CONFIG_SMP
776 bp = ((irq != 0) ?
777 __bucket(xchg32(irq_work(cpu, irq), 0)) :
778 &pil0_dummy_bucket);
779 #else
780 bp = __bucket(xchg32(irq_work(cpu, irq), 0));
781 #endif
782 for ( ; bp != NULL; bp = nbp) {
783 unsigned char flags = bp->flags;
784
785 nbp = __bucket(bp->irq_chain);
786 bp->irq_chain = 0;
787
788 if ((flags & IBF_ACTIVE) != 0) {
789 #ifdef CONFIG_PCI
790 if ((flags & IBF_DMA_SYNC) != 0) {
791 upa_readl(dma_sync_reg_table[bp->synctab_ent]);
792 upa_readq(pci_dma_wsync);
793 }
794 #endif
795 if ((flags & IBF_MULTI) == 0) {
796 struct irqaction *ap = bp->irq_info;
797 ap->handler(__irq(bp), ap->dev_id, regs);
798 } else {
799 void **vector = (void **)bp->irq_info;
800 int ent;
801 for (ent = 0; ent < 4; ent++) {
802 struct irqaction *ap = vector[ent];
803 if (ap != NULL)
804 ap->handler(__irq(bp), ap->dev_id, regs);
805 }
806 }
807 /* Only the dummy bucket lacks IMAP/ICLR. */
808 if (bp->pil != 0) {
809 #ifdef CONFIG_SMP
810 /* Ok, here is what is going on:
811 * 1) Retargeting IRQs on Starfire is very
812 * expensive so just forget about it on them.
813 * 2) Moving around very high priority interrupts
814 * is a losing game.
815 * 3) If the current cpu is idle, interrupts are
816 * useful work, so keep them here. But do not
817 * pass to our neighbour if he is not very idle.
818 */
819 if (should_forward != 0) {
820 /* Push it to our buddy. */
821 should_forward = 0;
822 upa_writel(buddy | IMAP_VALID, bp->imap);
823 }
824 #endif
825 upa_writel(ICLR_IDLE, bp->iclr);
826 }
827 } else
828 bp->pending = 1;
829 }
830 irq_exit(cpu, irq);
831 }
832
833 #ifdef CONFIG_BLK_DEV_FD
834 extern void floppy_interrupt(int irq, void *dev_cookie, struct pt_regs *regs);
835
836 void sparc_floppy_irq(int irq, void *dev_cookie, struct pt_regs *regs)
837 {
838 struct irqaction *action = *(irq + irq_action);
839 struct ino_bucket *bucket;
840 int cpu = smp_processor_id();
841
842 irq_enter(cpu, irq);
843 kstat.irqs[cpu][irq]++;
844
845 *(irq_work(cpu, irq)) = 0;
846 bucket = (struct ino_bucket *)action->mask;
847
848 floppy_interrupt(irq, dev_cookie, regs);
849 upa_writel(ICLR_IDLE, bucket->iclr);
850
851 irq_exit(cpu, irq);
852 }
853 #endif
854
855 /* The following assumes that the branch lies before the place we
856 * are branching to. This is the case for a trap vector...
857 * You have been warned.
858 */
859 #define SPARC_BRANCH(dest_addr, inst_addr) \
860 (0x10800000 | ((((dest_addr)-(inst_addr))>>2)&0x3fffff))
861
862 #define SPARC_NOP (0x01000000)
863
864 static void install_fast_irq(unsigned int cpu_irq,
865 void (*handler)(int, void *, struct pt_regs *))
866 {
867 extern unsigned long sparc64_ttable_tl0;
868 unsigned long ttent = (unsigned long) &sparc64_ttable_tl0;
869 unsigned int *insns;
870
871 ttent += 0x820;
872 ttent += (cpu_irq - 1) << 5;
873 insns = (unsigned int *) ttent;
874 insns[0] = SPARC_BRANCH(((unsigned long) handler),
875 ((unsigned long)&insns[0]));
876 insns[1] = SPARC_NOP;
877 __asm__ __volatile__("membar #StoreStore; flush %0" : : "r" (ttent));
878 }
879
880 int request_fast_irq(unsigned int irq,
881 void (*handler)(int, void *, struct pt_regs *),
882 unsigned long irqflags, const char *name, void *dev_id)
883 {
884 struct irqaction *action;
885 struct ino_bucket *bucket = __bucket(irq);
886 unsigned long flags;
887
888 /* No pil0 dummy buckets allowed here. */
889 if (bucket < &ivector_table[0] ||
890 bucket >= &ivector_table[NUM_IVECS]) {
891 unsigned int *caller;
892
893 __asm__ __volatile__("mov %%i7, %0" : "=r" (caller));
894 printk(KERN_CRIT "request_fast_irq: Old style IRQ registry attempt "
895 "from %p, irq %08x.\n", caller, irq);
896 return -EINVAL;
897 }
898
899 /* Only IMAP style interrupts can be registered as fast. */
900 if(bucket->pil == 0)
901 return -EINVAL;
902
903 if(!handler)
904 return -EINVAL;
905
906 if ((bucket->pil == 0) || (bucket->pil == 14)) {
907 printk("request_fast_irq: Trying to register shared IRQ 0 or 14.\n");
908 return -EBUSY;
909 }
910
911 action = *(bucket->pil + irq_action);
912 if(action) {
913 if(action->flags & SA_SHIRQ)
914 panic("Trying to register fast irq when already shared.\n");
915 if(irqflags & SA_SHIRQ)
916 panic("Trying to register fast irq as shared.\n");
917 printk("request_fast_irq: Trying to register yet already owned.\n");
918 return -EBUSY;
919 }
920
921 save_and_cli(flags);
922 if(irqflags & SA_STATIC_ALLOC) {
923 if(static_irq_count < MAX_STATIC_ALLOC)
924 action = &static_irqaction[static_irq_count++];
925 else
926 printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed "
927 "using kmalloc\n", bucket->pil, name);
928 }
929 if(action == NULL)
930 action = (struct irqaction *)kmalloc(sizeof(struct irqaction),
931 GFP_KERNEL);
932 if(!action) {
933 restore_flags(flags);
934 return -ENOMEM;
935 }
936 install_fast_irq(bucket->pil, handler);
937
938 bucket->irq_info = action;
939 bucket->flags |= IBF_ACTIVE;
940
941 action->mask = (unsigned long) bucket;
942 action->handler = handler;
943 action->flags = irqflags | SA_IMAP_MASKED;
944 action->dev_id = NULL;
945 action->name = name;
946 action->next = NULL;
947
948 *(bucket->pil + irq_action) = action;
949 enable_irq(irq);
950
951 restore_flags(flags);
952
953 #ifdef CONFIG_SMP
954 distribute_irqs();
955 #endif
956 return 0;
957 }
958
959 /* We really don't need these at all on the Sparc. We only have
960 * stubs here because they are exported to modules.
961 */
962 unsigned long probe_irq_on(void)
963 {
964 return 0;
965 }
966
967 int probe_irq_off(unsigned long mask)
968 {
969 return 0;
970 }
971
972 /* This is gets the master TICK_INT timer going. */
973 void init_timers(void (*cfunc)(int, void *, struct pt_regs *),
974 unsigned long *clock)
975 {
976 unsigned long pstate;
977 extern unsigned long timer_tick_offset;
978 int node, err;
979 #ifdef CONFIG_SMP
980 extern void smp_tick_init(void);
981 #endif
982
983 if (!SPARC64_USE_STICK) {
984 node = linux_cpus[0].prom_node;
985 *clock = prom_getint(node, "clock-frequency");
986 } else {
987 node = prom_root_node;
988 *clock = prom_getint(node, "stick-frequency");
989 }
990 timer_tick_offset = *clock / HZ;
991 #ifdef CONFIG_SMP
992 smp_tick_init();
993 #endif
994
995 /* Register IRQ handler. */
996 err = request_irq(build_irq(0, 0, 0UL, 0UL), cfunc, (SA_INTERRUPT | SA_STATIC_ALLOC),
997 "timer", NULL);
998
999 if(err) {
1000 prom_printf("Serious problem, cannot register TICK_INT\n");
1001 prom_halt();
1002 }
1003
1004 /* Guarentee that the following sequences execute
1005 * uninterrupted.
1006 */
1007 __asm__ __volatile__("rdpr %%pstate, %0\n\t"
1008 "wrpr %0, %1, %%pstate"
1009 : "=r" (pstate)
1010 : "i" (PSTATE_IE));
1011
1012 /* Set things up so user can access tick register for profiling
1013 * purposes. Also workaround BB_ERRATA_1 by doing a dummy
1014 * read back of %tick after writing it.
1015 */
1016 __asm__ __volatile__(
1017 " sethi %%hi(0x80000000), %%g1\n"
1018 " ba,pt %%xcc, 1f\n"
1019 " sllx %%g1, 32, %%g1\n"
1020 " .align 64\n"
1021 "1: rd %%tick, %%g2\n"
1022 " add %%g2, 6, %%g2\n"
1023 " andn %%g2, %%g1, %%g2\n"
1024 " wrpr %%g2, 0, %%tick\n"
1025 " rdpr %%tick, %%g0"
1026 : /* no outputs */
1027 : /* no inputs */
1028 : "g1", "g2");
1029
1030 /* Workaround for Spitfire Errata (#54 I think??), I discovered
1031 * this via Sun BugID 4008234, mentioned in Solaris-2.5.1 patch
1032 * number 103640.
1033 *
1034 * On Blackbird writes to %tick_cmpr can fail, the
1035 * workaround seems to be to execute the wr instruction
1036 * at the start of an I-cache line, and perform a dummy
1037 * read back from %tick_cmpr right after writing to it. -DaveM
1038 */
1039 if (!SPARC64_USE_STICK) {
1040 __asm__ __volatile__(
1041 " rd %%tick, %%g1\n"
1042 " ba,pt %%xcc, 1f\n"
1043 " add %%g1, %0, %%g1\n"
1044 " .align 64\n"
1045 "1: wr %%g1, 0x0, %%tick_cmpr\n"
1046 " rd %%tick_cmpr, %%g0"
1047 : /* no outputs */
1048 : "r" (timer_tick_offset)
1049 : "g1");
1050 } else {
1051 /* Let the user get at STICK too. */
1052 __asm__ __volatile__(
1053 " sethi %%hi(0x80000000), %%g1\n"
1054 " sllx %%g1, 32, %%g1\n"
1055 " rd %%asr24, %%g2\n"
1056 " andn %%g2, %%g1, %%g2\n"
1057 " wr %%g2, 0, %%asr24"
1058 : /* no outputs */
1059 : /* no inputs */
1060 : "g1", "g2");
1061
1062 __asm__ __volatile__(
1063 " rd %%asr24, %%g1\n"
1064 " add %%g1, %0, %%g1\n"
1065 " wr %%g1, 0x0, %%asr25"
1066 : /* no outputs */
1067 : "r" (timer_tick_offset)
1068 : "g1");
1069 }
1070
1071 /* Restore PSTATE_IE. */
1072 __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
1073 : /* no outputs */
1074 : "r" (pstate));
1075
1076 sti();
1077 }
1078
1079 #ifdef CONFIG_SMP
1080 static int retarget_one_irq(struct irqaction *p, int goal_cpu)
1081 {
1082 struct ino_bucket *bucket = __bucket(p->mask);
1083 unsigned long imap = bucket->imap;
1084 unsigned int tid;
1085
1086 /* Never change this, it causes problems on Ex000 systems. */
1087 if (bucket->pil == 12)
1088 return goal_cpu;
1089
1090 if (tlb_type == cheetah) {
1091 tid = __cpu_logical_map[goal_cpu] << 26;
1092 tid &= IMAP_AID_SAFARI;
1093 } else if (this_is_starfire == 0) {
1094 tid = __cpu_logical_map[goal_cpu] << 26;
1095 tid &= IMAP_TID_UPA;
1096 } else {
1097 tid = (starfire_translate(imap, __cpu_logical_map[goal_cpu]) << 26);
1098 tid &= IMAP_TID_UPA;
1099 }
1100 upa_writel(tid | IMAP_VALID, imap);
1101
1102 goal_cpu++;
1103 if(goal_cpu >= NR_CPUS ||
1104 __cpu_logical_map[goal_cpu] == -1)
1105 goal_cpu = 0;
1106 return goal_cpu;
1107 }
1108
1109 /* Called from request_irq. */
1110 static void distribute_irqs(void)
1111 {
1112 unsigned long flags;
1113 int cpu, level;
1114
1115 save_and_cli(flags);
1116 cpu = 0;
1117 for(level = 0; level < NR_IRQS; level++) {
1118 struct irqaction *p = irq_action[level];
1119 while(p) {
1120 if(p->flags & SA_IMAP_MASKED)
1121 cpu = retarget_one_irq(p, cpu);
1122 p = p->next;
1123 }
1124 }
1125 restore_flags(flags);
1126 }
1127 #endif
1128
1129
1130 struct sun5_timer *prom_timers;
1131 static u64 prom_limit0, prom_limit1;
1132
1133 static void map_prom_timers(void)
1134 {
1135 unsigned int addr[3];
1136 int tnode, err;
1137
1138 /* PROM timer node hangs out in the top level of device siblings... */
1139 tnode = prom_finddevice("/counter-timer");
1140
1141 /* Assume if node is not present, PROM uses different tick mechanism
1142 * which we should not care about.
1143 */
1144 if(tnode == 0 || tnode == -1) {
1145 prom_timers = (struct sun5_timer *) 0;
1146 return;
1147 }
1148
1149 /* If PROM is really using this, it must be mapped by him. */
1150 err = prom_getproperty(tnode, "address", (char *)addr, sizeof(addr));
1151 if(err == -1) {
1152 prom_printf("PROM does not have timer mapped, trying to continue.\n");
1153 prom_timers = (struct sun5_timer *) 0;
1154 return;
1155 }
1156 prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]);
1157 }
1158
1159 static void kill_prom_timer(void)
1160 {
1161 if(!prom_timers)
1162 return;
1163
1164 /* Save them away for later. */
1165 prom_limit0 = prom_timers->limit0;
1166 prom_limit1 = prom_timers->limit1;
1167
1168 /* Just as in sun4c/sun4m PROM uses timer which ticks at IRQ 14.
1169 * We turn both off here just to be paranoid.
1170 */
1171 prom_timers->limit0 = 0;
1172 prom_timers->limit1 = 0;
1173
1174 /* Wheee, eat the interrupt packet too... */
1175 __asm__ __volatile__(
1176 " mov 0x40, %%g2\n"
1177 " ldxa [%%g0] %0, %%g1\n"
1178 " ldxa [%%g2] %1, %%g1\n"
1179 " stxa %%g0, [%%g0] %0\n"
1180 " membar #Sync\n"
1181 : /* no outputs */
1182 : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R)
1183 : "g1", "g2");
1184 }
1185
1186 void enable_prom_timer(void)
1187 {
1188 if(!prom_timers)
1189 return;
1190
1191 /* Set it to whatever was there before. */
1192 prom_timers->limit1 = prom_limit1;
1193 prom_timers->count1 = 0;
1194 prom_timers->limit0 = prom_limit0;
1195 prom_timers->count0 = 0;
1196 }
1197
1198 void __init init_IRQ(void)
1199 {
1200 static int called = 0;
1201
1202 if (called == 0) {
1203 called = 1;
1204 map_prom_timers();
1205 kill_prom_timer();
1206 memset(&ivector_table[0], 0, sizeof(ivector_table));
1207 #ifndef CONFIG_SMP
1208 memset(&__up_workvec[0], 0, sizeof(__up_workvec));
1209 #endif
1210 }
1211
1212 /* We need to clear any IRQ's pending in the soft interrupt
1213 * registers, a spurious one could be left around from the
1214 * PROM timer which we just disabled.
1215 */
1216 clear_softint(get_softint());
1217
1218 /* Now that ivector table is initialized, it is safe
1219 * to receive IRQ vector traps. We will normally take
1220 * one or two right now, in case some device PROM used
1221 * to boot us wants to speak to us. We just ignore them.
1222 */
1223 __asm__ __volatile__("rdpr %%pstate, %%g1\n\t"
1224 "or %%g1, %0, %%g1\n\t"
1225 "wrpr %%g1, 0x0, %%pstate"
1226 : /* No outputs */
1227 : "i" (PSTATE_IE)
1228 : "g1");
1229 }
1230
1231 void init_irq_proc(void)
1232 {
1233 /* For now, nothing... */
1234 }
1235