File: /usr/src/linux/arch/ia64/kernel/unwind.c
1 /*
2 * Copyright (C) 1999-2001 Hewlett-Packard Co
3 * Copyright (C) 1999-2001 David Mosberger-Tang <davidm@hpl.hp.com>
4 */
5 /*
6 * This file implements call frame unwind support for the Linux
7 * kernel. Parsing and processing the unwind information is
8 * time-consuming, so this implementation translates the unwind
9 * descriptors into unwind scripts. These scripts are very simple
10 * (basically a sequence of assignments) and efficient to execute.
11 * They are cached for later re-use. Each script is specific for a
12 * given instruction pointer address and the set of predicate values
13 * that the script depends on (most unwind descriptors are
14 * unconditional and scripts often do not depend on predicates at
15 * all). This code is based on the unwind conventions described in
16 * the "IA-64 Software Conventions and Runtime Architecture" manual.
17 *
18 * SMP conventions:
19 * o updates to the global unwind data (in structure "unw") are serialized
20 * by the unw.lock spinlock
21 * o each unwind script has its own read-write lock; a thread must acquire
22 * a read lock before executing a script and must acquire a write lock
23 * before modifying a script
24 * o if both the unw.lock spinlock and a script's read-write lock must be
25 * acquired, then the read-write lock must be acquired first.
26 */
27 #include <linux/bootmem.h>
28 #include <linux/kernel.h>
29 #include <linux/sched.h>
30 #include <linux/slab.h>
31
32 #include <asm/unwind.h>
33
34 #include <asm/delay.h>
35 #include <asm/page.h>
36 #include <asm/ptrace.h>
37 #include <asm/ptrace_offsets.h>
38 #include <asm/rse.h>
39 #include <asm/system.h>
40 #include <asm/uaccess.h>
41
42 #include "entry.h"
43 #include "unwind_i.h"
44
45 #define MIN(a,b) ((a) < (b) ? (a) : (b))
46 #define p5 5
47
48 #define UNW_LOG_CACHE_SIZE 7 /* each unw_script is ~256 bytes in size */
49 #define UNW_CACHE_SIZE (1 << UNW_LOG_CACHE_SIZE)
50
51 #define UNW_LOG_HASH_SIZE (UNW_LOG_CACHE_SIZE + 1)
52 #define UNW_HASH_SIZE (1 << UNW_LOG_HASH_SIZE)
53
54 #define UNW_DEBUG 0
55 #define UNW_STATS 0 /* WARNING: this disabled interrupts for long time-spans!! */
56
57 #if UNW_DEBUG
58 static long unw_debug_level = 255;
59 # define debug(level,format...) if (unw_debug_level > level) printk(format)
60 # define dprintk(format...) printk(format)
61 # define inline
62 #else
63 # define debug(level,format...)
64 # define dprintk(format...)
65 #endif
66
67 #if UNW_STATS
68 # define STAT(x...) x
69 #else
70 # define STAT(x...)
71 #endif
72
73 #define alloc_reg_state() kmalloc(sizeof(struct unw_state_record), GFP_ATOMIC)
74 #define free_reg_state(usr) kfree(usr)
75
76 typedef unsigned long unw_word;
77 typedef unsigned char unw_hash_index_t;
78
79 #define struct_offset(str,fld) ((char *)&((str *)NULL)->fld - (char *) 0)
80
81 static struct {
82 spinlock_t lock; /* spinlock for unwind data */
83
84 /* list of unwind tables (one per load-module) */
85 struct unw_table *tables;
86
87 /* table of registers that prologues can save (and order in which they're saved): */
88 const unsigned char save_order[8];
89
90 /* maps a preserved register index (preg_index) to corresponding switch_stack offset: */
91 unsigned short sw_off[sizeof(struct unw_frame_info) / 8];
92
93 unsigned short lru_head; /* index of lead-recently used script */
94 unsigned short lru_tail; /* index of most-recently used script */
95
96 /* index into unw_frame_info for preserved register i */
97 unsigned short preg_index[UNW_NUM_REGS];
98
99 /* unwind table for the kernel: */
100 struct unw_table kernel_table;
101
102 /* unwind table describing the gate page (kernel code that is mapped into user space): */
103 size_t gate_table_size;
104 unsigned long *gate_table;
105
106 /* hash table that maps instruction pointer to script index: */
107 unsigned short hash[UNW_HASH_SIZE];
108
109 /* script cache: */
110 struct unw_script cache[UNW_CACHE_SIZE];
111
112 # if UNW_DEBUG
113 const char *preg_name[UNW_NUM_REGS];
114 # endif
115 # if UNW_STATS
116 struct {
117 struct {
118 int lookups;
119 int hinted_hits;
120 int normal_hits;
121 int collision_chain_traversals;
122 } cache;
123 struct {
124 unsigned long build_time;
125 unsigned long run_time;
126 unsigned long parse_time;
127 int builds;
128 int news;
129 int collisions;
130 int runs;
131 } script;
132 struct {
133 unsigned long init_time;
134 unsigned long unwind_time;
135 int inits;
136 int unwinds;
137 } api;
138 } stat;
139 # endif
140 } unw = {
141 tables: &unw.kernel_table,
142 lock: SPIN_LOCK_UNLOCKED,
143 save_order: {
144 UNW_REG_RP, UNW_REG_PFS, UNW_REG_PSP, UNW_REG_PR,
145 UNW_REG_UNAT, UNW_REG_LC, UNW_REG_FPSR, UNW_REG_PRI_UNAT_GR
146 },
147 preg_index: {
148 struct_offset(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_GR */
149 struct_offset(struct unw_frame_info, pri_unat_loc)/8, /* PRI_UNAT_MEM */
150 struct_offset(struct unw_frame_info, bsp_loc)/8,
151 struct_offset(struct unw_frame_info, bspstore_loc)/8,
152 struct_offset(struct unw_frame_info, pfs_loc)/8,
153 struct_offset(struct unw_frame_info, rnat_loc)/8,
154 struct_offset(struct unw_frame_info, psp)/8,
155 struct_offset(struct unw_frame_info, rp_loc)/8,
156 struct_offset(struct unw_frame_info, r4)/8,
157 struct_offset(struct unw_frame_info, r5)/8,
158 struct_offset(struct unw_frame_info, r6)/8,
159 struct_offset(struct unw_frame_info, r7)/8,
160 struct_offset(struct unw_frame_info, unat_loc)/8,
161 struct_offset(struct unw_frame_info, pr_loc)/8,
162 struct_offset(struct unw_frame_info, lc_loc)/8,
163 struct_offset(struct unw_frame_info, fpsr_loc)/8,
164 struct_offset(struct unw_frame_info, b1_loc)/8,
165 struct_offset(struct unw_frame_info, b2_loc)/8,
166 struct_offset(struct unw_frame_info, b3_loc)/8,
167 struct_offset(struct unw_frame_info, b4_loc)/8,
168 struct_offset(struct unw_frame_info, b5_loc)/8,
169 struct_offset(struct unw_frame_info, f2_loc)/8,
170 struct_offset(struct unw_frame_info, f3_loc)/8,
171 struct_offset(struct unw_frame_info, f4_loc)/8,
172 struct_offset(struct unw_frame_info, f5_loc)/8,
173 struct_offset(struct unw_frame_info, fr_loc[16 - 16])/8,
174 struct_offset(struct unw_frame_info, fr_loc[17 - 16])/8,
175 struct_offset(struct unw_frame_info, fr_loc[18 - 16])/8,
176 struct_offset(struct unw_frame_info, fr_loc[19 - 16])/8,
177 struct_offset(struct unw_frame_info, fr_loc[20 - 16])/8,
178 struct_offset(struct unw_frame_info, fr_loc[21 - 16])/8,
179 struct_offset(struct unw_frame_info, fr_loc[22 - 16])/8,
180 struct_offset(struct unw_frame_info, fr_loc[23 - 16])/8,
181 struct_offset(struct unw_frame_info, fr_loc[24 - 16])/8,
182 struct_offset(struct unw_frame_info, fr_loc[25 - 16])/8,
183 struct_offset(struct unw_frame_info, fr_loc[26 - 16])/8,
184 struct_offset(struct unw_frame_info, fr_loc[27 - 16])/8,
185 struct_offset(struct unw_frame_info, fr_loc[28 - 16])/8,
186 struct_offset(struct unw_frame_info, fr_loc[29 - 16])/8,
187 struct_offset(struct unw_frame_info, fr_loc[30 - 16])/8,
188 struct_offset(struct unw_frame_info, fr_loc[31 - 16])/8,
189 },
190 hash : { [0 ... UNW_HASH_SIZE - 1] = -1 },
191 #if UNW_DEBUG
192 preg_name: {
193 "pri_unat_gr", "pri_unat_mem", "bsp", "bspstore", "ar.pfs", "ar.rnat", "psp", "rp",
194 "r4", "r5", "r6", "r7",
195 "ar.unat", "pr", "ar.lc", "ar.fpsr",
196 "b1", "b2", "b3", "b4", "b5",
197 "f2", "f3", "f4", "f5",
198 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
199 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"
200 }
201 #endif
202 };
203
204
205 /* Unwind accessors. */
206
207 /*
208 * Returns offset of rREG in struct pt_regs.
209 */
210 static inline unsigned long
211 pt_regs_off (unsigned long reg)
212 {
213 unsigned long off =0;
214
215 if (reg >= 1 && reg <= 3)
216 off = struct_offset(struct pt_regs, r1) + 8*(reg - 1);
217 else if (reg <= 11)
218 off = struct_offset(struct pt_regs, r8) + 8*(reg - 8);
219 else if (reg <= 15)
220 off = struct_offset(struct pt_regs, r12) + 8*(reg - 12);
221 else if (reg <= 31)
222 off = struct_offset(struct pt_regs, r16) + 8*(reg - 16);
223 else
224 dprintk("unwind: bad scratch reg r%lu\n", reg);
225 return off;
226 }
227
228 int
229 unw_access_gr (struct unw_frame_info *info, int regnum, unsigned long *val, char *nat, int write)
230 {
231 unsigned long *addr, *nat_addr, nat_mask = 0, dummy_nat;
232 struct unw_ireg *ireg;
233 struct pt_regs *pt;
234
235 if ((unsigned) regnum - 1 >= 127) {
236 dprintk("unwind: trying to access non-existent r%u\n", regnum);
237 return -1;
238 }
239
240 if (regnum < 32) {
241 if (regnum >= 4 && regnum <= 7) {
242 /* access a preserved register */
243 ireg = &info->r4 + (regnum - 4);
244 addr = ireg->loc;
245 if (addr) {
246 nat_addr = addr + ireg->nat.off;
247 switch (ireg->nat.type) {
248 case UNW_NAT_VAL:
249 /* simulate getf.sig/setf.sig */
250 if (write) {
251 if (*nat) {
252 /* write NaTVal and be done with it */
253 addr[0] = 0;
254 addr[1] = 0x1fffe;
255 return 0;
256 }
257 addr[1] = 0x1003e;
258 } else {
259 if (addr[0] == 0 && addr[1] == 0x1ffe) {
260 /* return NaT and be done with it */
261 *val = 0;
262 *nat = 1;
263 return 0;
264 }
265 }
266 /* fall through */
267 case UNW_NAT_NONE:
268 dummy_nat = 0;
269 nat_addr = &dummy_nat;
270 break;
271
272 case UNW_NAT_MEMSTK:
273 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
274 break;
275
276 case UNW_NAT_REGSTK:
277 nat_addr = ia64_rse_rnat_addr(addr);
278 if ((unsigned long) addr < info->regstk.limit
279 || (unsigned long) addr >= info->regstk.top)
280 {
281 dprintk("unwind: %p outside of regstk "
282 "[0x%lx-0x%lx)\n", (void *) addr,
283 info->regstk.limit,
284 info->regstk.top);
285 return -1;
286 }
287 if ((unsigned long) nat_addr >= info->regstk.top)
288 nat_addr = &info->sw->ar_rnat;
289 nat_mask = (1UL << ia64_rse_slot_num(addr));
290 break;
291 }
292 } else {
293 addr = &info->sw->r4 + (regnum - 4);
294 nat_addr = &info->sw->ar_unat;
295 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
296 }
297 } else {
298 /* access a scratch register */
299 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
300 pt = (struct pt_regs *) info->psp - 1;
301 else
302 pt = (struct pt_regs *) info->sp - 1;
303 addr = (unsigned long *) ((long) pt + pt_regs_off(regnum));
304 if (info->pri_unat_loc)
305 nat_addr = info->pri_unat_loc;
306 else
307 nat_addr = &info->sw->ar_unat;
308 nat_mask = (1UL << ((long) addr & 0x1f8)/8);
309 }
310 } else {
311 /* access a stacked register */
312 addr = ia64_rse_skip_regs((unsigned long *) info->bsp, regnum - 32);
313 nat_addr = ia64_rse_rnat_addr(addr);
314 if ((unsigned long) addr < info->regstk.limit
315 || (unsigned long) addr >= info->regstk.top)
316 {
317 dprintk("unwind: ignoring attempt to access register outside of rbs\n");
318 return -1;
319 }
320 if ((unsigned long) nat_addr >= info->regstk.top)
321 nat_addr = &info->sw->ar_rnat;
322 nat_mask = (1UL << ia64_rse_slot_num(addr));
323 }
324
325 if (write) {
326 *addr = *val;
327 if (*nat)
328 *nat_addr |= nat_mask;
329 else
330 *nat_addr &= ~nat_mask;
331 } else {
332 if ((*nat_addr & nat_mask) == 0) {
333 *val = *addr;
334 *nat = 0;
335 } else {
336 *val = 0; /* if register is a NaT, *addr may contain kernel data! */
337 *nat = 1;
338 }
339 }
340 return 0;
341 }
342
343 int
344 unw_access_br (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
345 {
346 unsigned long *addr;
347 struct pt_regs *pt;
348
349 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
350 pt = (struct pt_regs *) info->psp - 1;
351 else
352 pt = (struct pt_regs *) info->sp - 1;
353 switch (regnum) {
354 /* scratch: */
355 case 0: addr = &pt->b0; break;
356 case 6: addr = &pt->b6; break;
357 case 7: addr = &pt->b7; break;
358
359 /* preserved: */
360 case 1: case 2: case 3: case 4: case 5:
361 addr = *(&info->b1_loc + (regnum - 1));
362 if (!addr)
363 addr = &info->sw->b1 + (regnum - 1);
364 break;
365
366 default:
367 dprintk("unwind: trying to access non-existent b%u\n", regnum);
368 return -1;
369 }
370 if (write)
371 *addr = *val;
372 else
373 *val = *addr;
374 return 0;
375 }
376
377 int
378 unw_access_fr (struct unw_frame_info *info, int regnum, struct ia64_fpreg *val, int write)
379 {
380 struct ia64_fpreg *addr = 0;
381 struct pt_regs *pt;
382
383 if ((unsigned) (regnum - 2) >= 126) {
384 dprintk("unwind: trying to access non-existent f%u\n", regnum);
385 return -1;
386 }
387
388 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
389 pt = (struct pt_regs *) info->psp - 1;
390 else
391 pt = (struct pt_regs *) info->sp - 1;
392
393 if (regnum <= 5) {
394 addr = *(&info->f2_loc + (regnum - 2));
395 if (!addr)
396 addr = &info->sw->f2 + (regnum - 2);
397 } else if (regnum <= 15) {
398 if (regnum <= 9)
399 addr = &pt->f6 + (regnum - 6);
400 else
401 addr = &info->sw->f10 + (regnum - 10);
402 } else if (regnum <= 31) {
403 addr = info->fr_loc[regnum - 16];
404 if (!addr)
405 addr = &info->sw->f16 + (regnum - 16);
406 } else {
407 struct task_struct *t = info->task;
408
409 if (write)
410 ia64_sync_fph(t);
411 else
412 ia64_flush_fph(t);
413 addr = t->thread.fph + (regnum - 32);
414 }
415
416 if (write)
417 *addr = *val;
418 else
419 *val = *addr;
420 return 0;
421 }
422
423 int
424 unw_access_ar (struct unw_frame_info *info, int regnum, unsigned long *val, int write)
425 {
426 unsigned long *addr;
427 struct pt_regs *pt;
428
429 if (info->flags & UNW_FLAG_INTERRUPT_FRAME)
430 pt = (struct pt_regs *) info->psp - 1;
431 else
432 pt = (struct pt_regs *) info->sp - 1;
433
434 switch (regnum) {
435 case UNW_AR_BSP:
436 addr = info->bsp_loc;
437 if (!addr)
438 addr = &info->sw->ar_bspstore;
439 break;
440
441 case UNW_AR_BSPSTORE:
442 addr = info->bspstore_loc;
443 if (!addr)
444 addr = &info->sw->ar_bspstore;
445 break;
446
447 case UNW_AR_PFS:
448 addr = info->pfs_loc;
449 if (!addr)
450 addr = &info->sw->ar_pfs;
451 break;
452
453 case UNW_AR_RNAT:
454 addr = info->rnat_loc;
455 if (!addr)
456 addr = &info->sw->ar_rnat;
457 break;
458
459 case UNW_AR_UNAT:
460 addr = info->unat_loc;
461 if (!addr)
462 addr = &info->sw->ar_unat;
463 break;
464
465 case UNW_AR_LC:
466 addr = info->lc_loc;
467 if (!addr)
468 addr = &info->sw->ar_lc;
469 break;
470
471 case UNW_AR_EC:
472 if (!info->cfm_loc)
473 return -1;
474 if (write)
475 *info->cfm_loc =
476 (*info->cfm_loc & ~(0x3fUL << 52)) | ((*val & 0x3f) << 52);
477 else
478 *val = (*info->cfm_loc >> 52) & 0x3f;
479 return 0;
480
481 case UNW_AR_FPSR:
482 addr = info->fpsr_loc;
483 if (!addr)
484 addr = &info->sw->ar_fpsr;
485 break;
486
487 case UNW_AR_RSC:
488 addr = &pt->ar_rsc;
489 break;
490
491 case UNW_AR_CCV:
492 addr = &pt->ar_ccv;
493 break;
494
495 default:
496 dprintk("unwind: trying to access non-existent ar%u\n", regnum);
497 return -1;
498 }
499
500 if (write)
501 *addr = *val;
502 else
503 *val = *addr;
504 return 0;
505 }
506
507 inline int
508 unw_access_pr (struct unw_frame_info *info, unsigned long *val, int write)
509 {
510 unsigned long *addr;
511
512 addr = info->pr_loc;
513 if (!addr)
514 addr = &info->sw->pr;
515
516 if (write)
517 *addr = *val;
518 else
519 *val = *addr;
520 return 0;
521 }
522
523
524 /* Unwind decoder routines */
525
526 static inline void
527 push (struct unw_state_record *sr)
528 {
529 struct unw_reg_state *rs;
530
531 rs = alloc_reg_state();
532 if (!rs) {
533 printk("unwind: cannot stack reg state!\n");
534 return;
535 }
536 memcpy(rs, &sr->curr, sizeof(*rs));
537 rs->next = sr->stack;
538 sr->stack = rs;
539 }
540
541 static void
542 pop (struct unw_state_record *sr)
543 {
544 struct unw_reg_state *rs;
545
546 if (!sr->stack) {
547 printk ("unwind: stack underflow!\n");
548 return;
549 }
550 rs = sr->stack;
551 sr->stack = rs->next;
552 free_reg_state(rs);
553 }
554
555 static enum unw_register_index __attribute__((const))
556 decode_abreg (unsigned char abreg, int memory)
557 {
558 switch (abreg) {
559 case 0x04 ... 0x07: return UNW_REG_R4 + (abreg - 0x04);
560 case 0x22 ... 0x25: return UNW_REG_F2 + (abreg - 0x22);
561 case 0x30 ... 0x3f: return UNW_REG_F16 + (abreg - 0x30);
562 case 0x41 ... 0x45: return UNW_REG_B1 + (abreg - 0x41);
563 case 0x60: return UNW_REG_PR;
564 case 0x61: return UNW_REG_PSP;
565 case 0x62: return memory ? UNW_REG_PRI_UNAT_MEM : UNW_REG_PRI_UNAT_GR;
566 case 0x63: return UNW_REG_RP;
567 case 0x64: return UNW_REG_BSP;
568 case 0x65: return UNW_REG_BSPSTORE;
569 case 0x66: return UNW_REG_RNAT;
570 case 0x67: return UNW_REG_UNAT;
571 case 0x68: return UNW_REG_FPSR;
572 case 0x69: return UNW_REG_PFS;
573 case 0x6a: return UNW_REG_LC;
574 default:
575 break;
576 }
577 dprintk("unwind: bad abreg=0x%x\n", abreg);
578 return UNW_REG_LC;
579 }
580
581 static void
582 set_reg (struct unw_reg_info *reg, enum unw_where where, int when, unsigned long val)
583 {
584 reg->val = val;
585 reg->where = where;
586 if (reg->when == UNW_WHEN_NEVER)
587 reg->when = when;
588 }
589
590 static void
591 alloc_spill_area (unsigned long *offp, unsigned long regsize,
592 struct unw_reg_info *lo, struct unw_reg_info *hi)
593 {
594 struct unw_reg_info *reg;
595
596 for (reg = hi; reg >= lo; --reg) {
597 if (reg->where == UNW_WHERE_SPILL_HOME) {
598 reg->where = UNW_WHERE_PSPREL;
599 reg->val = 0x10 - *offp;
600 *offp += regsize;
601 }
602 }
603 }
604
605 static inline void
606 spill_next_when (struct unw_reg_info **regp, struct unw_reg_info *lim, unw_word t)
607 {
608 struct unw_reg_info *reg;
609
610 for (reg = *regp; reg <= lim; ++reg) {
611 if (reg->where == UNW_WHERE_SPILL_HOME) {
612 reg->when = t;
613 *regp = reg + 1;
614 return;
615 }
616 }
617 dprintk("unwind: excess spill!\n");
618 }
619
620 static inline void
621 finish_prologue (struct unw_state_record *sr)
622 {
623 struct unw_reg_info *reg;
624 unsigned long off;
625 int i;
626
627 /*
628 * First, resolve implicit register save locations (see Section "11.4.2.3 Rules
629 * for Using Unwind Descriptors", rule 3):
630 */
631 for (i = 0; i < (int) sizeof(unw.save_order)/sizeof(unw.save_order[0]); ++i) {
632 reg = sr->curr.reg + unw.save_order[i];
633 if (reg->where == UNW_WHERE_GR_SAVE) {
634 reg->where = UNW_WHERE_GR;
635 reg->val = sr->gr_save_loc++;
636 }
637 }
638
639 /*
640 * Next, compute when the fp, general, and branch registers get
641 * saved. This must come before alloc_spill_area() because
642 * we need to know which registers are spilled to their home
643 * locations.
644 */
645 if (sr->imask) {
646 unsigned char kind, mask = 0, *cp = sr->imask;
647 unsigned long t;
648 static const unsigned char limit[3] = {
649 UNW_REG_F31, UNW_REG_R7, UNW_REG_B5
650 };
651 struct unw_reg_info *(regs[3]);
652
653 regs[0] = sr->curr.reg + UNW_REG_F2;
654 regs[1] = sr->curr.reg + UNW_REG_R4;
655 regs[2] = sr->curr.reg + UNW_REG_B1;
656
657 for (t = 0; t < sr->region_len; ++t) {
658 if ((t & 3) == 0)
659 mask = *cp++;
660 kind = (mask >> 2*(3-(t & 3))) & 3;
661 if (kind > 0)
662 spill_next_when(®s[kind - 1], sr->curr.reg + limit[kind - 1],
663 sr->region_start + t);
664 }
665 }
666 /*
667 * Next, lay out the memory stack spill area:
668 */
669 if (sr->any_spills) {
670 off = sr->spill_offset;
671 alloc_spill_area(&off, 16, sr->curr.reg + UNW_REG_F2, sr->curr.reg + UNW_REG_F31);
672 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_B1, sr->curr.reg + UNW_REG_B5);
673 alloc_spill_area(&off, 8, sr->curr.reg + UNW_REG_R4, sr->curr.reg + UNW_REG_R7);
674 }
675 }
676
677 /*
678 * Region header descriptors.
679 */
680
681 static void
682 desc_prologue (int body, unw_word rlen, unsigned char mask, unsigned char grsave,
683 struct unw_state_record *sr)
684 {
685 int i;
686
687 if (!(sr->in_body || sr->first_region))
688 finish_prologue(sr);
689 sr->first_region = 0;
690
691 /* check if we're done: */
692 if (body && sr->when_target < sr->region_start + sr->region_len) {
693 sr->done = 1;
694 return;
695 }
696
697 for (i = 0; i < sr->epilogue_count; ++i)
698 pop(sr);
699 sr->epilogue_count = 0;
700 sr->epilogue_start = UNW_WHEN_NEVER;
701
702 if (!body)
703 push(sr);
704
705 sr->region_start += sr->region_len;
706 sr->region_len = rlen;
707 sr->in_body = body;
708
709 if (!body) {
710 for (i = 0; i < 4; ++i) {
711 if (mask & 0x8)
712 set_reg(sr->curr.reg + unw.save_order[i], UNW_WHERE_GR,
713 sr->region_start + sr->region_len - 1, grsave++);
714 mask <<= 1;
715 }
716 sr->gr_save_loc = grsave;
717 sr->any_spills = 0;
718 sr->imask = 0;
719 sr->spill_offset = 0x10; /* default to psp+16 */
720 }
721 }
722
723 /*
724 * Prologue descriptors.
725 */
726
727 static inline void
728 desc_abi (unsigned char abi, unsigned char context, struct unw_state_record *sr)
729 {
730 if (abi == 0 && context == 'i')
731 sr->flags |= UNW_FLAG_INTERRUPT_FRAME;
732 else
733 dprintk("unwind: ignoring unwabi(abi=0x%x,context=0x%x)\n", abi, context);
734 }
735
736 static inline void
737 desc_br_gr (unsigned char brmask, unsigned char gr, struct unw_state_record *sr)
738 {
739 int i;
740
741 for (i = 0; i < 5; ++i) {
742 if (brmask & 1)
743 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_GR,
744 sr->region_start + sr->region_len - 1, gr++);
745 brmask >>= 1;
746 }
747 }
748
749 static inline void
750 desc_br_mem (unsigned char brmask, struct unw_state_record *sr)
751 {
752 int i;
753
754 for (i = 0; i < 5; ++i) {
755 if (brmask & 1) {
756 set_reg(sr->curr.reg + UNW_REG_B1 + i, UNW_WHERE_SPILL_HOME,
757 sr->region_start + sr->region_len - 1, 0);
758 sr->any_spills = 1;
759 }
760 brmask >>= 1;
761 }
762 }
763
764 static inline void
765 desc_frgr_mem (unsigned char grmask, unw_word frmask, struct unw_state_record *sr)
766 {
767 int i;
768
769 for (i = 0; i < 4; ++i) {
770 if ((grmask & 1) != 0) {
771 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
772 sr->region_start + sr->region_len - 1, 0);
773 sr->any_spills = 1;
774 }
775 grmask >>= 1;
776 }
777 for (i = 0; i < 20; ++i) {
778 if ((frmask & 1) != 0) {
779 set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
780 sr->region_start + sr->region_len - 1, 0);
781 sr->any_spills = 1;
782 }
783 frmask >>= 1;
784 }
785 }
786
787 static inline void
788 desc_fr_mem (unsigned char frmask, struct unw_state_record *sr)
789 {
790 int i;
791
792 for (i = 0; i < 4; ++i) {
793 if ((frmask & 1) != 0) {
794 set_reg(sr->curr.reg + UNW_REG_F2 + i, UNW_WHERE_SPILL_HOME,
795 sr->region_start + sr->region_len - 1, 0);
796 sr->any_spills = 1;
797 }
798 frmask >>= 1;
799 }
800 }
801
802 static inline void
803 desc_gr_gr (unsigned char grmask, unsigned char gr, struct unw_state_record *sr)
804 {
805 int i;
806
807 for (i = 0; i < 4; ++i) {
808 if ((grmask & 1) != 0)
809 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_GR,
810 sr->region_start + sr->region_len - 1, gr++);
811 grmask >>= 1;
812 }
813 }
814
815 static inline void
816 desc_gr_mem (unsigned char grmask, struct unw_state_record *sr)
817 {
818 int i;
819
820 for (i = 0; i < 4; ++i) {
821 if ((grmask & 1) != 0) {
822 set_reg(sr->curr.reg + UNW_REG_R4 + i, UNW_WHERE_SPILL_HOME,
823 sr->region_start + sr->region_len - 1, 0);
824 sr->any_spills = 1;
825 }
826 grmask >>= 1;
827 }
828 }
829
830 static inline void
831 desc_mem_stack_f (unw_word t, unw_word size, struct unw_state_record *sr)
832 {
833 set_reg(sr->curr.reg + UNW_REG_PSP, UNW_WHERE_NONE,
834 sr->region_start + MIN((int)t, sr->region_len - 1), 16*size);
835 }
836
837 static inline void
838 desc_mem_stack_v (unw_word t, struct unw_state_record *sr)
839 {
840 sr->curr.reg[UNW_REG_PSP].when = sr->region_start + MIN((int)t, sr->region_len - 1);
841 }
842
843 static inline void
844 desc_reg_gr (unsigned char reg, unsigned char dst, struct unw_state_record *sr)
845 {
846 set_reg(sr->curr.reg + reg, UNW_WHERE_GR, sr->region_start + sr->region_len - 1, dst);
847 }
848
849 static inline void
850 desc_reg_psprel (unsigned char reg, unw_word pspoff, struct unw_state_record *sr)
851 {
852 set_reg(sr->curr.reg + reg, UNW_WHERE_PSPREL, sr->region_start + sr->region_len - 1,
853 0x10 - 4*pspoff);
854 }
855
856 static inline void
857 desc_reg_sprel (unsigned char reg, unw_word spoff, struct unw_state_record *sr)
858 {
859 set_reg(sr->curr.reg + reg, UNW_WHERE_SPREL, sr->region_start + sr->region_len - 1,
860 4*spoff);
861 }
862
863 static inline void
864 desc_rp_br (unsigned char dst, struct unw_state_record *sr)
865 {
866 sr->return_link_reg = dst;
867 }
868
869 static inline void
870 desc_reg_when (unsigned char regnum, unw_word t, struct unw_state_record *sr)
871 {
872 struct unw_reg_info *reg = sr->curr.reg + regnum;
873
874 if (reg->where == UNW_WHERE_NONE)
875 reg->where = UNW_WHERE_GR_SAVE;
876 reg->when = sr->region_start + MIN((int)t, sr->region_len - 1);
877 }
878
879 static inline void
880 desc_spill_base (unw_word pspoff, struct unw_state_record *sr)
881 {
882 sr->spill_offset = 0x10 - 4*pspoff;
883 }
884
885 static inline unsigned char *
886 desc_spill_mask (unsigned char *imaskp, struct unw_state_record *sr)
887 {
888 sr->imask = imaskp;
889 return imaskp + (2*sr->region_len + 7)/8;
890 }
891
892 /*
893 * Body descriptors.
894 */
895 static inline void
896 desc_epilogue (unw_word t, unw_word ecount, struct unw_state_record *sr)
897 {
898 sr->epilogue_start = sr->region_start + sr->region_len - 1 - t;
899 sr->epilogue_count = ecount + 1;
900 }
901
902 static inline void
903 desc_copy_state (unw_word label, struct unw_state_record *sr)
904 {
905 struct unw_reg_state *rs;
906
907 for (rs = sr->reg_state_list; rs; rs = rs->next) {
908 if (rs->label == label) {
909 memcpy (&sr->curr, rs, sizeof(sr->curr));
910 return;
911 }
912 }
913 printk("unwind: failed to find state labelled 0x%lx\n", label);
914 }
915
916 static inline void
917 desc_label_state (unw_word label, struct unw_state_record *sr)
918 {
919 struct unw_reg_state *rs;
920
921 rs = alloc_reg_state();
922 if (!rs) {
923 printk("unwind: cannot stack!\n");
924 return;
925 }
926 memcpy(rs, &sr->curr, sizeof(*rs));
927 rs->label = label;
928 rs->next = sr->reg_state_list;
929 sr->reg_state_list = rs;
930 }
931
932 /*
933 * General descriptors.
934 */
935
936 static inline int
937 desc_is_active (unsigned char qp, unw_word t, struct unw_state_record *sr)
938 {
939 if (sr->when_target <= sr->region_start + MIN((int)t, sr->region_len - 1))
940 return 0;
941 if (qp > 0) {
942 if ((sr->pr_val & (1UL << qp)) == 0)
943 return 0;
944 sr->pr_mask |= (1UL << qp);
945 }
946 return 1;
947 }
948
949 static inline void
950 desc_restore_p (unsigned char qp, unw_word t, unsigned char abreg, struct unw_state_record *sr)
951 {
952 struct unw_reg_info *r;
953
954 if (!desc_is_active(qp, t, sr))
955 return;
956
957 r = sr->curr.reg + decode_abreg(abreg, 0);
958 r->where = UNW_WHERE_NONE;
959 r->when = UNW_WHEN_NEVER;
960 r->val = 0;
961 }
962
963 static inline void
964 desc_spill_reg_p (unsigned char qp, unw_word t, unsigned char abreg, unsigned char x,
965 unsigned char ytreg, struct unw_state_record *sr)
966 {
967 enum unw_where where = UNW_WHERE_GR;
968 struct unw_reg_info *r;
969
970 if (!desc_is_active(qp, t, sr))
971 return;
972
973 if (x)
974 where = UNW_WHERE_BR;
975 else if (ytreg & 0x80)
976 where = UNW_WHERE_FR;
977
978 r = sr->curr.reg + decode_abreg(abreg, 0);
979 r->where = where;
980 r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
981 r->val = (ytreg & 0x7f);
982 }
983
984 static inline void
985 desc_spill_psprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word pspoff,
986 struct unw_state_record *sr)
987 {
988 struct unw_reg_info *r;
989
990 if (!desc_is_active(qp, t, sr))
991 return;
992
993 r = sr->curr.reg + decode_abreg(abreg, 1);
994 r->where = UNW_WHERE_PSPREL;
995 r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
996 r->val = 0x10 - 4*pspoff;
997 }
998
999 static inline void
1000 desc_spill_sprel_p (unsigned char qp, unw_word t, unsigned char abreg, unw_word spoff,
1001 struct unw_state_record *sr)
1002 {
1003 struct unw_reg_info *r;
1004
1005 if (!desc_is_active(qp, t, sr))
1006 return;
1007
1008 r = sr->curr.reg + decode_abreg(abreg, 1);
1009 r->where = UNW_WHERE_SPREL;
1010 r->when = sr->region_start + MIN((int)t, sr->region_len - 1);
1011 r->val = 4*spoff;
1012 }
1013
1014 #define UNW_DEC_BAD_CODE(code) printk("unwind: unknown code 0x%02x\n", code);
1015
1016 /*
1017 * region headers:
1018 */
1019 #define UNW_DEC_PROLOGUE_GR(fmt,r,m,gr,arg) desc_prologue(0,r,m,gr,arg)
1020 #define UNW_DEC_PROLOGUE(fmt,b,r,arg) desc_prologue(b,r,0,32,arg)
1021 /*
1022 * prologue descriptors:
1023 */
1024 #define UNW_DEC_ABI(fmt,a,c,arg) desc_abi(a,c,arg)
1025 #define UNW_DEC_BR_GR(fmt,b,g,arg) desc_br_gr(b,g,arg)
1026 #define UNW_DEC_BR_MEM(fmt,b,arg) desc_br_mem(b,arg)
1027 #define UNW_DEC_FRGR_MEM(fmt,g,f,arg) desc_frgr_mem(g,f,arg)
1028 #define UNW_DEC_FR_MEM(fmt,f,arg) desc_fr_mem(f,arg)
1029 #define UNW_DEC_GR_GR(fmt,m,g,arg) desc_gr_gr(m,g,arg)
1030 #define UNW_DEC_GR_MEM(fmt,m,arg) desc_gr_mem(m,arg)
1031 #define UNW_DEC_MEM_STACK_F(fmt,t,s,arg) desc_mem_stack_f(t,s,arg)
1032 #define UNW_DEC_MEM_STACK_V(fmt,t,arg) desc_mem_stack_v(t,arg)
1033 #define UNW_DEC_REG_GR(fmt,r,d,arg) desc_reg_gr(r,d,arg)
1034 #define UNW_DEC_REG_PSPREL(fmt,r,o,arg) desc_reg_psprel(r,o,arg)
1035 #define UNW_DEC_REG_SPREL(fmt,r,o,arg) desc_reg_sprel(r,o,arg)
1036 #define UNW_DEC_REG_WHEN(fmt,r,t,arg) desc_reg_when(r,t,arg)
1037 #define UNW_DEC_PRIUNAT_WHEN_GR(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_GR,t,arg)
1038 #define UNW_DEC_PRIUNAT_WHEN_MEM(fmt,t,arg) desc_reg_when(UNW_REG_PRI_UNAT_MEM,t,arg)
1039 #define UNW_DEC_PRIUNAT_GR(fmt,r,arg) desc_reg_gr(UNW_REG_PRI_UNAT_GR,r,arg)
1040 #define UNW_DEC_PRIUNAT_PSPREL(fmt,o,arg) desc_reg_psprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1041 #define UNW_DEC_PRIUNAT_SPREL(fmt,o,arg) desc_reg_sprel(UNW_REG_PRI_UNAT_MEM,o,arg)
1042 #define UNW_DEC_RP_BR(fmt,d,arg) desc_rp_br(d,arg)
1043 #define UNW_DEC_SPILL_BASE(fmt,o,arg) desc_spill_base(o,arg)
1044 #define UNW_DEC_SPILL_MASK(fmt,m,arg) (m = desc_spill_mask(m,arg))
1045 /*
1046 * body descriptors:
1047 */
1048 #define UNW_DEC_EPILOGUE(fmt,t,c,arg) desc_epilogue(t,c,arg)
1049 #define UNW_DEC_COPY_STATE(fmt,l,arg) desc_copy_state(l,arg)
1050 #define UNW_DEC_LABEL_STATE(fmt,l,arg) desc_label_state(l,arg)
1051 /*
1052 * general unwind descriptors:
1053 */
1054 #define UNW_DEC_SPILL_REG_P(f,p,t,a,x,y,arg) desc_spill_reg_p(p,t,a,x,y,arg)
1055 #define UNW_DEC_SPILL_REG(f,t,a,x,y,arg) desc_spill_reg_p(0,t,a,x,y,arg)
1056 #define UNW_DEC_SPILL_PSPREL_P(f,p,t,a,o,arg) desc_spill_psprel_p(p,t,a,o,arg)
1057 #define UNW_DEC_SPILL_PSPREL(f,t,a,o,arg) desc_spill_psprel_p(0,t,a,o,arg)
1058 #define UNW_DEC_SPILL_SPREL_P(f,p,t,a,o,arg) desc_spill_sprel_p(p,t,a,o,arg)
1059 #define UNW_DEC_SPILL_SPREL(f,t,a,o,arg) desc_spill_sprel_p(0,t,a,o,arg)
1060 #define UNW_DEC_RESTORE_P(f,p,t,a,arg) desc_restore_p(p,t,a,arg)
1061 #define UNW_DEC_RESTORE(f,t,a,arg) desc_restore_p(0,t,a,arg)
1062
1063 #include "unwind_decoder.c"
1064
1065
1066 /* Unwind scripts. */
1067
1068 static inline unw_hash_index_t
1069 hash (unsigned long ip)
1070 {
1071 # define magic 0x9e3779b97f4a7c16 /* based on (sqrt(5)/2-1)*2^64 */
1072
1073 return (ip >> 4)*magic >> (64 - UNW_LOG_HASH_SIZE);
1074 }
1075
1076 static inline long
1077 cache_match (struct unw_script *script, unsigned long ip, unsigned long pr)
1078 {
1079 read_lock(&script->lock);
1080 if (ip == script->ip && ((pr ^ script->pr_val) & script->pr_mask) == 0)
1081 /* keep the read lock... */
1082 return 1;
1083 read_unlock(&script->lock);
1084 return 0;
1085 }
1086
1087 static inline struct unw_script *
1088 script_lookup (struct unw_frame_info *info)
1089 {
1090 struct unw_script *script = unw.cache + info->hint;
1091 unsigned short index;
1092 unsigned long ip, pr;
1093
1094 STAT(++unw.stat.cache.lookups);
1095
1096 ip = info->ip;
1097 pr = info->pr;
1098
1099 if (cache_match(script, ip, pr)) {
1100 STAT(++unw.stat.cache.hinted_hits);
1101 return script;
1102 }
1103
1104 index = unw.hash[hash(ip)];
1105 if (index >= UNW_CACHE_SIZE)
1106 return 0;
1107
1108 script = unw.cache + index;
1109 while (1) {
1110 if (cache_match(script, ip, pr)) {
1111 /* update hint; no locking required as single-word writes are atomic */
1112 STAT(++unw.stat.cache.normal_hits);
1113 unw.cache[info->prev_script].hint = script - unw.cache;
1114 return script;
1115 }
1116 if (script->coll_chain >= UNW_HASH_SIZE)
1117 return 0;
1118 script = unw.cache + script->coll_chain;
1119 STAT(++unw.stat.cache.collision_chain_traversals);
1120 }
1121 }
1122
1123 /*
1124 * On returning, a write lock for the SCRIPT is still being held.
1125 */
1126 static inline struct unw_script *
1127 script_new (unsigned long ip)
1128 {
1129 struct unw_script *script, *prev, *tmp;
1130 unw_hash_index_t index;
1131 unsigned long flags;
1132 unsigned short head;
1133
1134 STAT(++unw.stat.script.news);
1135
1136 /*
1137 * Can't (easily) use cmpxchg() here because of ABA problem
1138 * that is intrinsic in cmpxchg()...
1139 */
1140 spin_lock_irqsave(&unw.lock, flags);
1141 {
1142 head = unw.lru_head;
1143 script = unw.cache + head;
1144 unw.lru_head = script->lru_chain;
1145 }
1146 spin_unlock(&unw.lock);
1147
1148 /*
1149 * XXX We'll deadlock here if we interrupt a thread that is
1150 * holding a read lock on script->lock. A try_write_lock()
1151 * might be mighty handy here... Alternatively, we could
1152 * disable interrupts whenever we hold a read-lock, but that
1153 * seems silly.
1154 */
1155 write_lock(&script->lock);
1156
1157 spin_lock(&unw.lock);
1158 {
1159 /* re-insert script at the tail of the LRU chain: */
1160 unw.cache[unw.lru_tail].lru_chain = head;
1161 unw.lru_tail = head;
1162
1163 /* remove the old script from the hash table (if it's there): */
1164 if (script->ip) {
1165 index = hash(script->ip);
1166 tmp = unw.cache + unw.hash[index];
1167 prev = 0;
1168 while (1) {
1169 if (tmp == script) {
1170 if (prev)
1171 prev->coll_chain = tmp->coll_chain;
1172 else
1173 unw.hash[index] = tmp->coll_chain;
1174 break;
1175 } else
1176 prev = tmp;
1177 if (tmp->coll_chain >= UNW_CACHE_SIZE)
1178 /* old script wasn't in the hash-table */
1179 break;
1180 tmp = unw.cache + tmp->coll_chain;
1181 }
1182 }
1183
1184 /* enter new script in the hash table */
1185 index = hash(ip);
1186 script->coll_chain = unw.hash[index];
1187 unw.hash[index] = script - unw.cache;
1188
1189 script->ip = ip; /* set new IP while we're holding the locks */
1190
1191 STAT(if (script->coll_chain < UNW_CACHE_SIZE) ++unw.stat.script.collisions);
1192 }
1193 spin_unlock_irqrestore(&unw.lock, flags);
1194
1195 script->flags = 0;
1196 script->hint = 0;
1197 script->count = 0;
1198 return script;
1199 }
1200
1201 static void
1202 script_finalize (struct unw_script *script, struct unw_state_record *sr)
1203 {
1204 script->pr_mask = sr->pr_mask;
1205 script->pr_val = sr->pr_val;
1206 /*
1207 * We could down-grade our write-lock on script->lock here but
1208 * the rwlock API doesn't offer atomic lock downgrading, so
1209 * we'll just keep the write-lock and release it later when
1210 * we're done using the script.
1211 */
1212 }
1213
1214 static inline void
1215 script_emit (struct unw_script *script, struct unw_insn insn)
1216 {
1217 if (script->count >= UNW_MAX_SCRIPT_LEN) {
1218 dprintk("unwind: script exceeds maximum size of %u instructions!\n",
1219 UNW_MAX_SCRIPT_LEN);
1220 return;
1221 }
1222 script->insn[script->count++] = insn;
1223 }
1224
1225 static inline void
1226 emit_nat_info (struct unw_state_record *sr, int i, struct unw_script *script)
1227 {
1228 struct unw_reg_info *r = sr->curr.reg + i;
1229 enum unw_insn_opcode opc;
1230 struct unw_insn insn;
1231 unsigned long val = 0;
1232
1233 switch (r->where) {
1234 case UNW_WHERE_GR:
1235 if (r->val >= 32) {
1236 /* register got spilled to a stacked register */
1237 opc = UNW_INSN_SETNAT_TYPE;
1238 val = UNW_NAT_REGSTK;
1239 } else
1240 /* register got spilled to a scratch register */
1241 opc = UNW_INSN_SETNAT_MEMSTK;
1242 break;
1243
1244 case UNW_WHERE_FR:
1245 opc = UNW_INSN_SETNAT_TYPE;
1246 val = UNW_NAT_VAL;
1247 break;
1248
1249 case UNW_WHERE_BR:
1250 opc = UNW_INSN_SETNAT_TYPE;
1251 val = UNW_NAT_NONE;
1252 break;
1253
1254 case UNW_WHERE_PSPREL:
1255 case UNW_WHERE_SPREL:
1256 opc = UNW_INSN_SETNAT_MEMSTK;
1257 break;
1258
1259 default:
1260 dprintk("unwind: don't know how to emit nat info for where = %u\n", r->where);
1261 return;
1262 }
1263 insn.opc = opc;
1264 insn.dst = unw.preg_index[i];
1265 insn.val = val;
1266 script_emit(script, insn);
1267 }
1268
1269 static void
1270 compile_reg (struct unw_state_record *sr, int i, struct unw_script *script)
1271 {
1272 struct unw_reg_info *r = sr->curr.reg + i;
1273 enum unw_insn_opcode opc;
1274 unsigned long val, rval;
1275 struct unw_insn insn;
1276 long need_nat_info;
1277
1278 if (r->where == UNW_WHERE_NONE || r->when >= sr->when_target)
1279 return;
1280
1281 opc = UNW_INSN_MOVE;
1282 val = rval = r->val;
1283 need_nat_info = (i >= UNW_REG_R4 && i <= UNW_REG_R7);
1284
1285 switch (r->where) {
1286 case UNW_WHERE_GR:
1287 if (rval >= 32) {
1288 opc = UNW_INSN_MOVE_STACKED;
1289 val = rval - 32;
1290 } else if (rval >= 4 && rval <= 7) {
1291 if (need_nat_info) {
1292 opc = UNW_INSN_MOVE2;
1293 need_nat_info = 0;
1294 }
1295 val = unw.preg_index[UNW_REG_R4 + (rval - 4)];
1296 } else {
1297 opc = UNW_INSN_ADD_SP;
1298 val = -sizeof(struct pt_regs) + pt_regs_off(rval);
1299 }
1300 break;
1301
1302 case UNW_WHERE_FR:
1303 if (rval <= 5)
1304 val = unw.preg_index[UNW_REG_F2 + (rval - 1)];
1305 else if (rval >= 16 && rval <= 31)
1306 val = unw.preg_index[UNW_REG_F16 + (rval - 16)];
1307 else {
1308 opc = UNW_INSN_ADD_SP;
1309 val = -sizeof(struct pt_regs);
1310 if (rval <= 9)
1311 val += struct_offset(struct pt_regs, f6) + 16*(rval - 6);
1312 else
1313 dprintk("unwind: kernel may not touch f%lu\n", rval);
1314 }
1315 break;
1316
1317 case UNW_WHERE_BR:
1318 if (rval >= 1 && rval <= 5)
1319 val = unw.preg_index[UNW_REG_B1 + (rval - 1)];
1320 else {
1321 opc = UNW_INSN_ADD_SP;
1322 val = -sizeof(struct pt_regs);
1323 if (rval == 0)
1324 val += struct_offset(struct pt_regs, b0);
1325 else if (rval == 6)
1326 val += struct_offset(struct pt_regs, b6);
1327 else
1328 val += struct_offset(struct pt_regs, b7);
1329 }
1330 break;
1331
1332 case UNW_WHERE_SPREL:
1333 opc = UNW_INSN_ADD_SP;
1334 break;
1335
1336 case UNW_WHERE_PSPREL:
1337 opc = UNW_INSN_ADD_PSP;
1338 break;
1339
1340 default:
1341 dprintk("unwind: register %u has unexpected `where' value of %u\n", i, r->where);
1342 break;
1343 }
1344 insn.opc = opc;
1345 insn.dst = unw.preg_index[i];
1346 insn.val = val;
1347 script_emit(script, insn);
1348 if (need_nat_info)
1349 emit_nat_info(sr, i, script);
1350
1351 if (i == UNW_REG_PSP) {
1352 /*
1353 * info->psp must contain the _value_ of the previous
1354 * sp, not it's save location. We get this by
1355 * dereferencing the value we just stored in
1356 * info->psp:
1357 */
1358 insn.opc = UNW_INSN_LOAD;
1359 insn.dst = insn.val = unw.preg_index[UNW_REG_PSP];
1360 script_emit(script, insn);
1361 }
1362 }
1363
1364 static inline const struct unw_table_entry *
1365 lookup (struct unw_table *table, unsigned long rel_ip)
1366 {
1367 const struct unw_table_entry *e = 0;
1368 unsigned long lo, hi, mid;
1369
1370 /* do a binary search for right entry: */
1371 for (lo = 0, hi = table->length; lo < hi; ) {
1372 mid = (lo + hi) / 2;
1373 e = &table->array[mid];
1374 if (rel_ip < e->start_offset)
1375 hi = mid;
1376 else if (rel_ip >= e->end_offset)
1377 lo = mid + 1;
1378 else
1379 break;
1380 }
1381 return e;
1382 }
1383
1384 /*
1385 * Build an unwind script that unwinds from state OLD_STATE to the
1386 * entrypoint of the function that called OLD_STATE.
1387 */
1388 static inline struct unw_script *
1389 build_script (struct unw_frame_info *info)
1390 {
1391 struct unw_reg_state *rs, *next;
1392 const struct unw_table_entry *e = 0;
1393 struct unw_script *script = 0;
1394 unsigned long ip = info->ip;
1395 struct unw_state_record sr;
1396 struct unw_table *table;
1397 struct unw_reg_info *r;
1398 struct unw_insn insn;
1399 u8 *dp, *desc_end;
1400 u64 hdr;
1401 int i;
1402 STAT(unsigned long start, parse_start;)
1403
1404 STAT(++unw.stat.script.builds; start = ia64_get_itc());
1405
1406 /* build state record */
1407 memset(&sr, 0, sizeof(sr));
1408 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1409 r->when = UNW_WHEN_NEVER;
1410 sr.pr_val = info->pr;
1411
1412 script = script_new(ip);
1413 if (!script) {
1414 dprintk("unwind: failed to create unwind script\n");
1415 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1416 return 0;
1417 }
1418 unw.cache[info->prev_script].hint = script - unw.cache;
1419
1420 /* search the kernels and the modules' unwind tables for IP: */
1421
1422 STAT(parse_start = ia64_get_itc());
1423
1424 for (table = unw.tables; table; table = table->next) {
1425 if (ip >= table->start && ip < table->end) {
1426 e = lookup(table, ip - table->segment_base);
1427 break;
1428 }
1429 }
1430 if (!e) {
1431 /* no info, return default unwinder (leaf proc, no mem stack, no saved regs) */
1432 dprintk("unwind: no unwind info for ip=0x%lx (prev ip=0x%lx)\n", ip,
1433 unw.cache[info->prev_script].ip);
1434 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1435 sr.curr.reg[UNW_REG_RP].when = -1;
1436 sr.curr.reg[UNW_REG_RP].val = 0;
1437 compile_reg(&sr, UNW_REG_RP, script);
1438 script_finalize(script, &sr);
1439 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1440 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1441 return script;
1442 }
1443
1444 sr.when_target = (3*((ip & ~0xfUL) - (table->segment_base + e->start_offset))/16
1445 + (ip & 0xfUL));
1446 hdr = *(u64 *) (table->segment_base + e->info_offset);
1447 dp = (u8 *) (table->segment_base + e->info_offset + 8);
1448 desc_end = dp + 8*UNW_LENGTH(hdr);
1449
1450 while (!sr.done && dp < desc_end)
1451 dp = unw_decode(dp, sr.in_body, &sr);
1452
1453 if (sr.when_target > sr.epilogue_start) {
1454 /*
1455 * sp has been restored and all values on the memory stack below
1456 * psp also have been restored.
1457 */
1458 sr.curr.reg[UNW_REG_PSP].val = 0;
1459 sr.curr.reg[UNW_REG_PSP].where = UNW_WHERE_NONE;
1460 sr.curr.reg[UNW_REG_PSP].when = UNW_WHEN_NEVER;
1461 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r)
1462 if ((r->where == UNW_WHERE_PSPREL && r->val <= 0x10)
1463 || r->where == UNW_WHERE_SPREL)
1464 {
1465 r->val = 0;
1466 r->where = UNW_WHERE_NONE;
1467 r->when = UNW_WHEN_NEVER;
1468 }
1469 }
1470
1471 script->flags = sr.flags;
1472
1473 /*
1474 * If RP did't get saved, generate entry for the return link
1475 * register.
1476 */
1477 if (sr.curr.reg[UNW_REG_RP].when >= sr.when_target) {
1478 sr.curr.reg[UNW_REG_RP].where = UNW_WHERE_BR;
1479 sr.curr.reg[UNW_REG_RP].when = -1;
1480 sr.curr.reg[UNW_REG_RP].val = sr.return_link_reg;
1481 }
1482
1483 #if UNW_DEBUG
1484 printk("unwind: state record for func 0x%lx, t=%u:\n",
1485 table->segment_base + e->start_offset, sr.when_target);
1486 for (r = sr.curr.reg; r < sr.curr.reg + UNW_NUM_REGS; ++r) {
1487 if (r->where != UNW_WHERE_NONE || r->when != UNW_WHEN_NEVER) {
1488 printk(" %s <- ", unw.preg_name[r - sr.curr.reg]);
1489 switch (r->where) {
1490 case UNW_WHERE_GR: printk("r%lu", r->val); break;
1491 case UNW_WHERE_FR: printk("f%lu", r->val); break;
1492 case UNW_WHERE_BR: printk("b%lu", r->val); break;
1493 case UNW_WHERE_SPREL: printk("[sp+0x%lx]", r->val); break;
1494 case UNW_WHERE_PSPREL: printk("[psp+0x%lx]", r->val); break;
1495 case UNW_WHERE_NONE:
1496 printk("%s+0x%lx", unw.preg_name[r - sr.curr.reg], r->val);
1497 break;
1498 default: printk("BADWHERE(%d)", r->where); break;
1499 }
1500 printk("\t\t%d\n", r->when);
1501 }
1502 }
1503 #endif
1504
1505 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1506
1507 /* translate state record into unwinder instructions: */
1508
1509 /*
1510 * First, set psp if we're dealing with a fixed-size frame;
1511 * subsequent instructions may depend on this value.
1512 */
1513 if (sr.when_target > sr.curr.reg[UNW_REG_PSP].when
1514 && (sr.curr.reg[UNW_REG_PSP].where == UNW_WHERE_NONE)
1515 && sr.curr.reg[UNW_REG_PSP].val != 0) {
1516 /* new psp is sp plus frame size */
1517 insn.opc = UNW_INSN_ADD;
1518 insn.dst = struct_offset(struct unw_frame_info, psp)/8;
1519 insn.val = sr.curr.reg[UNW_REG_PSP].val; /* frame size */
1520 script_emit(script, insn);
1521 }
1522
1523 /* determine where the primary UNaT is: */
1524 if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1525 i = UNW_REG_PRI_UNAT_MEM;
1526 else if (sr.when_target < sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when)
1527 i = UNW_REG_PRI_UNAT_GR;
1528 else if (sr.curr.reg[UNW_REG_PRI_UNAT_MEM].when > sr.curr.reg[UNW_REG_PRI_UNAT_GR].when)
1529 i = UNW_REG_PRI_UNAT_MEM;
1530 else
1531 i = UNW_REG_PRI_UNAT_GR;
1532
1533 compile_reg(&sr, i, script);
1534
1535 for (i = UNW_REG_BSP; i < UNW_NUM_REGS; ++i)
1536 compile_reg(&sr, i, script);
1537
1538 /* free labelled register states & stack: */
1539
1540 STAT(parse_start = ia64_get_itc());
1541 for (rs = sr.reg_state_list; rs; rs = next) {
1542 next = rs->next;
1543 free_reg_state(rs);
1544 }
1545 while (sr.stack)
1546 pop(&sr);
1547 STAT(unw.stat.script.parse_time += ia64_get_itc() - parse_start);
1548
1549 script_finalize(script, &sr);
1550 STAT(unw.stat.script.build_time += ia64_get_itc() - start);
1551 return script;
1552 }
1553
1554 /*
1555 * Apply the unwinding actions represented by OPS and update SR to
1556 * reflect the state that existed upon entry to the function that this
1557 * unwinder represents.
1558 */
1559 static inline void
1560 run_script (struct unw_script *script, struct unw_frame_info *state)
1561 {
1562 struct unw_insn *ip, *limit, next_insn;
1563 unsigned long opc, dst, val, off;
1564 unsigned long *s = (unsigned long *) state;
1565 STAT(unsigned long start;)
1566
1567 STAT(++unw.stat.script.runs; start = ia64_get_itc());
1568 state->flags = script->flags;
1569 ip = script->insn;
1570 limit = script->insn + script->count;
1571 next_insn = *ip;
1572
1573 while (ip++ < limit) {
1574 opc = next_insn.opc;
1575 dst = next_insn.dst;
1576 val = next_insn.val;
1577 next_insn = *ip;
1578
1579 redo:
1580 switch (opc) {
1581 case UNW_INSN_ADD:
1582 s[dst] += val;
1583 break;
1584
1585 case UNW_INSN_MOVE2:
1586 if (!s[val])
1587 goto lazy_init;
1588 s[dst+1] = s[val+1];
1589 s[dst] = s[val];
1590 break;
1591
1592 case UNW_INSN_MOVE:
1593 if (!s[val])
1594 goto lazy_init;
1595 s[dst] = s[val];
1596 break;
1597
1598 case UNW_INSN_MOVE_STACKED:
1599 s[dst] = (unsigned long) ia64_rse_skip_regs((unsigned long *)state->bsp,
1600 val);
1601 break;
1602
1603 case UNW_INSN_ADD_PSP:
1604 s[dst] = state->psp + val;
1605 break;
1606
1607 case UNW_INSN_ADD_SP:
1608 s[dst] = state->sp + val;
1609 break;
1610
1611 case UNW_INSN_SETNAT_MEMSTK:
1612 if (!state->pri_unat_loc)
1613 state->pri_unat_loc = &state->sw->ar_unat;
1614 /* register off. is a multiple of 8, so the least 3 bits (type) are 0 */
1615 s[dst+1] = (*state->pri_unat_loc - s[dst]) | UNW_NAT_MEMSTK;
1616 break;
1617
1618 case UNW_INSN_SETNAT_TYPE:
1619 s[dst+1] = val;
1620 break;
1621
1622 case UNW_INSN_LOAD:
1623 #if UNW_DEBUG
1624 if ((s[val] & (local_cpu_data->unimpl_va_mask | 0x7)) != 0
1625 || s[val] < TASK_SIZE)
1626 {
1627 debug(1, "unwind: rejecting bad psp=0x%lx\n", s[val]);
1628 break;
1629 }
1630 #endif
1631 s[dst] = *(unsigned long *) s[val];
1632 break;
1633 }
1634 }
1635 STAT(unw.stat.script.run_time += ia64_get_itc() - start);
1636 return;
1637
1638 lazy_init:
1639 off = unw.sw_off[val];
1640 s[val] = (unsigned long) state->sw + off;
1641 if (off >= struct_offset(struct switch_stack, r4)
1642 && off <= struct_offset(struct switch_stack, r7))
1643 /*
1644 * We're initializing a general register: init NaT info, too. Note that
1645 * the offset is a multiple of 8 which gives us the 3 bits needed for
1646 * the type field.
1647 */
1648 s[val+1] = (struct_offset(struct switch_stack, ar_unat) - off) | UNW_NAT_MEMSTK;
1649 goto redo;
1650 }
1651
1652 static int
1653 find_save_locs (struct unw_frame_info *info)
1654 {
1655 int have_write_lock = 0;
1656 struct unw_script *scr;
1657
1658 if ((info->ip & (local_cpu_data->unimpl_va_mask | 0xf)) || info->ip < TASK_SIZE) {
1659 /* don't let obviously bad addresses pollute the cache */
1660 debug(1, "unwind: rejecting bad ip=0x%lx\n", info->ip);
1661 info->rp_loc = 0;
1662 return -1;
1663 }
1664
1665 scr = script_lookup(info);
1666 if (!scr) {
1667 scr = build_script(info);
1668 if (!scr) {
1669 dprintk("unwind: failed to locate/build unwind script for ip %lx\n",
1670 info->ip);
1671 return -1;
1672 }
1673 have_write_lock = 1;
1674 }
1675 info->hint = scr->hint;
1676 info->prev_script = scr - unw.cache;
1677
1678 run_script(scr, info);
1679
1680 if (have_write_lock)
1681 write_unlock(&scr->lock);
1682 else
1683 read_unlock(&scr->lock);
1684 return 0;
1685 }
1686
1687 int
1688 unw_unwind (struct unw_frame_info *info)
1689 {
1690 unsigned long prev_ip, prev_sp, prev_bsp;
1691 unsigned long ip, pr, num_regs;
1692 STAT(unsigned long start, flags;)
1693 int retval;
1694
1695 STAT(local_irq_save(flags); ++unw.stat.api.unwinds; start = ia64_get_itc());
1696
1697 prev_ip = info->ip;
1698 prev_sp = info->sp;
1699 prev_bsp = info->bsp;
1700
1701 /* restore the ip */
1702 if (!info->rp_loc) {
1703 debug(1, "unwind: failed to locate return link (ip=0x%lx)!\n", info->ip);
1704 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1705 return -1;
1706 }
1707 ip = info->ip = *info->rp_loc;
1708 if (ip < GATE_ADDR + PAGE_SIZE) {
1709 /*
1710 * We don't have unwind info for the gate page, so we consider that part
1711 * of user-space for the purpose of unwinding.
1712 */
1713 debug(1, "unwind: reached user-space (ip=0x%lx)\n", ip);
1714 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1715 return -1;
1716 }
1717
1718 /* restore the cfm: */
1719 if (!info->pfs_loc) {
1720 dprintk("unwind: failed to locate ar.pfs!\n");
1721 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1722 return -1;
1723 }
1724 info->cfm_loc = info->pfs_loc;
1725
1726 /* restore the bsp: */
1727 pr = info->pr;
1728 num_regs = 0;
1729 if ((info->flags & UNW_FLAG_INTERRUPT_FRAME)) {
1730 if ((pr & (1UL << pNonSys)) != 0)
1731 num_regs = *info->cfm_loc & 0x7f; /* size of frame */
1732 info->pfs_loc =
1733 (unsigned long *) (info->sp + 16 + struct_offset(struct pt_regs, ar_pfs));
1734 } else
1735 num_regs = (*info->cfm_loc >> 7) & 0x7f; /* size of locals */
1736 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->bsp, -num_regs);
1737 if (info->bsp < info->regstk.limit || info->bsp > info->regstk.top) {
1738 dprintk("unwind: bsp (0x%lx) out of range [0x%lx-0x%lx]\n",
1739 info->bsp, info->regstk.limit, info->regstk.top);
1740 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1741 return -1;
1742 }
1743
1744 /* restore the sp: */
1745 info->sp = info->psp;
1746 if (info->sp < info->memstk.top || info->sp > info->memstk.limit) {
1747 dprintk("unwind: sp (0x%lx) out of range [0x%lx-0x%lx]\n",
1748 info->sp, info->memstk.top, info->memstk.limit);
1749 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1750 return -1;
1751 }
1752
1753 if (info->ip == prev_ip && info->sp == prev_sp && info->bsp == prev_bsp) {
1754 dprintk("unwind: ip, sp, bsp remain unchanged; stopping here (ip=0x%lx)\n", ip);
1755 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1756 return -1;
1757 }
1758
1759 /* as we unwind, the saved ar.unat becomes the primary unat: */
1760 info->pri_unat_loc = info->unat_loc;
1761
1762 /* finally, restore the predicates: */
1763 unw_get_pr(info, &info->pr);
1764
1765 retval = find_save_locs(info);
1766 STAT(unw.stat.api.unwind_time += ia64_get_itc() - start; local_irq_restore(flags));
1767 return retval;
1768 }
1769
1770 int
1771 unw_unwind_to_user (struct unw_frame_info *info)
1772 {
1773 unsigned long ip;
1774
1775 while (unw_unwind(info) >= 0) {
1776 if (unw_get_rp(info, &ip) < 0) {
1777 unw_get_ip(info, &ip);
1778 dprintk("unwind: failed to read return pointer (ip=0x%lx)\n", ip);
1779 return -1;
1780 }
1781 /*
1782 * We don't have unwind info for the gate page, so we consider that part
1783 * of user-space for the purpose of unwinding.
1784 */
1785 if (ip < GATE_ADDR + PAGE_SIZE)
1786 return 0;
1787 }
1788 unw_get_ip(info, &ip);
1789 dprintk("unwind: failed to unwind to user-level (ip=0x%lx)\n", ip);
1790 return -1;
1791 }
1792
1793 void
1794 unw_init_frame_info (struct unw_frame_info *info, struct task_struct *t, struct switch_stack *sw)
1795 {
1796 unsigned long rbslimit, rbstop, stklimit, stktop, sol;
1797 STAT(unsigned long start, flags;)
1798
1799 STAT(local_irq_save(flags); ++unw.stat.api.inits; start = ia64_get_itc());
1800
1801 /*
1802 * Subtle stuff here: we _could_ unwind through the
1803 * switch_stack frame but we don't want to do that because it
1804 * would be slow as each preserved register would have to be
1805 * processed. Instead, what we do here is zero out the frame
1806 * info and start the unwind process at the function that
1807 * created the switch_stack frame. When a preserved value in
1808 * switch_stack needs to be accessed, run_script() will
1809 * initialize the appropriate pointer on demand.
1810 */
1811 memset(info, 0, sizeof(*info));
1812
1813 rbslimit = (unsigned long) t + IA64_RBS_OFFSET;
1814 rbstop = sw->ar_bspstore;
1815 if (rbstop - (unsigned long) t >= IA64_STK_OFFSET)
1816 rbstop = rbslimit;
1817
1818 stklimit = (unsigned long) t + IA64_STK_OFFSET;
1819 stktop = (unsigned long) sw - 16;
1820 if (stktop <= rbstop)
1821 stktop = rbstop;
1822
1823 info->regstk.limit = rbslimit;
1824 info->regstk.top = rbstop;
1825 info->memstk.limit = stklimit;
1826 info->memstk.top = stktop;
1827 info->task = t;
1828 info->sw = sw;
1829 info->sp = info->psp = (unsigned long) (sw + 1) - 16;
1830 info->cfm_loc = &sw->ar_pfs;
1831 sol = (*info->cfm_loc >> 7) & 0x7f;
1832 info->bsp = (unsigned long) ia64_rse_skip_regs((unsigned long *) info->regstk.top, -sol);
1833 info->ip = sw->b0;
1834 info->pr = sw->pr;
1835
1836 find_save_locs(info);
1837 STAT(unw.stat.api.init_time += ia64_get_itc() - start; local_irq_restore(flags));
1838 }
1839
1840 void
1841 unw_init_from_blocked_task (struct unw_frame_info *info, struct task_struct *t)
1842 {
1843 struct switch_stack *sw = (struct switch_stack *) (t->thread.ksp + 16);
1844
1845 unw_init_frame_info(info, t, sw);
1846 }
1847
1848 static void
1849 init_unwind_table (struct unw_table *table, const char *name, unsigned long segment_base,
1850 unsigned long gp, const void *table_start, const void *table_end)
1851 {
1852 const struct unw_table_entry *start = table_start, *end = table_end;
1853
1854 table->name = name;
1855 table->segment_base = segment_base;
1856 table->gp = gp;
1857 table->start = segment_base + start[0].start_offset;
1858 table->end = segment_base + end[-1].end_offset;
1859 table->array = start;
1860 table->length = end - start;
1861 }
1862
1863 void *
1864 unw_add_unwind_table (const char *name, unsigned long segment_base, unsigned long gp,
1865 const void *table_start, const void *table_end)
1866 {
1867 const struct unw_table_entry *start = table_start, *end = table_end;
1868 struct unw_table *table;
1869 unsigned long flags;
1870
1871 if (end - start <= 0) {
1872 dprintk("unwind: ignoring attempt to insert empty unwind table\n");
1873 return 0;
1874 }
1875
1876 table = kmalloc(sizeof(*table), GFP_USER);
1877 if (!table)
1878 return 0;
1879
1880 init_unwind_table(table, name, segment_base, gp, table_start, table_end);
1881
1882 spin_lock_irqsave(&unw.lock, flags);
1883 {
1884 /* keep kernel unwind table at the front (it's searched most commonly): */
1885 table->next = unw.tables->next;
1886 unw.tables->next = table;
1887 }
1888 spin_unlock_irqrestore(&unw.lock, flags);
1889
1890 return table;
1891 }
1892
1893 void
1894 unw_remove_unwind_table (void *handle)
1895 {
1896 struct unw_table *table, *prev;
1897 struct unw_script *tmp;
1898 unsigned long flags;
1899 long index;
1900
1901 if (!handle) {
1902 dprintk("unwind: ignoring attempt to remove non-existent unwind table\n");
1903 return;
1904 }
1905
1906 table = handle;
1907 if (table == &unw.kernel_table) {
1908 dprintk("unwind: sorry, freeing the kernel's unwind table is a no-can-do!\n");
1909 return;
1910 }
1911
1912 spin_lock_irqsave(&unw.lock, flags);
1913 {
1914 /* first, delete the table: */
1915
1916 for (prev = (struct unw_table *) &unw.tables; prev; prev = prev->next)
1917 if (prev->next == table)
1918 break;
1919 if (!prev) {
1920 dprintk("unwind: failed to find unwind table %p\n", (void *) table);
1921 spin_unlock_irqrestore(&unw.lock, flags);
1922 return;
1923 }
1924 prev->next = table->next;
1925 }
1926 spin_unlock_irqrestore(&unw.lock, flags);
1927
1928 /* next, remove hash table entries for this table */
1929
1930 for (index = 0; index <= UNW_HASH_SIZE; ++index) {
1931 tmp = unw.cache + unw.hash[index];
1932 if (unw.hash[index] >= UNW_CACHE_SIZE
1933 || tmp->ip < table->start || tmp->ip >= table->end)
1934 continue;
1935
1936 write_lock(&tmp->lock);
1937 {
1938 if (tmp->ip >= table->start && tmp->ip < table->end) {
1939 unw.hash[index] = tmp->coll_chain;
1940 tmp->ip = 0;
1941 }
1942 }
1943 write_unlock(&tmp->lock);
1944 }
1945
1946 kfree(table);
1947 }
1948
1949 void
1950 unw_create_gate_table (void)
1951 {
1952 extern char __start_gate_section[], __stop_gate_section[];
1953 unsigned long *lp, start, end, segbase = unw.kernel_table.segment_base;
1954 const struct unw_table_entry *entry, *first;
1955 size_t info_size, size;
1956 char *info;
1957
1958 start = (unsigned long) __start_gate_section - segbase;
1959 end = (unsigned long) __stop_gate_section - segbase;
1960 size = 0;
1961 first = lookup(&unw.kernel_table, start);
1962
1963 for (entry = first; entry->start_offset < end; ++entry)
1964 size += 3*8 + 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
1965 size += 8; /* reserve space for "end of table" marker */
1966
1967 unw.gate_table = alloc_bootmem(size);
1968 if (!unw.gate_table) {
1969 unw.gate_table_size = 0;
1970 printk("unwind: unable to create unwind data for gate page!\n");
1971 return;
1972 }
1973 unw.gate_table_size = size;
1974
1975 lp = unw.gate_table;
1976 info = (char *) unw.gate_table + size;
1977
1978 for (entry = first; entry->start_offset < end; ++entry, lp += 3) {
1979 info_size = 8 + 8*UNW_LENGTH(*(u64 *) (segbase + entry->info_offset));
1980 info -= info_size;
1981 memcpy(info, (char *) segbase + entry->info_offset, info_size);
1982
1983 lp[0] = entry->start_offset - start + GATE_ADDR; /* start */
1984 lp[1] = entry->end_offset - start + GATE_ADDR; /* end */
1985 lp[2] = info - (char *) unw.gate_table; /* info */
1986 }
1987 *lp = 0; /* end-of-table marker */
1988 }
1989
1990 void
1991 unw_init (void)
1992 {
1993 extern int ia64_unw_start, ia64_unw_end, __gp;
1994 extern void unw_hash_index_t_is_too_narrow (void);
1995 long i, off;
1996
1997 if (8*sizeof(unw_hash_index_t) < UNW_LOG_HASH_SIZE)
1998 unw_hash_index_t_is_too_narrow();
1999
2000 unw.sw_off[unw.preg_index[UNW_REG_PRI_UNAT_GR]] = SW(AR_UNAT);
2001 unw.sw_off[unw.preg_index[UNW_REG_BSPSTORE]] = SW(AR_BSPSTORE);
2002 unw.sw_off[unw.preg_index[UNW_REG_PFS]] = SW(AR_UNAT);
2003 unw.sw_off[unw.preg_index[UNW_REG_RP]] = SW(B0);
2004 unw.sw_off[unw.preg_index[UNW_REG_UNAT]] = SW(AR_UNAT);
2005 unw.sw_off[unw.preg_index[UNW_REG_PR]] = SW(PR);
2006 unw.sw_off[unw.preg_index[UNW_REG_LC]] = SW(AR_LC);
2007 unw.sw_off[unw.preg_index[UNW_REG_FPSR]] = SW(AR_FPSR);
2008 for (i = UNW_REG_R4, off = SW(R4); i <= UNW_REG_R7; ++i, off += 8)
2009 unw.sw_off[unw.preg_index[i]] = off;
2010 for (i = UNW_REG_B1, off = SW(B1); i <= UNW_REG_B5; ++i, off += 8)
2011 unw.sw_off[unw.preg_index[i]] = off;
2012 for (i = UNW_REG_F2, off = SW(F2); i <= UNW_REG_F5; ++i, off += 16)
2013 unw.sw_off[unw.preg_index[i]] = off;
2014 for (i = UNW_REG_F16, off = SW(F16); i <= UNW_REG_F31; ++i, off += 16)
2015 unw.sw_off[unw.preg_index[i]] = off;
2016
2017 for (i = 0; i < UNW_CACHE_SIZE; ++i) {
2018 if (i > 0)
2019 unw.cache[i].lru_chain = (i - 1);
2020 unw.cache[i].coll_chain = -1;
2021 unw.cache[i].lock = RW_LOCK_UNLOCKED;
2022 }
2023 unw.lru_head = UNW_CACHE_SIZE - 1;
2024 unw.lru_tail = 0;
2025
2026 init_unwind_table(&unw.kernel_table, "kernel", KERNEL_START, (unsigned long) &__gp,
2027 &ia64_unw_start, &ia64_unw_end);
2028 }
2029
2030 /*
2031 * This system call copies the unwind data into the buffer pointed to by BUF and returns
2032 * the size of the unwind data. If BUF_SIZE is smaller than the size of the unwind data
2033 * or if BUF is NULL, nothing is copied, but the system call still returns the size of the
2034 * unwind data.
2035 *
2036 * The first portion of the unwind data contains an unwind table and rest contains the
2037 * associated unwind info (in no particular order). The unwind table consists of a table
2038 * of entries of the form:
2039 *
2040 * u64 start; (64-bit address of start of function)
2041 * u64 end; (64-bit address of start of function)
2042 * u64 info; (BUF-relative offset to unwind info)
2043 *
2044 * The end of the unwind table is indicated by an entry with a START address of zero.
2045 *
2046 * Please see the IA-64 Software Conventions and Runtime Architecture manual for details
2047 * on the format of the unwind info.
2048 *
2049 * ERRORS
2050 * EFAULT BUF points outside your accessible address space.
2051 */
2052 asmlinkage long
2053 sys_getunwind (void *buf, size_t buf_size)
2054 {
2055 if (buf && buf_size >= unw.gate_table_size)
2056 if (copy_to_user(buf, unw.gate_table, unw.gate_table_size) != 0)
2057 return -EFAULT;
2058 return unw.gate_table_size;
2059 }
2060