File: /usr/src/linux/arch/arm/mm/fault-armv.c
1 /*
2 * linux/arch/arm/mm/fault-armv.c
3 *
4 * Copyright (C) 1995 Linus Torvalds
5 * Modifications for ARM processor (c) 1995-2001 Russell King
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/config.h>
12 #include <linux/signal.h>
13 #include <linux/sched.h>
14 #include <linux/kernel.h>
15 #include <linux/errno.h>
16 #include <linux/string.h>
17 #include <linux/types.h>
18 #include <linux/ptrace.h>
19 #include <linux/mman.h>
20 #include <linux/mm.h>
21 #include <linux/interrupt.h>
22 #include <linux/proc_fs.h>
23 #include <linux/bitops.h>
24 #include <linux/init.h>
25
26 #include <asm/system.h>
27 #include <asm/uaccess.h>
28 #include <asm/pgalloc.h>
29 #include <asm/pgtable.h>
30 #include <asm/unaligned.h>
31
32 extern void die_if_kernel(const char *str, struct pt_regs *regs, int err);
33 extern void show_pte(struct mm_struct *mm, unsigned long addr);
34 extern int do_page_fault(unsigned long addr, int error_code,
35 struct pt_regs *regs);
36 extern int do_translation_fault(unsigned long addr, int error_code,
37 struct pt_regs *regs);
38 extern void do_bad_area(struct task_struct *tsk, struct mm_struct *mm,
39 unsigned long addr, int error_code,
40 struct pt_regs *regs);
41
42 #ifdef CONFIG_ALIGNMENT_TRAP
43 /*
44 * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998
45 * /proc/sys/debug/alignment, modified and integrated into
46 * Linux 2.1 by Russell King
47 *
48 * Speed optimisations and better fault handling by Russell King.
49 *
50 * *** NOTE ***
51 * This code is not portable to processors with late data abort handling.
52 */
53 #define CODING_BITS(i) (i & 0x0e000000)
54
55 #define LDST_I_BIT(i) (i & (1 << 26)) /* Immediate constant */
56 #define LDST_P_BIT(i) (i & (1 << 24)) /* Preindex */
57 #define LDST_U_BIT(i) (i & (1 << 23)) /* Add offset */
58 #define LDST_W_BIT(i) (i & (1 << 21)) /* Writeback */
59 #define LDST_L_BIT(i) (i & (1 << 20)) /* Load */
60
61 #define LDST_P_EQ_U(i) ((((i) ^ ((i) >> 1)) & (1 << 23)) == 0)
62
63 #define LDSTH_I_BIT(i) (i & (1 << 22)) /* half-word immed */
64 #define LDM_S_BIT(i) (i & (1 << 22)) /* write CPSR from SPSR */
65
66 #define RN_BITS(i) ((i >> 16) & 15) /* Rn */
67 #define RD_BITS(i) ((i >> 12) & 15) /* Rd */
68 #define RM_BITS(i) (i & 15) /* Rm */
69
70 #define REGMASK_BITS(i) (i & 0xffff)
71 #define OFFSET_BITS(i) (i & 0x0fff)
72
73 #define IS_SHIFT(i) (i & 0x0ff0)
74 #define SHIFT_BITS(i) ((i >> 7) & 0x1f)
75 #define SHIFT_TYPE(i) (i & 0x60)
76 #define SHIFT_LSL 0x00
77 #define SHIFT_LSR 0x20
78 #define SHIFT_ASR 0x40
79 #define SHIFT_RORRRX 0x60
80
81 static unsigned long ai_user;
82 static unsigned long ai_sys;
83 static unsigned long ai_skipped;
84 static unsigned long ai_half;
85 static unsigned long ai_word;
86 static unsigned long ai_multi;
87
88 #ifdef CONFIG_SYSCTL
89 static int proc_alignment_read(char *page, char **start, off_t off,
90 int count, int *eof, void *data)
91 {
92 char *p = page;
93 int len;
94
95 p += sprintf(p, "User:\t\t%li\n", ai_user);
96 p += sprintf(p, "System:\t\t%li\n", ai_sys);
97 p += sprintf(p, "Skipped:\t%li\n", ai_skipped);
98 p += sprintf(p, "Half:\t\t%li\n", ai_half);
99 p += sprintf(p, "Word:\t\t%li\n", ai_word);
100 p += sprintf(p, "Multi:\t\t%li\n", ai_multi);
101
102 len = (p - page) - off;
103 if (len < 0)
104 len = 0;
105
106 *eof = (len <= count) ? 1 : 0;
107 *start = page + off;
108
109 return len;
110 }
111
112 /*
113 * This needs to be done after sysctl_init, otherwise sys/
114 * will be overwritten.
115 */
116 static int __init alignment_init(void)
117 {
118 create_proc_read_entry("sys/debug/alignment", 0, NULL,
119 proc_alignment_read, NULL);
120 return 0;
121 }
122
123 __initcall(alignment_init);
124 #endif /* CONFIG_SYSCTL */
125
126 union offset_union {
127 unsigned long un;
128 signed long sn;
129 };
130
131 #define TYPE_ERROR 0
132 #define TYPE_FAULT 1
133 #define TYPE_LDST 2
134 #define TYPE_DONE 3
135
136 #define get8_unaligned_check(val,addr,err) \
137 __asm__( \
138 "1: ldrb %1, [%2], #1\n" \
139 "2:\n" \
140 " .section .fixup,\"ax\"\n" \
141 " .align 2\n" \
142 "3: mov %0, #1\n" \
143 " b 2b\n" \
144 " .previous\n" \
145 " .section __ex_table,\"a\"\n" \
146 " .align 3\n" \
147 " .long 1b, 3b\n" \
148 " .previous\n" \
149 : "=r" (err), "=&r" (val), "=r" (addr) \
150 : "0" (err), "2" (addr))
151
152 #define get8t_unaligned_check(val,addr,err) \
153 __asm__( \
154 "1: ldrbt %1, [%2], #1\n" \
155 "2:\n" \
156 " .section .fixup,\"ax\"\n" \
157 " .align 2\n" \
158 "3: mov %0, #1\n" \
159 " b 2b\n" \
160 " .previous\n" \
161 " .section __ex_table,\"a\"\n" \
162 " .align 3\n" \
163 " .long 1b, 3b\n" \
164 " .previous\n" \
165 : "=r" (err), "=&r" (val), "=r" (addr) \
166 : "0" (err), "2" (addr))
167
168 #define get16_unaligned_check(val,addr) \
169 do { \
170 unsigned int err = 0, v, a = addr; \
171 get8_unaligned_check(val,a,err); \
172 get8_unaligned_check(v,a,err); \
173 val |= v << 8; \
174 if (err) \
175 goto fault; \
176 } while (0)
177
178 #define put16_unaligned_check(val,addr) \
179 do { \
180 unsigned int err = 0, v = val, a = addr; \
181 __asm__( \
182 "1: strb %1, [%2], #1\n" \
183 " mov %1, %1, lsr #8\n" \
184 "2: strb %1, [%2]\n" \
185 "3:\n" \
186 " .section .fixup,\"ax\"\n" \
187 " .align 2\n" \
188 "4: mov %0, #1\n" \
189 " b 3b\n" \
190 " .previous\n" \
191 " .section __ex_table,\"a\"\n" \
192 " .align 3\n" \
193 " .long 1b, 4b\n" \
194 " .long 2b, 4b\n" \
195 " .previous\n" \
196 : "=r" (err), "=&r" (v), "=&r" (a) \
197 : "0" (err), "1" (v), "2" (a)); \
198 if (err) \
199 goto fault; \
200 } while (0)
201
202 #define __put32_unaligned_check(ins,val,addr) \
203 do { \
204 unsigned int err = 0, v = val, a = addr; \
205 __asm__( \
206 "1: "ins" %1, [%2], #1\n" \
207 " mov %1, %1, lsr #8\n" \
208 "2: "ins" %1, [%2], #1\n" \
209 " mov %1, %1, lsr #8\n" \
210 "3: "ins" %1, [%2], #1\n" \
211 " mov %1, %1, lsr #8\n" \
212 "4: "ins" %1, [%2]\n" \
213 "5:\n" \
214 " .section .fixup,\"ax\"\n" \
215 " .align 2\n" \
216 "6: mov %0, #1\n" \
217 " b 5b\n" \
218 " .previous\n" \
219 " .section __ex_table,\"a\"\n" \
220 " .align 3\n" \
221 " .long 1b, 6b\n" \
222 " .long 2b, 6b\n" \
223 " .long 3b, 6b\n" \
224 " .long 4b, 6b\n" \
225 " .previous\n" \
226 : "=r" (err), "=&r" (v), "=&r" (a) \
227 : "0" (err), "1" (v), "2" (a)); \
228 if (err) \
229 goto fault; \
230 } while (0)
231
232 #define get32_unaligned_check(val,addr) \
233 do { \
234 unsigned int err = 0, v, a = addr; \
235 get8_unaligned_check(val,a,err); \
236 get8_unaligned_check(v,a,err); \
237 val |= v << 8; \
238 get8_unaligned_check(v,a,err); \
239 val |= v << 16; \
240 get8_unaligned_check(v,a,err); \
241 val |= v << 24; \
242 if (err) \
243 goto fault; \
244 } while (0)
245
246 #define put32_unaligned_check(val,addr) \
247 __put32_unaligned_check("strb", val, addr)
248
249 #define get32t_unaligned_check(val,addr) \
250 do { \
251 unsigned int err = 0, v, a = addr; \
252 get8t_unaligned_check(val,a,err); \
253 get8t_unaligned_check(v,a,err); \
254 val |= v << 8; \
255 get8t_unaligned_check(v,a,err); \
256 val |= v << 16; \
257 get8t_unaligned_check(v,a,err); \
258 val |= v << 24; \
259 if (err) \
260 goto fault; \
261 } while (0)
262
263 #define put32t_unaligned_check(val,addr) \
264 __put32_unaligned_check("strbt", val, addr)
265
266 static void
267 do_alignment_finish_ldst(unsigned long addr, unsigned long instr, struct pt_regs *regs, union offset_union offset)
268 {
269 if (!LDST_U_BIT(instr))
270 offset.un = -offset.un;
271
272 if (!LDST_P_BIT(instr))
273 addr += offset.un;
274
275 if (!LDST_P_BIT(instr) || LDST_W_BIT(instr))
276 regs->uregs[RN_BITS(instr)] = addr;
277 }
278
279 static int
280 do_alignment_ldrhstrh(unsigned long addr, unsigned long instr, struct pt_regs *regs)
281 {
282 unsigned int rd = RD_BITS(instr);
283
284 if ((instr & 0x01f00ff0) == 0x01000090)
285 goto swp;
286
287 if ((instr & 0x90) != 0x90 || (instr & 0x60) == 0)
288 goto bad;
289
290 ai_half += 1;
291
292 if (LDST_L_BIT(instr)) {
293 unsigned long val;
294 get16_unaligned_check(val, addr);
295
296 /* signed half-word? */
297 if (instr & 0x40)
298 val = (signed long)((signed short) val);
299
300 regs->uregs[rd] = val;
301 } else
302 put16_unaligned_check(regs->uregs[rd], addr);
303
304 return TYPE_LDST;
305
306 swp:
307 printk(KERN_ERR "Alignment trap: not handling swp instruction\n");
308 bad:
309 return TYPE_ERROR;
310
311 fault:
312 return TYPE_FAULT;
313 }
314
315 static int
316 do_alignment_ldrstr(unsigned long addr, unsigned long instr, struct pt_regs *regs)
317 {
318 unsigned int rd = RD_BITS(instr);
319
320 ai_word += 1;
321
322 if (!LDST_P_BIT(instr) && LDST_W_BIT(instr))
323 goto trans;
324
325 if (LDST_L_BIT(instr))
326 get32_unaligned_check(regs->uregs[rd], addr);
327 else
328 put32_unaligned_check(regs->uregs[rd], addr);
329 return TYPE_LDST;
330
331 trans:
332 if (LDST_L_BIT(instr))
333 get32t_unaligned_check(regs->uregs[rd], addr);
334 else
335 put32t_unaligned_check(regs->uregs[rd], addr);
336 return TYPE_LDST;
337
338 fault:
339 return TYPE_FAULT;
340 }
341
342 /*
343 * LDM/STM alignment handler.
344 *
345 * There are 4 variants of this instruction:
346 *
347 * B = rn pointer before instruction, A = rn pointer after instruction
348 * ------ increasing address ----->
349 * | | r0 | r1 | ... | rx | |
350 * PU = 01 B A
351 * PU = 11 B A
352 * PU = 00 A B
353 * PU = 10 A B
354 */
355 static int
356 do_alignment_ldmstm(unsigned long addr, unsigned long instr, struct pt_regs *regs)
357 {
358 unsigned int rd, rn, correction, nr_regs, regbits;
359 unsigned long eaddr, newaddr;
360
361 if (LDM_S_BIT(instr))
362 goto bad;
363
364 correction = 4; /* processor implementation defined */
365 regs->ARM_pc += correction;
366
367 ai_multi += 1;
368
369 /* count the number of registers in the mask to be transferred */
370 nr_regs = hweight16(REGMASK_BITS(instr)) * 4;
371
372 rn = RN_BITS(instr);
373 newaddr = eaddr = regs->uregs[rn];
374
375 if (!LDST_U_BIT(instr))
376 nr_regs = -nr_regs;
377 newaddr += nr_regs;
378 if (!LDST_U_BIT(instr))
379 eaddr = newaddr;
380
381 if (LDST_P_EQ_U(instr)) /* U = P */
382 eaddr += 4;
383
384 /*
385 * This is a "hint" - we already have eaddr worked out by the
386 * processor for us.
387 */
388 if (addr != eaddr) {
389 printk(KERN_ERR "LDMSTM: PC = %08lx, instr = %08lx, "
390 "addr = %08lx, eaddr = %08lx\n",
391 instruction_pointer(regs), instr, addr, eaddr);
392 show_regs(regs);
393 }
394
395 for (regbits = REGMASK_BITS(instr), rd = 0; regbits; regbits >>= 1, rd += 1)
396 if (regbits & 1) {
397 if (LDST_L_BIT(instr))
398 get32_unaligned_check(regs->uregs[rd], eaddr);
399 else
400 put32_unaligned_check(regs->uregs[rd], eaddr);
401 eaddr += 4;
402 }
403
404 if (LDST_W_BIT(instr))
405 regs->uregs[rn] = newaddr;
406 if (!LDST_L_BIT(instr) || !(REGMASK_BITS(instr) & (1 << 15)))
407 regs->ARM_pc -= correction;
408 return TYPE_DONE;
409
410 fault:
411 regs->ARM_pc -= correction;
412 return TYPE_FAULT;
413
414 bad:
415 printk(KERN_ERR "Alignment trap: not handling ldm with s-bit set\n");
416 return TYPE_ERROR;
417 }
418
419 static int
420 do_alignment(unsigned long addr, int error_code, struct pt_regs *regs)
421 {
422 union offset_union offset;
423 unsigned long instr, instrptr;
424 int (*handler)(unsigned long addr, unsigned long instr, struct pt_regs *regs);
425 unsigned int type;
426
427 if (user_mode(regs))
428 goto user;
429
430 ai_sys += 1;
431
432 instrptr = instruction_pointer(regs);
433 instr = *(unsigned long *)instrptr;
434
435 regs->ARM_pc += 4;
436
437 switch (CODING_BITS(instr)) {
438 case 0x00000000: /* ldrh or strh */
439 if (LDSTH_I_BIT(instr))
440 offset.un = (instr & 0xf00) >> 4 | (instr & 15);
441 else
442 offset.un = regs->uregs[RM_BITS(instr)];
443 handler = do_alignment_ldrhstrh;
444 break;
445
446 case 0x04000000: /* ldr or str immediate */
447 offset.un = OFFSET_BITS(instr);
448 handler = do_alignment_ldrstr;
449 break;
450
451 case 0x06000000: /* ldr or str register */
452 offset.un = regs->uregs[RM_BITS(instr)];
453
454 if (IS_SHIFT(instr)) {
455 unsigned int shiftval = SHIFT_BITS(instr);
456
457 switch(SHIFT_TYPE(instr)) {
458 case SHIFT_LSL:
459 offset.un <<= shiftval;
460 break;
461
462 case SHIFT_LSR:
463 offset.un >>= shiftval;
464 break;
465
466 case SHIFT_ASR:
467 offset.sn >>= shiftval;
468 break;
469
470 case SHIFT_RORRRX:
471 if (shiftval == 0) {
472 offset.un >>= 1;
473 if (regs->ARM_cpsr & CC_C_BIT)
474 offset.un |= 1 << 31;
475 } else
476 offset.un = offset.un >> shiftval |
477 offset.un << (32 - shiftval);
478 break;
479 }
480 }
481 handler = do_alignment_ldrstr;
482 break;
483
484 case 0x08000000: /* ldm or stm */
485 handler = do_alignment_ldmstm;
486 break;
487
488 default:
489 goto bad;
490 }
491
492 type = handler(addr, instr, regs);
493
494 if (type == TYPE_ERROR || type == TYPE_FAULT)
495 goto bad_or_fault;
496
497 if (type == TYPE_LDST)
498 do_alignment_finish_ldst(addr, instr, regs, offset);
499
500 return 0;
501
502 bad_or_fault:
503 if (type == TYPE_ERROR)
504 goto bad;
505 regs->ARM_pc -= 4;
506 /*
507 * We got a fault - fix it up, or die.
508 */
509 do_bad_area(current, current->mm, addr, error_code, regs);
510 return 0;
511
512 bad:
513 /*
514 * Oops, we didn't handle the instruction.
515 */
516 printk(KERN_ERR "Alignment trap: not handling instruction "
517 "%08lx at [<%08lx>]", instr, instrptr);
518 ai_skipped += 1;
519 return 1;
520
521 user:
522 set_cr(cr_no_alignment);
523 ai_user += 1;
524 return 0;
525 }
526
527 #else
528
529 #define do_alignment NULL
530
531 #endif
532
533 /*
534 * Some section permission faults need to be handled gracefully, for
535 * instance, when they happen due to a __{get,put}_user during an oops).
536 */
537 static int
538 do_sect_fault(unsigned long addr, int error_code, struct pt_regs *regs)
539 {
540 struct task_struct *tsk = current;
541 do_bad_area(tsk, tsk->active_mm, addr, error_code, regs);
542 return 0;
543 }
544
545 /*
546 * Hook for things that need to trap external faults. Note that
547 * we don't guarantee that this will be the final version of the
548 * interface.
549 */
550 int (*external_fault)(unsigned long addr, struct pt_regs *regs);
551
552 static int
553 do_external_fault(unsigned long addr, int error_code, struct pt_regs *regs)
554 {
555 if (external_fault)
556 return external_fault(addr, regs);
557 return 1;
558 }
559
560 static const struct fsr_info {
561 int (*fn)(unsigned long addr, int error_code, struct pt_regs *regs);
562 int sig;
563 char *name;
564 } fsr_info[] = {
565 { NULL, SIGSEGV, "vector exception" },
566 { do_alignment, SIGILL, "alignment exception" },
567 { NULL, SIGKILL, "terminal exception" },
568 { do_alignment, SIGILL, "alignment exception" },
569 { do_external_fault, SIGBUS, "external abort on linefetch" },
570 { do_translation_fault, SIGSEGV, "section translation fault" },
571 { do_external_fault, SIGBUS, "external abort on linefetch" },
572 { do_page_fault, SIGSEGV, "page translation fault" },
573 { do_external_fault, SIGBUS, "external abort on non-linefetch" },
574 { NULL, SIGSEGV, "section domain fault" },
575 { do_external_fault, SIGBUS, "external abort on non-linefetch" },
576 { NULL, SIGSEGV, "page domain fault" },
577 { NULL, SIGBUS, "external abort on translation" },
578 { do_sect_fault, SIGSEGV, "section permission fault" },
579 { NULL, SIGBUS, "external abort on translation" },
580 { do_page_fault, SIGSEGV, "page permission fault" }
581 };
582
583 /*
584 * Currently dropped down to debug level
585 */
586 asmlinkage void
587 do_DataAbort(unsigned long addr, int error_code, struct pt_regs *regs, int fsr)
588 {
589 const struct fsr_info *inf = fsr_info + (fsr & 15);
590
591 #if defined(CONFIG_CPU_SA110) || defined(CONFIG_CPU_SA1100) || defined(CONFIG_DEBUG_ERRORS)
592 if (addr == regs->ARM_pc)
593 goto sa1_weirdness;
594 #endif
595
596 if (!inf->fn)
597 goto bad;
598
599 if (!inf->fn(addr, error_code, regs))
600 return;
601 bad:
602 force_sig(inf->sig, current);
603 printk(KERN_ALERT "Unhandled fault: %s (%X) at 0x%08lx\n",
604 inf->name, fsr, addr);
605 show_pte(current->mm, addr);
606 die_if_kernel("Oops", regs, 0);
607 return;
608
609 #if defined(CONFIG_CPU_SA110) || defined(CONFIG_CPU_SA1100) || defined(CONFIG_DEBUG_ERRORS)
610 sa1_weirdness:
611 if (user_mode(regs)) {
612 static int first = 1;
613 if (first) {
614 printk(KERN_DEBUG "Fixing up bad data abort at %08lx\n", addr);
615 #ifdef CONFIG_DEBUG_ERRORS
616 show_pte(current->mm, addr);
617 #endif
618 }
619 first = 0;
620 return;
621 }
622
623 if (!inf->fn || inf->fn(addr, error_code, regs))
624 goto bad;
625 return;
626 #endif
627 }
628
629 asmlinkage void
630 do_PrefetchAbort(unsigned long addr, struct pt_regs *regs)
631 {
632 do_translation_fault(addr, 0, regs);
633 }
634
635 /*
636 * We take the easy way out of this problem - we make the
637 * PTE uncacheable. However, we leave the write buffer on.
638 */
639 static void adjust_pte(struct vm_area_struct *vma, unsigned long address)
640 {
641 pgd_t *pgd;
642 pmd_t *pmd;
643 pte_t *pte, entry;
644
645 pgd = pgd_offset(vma->vm_mm, address);
646 if (pgd_none(*pgd))
647 return;
648 if (pgd_bad(*pgd))
649 goto bad_pgd;
650
651 pmd = pmd_offset(pgd, address);
652 if (pmd_none(*pmd))
653 return;
654 if (pmd_bad(*pmd))
655 goto bad_pmd;
656
657 pte = pte_offset(pmd, address);
658 entry = *pte;
659
660 /*
661 * If this page isn't present, or is already setup to
662 * fault (ie, is old), we can safely ignore any issues.
663 */
664 if (pte_present(entry) && pte_val(entry) & L_PTE_CACHEABLE) {
665 flush_cache_page(vma, address);
666 pte_val(entry) &= ~L_PTE_CACHEABLE;
667 set_pte(pte, entry);
668 flush_tlb_page(vma, address);
669 }
670 return;
671
672 bad_pgd:
673 pgd_ERROR(*pgd);
674 pgd_clear(pgd);
675 return;
676
677 bad_pmd:
678 pmd_ERROR(*pmd);
679 pmd_clear(pmd);
680 return;
681 }
682
683 /*
684 * Take care of architecture specific things when placing a new PTE into
685 * a page table, or changing an existing PTE. Basically, there are two
686 * things that we need to take care of:
687 *
688 * 1. If PG_dcache_dirty is set for the page, we need to ensure
689 * that any cache entries for the kernels virtual memory
690 * range are written back to the page.
691 * 2. If we have multiple shared mappings of the same space in
692 * an object, we need to deal with the cache aliasing issues.
693 *
694 * Note that the page_table_lock will be held.
695 */
696 void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
697 {
698 struct page *page = pte_page(pte);
699 struct vm_area_struct *mpnt;
700 struct mm_struct *mm;
701 unsigned long pgoff;
702 int aliases;
703
704 if (!VALID_PAGE(page) || !page->mapping)
705 return;
706
707 if (test_and_clear_bit(PG_dcache_dirty, &page->flags)) {
708 unsigned long kvirt = (unsigned long)page_address(page);
709 cpu_cache_clean_invalidate_range(kvirt, kvirt + PAGE_SIZE, 0);
710 }
711
712 mm = vma->vm_mm;
713 pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
714 aliases = 0;
715
716 /*
717 * If we have any shared mappings that are in the same mm
718 * space, then we need to handle them specially to maintain
719 * cache coherency.
720 */
721 for (mpnt = page->mapping->i_mmap_shared; mpnt;
722 mpnt = mpnt->vm_next_share) {
723 unsigned long off;
724
725 /*
726 * If this VMA is not in our MM, we can ignore it.
727 * Note that we intentionally don't mask out the VMA
728 * that we are fixing up.
729 */
730 if (mpnt->vm_mm != mm && mpnt != vma)
731 continue;
732
733 /*
734 * If the page isn't in this VMA, we can also ignore it.
735 */
736 if (pgoff < mpnt->vm_pgoff)
737 continue;
738
739 off = pgoff - mpnt->vm_pgoff;
740 if (off >= (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT)
741 continue;
742
743 /*
744 * Ok, it is within mpnt. Fix it up.
745 */
746 adjust_pte(mpnt, mpnt->vm_start + (off << PAGE_SHIFT));
747 aliases ++;
748 }
749 if (aliases)
750 adjust_pte(vma, addr);
751 }
752