File: /usr/src/linux/arch/sparc/mm/srmmu.c
1 /* $Id: srmmu.c,v 1.231 2001/09/20 00:35:31 davem Exp $
2 * srmmu.c: SRMMU specific routines for memory management.
3 *
4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1995 Pete Zaitcev
6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
9 */
10
11 #include <linux/config.h>
12 #include <linux/kernel.h>
13 #include <linux/mm.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/pagemap.h>
17 #include <linux/init.h>
18 #include <linux/blk.h>
19 #include <linux/spinlock.h>
20 #include <linux/bootmem.h>
21
22 #include <asm/page.h>
23 #include <asm/pgalloc.h>
24 #include <asm/pgtable.h>
25 #include <asm/io.h>
26 #include <asm/kdebug.h>
27 #include <asm/vaddrs.h>
28 #include <asm/traps.h>
29 #include <asm/smp.h>
30 #include <asm/mbus.h>
31 #include <asm/cache.h>
32 #include <asm/oplib.h>
33 #include <asm/sbus.h>
34 #include <asm/asi.h>
35 #include <asm/msi.h>
36 #include <asm/a.out.h>
37 #include <asm/mmu_context.h>
38 #include <asm/io-unit.h>
39
40 /* Now the cpu specific definitions. */
41 #include <asm/viking.h>
42 #include <asm/mxcc.h>
43 #include <asm/ross.h>
44 #include <asm/tsunami.h>
45 #include <asm/swift.h>
46 #include <asm/turbosparc.h>
47
48 #include <asm/btfixup.h>
49
50 enum mbus_module srmmu_modtype;
51 unsigned int hwbug_bitmask;
52 int vac_cache_size;
53 int vac_line_size;
54
55 extern struct resource sparc_iomap;
56
57 extern unsigned long last_valid_pfn;
58
59 extern unsigned long page_kernel;
60
61 pgd_t *srmmu_swapper_pg_dir;
62
63 #ifdef CONFIG_SMP
64 #define FLUSH_BEGIN(mm)
65 #define FLUSH_END
66 #else
67 #define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) {
68 #define FLUSH_END }
69 #endif
70
71 BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
72 #define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
73
74 int flush_page_for_dma_global = 1;
75
76 #ifdef CONFIG_SMP
77 BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long)
78 #define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page)
79 #endif
80
81 char *srmmu_name;
82
83 ctxd_t *srmmu_ctx_table_phys;
84 ctxd_t *srmmu_context_table;
85
86 int viking_mxcc_present;
87 spinlock_t srmmu_context_spinlock = SPIN_LOCK_UNLOCKED;
88
89 int is_hypersparc;
90
91 /*
92 * In general all page table modifications should use the V8 atomic
93 * swap instruction. This insures the mmu and the cpu are in sync
94 * with respect to ref/mod bits in the page tables.
95 */
96 static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value)
97 {
98 __asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr));
99 return value;
100 }
101
102 static inline void srmmu_set_pte(pte_t *ptep, pte_t pteval)
103 {
104 srmmu_swap((unsigned long *)ptep, pte_val(pteval));
105 }
106
107 /* The very generic SRMMU page table operations. */
108 static inline int srmmu_device_memory(unsigned long x)
109 {
110 return ((x & 0xF0000000) != 0);
111 }
112
113 int srmmu_cache_pagetables;
114
115 /* XXX Make this dynamic based on ram size - Anton */
116 #define SRMMU_NOCACHE_BITMAP_SIZE (SRMMU_NOCACHE_NPAGES * 16)
117 #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4)
118
119 void *srmmu_nocache_pool;
120 void *srmmu_nocache_bitmap;
121 int srmmu_nocache_low;
122 int srmmu_nocache_used;
123 spinlock_t srmmu_nocache_spinlock;
124
125 /* This makes sense. Honest it does - Anton */
126 #define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool))
127 #define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR)
128 #define __nocache_fix(VADDR) __va(__nocache_pa(VADDR))
129
130 static inline unsigned long srmmu_pgd_page(pgd_t pgd)
131 { return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); }
132
133 static inline unsigned long srmmu_pmd_page(pmd_t pmd)
134 { return srmmu_device_memory(pmd_val(pmd))?~0:(unsigned long)__nocache_va((pmd_val(pmd) & SRMMU_PTD_PMASK) << 4); }
135
136 static inline struct page *srmmu_pte_page(pte_t pte)
137 { return (mem_map + (unsigned long)(srmmu_device_memory(pte_val(pte))?~0:(((pte_val(pte) & SRMMU_PTE_PMASK) << 4) >> PAGE_SHIFT))); }
138
139 static inline int srmmu_pte_none(pte_t pte)
140 { return !(pte_val(pte) & 0xFFFFFFF); }
141
142 static inline int srmmu_pte_present(pte_t pte)
143 { return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); }
144
145 static inline void srmmu_pte_clear(pte_t *ptep)
146 { srmmu_set_pte(ptep, __pte(0)); }
147
148 static inline int srmmu_pmd_none(pmd_t pmd)
149 { return !(pmd_val(pmd) & 0xFFFFFFF); }
150
151 static inline int srmmu_pmd_bad(pmd_t pmd)
152 { return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }
153
154 static inline int srmmu_pmd_present(pmd_t pmd)
155 { return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }
156
157 static inline void srmmu_pmd_clear(pmd_t *pmdp)
158 { srmmu_set_pte((pte_t *)pmdp, __pte(0)); }
159
160 static inline int srmmu_pgd_none(pgd_t pgd)
161 { return !(pgd_val(pgd) & 0xFFFFFFF); }
162
163 static inline int srmmu_pgd_bad(pgd_t pgd)
164 { return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; }
165
166 static inline int srmmu_pgd_present(pgd_t pgd)
167 { return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); }
168
169 static inline void srmmu_pgd_clear(pgd_t * pgdp)
170 { srmmu_set_pte((pte_t *)pgdp, __pte(0)); }
171
172 static inline int srmmu_pte_write(pte_t pte)
173 { return pte_val(pte) & SRMMU_WRITE; }
174
175 static inline int srmmu_pte_dirty(pte_t pte)
176 { return pte_val(pte) & SRMMU_DIRTY; }
177
178 static inline int srmmu_pte_young(pte_t pte)
179 { return pte_val(pte) & SRMMU_REF; }
180
181 static inline pte_t srmmu_pte_wrprotect(pte_t pte)
182 { return __pte(pte_val(pte) & ~SRMMU_WRITE);}
183
184 static inline pte_t srmmu_pte_mkclean(pte_t pte)
185 { return __pte(pte_val(pte) & ~SRMMU_DIRTY);}
186
187 static inline pte_t srmmu_pte_mkold(pte_t pte)
188 { return __pte(pte_val(pte) & ~SRMMU_REF);}
189
190 static inline pte_t srmmu_pte_mkwrite(pte_t pte)
191 { return __pte(pte_val(pte) | SRMMU_WRITE);}
192
193 static inline pte_t srmmu_pte_mkdirty(pte_t pte)
194 { return __pte(pte_val(pte) | SRMMU_DIRTY);}
195
196 static inline pte_t srmmu_pte_mkyoung(pte_t pte)
197 { return __pte(pte_val(pte) | SRMMU_REF);}
198
199 /*
200 * Conversion functions: convert a page and protection to a page entry,
201 * and a page entry and page directory to the page they refer to.
202 */
203 static pte_t srmmu_mk_pte(struct page *page, pgprot_t pgprot)
204 { return __pte((((page - mem_map) << PAGE_SHIFT) >> 4) | pgprot_val(pgprot)); }
205
206 static pte_t srmmu_mk_pte_phys(unsigned long page, pgprot_t pgprot)
207 { return __pte(((page) >> 4) | pgprot_val(pgprot)); }
208
209 static pte_t srmmu_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
210 { return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); }
211
212 /* XXX should we hyper_flush_whole_icache here - Anton */
213 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp)
214 { srmmu_set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); }
215
216 static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
217 { srmmu_set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); }
218
219 static inline void srmmu_pmd_set(pmd_t * pmdp, pte_t * ptep)
220 { srmmu_set_pte((pte_t *)pmdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) ptep) >> 4))); }
221
222 static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot)
223 { return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); }
224
225 /* to find an entry in a top-level page table... */
226 extern inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address)
227 { return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); }
228
229 /* Find an entry in the second-level page table.. */
230 static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address)
231 { return (pmd_t *) srmmu_pgd_page(*dir) + ((address >> SRMMU_PMD_SHIFT) & (SRMMU_PTRS_PER_PMD - 1)); }
232
233 /* Find an entry in the third-level page table.. */
234 static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address)
235 { return (pte_t *) srmmu_pmd_page(*dir) + ((address >> PAGE_SHIFT) & (SRMMU_PTRS_PER_PTE - 1)); }
236
237 unsigned long __srmmu_get_nocache(int size, int align)
238 {
239 int offset = srmmu_nocache_low;
240 int i;
241 unsigned long va_tmp, phys_tmp;
242 int lowest_failed = 0;
243
244 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
245
246 spin_lock(&srmmu_nocache_spinlock);
247
248 repeat:
249 offset = find_next_zero_bit(srmmu_nocache_bitmap, SRMMU_NOCACHE_BITMAP_SIZE, offset);
250
251 /* we align on physical address */
252 if (align) {
253 va_tmp = (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
254 phys_tmp = (__nocache_pa(va_tmp) + align - 1) & ~(align - 1);
255 va_tmp = (unsigned long)__nocache_va(phys_tmp);
256 offset = (va_tmp - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
257 }
258
259 if ((SRMMU_NOCACHE_BITMAP_SIZE - offset) < size) {
260 printk("Run out of nocached RAM!\n");
261 spin_unlock(&srmmu_nocache_spinlock);
262 return 0;
263 }
264
265 i = 0;
266 while(i < size) {
267 if (test_bit(offset + i, srmmu_nocache_bitmap)) {
268 lowest_failed = 1;
269 offset = offset + i + 1;
270 goto repeat;
271 }
272 i++;
273 }
274
275 i = 0;
276 while(i < size) {
277 set_bit(offset + i, srmmu_nocache_bitmap);
278 i++;
279 srmmu_nocache_used++;
280 }
281
282 if (!lowest_failed && ((align >> SRMMU_NOCACHE_BITMAP_SHIFT) <= 1) && (offset > srmmu_nocache_low))
283 srmmu_nocache_low = offset;
284
285 spin_unlock(&srmmu_nocache_spinlock);
286
287 return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
288 }
289
290 unsigned inline long srmmu_get_nocache(int size, int align)
291 {
292 unsigned long tmp;
293
294 tmp = __srmmu_get_nocache(size, align);
295
296 if (tmp)
297 memset((void *)tmp, 0, size);
298
299 return tmp;
300 }
301
302 void srmmu_free_nocache(unsigned long vaddr, int size)
303 {
304 int offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT;
305
306 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT;
307
308 spin_lock(&srmmu_nocache_spinlock);
309
310 while(size--) {
311 clear_bit(offset + size, srmmu_nocache_bitmap);
312 srmmu_nocache_used--;
313 }
314
315 if (offset < srmmu_nocache_low)
316 srmmu_nocache_low = offset;
317
318 spin_unlock(&srmmu_nocache_spinlock);
319 }
320
321 void srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end);
322
323 void srmmu_nocache_init(void)
324 {
325 pgd_t *pgd;
326 pmd_t *pmd;
327 pte_t *pte;
328 unsigned long paddr, vaddr;
329 unsigned long pteval;
330
331 srmmu_nocache_pool = __alloc_bootmem(SRMMU_NOCACHE_SIZE, PAGE_SIZE, 0UL);
332 memset(srmmu_nocache_pool, 0, SRMMU_NOCACHE_SIZE);
333
334 srmmu_nocache_bitmap = __alloc_bootmem(SRMMU_NOCACHE_BITMAP_SIZE, SMP_CACHE_BYTES, 0UL);
335 memset(srmmu_nocache_bitmap, 0, SRMMU_NOCACHE_BITMAP_SIZE);
336
337 srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
338 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
339 init_mm.pgd = srmmu_swapper_pg_dir;
340
341 srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, SRMMU_NOCACHE_END);
342
343 spin_lock_init(&srmmu_nocache_spinlock);
344
345 paddr = __pa((unsigned long)srmmu_nocache_pool);
346 vaddr = SRMMU_NOCACHE_VADDR;
347
348 while (vaddr < SRMMU_NOCACHE_END) {
349 pgd = pgd_offset_k(vaddr);
350 pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr);
351 pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr);
352
353 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV);
354
355 if (srmmu_cache_pagetables)
356 pteval |= SRMMU_CACHE;
357
358 srmmu_set_pte(__nocache_fix(pte), pteval);
359
360 vaddr += PAGE_SIZE;
361 paddr += PAGE_SIZE;
362 }
363
364 flush_cache_all();
365 flush_tlb_all();
366 }
367
368 static inline pgd_t *srmmu_get_pgd_fast(void)
369 {
370 pgd_t *pgd = NULL;
371
372 pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
373 if (pgd) {
374 pgd_t *init = pgd_offset_k(0);
375 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
376 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
377 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
378 }
379
380 return pgd;
381 }
382
383 static void srmmu_free_pgd_fast(pgd_t *pgd)
384 {
385 srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE);
386 }
387
388 static pte_t *srmmu_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
389 {
390 return (pte_t *)srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, SRMMU_PTE_TABLE_SIZE);
391 }
392
393 static pte_t *srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address)
394 {
395 BUG();
396 }
397
398 static void srmmu_free_pte_fast(pte_t *pte)
399 {
400 srmmu_free_nocache((unsigned long)pte, SRMMU_PTE_TABLE_SIZE);
401 }
402
403 static pmd_t *srmmu_pmd_alloc_one_fast(struct mm_struct *mm, unsigned long address)
404 {
405 return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
406 }
407
408 static void srmmu_free_pmd_fast(pmd_t * pmd)
409 {
410 srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE);
411 }
412
413 static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
414 {
415 struct ctx_list *ctxp;
416
417 ctxp = ctx_free.next;
418 if(ctxp != &ctx_free) {
419 remove_from_ctx_list(ctxp);
420 add_to_used_ctxlist(ctxp);
421 mm->context = ctxp->ctx_number;
422 ctxp->ctx_mm = mm;
423 return;
424 }
425 ctxp = ctx_used.next;
426 if(ctxp->ctx_mm == old_mm)
427 ctxp = ctxp->next;
428 if(ctxp == &ctx_used)
429 panic("out of mmu contexts");
430 flush_cache_mm(ctxp->ctx_mm);
431 flush_tlb_mm(ctxp->ctx_mm);
432 remove_from_ctx_list(ctxp);
433 add_to_used_ctxlist(ctxp);
434 ctxp->ctx_mm->context = NO_CONTEXT;
435 ctxp->ctx_mm = mm;
436 mm->context = ctxp->ctx_number;
437 }
438
439 static inline void free_context(int context)
440 {
441 struct ctx_list *ctx_old;
442
443 ctx_old = ctx_list_pool + context;
444 remove_from_ctx_list(ctx_old);
445 add_to_free_ctxlist(ctx_old);
446 }
447
448
449 static void srmmu_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
450 struct task_struct *tsk, int cpu)
451 {
452 if(mm->context == NO_CONTEXT) {
453 spin_lock(&srmmu_context_spinlock);
454 alloc_context(old_mm, mm);
455 spin_unlock(&srmmu_context_spinlock);
456 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd);
457 }
458
459 if (is_hypersparc)
460 hyper_flush_whole_icache();
461
462 srmmu_set_context(mm->context);
463 }
464
465 /* Low level IO area allocation on the SRMMU. */
466 void srmmu_mapioaddr(unsigned long physaddr, unsigned long virt_addr, int bus_type, int rdonly)
467 {
468 pgd_t *pgdp;
469 pmd_t *pmdp;
470 pte_t *ptep;
471 unsigned long tmp;
472
473 physaddr &= PAGE_MASK;
474 pgdp = pgd_offset_k(virt_addr);
475 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
476 ptep = srmmu_pte_offset(pmdp, virt_addr);
477 tmp = (physaddr >> 4) | SRMMU_ET_PTE;
478
479 /*
480 * I need to test whether this is consistent over all
481 * sun4m's. The bus_type represents the upper 4 bits of
482 * 36-bit physical address on the I/O space lines...
483 */
484 tmp |= (bus_type << 28);
485 if(rdonly)
486 tmp |= SRMMU_PRIV_RDONLY;
487 else
488 tmp |= SRMMU_PRIV;
489 __flush_page_to_ram(virt_addr);
490 srmmu_set_pte(ptep, __pte(tmp));
491 flush_tlb_all();
492 }
493
494 void srmmu_unmapioaddr(unsigned long virt_addr)
495 {
496 pgd_t *pgdp;
497 pmd_t *pmdp;
498 pte_t *ptep;
499
500 pgdp = pgd_offset_k(virt_addr);
501 pmdp = srmmu_pmd_offset(pgdp, virt_addr);
502 ptep = srmmu_pte_offset(pmdp, virt_addr);
503
504 /* No need to flush uncacheable page. */
505 srmmu_pte_clear(ptep);
506 flush_tlb_all();
507 }
508
509 /*
510 * On the SRMMU we do not have the problems with limited tlb entries
511 * for mapping kernel pages, so we just take things from the free page
512 * pool. As a side effect we are putting a little too much pressure
513 * on the gfp() subsystem. This setup also makes the logic of the
514 * iommu mapping code a lot easier as we can transparently handle
515 * mappings on the kernel stack without any special code as we did
516 * need on the sun4c.
517 */
518 struct task_struct *srmmu_alloc_task_struct(void)
519 {
520 return (struct task_struct *) __get_free_pages(GFP_KERNEL, 1);
521 }
522
523 static void srmmu_free_task_struct(struct task_struct *tsk)
524 {
525 free_pages((unsigned long)tsk, 1);
526 }
527
528 static void srmmu_get_task_struct(struct task_struct *tsk)
529 {
530 atomic_inc(&virt_to_page(tsk)->count);
531 }
532
533 /* tsunami.S */
534 extern void tsunami_flush_cache_all(void);
535 extern void tsunami_flush_cache_mm(struct mm_struct *mm);
536 extern void tsunami_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end);
537 extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
538 extern void tsunami_flush_page_to_ram(unsigned long page);
539 extern void tsunami_flush_page_for_dma(unsigned long page);
540 extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
541 extern void tsunami_flush_tlb_all(void);
542 extern void tsunami_flush_tlb_mm(struct mm_struct *mm);
543 extern void tsunami_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end);
544 extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
545 extern void tsunami_setup_blockops(void);
546
547 /*
548 * Workaround, until we find what's going on with Swift. When low on memory,
549 * it sometimes loops in fault/handle_mm_fault incl. flush_tlb_page to find
550 * out it is already in page tables/ fault again on the same instruction.
551 * I really don't understand it, have checked it and contexts
552 * are right, flush_tlb_all is done as well, and it faults again...
553 * Strange. -jj
554 *
555 * The following code is a deadwood that may be necessary when
556 * we start to make precise page flushes again. --zaitcev
557 */
558 static void swift_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte)
559 {
560 #if 0
561 static unsigned long last;
562 unsigned int val;
563 /* unsigned int n; */
564
565 if (address == last) {
566 val = srmmu_hwprobe(address);
567 if (val != 0 && pte_val(pte) != val) {
568 printk("swift_update_mmu_cache: "
569 "addr %lx put %08x probed %08x from %p\n",
570 address, pte_val(pte), val,
571 __builtin_return_address(0));
572 srmmu_flush_whole_tlb();
573 }
574 }
575 last = address;
576 #endif
577 }
578
579 /* swift.S */
580 extern void swift_flush_cache_all(void);
581 extern void swift_flush_cache_mm(struct mm_struct *mm);
582 extern void swift_flush_cache_range(struct mm_struct *mm,
583 unsigned long start, unsigned long end);
584 extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
585 extern void swift_flush_page_to_ram(unsigned long page);
586 extern void swift_flush_page_for_dma(unsigned long page);
587 extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
588 extern void swift_flush_tlb_all(void);
589 extern void swift_flush_tlb_mm(struct mm_struct *mm);
590 extern void swift_flush_tlb_range(struct mm_struct *mm,
591 unsigned long start, unsigned long end);
592 extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
593
594 #if 0 /* P3: deadwood to debug precise flushes on Swift. */
595 void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
596 {
597 int cctx, ctx1;
598
599 page &= PAGE_MASK;
600 if ((ctx1 = vma->vm_mm->context) != -1) {
601 cctx = srmmu_get_context();
602 /* Is context # ever different from current context? P3 */
603 if (cctx != ctx1) {
604 printk("flush ctx %02x curr %02x\n", ctx1, cctx);
605 srmmu_set_context(ctx1);
606 swift_flush_page(page);
607 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
608 "r" (page), "i" (ASI_M_FLUSH_PROBE));
609 srmmu_set_context(cctx);
610 } else {
611 /* Rm. prot. bits from virt. c. */
612 /* swift_flush_cache_all(); */
613 /* swift_flush_cache_page(vma, page); */
614 swift_flush_page(page);
615
616 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
617 "r" (page), "i" (ASI_M_FLUSH_PROBE));
618 /* same as above: srmmu_flush_tlb_page() */
619 }
620 }
621 }
622 #endif
623
624 /*
625 * The following are all MBUS based SRMMU modules, and therefore could
626 * be found in a multiprocessor configuration. On the whole, these
627 * chips seems to be much more touchy about DVMA and page tables
628 * with respect to cache coherency.
629 */
630
631 /* Cypress flushes. */
632 static void cypress_flush_cache_all(void)
633 {
634 volatile unsigned long cypress_sucks;
635 unsigned long faddr, tagval;
636
637 flush_user_windows();
638 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
639 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
640 "=r" (tagval) :
641 "r" (faddr), "r" (0x40000),
642 "i" (ASI_M_DATAC_TAG));
643
644 /* If modified and valid, kick it. */
645 if((tagval & 0x60) == 0x60)
646 cypress_sucks = *(unsigned long *)(0xf0020000 + faddr);
647 }
648 }
649
650 static void cypress_flush_cache_mm(struct mm_struct *mm)
651 {
652 register unsigned long a, b, c, d, e, f, g;
653 unsigned long flags, faddr;
654 int octx;
655
656 FLUSH_BEGIN(mm)
657 flush_user_windows();
658 __save_and_cli(flags);
659 octx = srmmu_get_context();
660 srmmu_set_context(mm->context);
661 a = 0x20; b = 0x40; c = 0x60;
662 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
663
664 faddr = (0x10000 - 0x100);
665 goto inside;
666 do {
667 faddr -= 0x100;
668 inside:
669 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
670 "sta %%g0, [%0 + %2] %1\n\t"
671 "sta %%g0, [%0 + %3] %1\n\t"
672 "sta %%g0, [%0 + %4] %1\n\t"
673 "sta %%g0, [%0 + %5] %1\n\t"
674 "sta %%g0, [%0 + %6] %1\n\t"
675 "sta %%g0, [%0 + %7] %1\n\t"
676 "sta %%g0, [%0 + %8] %1\n\t" : :
677 "r" (faddr), "i" (ASI_M_FLUSH_CTX),
678 "r" (a), "r" (b), "r" (c), "r" (d),
679 "r" (e), "r" (f), "r" (g));
680 } while(faddr);
681 srmmu_set_context(octx);
682 __restore_flags(flags);
683 FLUSH_END
684 }
685
686 static void cypress_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end)
687 {
688 register unsigned long a, b, c, d, e, f, g;
689 unsigned long flags, faddr;
690 int octx;
691
692 FLUSH_BEGIN(mm)
693 flush_user_windows();
694 __save_and_cli(flags);
695 octx = srmmu_get_context();
696 srmmu_set_context(mm->context);
697 a = 0x20; b = 0x40; c = 0x60;
698 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
699
700 start &= SRMMU_PMD_MASK;
701 while(start < end) {
702 faddr = (start + (0x10000 - 0x100));
703 goto inside;
704 do {
705 faddr -= 0x100;
706 inside:
707 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
708 "sta %%g0, [%0 + %2] %1\n\t"
709 "sta %%g0, [%0 + %3] %1\n\t"
710 "sta %%g0, [%0 + %4] %1\n\t"
711 "sta %%g0, [%0 + %5] %1\n\t"
712 "sta %%g0, [%0 + %6] %1\n\t"
713 "sta %%g0, [%0 + %7] %1\n\t"
714 "sta %%g0, [%0 + %8] %1\n\t" : :
715 "r" (faddr),
716 "i" (ASI_M_FLUSH_SEG),
717 "r" (a), "r" (b), "r" (c), "r" (d),
718 "r" (e), "r" (f), "r" (g));
719 } while (faddr != start);
720 start += SRMMU_PMD_SIZE;
721 }
722 srmmu_set_context(octx);
723 __restore_flags(flags);
724 FLUSH_END
725 }
726
727 static void cypress_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
728 {
729 register unsigned long a, b, c, d, e, f, g;
730 struct mm_struct *mm = vma->vm_mm;
731 unsigned long flags, line;
732 int octx;
733
734 FLUSH_BEGIN(mm)
735 flush_user_windows();
736 __save_and_cli(flags);
737 octx = srmmu_get_context();
738 srmmu_set_context(mm->context);
739 a = 0x20; b = 0x40; c = 0x60;
740 d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
741
742 page &= PAGE_MASK;
743 line = (page + PAGE_SIZE) - 0x100;
744 goto inside;
745 do {
746 line -= 0x100;
747 inside:
748 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
749 "sta %%g0, [%0 + %2] %1\n\t"
750 "sta %%g0, [%0 + %3] %1\n\t"
751 "sta %%g0, [%0 + %4] %1\n\t"
752 "sta %%g0, [%0 + %5] %1\n\t"
753 "sta %%g0, [%0 + %6] %1\n\t"
754 "sta %%g0, [%0 + %7] %1\n\t"
755 "sta %%g0, [%0 + %8] %1\n\t" : :
756 "r" (line),
757 "i" (ASI_M_FLUSH_PAGE),
758 "r" (a), "r" (b), "r" (c), "r" (d),
759 "r" (e), "r" (f), "r" (g));
760 } while(line != page);
761 srmmu_set_context(octx);
762 __restore_flags(flags);
763 FLUSH_END
764 }
765
766 /* Cypress is copy-back, at least that is how we configure it. */
767 static void cypress_flush_page_to_ram(unsigned long page)
768 {
769 register unsigned long a, b, c, d, e, f, g;
770 unsigned long line;
771
772 a = 0x20; b = 0x40; c = 0x60; d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0;
773 page &= PAGE_MASK;
774 line = (page + PAGE_SIZE) - 0x100;
775 goto inside;
776 do {
777 line -= 0x100;
778 inside:
779 __asm__ __volatile__("sta %%g0, [%0] %1\n\t"
780 "sta %%g0, [%0 + %2] %1\n\t"
781 "sta %%g0, [%0 + %3] %1\n\t"
782 "sta %%g0, [%0 + %4] %1\n\t"
783 "sta %%g0, [%0 + %5] %1\n\t"
784 "sta %%g0, [%0 + %6] %1\n\t"
785 "sta %%g0, [%0 + %7] %1\n\t"
786 "sta %%g0, [%0 + %8] %1\n\t" : :
787 "r" (line),
788 "i" (ASI_M_FLUSH_PAGE),
789 "r" (a), "r" (b), "r" (c), "r" (d),
790 "r" (e), "r" (f), "r" (g));
791 } while(line != page);
792 }
793
794 /* Cypress is also IO cache coherent. */
795 static void cypress_flush_page_for_dma(unsigned long page)
796 {
797 }
798
799 /* Cypress has unified L2 VIPT, from which both instructions and data
800 * are stored. It does not have an onboard icache of any sort, therefore
801 * no flush is necessary.
802 */
803 static void cypress_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
804 {
805 }
806
807 static void cypress_flush_tlb_all(void)
808 {
809 srmmu_flush_whole_tlb();
810 }
811
812 static void cypress_flush_tlb_mm(struct mm_struct *mm)
813 {
814 FLUSH_BEGIN(mm)
815 __asm__ __volatile__("
816 lda [%0] %3, %%g5
817 sta %2, [%0] %3
818 sta %%g0, [%1] %4
819 sta %%g5, [%0] %3"
820 : /* no outputs */
821 : "r" (SRMMU_CTX_REG), "r" (0x300), "r" (mm->context),
822 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
823 : "g5");
824 FLUSH_END
825 }
826
827 static void cypress_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
828 {
829 unsigned long size;
830
831 FLUSH_BEGIN(mm)
832 start &= SRMMU_PGDIR_MASK;
833 size = SRMMU_PGDIR_ALIGN(end) - start;
834 __asm__ __volatile__("
835 lda [%0] %5, %%g5
836 sta %1, [%0] %5
837 1: subcc %3, %4, %3
838 bne 1b
839 sta %%g0, [%2 + %3] %6
840 sta %%g5, [%0] %5"
841 : /* no outputs */
842 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (start | 0x200),
843 "r" (size), "r" (SRMMU_PGDIR_SIZE), "i" (ASI_M_MMUREGS),
844 "i" (ASI_M_FLUSH_PROBE)
845 : "g5", "cc");
846 FLUSH_END
847 }
848
849 static void cypress_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
850 {
851 struct mm_struct *mm = vma->vm_mm;
852
853 FLUSH_BEGIN(mm)
854 __asm__ __volatile__("
855 lda [%0] %3, %%g5
856 sta %1, [%0] %3
857 sta %%g0, [%2] %4
858 sta %%g5, [%0] %3"
859 : /* no outputs */
860 : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (page & PAGE_MASK),
861 "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE)
862 : "g5");
863 FLUSH_END
864 }
865
866 /* viking.S */
867 extern void viking_flush_cache_all(void);
868 extern void viking_flush_cache_mm(struct mm_struct *mm);
869 extern void viking_flush_cache_range(struct mm_struct *mm, unsigned long start,
870 unsigned long end);
871 extern void viking_flush_cache_page(struct vm_area_struct *vma,
872 unsigned long page);
873 extern void viking_flush_page_to_ram(unsigned long page);
874 extern void viking_flush_page_for_dma(unsigned long page);
875 extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr);
876 extern void viking_flush_page(unsigned long page);
877 extern void viking_mxcc_flush_page(unsigned long page);
878 extern void viking_flush_tlb_all(void);
879 extern void viking_flush_tlb_mm(struct mm_struct *mm);
880 extern void viking_flush_tlb_range(struct mm_struct *mm, unsigned long start,
881 unsigned long end);
882 extern void viking_flush_tlb_page(struct vm_area_struct *vma,
883 unsigned long page);
884 extern void sun4dsmp_flush_tlb_all(void);
885 extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm);
886 extern void sun4dsmp_flush_tlb_range(struct mm_struct *mm, unsigned long start,
887 unsigned long end);
888 extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma,
889 unsigned long page);
890
891 /* hypersparc.S */
892 extern void hypersparc_flush_cache_all(void);
893 extern void hypersparc_flush_cache_mm(struct mm_struct *mm);
894 extern void hypersparc_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end);
895 extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page);
896 extern void hypersparc_flush_page_to_ram(unsigned long page);
897 extern void hypersparc_flush_page_for_dma(unsigned long page);
898 extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr);
899 extern void hypersparc_flush_tlb_all(void);
900 extern void hypersparc_flush_tlb_mm(struct mm_struct *mm);
901 extern void hypersparc_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end);
902 extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
903 extern void hypersparc_setup_blockops(void);
904
905 /*
906 * NOTE: All of this startup code assumes the low 16mb (approx.) of
907 * kernel mappings are done with one single contiguous chunk of
908 * ram. On small ram machines (classics mainly) we only get
909 * around 8mb mapped for us.
910 */
911
912 void __init early_pgtable_allocfail(char *type)
913 {
914 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
915 prom_halt();
916 }
917
918 void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end)
919 {
920 pgd_t *pgdp;
921 pmd_t *pmdp;
922 pte_t *ptep;
923
924 while(start < end) {
925 pgdp = pgd_offset_k(start);
926 if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
927 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
928 if (pmdp == NULL)
929 early_pgtable_allocfail("pmd");
930 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
931 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
932 }
933 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
934 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
935 ptep = (pte_t *)__srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, SRMMU_PTE_TABLE_SIZE);
936 if (ptep == NULL)
937 early_pgtable_allocfail("pte");
938 memset(__nocache_fix(ptep), 0, SRMMU_PTE_TABLE_SIZE);
939 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
940 }
941 start = (start + SRMMU_PMD_SIZE) & SRMMU_PMD_MASK;
942 }
943 }
944
945 void __init srmmu_allocate_ptable_skeleton(unsigned long start, unsigned long end)
946 {
947 pgd_t *pgdp;
948 pmd_t *pmdp;
949 pte_t *ptep;
950
951 while(start < end) {
952 pgdp = pgd_offset_k(start);
953 if(srmmu_pgd_none(*pgdp)) {
954 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
955 if (pmdp == NULL)
956 early_pgtable_allocfail("pmd");
957 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
958 srmmu_pgd_set(pgdp, pmdp);
959 }
960 pmdp = srmmu_pmd_offset(pgdp, start);
961 if(srmmu_pmd_none(*pmdp)) {
962 ptep = (pte_t *)__srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, SRMMU_PTE_TABLE_SIZE);
963 if (ptep == NULL)
964 early_pgtable_allocfail("pte");
965 memset(ptep, 0, SRMMU_PTE_TABLE_SIZE);
966 srmmu_pmd_set(pmdp, ptep);
967 }
968 start = (start + SRMMU_PMD_SIZE) & SRMMU_PMD_MASK;
969 }
970 }
971
972 /*
973 * This is much cleaner than poking around physical address space
974 * looking at the prom's page table directly which is what most
975 * other OS's do. Yuck... this is much better.
976 */
977 void __init srmmu_inherit_prom_mappings(unsigned long start,unsigned long end)
978 {
979 pgd_t *pgdp;
980 pmd_t *pmdp;
981 pte_t *ptep;
982 int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
983 unsigned long prompte;
984
985 while(start <= end) {
986 if (start == 0)
987 break; /* probably wrap around */
988 if(start == 0xfef00000)
989 start = KADB_DEBUGGER_BEGVM;
990 if(!(prompte = srmmu_hwprobe(start))) {
991 start += PAGE_SIZE;
992 continue;
993 }
994
995 /* A red snapper, see what it really is. */
996 what = 0;
997
998 if(!(start & ~(SRMMU_PMD_MASK))) {
999 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PMD_SIZE) == prompte)
1000 what = 1;
1001 }
1002
1003 if(!(start & ~(SRMMU_PGDIR_MASK))) {
1004 if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) ==
1005 prompte)
1006 what = 2;
1007 }
1008
1009 pgdp = pgd_offset_k(start);
1010 if(what == 2) {
1011 *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte);
1012 start += SRMMU_PGDIR_SIZE;
1013 continue;
1014 }
1015 if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
1016 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
1017 if (pmdp == NULL)
1018 early_pgtable_allocfail("pmd");
1019 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
1020 srmmu_pgd_set(__nocache_fix(pgdp), pmdp);
1021 }
1022 pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start);
1023 if(what == 1) {
1024 *(pmd_t *)__nocache_fix(pmdp) = __pmd(prompte);
1025 start += SRMMU_PMD_SIZE;
1026 continue;
1027 }
1028 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
1029 ptep = (pte_t *)__srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, SRMMU_PTE_TABLE_SIZE);
1030 if (ptep == NULL)
1031 early_pgtable_allocfail("pte");
1032 memset(__nocache_fix(ptep), 0, SRMMU_PTE_TABLE_SIZE);
1033 srmmu_pmd_set(__nocache_fix(pmdp), ptep);
1034 }
1035 ptep = srmmu_pte_offset(__nocache_fix(pmdp), start);
1036 *(pte_t *)__nocache_fix(ptep) = __pte(prompte);
1037 start += PAGE_SIZE;
1038 }
1039 }
1040
1041 #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID)
1042
1043 /* Create a third-level SRMMU 16MB page mapping. */
1044 static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base)
1045 {
1046 pgd_t *pgdp = pgd_offset_k(vaddr);
1047 unsigned long big_pte;
1048
1049 big_pte = KERNEL_PTE(phys_base >> 4);
1050 *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte);
1051 }
1052
1053 /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */
1054 static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
1055 {
1056 unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK);
1057 unsigned long vstart = (vbase & SRMMU_PGDIR_MASK);
1058 unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes);
1059 /* Map "low" memory only */
1060 const unsigned long min_vaddr = PAGE_OFFSET;
1061 const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM;
1062
1063 if (vstart < min_vaddr || vstart >= max_vaddr)
1064 return vstart;
1065
1066 if (vend > max_vaddr || vend < min_vaddr)
1067 vend = max_vaddr;
1068
1069 while(vstart < vend) {
1070 do_large_mapping(vstart, pstart);
1071 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
1072 }
1073 return vstart;
1074 }
1075
1076 static inline void memprobe_error(char *msg)
1077 {
1078 prom_printf(msg);
1079 prom_printf("Halting now...\n");
1080 prom_halt();
1081 }
1082
1083 static inline void map_kernel(void)
1084 {
1085 int i;
1086
1087 if (phys_base > 0) {
1088 do_large_mapping(PAGE_OFFSET, phys_base);
1089 }
1090
1091 for (i = 0; sp_banks[i].num_bytes != 0; i++) {
1092 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i);
1093 }
1094
1095 BTFIXUPSET_SIMM13(user_ptrs_per_pgd, PAGE_OFFSET / SRMMU_PGDIR_SIZE);
1096 }
1097
1098 /* Paging initialization on the Sparc Reference MMU. */
1099 extern void sparc_context_init(int);
1100
1101 extern int linux_num_cpus;
1102 extern unsigned long totalhigh_pages;
1103
1104 void (*poke_srmmu)(void) __initdata = NULL;
1105
1106 extern unsigned long bootmem_init(unsigned long *pages_avail);
1107 extern void sun_serial_setup(void);
1108
1109 void __init srmmu_paging_init(void)
1110 {
1111 int i, cpunode;
1112 char node_str[128];
1113 pgd_t *pgd;
1114 pmd_t *pmd;
1115 pte_t *pte;
1116 unsigned long pages_avail;
1117
1118 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
1119
1120 if (sparc_cpu_model == sun4d)
1121 num_contexts = 65536; /* We know it is Viking */
1122 else {
1123 /* Find the number of contexts on the srmmu. */
1124 cpunode = prom_getchild(prom_root_node);
1125 num_contexts = 0;
1126 while(cpunode != 0) {
1127 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1128 if(!strcmp(node_str, "cpu")) {
1129 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
1130 break;
1131 }
1132 cpunode = prom_getsibling(cpunode);
1133 }
1134 }
1135
1136 if(!num_contexts) {
1137 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
1138 prom_halt();
1139 }
1140
1141 pages_avail = 0;
1142 last_valid_pfn = bootmem_init(&pages_avail);
1143
1144 srmmu_nocache_init();
1145 srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE));
1146 map_kernel();
1147
1148 /* ctx table has to be physically aligned to its size */
1149 srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t));
1150 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
1151
1152 for(i = 0; i < num_contexts; i++)
1153 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
1154
1155 flush_cache_all();
1156 srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys);
1157 flush_tlb_all();
1158 poke_srmmu();
1159
1160 #if CONFIG_SUN_IO
1161 srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END);
1162 srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END);
1163 #endif
1164
1165 srmmu_allocate_ptable_skeleton(FIX_KMAP_BEGIN, FIX_KMAP_END);
1166 srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_BASE_END);
1167
1168 pgd = pgd_offset_k(PKMAP_BASE);
1169 pmd = pmd_offset(pgd, PKMAP_BASE);
1170 pte = pte_offset(pmd, PKMAP_BASE);
1171 pkmap_page_table = pte;
1172
1173 flush_cache_all();
1174 flush_tlb_all();
1175
1176 /*
1177 * This does not logically belong here, but we need to
1178 * call it at the moment we are able to use the bootmem
1179 * allocator.
1180 */
1181 sun_serial_setup();
1182
1183 sparc_context_init(num_contexts);
1184
1185 kmap_init();
1186
1187 {
1188 unsigned long zones_size[MAX_NR_ZONES];
1189 unsigned long zholes_size[MAX_NR_ZONES];
1190 unsigned long npages;
1191 int znum;
1192
1193 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1194 zones_size[znum] = zholes_size[znum] = 0;
1195
1196 npages = max_low_pfn - (phys_base >> PAGE_SHIFT);
1197
1198 zones_size[ZONE_DMA] = npages;
1199 zholes_size[ZONE_DMA] = npages - pages_avail;
1200
1201 npages = highend_pfn - max_low_pfn;
1202 zones_size[ZONE_HIGHMEM] = npages;
1203 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
1204
1205 free_area_init_node(0, NULL, NULL, zones_size,
1206 phys_base, zholes_size);
1207 }
1208 }
1209
1210 static int srmmu_mmu_info(char *buf)
1211 {
1212 return sprintf(buf,
1213 "MMU type\t: %s\n"
1214 "contexts\t: %d\n"
1215 "nocache total\t: %ld\n"
1216 "nocache used\t: %d\n"
1217 , srmmu_name,
1218 num_contexts,
1219 SRMMU_NOCACHE_SIZE,
1220 (srmmu_nocache_used << SRMMU_NOCACHE_BITMAP_SHIFT)
1221 );
1222 }
1223
1224 static void srmmu_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte)
1225 {
1226 }
1227
1228 static void srmmu_destroy_context(struct mm_struct *mm)
1229 {
1230
1231 if(mm->context != NO_CONTEXT) {
1232 flush_cache_mm(mm);
1233 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
1234 flush_tlb_mm(mm);
1235 spin_lock(&srmmu_context_spinlock);
1236 free_context(mm->context);
1237 spin_unlock(&srmmu_context_spinlock);
1238 mm->context = NO_CONTEXT;
1239 }
1240 }
1241
1242 /* Init various srmmu chip types. */
1243 static void __init srmmu_is_bad(void)
1244 {
1245 prom_printf("Could not determine SRMMU chip type.\n");
1246 prom_halt();
1247 }
1248
1249 static void __init init_vac_layout(void)
1250 {
1251 int nd, cache_lines;
1252 char node_str[128];
1253 #ifdef CONFIG_SMP
1254 int cpu = 0;
1255 unsigned long max_size = 0;
1256 unsigned long min_line_size = 0x10000000;
1257 #endif
1258
1259 nd = prom_getchild(prom_root_node);
1260 while((nd = prom_getsibling(nd)) != 0) {
1261 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
1262 if(!strcmp(node_str, "cpu")) {
1263 vac_line_size = prom_getint(nd, "cache-line-size");
1264 if (vac_line_size == -1) {
1265 prom_printf("can't determine cache-line-size, "
1266 "halting.\n");
1267 prom_halt();
1268 }
1269 cache_lines = prom_getint(nd, "cache-nlines");
1270 if (cache_lines == -1) {
1271 prom_printf("can't determine cache-nlines, halting.\n");
1272 prom_halt();
1273 }
1274
1275 vac_cache_size = cache_lines * vac_line_size;
1276 #ifdef CONFIG_SMP
1277 if(vac_cache_size > max_size)
1278 max_size = vac_cache_size;
1279 if(vac_line_size < min_line_size)
1280 min_line_size = vac_line_size;
1281 cpu++;
1282 if(cpu == smp_num_cpus)
1283 break;
1284 #else
1285 break;
1286 #endif
1287 }
1288 }
1289 if(nd == 0) {
1290 prom_printf("No CPU nodes found, halting.\n");
1291 prom_halt();
1292 }
1293 #ifdef CONFIG_SMP
1294 vac_cache_size = max_size;
1295 vac_line_size = min_line_size;
1296 #endif
1297 printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n",
1298 (int)vac_cache_size, (int)vac_line_size);
1299 }
1300
1301 static void __init poke_hypersparc(void)
1302 {
1303 volatile unsigned long clear;
1304 unsigned long mreg = srmmu_get_mmureg();
1305
1306 hyper_flush_unconditional_combined();
1307
1308 mreg &= ~(HYPERSPARC_CWENABLE);
1309 mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE);
1310 mreg |= (HYPERSPARC_CMODE);
1311
1312 srmmu_set_mmureg(mreg);
1313
1314 #if 0 /* XXX I think this is bad news... -DaveM */
1315 hyper_clear_all_tags();
1316 #endif
1317
1318 put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE);
1319 hyper_flush_whole_icache();
1320 clear = srmmu_get_faddr();
1321 clear = srmmu_get_fstatus();
1322 }
1323
1324 static void __init init_hypersparc(void)
1325 {
1326 srmmu_name = "ROSS HyperSparc";
1327
1328 init_vac_layout();
1329
1330 is_hypersparc = 1;
1331
1332 BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM);
1333 BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM);
1334 BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM);
1335 BTFIXUPSET_CALL(flush_cache_all, hypersparc_flush_cache_all, BTFIXUPCALL_NORM);
1336 BTFIXUPSET_CALL(flush_cache_mm, hypersparc_flush_cache_mm, BTFIXUPCALL_NORM);
1337 BTFIXUPSET_CALL(flush_cache_range, hypersparc_flush_cache_range, BTFIXUPCALL_NORM);
1338 BTFIXUPSET_CALL(flush_cache_page, hypersparc_flush_cache_page, BTFIXUPCALL_NORM);
1339
1340 BTFIXUPSET_CALL(flush_tlb_all, hypersparc_flush_tlb_all, BTFIXUPCALL_NORM);
1341 BTFIXUPSET_CALL(flush_tlb_mm, hypersparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1342 BTFIXUPSET_CALL(flush_tlb_range, hypersparc_flush_tlb_range, BTFIXUPCALL_NORM);
1343 BTFIXUPSET_CALL(flush_tlb_page, hypersparc_flush_tlb_page, BTFIXUPCALL_NORM);
1344
1345 BTFIXUPSET_CALL(__flush_page_to_ram, hypersparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1346 BTFIXUPSET_CALL(flush_sig_insns, hypersparc_flush_sig_insns, BTFIXUPCALL_NORM);
1347 BTFIXUPSET_CALL(flush_page_for_dma, hypersparc_flush_page_for_dma, BTFIXUPCALL_NOP);
1348
1349
1350 poke_srmmu = poke_hypersparc;
1351
1352 hypersparc_setup_blockops();
1353 }
1354
1355 static void __init poke_cypress(void)
1356 {
1357 unsigned long mreg = srmmu_get_mmureg();
1358 unsigned long faddr, tagval;
1359 volatile unsigned long cypress_sucks;
1360 volatile unsigned long clear;
1361
1362 clear = srmmu_get_faddr();
1363 clear = srmmu_get_fstatus();
1364
1365 if (!(mreg & CYPRESS_CENABLE)) {
1366 for(faddr = 0x0; faddr < 0x10000; faddr += 20) {
1367 __asm__ __volatile__("sta %%g0, [%0 + %1] %2\n\t"
1368 "sta %%g0, [%0] %2\n\t" : :
1369 "r" (faddr), "r" (0x40000),
1370 "i" (ASI_M_DATAC_TAG));
1371 }
1372 } else {
1373 for(faddr = 0; faddr < 0x10000; faddr += 0x20) {
1374 __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" :
1375 "=r" (tagval) :
1376 "r" (faddr), "r" (0x40000),
1377 "i" (ASI_M_DATAC_TAG));
1378
1379 /* If modified and valid, kick it. */
1380 if((tagval & 0x60) == 0x60)
1381 cypress_sucks = *(unsigned long *)
1382 (0xf0020000 + faddr);
1383 }
1384 }
1385
1386 /* And one more, for our good neighbor, Mr. Broken Cypress. */
1387 clear = srmmu_get_faddr();
1388 clear = srmmu_get_fstatus();
1389
1390 mreg |= (CYPRESS_CENABLE | CYPRESS_CMODE);
1391 srmmu_set_mmureg(mreg);
1392 }
1393
1394 static void __init init_cypress_common(void)
1395 {
1396 init_vac_layout();
1397
1398 BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM);
1399 BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM);
1400 BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM);
1401 BTFIXUPSET_CALL(flush_cache_all, cypress_flush_cache_all, BTFIXUPCALL_NORM);
1402 BTFIXUPSET_CALL(flush_cache_mm, cypress_flush_cache_mm, BTFIXUPCALL_NORM);
1403 BTFIXUPSET_CALL(flush_cache_range, cypress_flush_cache_range, BTFIXUPCALL_NORM);
1404 BTFIXUPSET_CALL(flush_cache_page, cypress_flush_cache_page, BTFIXUPCALL_NORM);
1405
1406 BTFIXUPSET_CALL(flush_tlb_all, cypress_flush_tlb_all, BTFIXUPCALL_NORM);
1407 BTFIXUPSET_CALL(flush_tlb_mm, cypress_flush_tlb_mm, BTFIXUPCALL_NORM);
1408 BTFIXUPSET_CALL(flush_tlb_page, cypress_flush_tlb_page, BTFIXUPCALL_NORM);
1409 BTFIXUPSET_CALL(flush_tlb_range, cypress_flush_tlb_range, BTFIXUPCALL_NORM);
1410
1411
1412 BTFIXUPSET_CALL(__flush_page_to_ram, cypress_flush_page_to_ram, BTFIXUPCALL_NORM);
1413 BTFIXUPSET_CALL(flush_sig_insns, cypress_flush_sig_insns, BTFIXUPCALL_NOP);
1414 BTFIXUPSET_CALL(flush_page_for_dma, cypress_flush_page_for_dma, BTFIXUPCALL_NOP);
1415
1416 poke_srmmu = poke_cypress;
1417 }
1418
1419 static void __init init_cypress_604(void)
1420 {
1421 srmmu_name = "ROSS Cypress-604(UP)";
1422 srmmu_modtype = Cypress;
1423 init_cypress_common();
1424 }
1425
1426 static void __init init_cypress_605(unsigned long mrev)
1427 {
1428 srmmu_name = "ROSS Cypress-605(MP)";
1429 if(mrev == 0xe) {
1430 srmmu_modtype = Cypress_vE;
1431 hwbug_bitmask |= HWBUG_COPYBACK_BROKEN;
1432 } else {
1433 if(mrev == 0xd) {
1434 srmmu_modtype = Cypress_vD;
1435 hwbug_bitmask |= HWBUG_ASIFLUSH_BROKEN;
1436 } else {
1437 srmmu_modtype = Cypress;
1438 }
1439 }
1440 init_cypress_common();
1441 }
1442
1443 static void __init poke_swift(void)
1444 {
1445 unsigned long mreg;
1446
1447 /* Clear any crap from the cache or else... */
1448 swift_flush_cache_all();
1449
1450 /* Enable I & D caches */
1451 mreg = srmmu_get_mmureg();
1452 mreg |= (SWIFT_IE | SWIFT_DE);
1453 /*
1454 * The Swift branch folding logic is completely broken. At
1455 * trap time, if things are just right, if can mistakenly
1456 * think that a trap is coming from kernel mode when in fact
1457 * it is coming from user mode (it mis-executes the branch in
1458 * the trap code). So you see things like crashme completely
1459 * hosing your machine which is completely unacceptable. Turn
1460 * this shit off... nice job Fujitsu.
1461 */
1462 mreg &= ~(SWIFT_BF);
1463 srmmu_set_mmureg(mreg);
1464 }
1465
1466 #define SWIFT_MASKID_ADDR 0x10003018
1467 static void __init init_swift(void)
1468 {
1469 unsigned long swift_rev;
1470
1471 __asm__ __volatile__("lda [%1] %2, %0\n\t"
1472 "srl %0, 0x18, %0\n\t" :
1473 "=r" (swift_rev) :
1474 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1475 srmmu_name = "Fujitsu Swift";
1476 switch(swift_rev) {
1477 case 0x11:
1478 case 0x20:
1479 case 0x23:
1480 case 0x30:
1481 srmmu_modtype = Swift_lots_o_bugs;
1482 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN);
1483 /*
1484 * Gee george, I wonder why Sun is so hush hush about
1485 * this hardware bug... really braindamage stuff going
1486 * on here. However I think we can find a way to avoid
1487 * all of the workaround overhead under Linux. Basically,
1488 * any page fault can cause kernel pages to become user
1489 * accessible (the mmu gets confused and clears some of
1490 * the ACC bits in kernel ptes). Aha, sounds pretty
1491 * horrible eh? But wait, after extensive testing it appears
1492 * that if you use pgd_t level large kernel pte's (like the
1493 * 4MB pages on the Pentium) the bug does not get tripped
1494 * at all. This avoids almost all of the major overhead.
1495 * Welcome to a world where your vendor tells you to,
1496 * "apply this kernel patch" instead of "sorry for the
1497 * broken hardware, send it back and we'll give you
1498 * properly functioning parts"
1499 */
1500 break;
1501 case 0x25:
1502 case 0x31:
1503 srmmu_modtype = Swift_bad_c;
1504 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN;
1505 /*
1506 * You see Sun allude to this hardware bug but never
1507 * admit things directly, they'll say things like,
1508 * "the Swift chip cache problems" or similar.
1509 */
1510 break;
1511 default:
1512 srmmu_modtype = Swift_ok;
1513 break;
1514 };
1515
1516 BTFIXUPSET_CALL(flush_cache_all, swift_flush_cache_all, BTFIXUPCALL_NORM);
1517 BTFIXUPSET_CALL(flush_cache_mm, swift_flush_cache_mm, BTFIXUPCALL_NORM);
1518 BTFIXUPSET_CALL(flush_cache_page, swift_flush_cache_page, BTFIXUPCALL_NORM);
1519 BTFIXUPSET_CALL(flush_cache_range, swift_flush_cache_range, BTFIXUPCALL_NORM);
1520
1521
1522 BTFIXUPSET_CALL(flush_tlb_all, swift_flush_tlb_all, BTFIXUPCALL_NORM);
1523 BTFIXUPSET_CALL(flush_tlb_mm, swift_flush_tlb_mm, BTFIXUPCALL_NORM);
1524 BTFIXUPSET_CALL(flush_tlb_page, swift_flush_tlb_page, BTFIXUPCALL_NORM);
1525 BTFIXUPSET_CALL(flush_tlb_range, swift_flush_tlb_range, BTFIXUPCALL_NORM);
1526
1527 BTFIXUPSET_CALL(__flush_page_to_ram, swift_flush_page_to_ram, BTFIXUPCALL_NORM);
1528 BTFIXUPSET_CALL(flush_sig_insns, swift_flush_sig_insns, BTFIXUPCALL_NORM);
1529 BTFIXUPSET_CALL(flush_page_for_dma, swift_flush_page_for_dma, BTFIXUPCALL_NORM);
1530
1531 BTFIXUPSET_CALL(update_mmu_cache, swift_update_mmu_cache, BTFIXUPCALL_NORM);
1532
1533 flush_page_for_dma_global = 0;
1534
1535 /*
1536 * Are you now convinced that the Swift is one of the
1537 * biggest VLSI abortions of all time? Bravo Fujitsu!
1538 * Fujitsu, the !#?!%$'d up processor people. I bet if
1539 * you examined the microcode of the Swift you'd find
1540 * XXX's all over the place.
1541 */
1542 poke_srmmu = poke_swift;
1543 }
1544
1545 static void turbosparc_flush_cache_all(void)
1546 {
1547 flush_user_windows();
1548 turbosparc_idflash_clear();
1549 }
1550
1551 static void turbosparc_flush_cache_mm(struct mm_struct *mm)
1552 {
1553 FLUSH_BEGIN(mm)
1554 flush_user_windows();
1555 turbosparc_idflash_clear();
1556 FLUSH_END
1557 }
1558
1559 static void turbosparc_flush_cache_range(struct mm_struct *mm, unsigned long start, unsigned long end)
1560 {
1561 FLUSH_BEGIN(mm)
1562 flush_user_windows();
1563 turbosparc_idflash_clear();
1564 FLUSH_END
1565 }
1566
1567 static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
1568 {
1569 FLUSH_BEGIN(vma->vm_mm)
1570 flush_user_windows();
1571 if (vma->vm_flags & VM_EXEC)
1572 turbosparc_flush_icache();
1573 turbosparc_flush_dcache();
1574 FLUSH_END
1575 }
1576
1577 /* TurboSparc is copy-back, if we turn it on, but this does not work. */
1578 static void turbosparc_flush_page_to_ram(unsigned long page)
1579 {
1580 #ifdef TURBOSPARC_WRITEBACK
1581 volatile unsigned long clear;
1582
1583 if (srmmu_hwprobe(page))
1584 turbosparc_flush_page_cache(page);
1585 clear = srmmu_get_fstatus();
1586 #endif
1587 }
1588
1589 static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1590 {
1591 }
1592
1593 static void turbosparc_flush_page_for_dma(unsigned long page)
1594 {
1595 turbosparc_flush_dcache();
1596 }
1597
1598 static void turbosparc_flush_tlb_all(void)
1599 {
1600 srmmu_flush_whole_tlb();
1601 }
1602
1603 static void turbosparc_flush_tlb_mm(struct mm_struct *mm)
1604 {
1605 FLUSH_BEGIN(mm)
1606 srmmu_flush_whole_tlb();
1607 FLUSH_END
1608 }
1609
1610 static void turbosparc_flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long end)
1611 {
1612 FLUSH_BEGIN(mm)
1613 srmmu_flush_whole_tlb();
1614 FLUSH_END
1615 }
1616
1617 static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1618 {
1619 FLUSH_BEGIN(vma->vm_mm)
1620 srmmu_flush_whole_tlb();
1621 FLUSH_END
1622 }
1623
1624
1625 static void __init poke_turbosparc(void)
1626 {
1627 unsigned long mreg = srmmu_get_mmureg();
1628 unsigned long ccreg;
1629
1630 /* Clear any crap from the cache or else... */
1631 turbosparc_flush_cache_all();
1632 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */
1633 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
1634 srmmu_set_mmureg(mreg);
1635
1636 ccreg = turbosparc_get_ccreg();
1637
1638 #ifdef TURBOSPARC_WRITEBACK
1639 ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */
1640 ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE);
1641 /* Write-back D-cache, emulate VLSI
1642 * abortion number three, not number one */
1643 #else
1644 /* For now let's play safe, optimize later */
1645 ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE);
1646 /* Do DVMA snooping in Dcache, Write-thru D-cache */
1647 ccreg &= ~(TURBOSPARC_uS2);
1648 /* Emulate VLSI abortion number three, not number one */
1649 #endif
1650
1651 switch (ccreg & 7) {
1652 case 0: /* No SE cache */
1653 case 7: /* Test mode */
1654 break;
1655 default:
1656 ccreg |= (TURBOSPARC_SCENABLE);
1657 }
1658 turbosparc_set_ccreg (ccreg);
1659
1660 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1661 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
1662 srmmu_set_mmureg(mreg);
1663 }
1664
1665 static void __init init_turbosparc(void)
1666 {
1667 srmmu_name = "Fujitsu TurboSparc";
1668 srmmu_modtype = TurboSparc;
1669
1670 BTFIXUPSET_CALL(flush_cache_all, turbosparc_flush_cache_all, BTFIXUPCALL_NORM);
1671 BTFIXUPSET_CALL(flush_cache_mm, turbosparc_flush_cache_mm, BTFIXUPCALL_NORM);
1672 BTFIXUPSET_CALL(flush_cache_page, turbosparc_flush_cache_page, BTFIXUPCALL_NORM);
1673 BTFIXUPSET_CALL(flush_cache_range, turbosparc_flush_cache_range, BTFIXUPCALL_NORM);
1674
1675 BTFIXUPSET_CALL(flush_tlb_all, turbosparc_flush_tlb_all, BTFIXUPCALL_NORM);
1676 BTFIXUPSET_CALL(flush_tlb_mm, turbosparc_flush_tlb_mm, BTFIXUPCALL_NORM);
1677 BTFIXUPSET_CALL(flush_tlb_page, turbosparc_flush_tlb_page, BTFIXUPCALL_NORM);
1678 BTFIXUPSET_CALL(flush_tlb_range, turbosparc_flush_tlb_range, BTFIXUPCALL_NORM);
1679
1680 BTFIXUPSET_CALL(__flush_page_to_ram, turbosparc_flush_page_to_ram, BTFIXUPCALL_NORM);
1681
1682 BTFIXUPSET_CALL(flush_sig_insns, turbosparc_flush_sig_insns, BTFIXUPCALL_NOP);
1683 BTFIXUPSET_CALL(flush_page_for_dma, turbosparc_flush_page_for_dma, BTFIXUPCALL_NORM);
1684
1685 poke_srmmu = poke_turbosparc;
1686 }
1687
1688 static void __init poke_tsunami(void)
1689 {
1690 unsigned long mreg = srmmu_get_mmureg();
1691
1692 tsunami_flush_icache();
1693 tsunami_flush_dcache();
1694 mreg &= ~TSUNAMI_ITD;
1695 mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB);
1696 srmmu_set_mmureg(mreg);
1697 }
1698
1699 static void __init init_tsunami(void)
1700 {
1701 /*
1702 * Tsunami's pretty sane, Sun and TI actually got it
1703 * somewhat right this time. Fujitsu should have
1704 * taken some lessons from them.
1705 */
1706
1707 srmmu_name = "TI Tsunami";
1708 srmmu_modtype = Tsunami;
1709
1710 BTFIXUPSET_CALL(flush_cache_all, tsunami_flush_cache_all, BTFIXUPCALL_NORM);
1711 BTFIXUPSET_CALL(flush_cache_mm, tsunami_flush_cache_mm, BTFIXUPCALL_NORM);
1712 BTFIXUPSET_CALL(flush_cache_page, tsunami_flush_cache_page, BTFIXUPCALL_NORM);
1713 BTFIXUPSET_CALL(flush_cache_range, tsunami_flush_cache_range, BTFIXUPCALL_NORM);
1714
1715
1716 BTFIXUPSET_CALL(flush_tlb_all, tsunami_flush_tlb_all, BTFIXUPCALL_NORM);
1717 BTFIXUPSET_CALL(flush_tlb_mm, tsunami_flush_tlb_mm, BTFIXUPCALL_NORM);
1718 BTFIXUPSET_CALL(flush_tlb_page, tsunami_flush_tlb_page, BTFIXUPCALL_NORM);
1719 BTFIXUPSET_CALL(flush_tlb_range, tsunami_flush_tlb_range, BTFIXUPCALL_NORM);
1720
1721 BTFIXUPSET_CALL(__flush_page_to_ram, tsunami_flush_page_to_ram, BTFIXUPCALL_NOP);
1722 BTFIXUPSET_CALL(flush_sig_insns, tsunami_flush_sig_insns, BTFIXUPCALL_NORM);
1723 BTFIXUPSET_CALL(flush_page_for_dma, tsunami_flush_page_for_dma, BTFIXUPCALL_NORM);
1724
1725 poke_srmmu = poke_tsunami;
1726
1727 tsunami_setup_blockops();
1728 }
1729
1730 static void __init poke_viking(void)
1731 {
1732 unsigned long mreg = srmmu_get_mmureg();
1733 static int smp_catch = 0;
1734
1735 if(viking_mxcc_present) {
1736 unsigned long mxcc_control = mxcc_get_creg();
1737
1738 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE);
1739 mxcc_control &= ~(MXCC_CTL_RRC);
1740 mxcc_set_creg(mxcc_control);
1741
1742 /*
1743 * We don't need memory parity checks.
1744 * XXX This is a mess, have to dig out later. ecd.
1745 viking_mxcc_turn_off_parity(&mreg, &mxcc_control);
1746 */
1747
1748 /* We do cache ptables on MXCC. */
1749 mreg |= VIKING_TCENABLE;
1750 } else {
1751 unsigned long bpreg;
1752
1753 mreg &= ~(VIKING_TCENABLE);
1754 if(smp_catch++) {
1755 /* Must disable mixed-cmd mode here for other cpu's. */
1756 bpreg = viking_get_bpreg();
1757 bpreg &= ~(VIKING_ACTION_MIX);
1758 viking_set_bpreg(bpreg);
1759
1760 /* Just in case PROM does something funny. */
1761 msi_set_sync();
1762 }
1763 }
1764
1765 mreg |= VIKING_SPENABLE;
1766 mreg |= (VIKING_ICENABLE | VIKING_DCENABLE);
1767 mreg |= VIKING_SBENABLE;
1768 mreg &= ~(VIKING_ACENABLE);
1769 srmmu_set_mmureg(mreg);
1770
1771 #ifdef CONFIG_SMP
1772 /* Avoid unnecessary cross calls. */
1773 BTFIXUPCOPY_CALL(flush_cache_all, local_flush_cache_all);
1774 BTFIXUPCOPY_CALL(flush_cache_mm, local_flush_cache_mm);
1775 BTFIXUPCOPY_CALL(flush_cache_range, local_flush_cache_range);
1776 BTFIXUPCOPY_CALL(flush_cache_page, local_flush_cache_page);
1777 BTFIXUPCOPY_CALL(__flush_page_to_ram, local_flush_page_to_ram);
1778 BTFIXUPCOPY_CALL(flush_sig_insns, local_flush_sig_insns);
1779 BTFIXUPCOPY_CALL(flush_page_for_dma, local_flush_page_for_dma);
1780 btfixup();
1781 #endif
1782 }
1783
1784 static void __init init_viking(void)
1785 {
1786 unsigned long mreg = srmmu_get_mmureg();
1787
1788 /* Ahhh, the viking. SRMMU VLSI abortion number two... */
1789 if(mreg & VIKING_MMODE) {
1790 srmmu_name = "TI Viking";
1791 viking_mxcc_present = 0;
1792 msi_set_sync();
1793
1794 BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM);
1795 BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM);
1796 BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM);
1797
1798 /*
1799 * We need this to make sure old viking takes no hits
1800 * on it's cache for dma snoops to workaround the
1801 * "load from non-cacheable memory" interrupt bug.
1802 * This is only necessary because of the new way in
1803 * which we use the IOMMU.
1804 */
1805 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page, BTFIXUPCALL_NORM);
1806
1807 flush_page_for_dma_global = 0;
1808 } else {
1809 srmmu_name = "TI Viking/MXCC";
1810 viking_mxcc_present = 1;
1811
1812 srmmu_cache_pagetables = 1;
1813
1814 /* MXCC vikings lack the DMA snooping bug. */
1815 BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page_for_dma, BTFIXUPCALL_NOP);
1816 }
1817
1818 BTFIXUPSET_CALL(flush_cache_all, viking_flush_cache_all, BTFIXUPCALL_NORM);
1819 BTFIXUPSET_CALL(flush_cache_mm, viking_flush_cache_mm, BTFIXUPCALL_NORM);
1820 BTFIXUPSET_CALL(flush_cache_page, viking_flush_cache_page, BTFIXUPCALL_NORM);
1821 BTFIXUPSET_CALL(flush_cache_range, viking_flush_cache_range, BTFIXUPCALL_NORM);
1822
1823 #ifdef CONFIG_SMP
1824 if (sparc_cpu_model == sun4d) {
1825 BTFIXUPSET_CALL(flush_tlb_all, sun4dsmp_flush_tlb_all, BTFIXUPCALL_NORM);
1826 BTFIXUPSET_CALL(flush_tlb_mm, sun4dsmp_flush_tlb_mm, BTFIXUPCALL_NORM);
1827 BTFIXUPSET_CALL(flush_tlb_page, sun4dsmp_flush_tlb_page, BTFIXUPCALL_NORM);
1828 BTFIXUPSET_CALL(flush_tlb_range, sun4dsmp_flush_tlb_range, BTFIXUPCALL_NORM);
1829 } else
1830 #endif
1831 {
1832 BTFIXUPSET_CALL(flush_tlb_all, viking_flush_tlb_all, BTFIXUPCALL_NORM);
1833 BTFIXUPSET_CALL(flush_tlb_mm, viking_flush_tlb_mm, BTFIXUPCALL_NORM);
1834 BTFIXUPSET_CALL(flush_tlb_page, viking_flush_tlb_page, BTFIXUPCALL_NORM);
1835 BTFIXUPSET_CALL(flush_tlb_range, viking_flush_tlb_range, BTFIXUPCALL_NORM);
1836 }
1837
1838 BTFIXUPSET_CALL(__flush_page_to_ram, viking_flush_page_to_ram, BTFIXUPCALL_NOP);
1839 BTFIXUPSET_CALL(flush_sig_insns, viking_flush_sig_insns, BTFIXUPCALL_NOP);
1840
1841 poke_srmmu = poke_viking;
1842 }
1843
1844 /* Probe for the srmmu chip version. */
1845 static void __init get_srmmu_type(void)
1846 {
1847 unsigned long mreg, psr;
1848 unsigned long mod_typ, mod_rev, psr_typ, psr_vers;
1849
1850 srmmu_modtype = SRMMU_INVAL_MOD;
1851 hwbug_bitmask = 0;
1852
1853 mreg = srmmu_get_mmureg(); psr = get_psr();
1854 mod_typ = (mreg & 0xf0000000) >> 28;
1855 mod_rev = (mreg & 0x0f000000) >> 24;
1856 psr_typ = (psr >> 28) & 0xf;
1857 psr_vers = (psr >> 24) & 0xf;
1858
1859 /* First, check for HyperSparc or Cypress. */
1860 if(mod_typ == 1) {
1861 switch(mod_rev) {
1862 case 7:
1863 /* UP or MP Hypersparc */
1864 init_hypersparc();
1865 break;
1866 case 0:
1867 case 2:
1868 /* Uniprocessor Cypress */
1869 init_cypress_604();
1870 break;
1871 case 10:
1872 case 11:
1873 case 12:
1874 /* _REALLY OLD_ Cypress MP chips... */
1875 case 13:
1876 case 14:
1877 case 15:
1878 /* MP Cypress mmu/cache-controller */
1879 init_cypress_605(mod_rev);
1880 break;
1881 default:
1882 /* Some other Cypress revision, assume a 605. */
1883 init_cypress_605(mod_rev);
1884 break;
1885 };
1886 return;
1887 }
1888
1889 /*
1890 * Now Fujitsu TurboSparc. It might happen that it is
1891 * in Swift emulation mode, so we will check later...
1892 */
1893 if (psr_typ == 0 && psr_vers == 5) {
1894 init_turbosparc();
1895 return;
1896 }
1897
1898 /* Next check for Fujitsu Swift. */
1899 if(psr_typ == 0 && psr_vers == 4) {
1900 int cpunode;
1901 char node_str[128];
1902
1903 /* Look if it is not a TurboSparc emulating Swift... */
1904 cpunode = prom_getchild(prom_root_node);
1905 while((cpunode = prom_getsibling(cpunode)) != 0) {
1906 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1907 if(!strcmp(node_str, "cpu")) {
1908 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1909 prom_getintdefault(cpunode, "psr-version", 1) == 5) {
1910 init_turbosparc();
1911 return;
1912 }
1913 break;
1914 }
1915 }
1916
1917 init_swift();
1918 return;
1919 }
1920
1921 /* Now the Viking family of srmmu. */
1922 if(psr_typ == 4 &&
1923 ((psr_vers == 0) ||
1924 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
1925 init_viking();
1926 return;
1927 }
1928
1929 /* Finally the Tsunami. */
1930 if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
1931 init_tsunami();
1932 return;
1933 }
1934
1935 /* Oh well */
1936 srmmu_is_bad();
1937 }
1938
1939 /* dont laugh, static pagetables */
1940 static int srmmu_check_pgt_cache(int low, int high)
1941 {
1942 return 0;
1943 }
1944
1945 extern unsigned long spwin_mmu_patchme, fwin_mmu_patchme,
1946 tsetup_mmu_patchme, rtrap_mmu_patchme;
1947
1948 extern unsigned long spwin_srmmu_stackchk, srmmu_fwin_stackchk,
1949 tsetup_srmmu_stackchk, srmmu_rett_stackchk;
1950
1951 extern unsigned long srmmu_fault;
1952
1953 #define PATCH_BRANCH(insn, dest) do { \
1954 iaddr = &(insn); \
1955 daddr = &(dest); \
1956 *iaddr = SPARC_BRANCH((unsigned long) daddr, (unsigned long) iaddr); \
1957 } while(0);
1958
1959 static void __init patch_window_trap_handlers(void)
1960 {
1961 unsigned long *iaddr, *daddr;
1962
1963 PATCH_BRANCH(spwin_mmu_patchme, spwin_srmmu_stackchk);
1964 PATCH_BRANCH(fwin_mmu_patchme, srmmu_fwin_stackchk);
1965 PATCH_BRANCH(tsetup_mmu_patchme, tsetup_srmmu_stackchk);
1966 PATCH_BRANCH(rtrap_mmu_patchme, srmmu_rett_stackchk);
1967 PATCH_BRANCH(sparc_ttable[SP_TRAP_TFLT].inst_three, srmmu_fault);
1968 PATCH_BRANCH(sparc_ttable[SP_TRAP_DFLT].inst_three, srmmu_fault);
1969 PATCH_BRANCH(sparc_ttable[SP_TRAP_DACC].inst_three, srmmu_fault);
1970 }
1971
1972 #ifdef CONFIG_SMP
1973 /* Local cross-calls. */
1974 static void smp_flush_page_for_dma(unsigned long page)
1975 {
1976 xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_for_dma), page);
1977 local_flush_page_for_dma(page);
1978 }
1979
1980 #endif
1981
1982 /* Load up routines and constants for sun4m and sun4d mmu */
1983 void __init ld_mmu_srmmu(void)
1984 {
1985 extern void ld_mmu_iommu(void);
1986 extern void ld_mmu_iounit(void);
1987 extern void ___xchg32_sun4md(void);
1988
1989 /* First the constants */
1990 BTFIXUPSET_SIMM13(pmd_shift, SRMMU_PMD_SHIFT);
1991 BTFIXUPSET_SETHI(pmd_size, SRMMU_PMD_SIZE);
1992 BTFIXUPSET_SETHI(pmd_mask, SRMMU_PMD_MASK);
1993 BTFIXUPSET_SIMM13(pgdir_shift, SRMMU_PGDIR_SHIFT);
1994 BTFIXUPSET_SETHI(pgdir_size, SRMMU_PGDIR_SIZE);
1995 BTFIXUPSET_SETHI(pgdir_mask, SRMMU_PGDIR_MASK);
1996
1997 BTFIXUPSET_SIMM13(ptrs_per_pte, SRMMU_PTRS_PER_PTE);
1998 BTFIXUPSET_SIMM13(ptrs_per_pmd, SRMMU_PTRS_PER_PMD);
1999 BTFIXUPSET_SIMM13(ptrs_per_pgd, SRMMU_PTRS_PER_PGD);
2000
2001 BTFIXUPSET_INT(page_none, pgprot_val(SRMMU_PAGE_NONE));
2002 BTFIXUPSET_INT(page_shared, pgprot_val(SRMMU_PAGE_SHARED));
2003 BTFIXUPSET_INT(page_copy, pgprot_val(SRMMU_PAGE_COPY));
2004 BTFIXUPSET_INT(page_readonly, pgprot_val(SRMMU_PAGE_RDONLY));
2005 BTFIXUPSET_INT(page_kernel, pgprot_val(SRMMU_PAGE_KERNEL));
2006 page_kernel = pgprot_val(SRMMU_PAGE_KERNEL);
2007 pg_iobits = SRMMU_VALID | SRMMU_WRITE | SRMMU_REF;
2008
2009 /* Functions */
2010 #ifndef CONFIG_SMP
2011 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4md, BTFIXUPCALL_SWAPG1G2);
2012 #endif
2013 BTFIXUPSET_CALL(do_check_pgt_cache, srmmu_check_pgt_cache, BTFIXUPCALL_NORM);
2014
2015 BTFIXUPSET_CALL(set_pte, srmmu_set_pte, BTFIXUPCALL_SWAPO0O1);
2016 BTFIXUPSET_CALL(switch_mm, srmmu_switch_mm, BTFIXUPCALL_NORM);
2017
2018 BTFIXUPSET_CALL(pte_page, srmmu_pte_page, BTFIXUPCALL_NORM);
2019 BTFIXUPSET_CALL(pmd_page, srmmu_pmd_page, BTFIXUPCALL_NORM);
2020 BTFIXUPSET_CALL(pgd_page, srmmu_pgd_page, BTFIXUPCALL_NORM);
2021
2022 BTFIXUPSET_SETHI(none_mask, 0xF0000000); /* XXX P3: is it used? */
2023
2024 BTFIXUPSET_CALL(pte_present, srmmu_pte_present, BTFIXUPCALL_NORM);
2025 BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_SWAPO0G0);
2026
2027 BTFIXUPSET_CALL(pmd_bad, srmmu_pmd_bad, BTFIXUPCALL_NORM);
2028 BTFIXUPSET_CALL(pmd_present, srmmu_pmd_present, BTFIXUPCALL_NORM);
2029 BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_SWAPO0G0);
2030
2031 BTFIXUPSET_CALL(pgd_none, srmmu_pgd_none, BTFIXUPCALL_NORM);
2032 BTFIXUPSET_CALL(pgd_bad, srmmu_pgd_bad, BTFIXUPCALL_NORM);
2033 BTFIXUPSET_CALL(pgd_present, srmmu_pgd_present, BTFIXUPCALL_NORM);
2034 BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_SWAPO0G0);
2035
2036 BTFIXUPSET_CALL(mk_pte, srmmu_mk_pte, BTFIXUPCALL_NORM);
2037 BTFIXUPSET_CALL(mk_pte_phys, srmmu_mk_pte_phys, BTFIXUPCALL_NORM);
2038 BTFIXUPSET_CALL(mk_pte_io, srmmu_mk_pte_io, BTFIXUPCALL_NORM);
2039 BTFIXUPSET_CALL(pgd_set, srmmu_pgd_set, BTFIXUPCALL_NORM);
2040 BTFIXUPSET_CALL(pmd_set, srmmu_pmd_set, BTFIXUPCALL_NORM);
2041
2042 BTFIXUPSET_INT(pte_modify_mask, SRMMU_CHG_MASK);
2043 BTFIXUPSET_CALL(pmd_offset, srmmu_pmd_offset, BTFIXUPCALL_NORM);
2044 BTFIXUPSET_CALL(pte_offset, srmmu_pte_offset, BTFIXUPCALL_NORM);
2045 BTFIXUPSET_CALL(free_pte_fast, srmmu_free_pte_fast, BTFIXUPCALL_NORM);
2046 BTFIXUPSET_CALL(pte_alloc_one_fast, srmmu_pte_alloc_one_fast, BTFIXUPCALL_NORM);
2047 BTFIXUPSET_CALL(pte_alloc_one, srmmu_pte_alloc_one, BTFIXUPCALL_NORM);
2048 BTFIXUPSET_CALL(free_pmd_fast, srmmu_free_pmd_fast, BTFIXUPCALL_NORM);
2049 BTFIXUPSET_CALL(pmd_alloc_one_fast, srmmu_pmd_alloc_one_fast, BTFIXUPCALL_NORM);
2050 BTFIXUPSET_CALL(free_pgd_fast, srmmu_free_pgd_fast, BTFIXUPCALL_NORM);
2051 BTFIXUPSET_CALL(get_pgd_fast, srmmu_get_pgd_fast, BTFIXUPCALL_NORM);
2052
2053 BTFIXUPSET_HALF(pte_writei, SRMMU_WRITE);
2054 BTFIXUPSET_HALF(pte_dirtyi, SRMMU_DIRTY);
2055 BTFIXUPSET_HALF(pte_youngi, SRMMU_REF);
2056 BTFIXUPSET_HALF(pte_wrprotecti, SRMMU_WRITE);
2057 BTFIXUPSET_HALF(pte_mkcleani, SRMMU_DIRTY);
2058 BTFIXUPSET_HALF(pte_mkoldi, SRMMU_REF);
2059 BTFIXUPSET_CALL(pte_mkwrite, srmmu_pte_mkwrite, BTFIXUPCALL_ORINT(SRMMU_WRITE));
2060 BTFIXUPSET_CALL(pte_mkdirty, srmmu_pte_mkdirty, BTFIXUPCALL_ORINT(SRMMU_DIRTY));
2061 BTFIXUPSET_CALL(pte_mkyoung, srmmu_pte_mkyoung, BTFIXUPCALL_ORINT(SRMMU_REF));
2062 BTFIXUPSET_CALL(update_mmu_cache, srmmu_update_mmu_cache, BTFIXUPCALL_NOP);
2063 BTFIXUPSET_CALL(destroy_context, srmmu_destroy_context, BTFIXUPCALL_NORM);
2064
2065 BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM);
2066
2067 /* Task struct and kernel stack allocating/freeing. */
2068 BTFIXUPSET_CALL(alloc_task_struct, srmmu_alloc_task_struct, BTFIXUPCALL_NORM);
2069 BTFIXUPSET_CALL(free_task_struct, srmmu_free_task_struct, BTFIXUPCALL_NORM);
2070 BTFIXUPSET_CALL(get_task_struct, srmmu_get_task_struct, BTFIXUPCALL_NORM);
2071
2072 get_srmmu_type();
2073 patch_window_trap_handlers();
2074
2075 #ifdef CONFIG_SMP
2076 /* El switcheroo... */
2077
2078 BTFIXUPCOPY_CALL(local_flush_cache_all, flush_cache_all);
2079 BTFIXUPCOPY_CALL(local_flush_cache_mm, flush_cache_mm);
2080 BTFIXUPCOPY_CALL(local_flush_cache_range, flush_cache_range);
2081 BTFIXUPCOPY_CALL(local_flush_cache_page, flush_cache_page);
2082 BTFIXUPCOPY_CALL(local_flush_tlb_all, flush_tlb_all);
2083 BTFIXUPCOPY_CALL(local_flush_tlb_mm, flush_tlb_mm);
2084 BTFIXUPCOPY_CALL(local_flush_tlb_range, flush_tlb_range);
2085 BTFIXUPCOPY_CALL(local_flush_tlb_page, flush_tlb_page);
2086 BTFIXUPCOPY_CALL(local_flush_page_to_ram, __flush_page_to_ram);
2087 BTFIXUPCOPY_CALL(local_flush_sig_insns, flush_sig_insns);
2088 BTFIXUPCOPY_CALL(local_flush_page_for_dma, flush_page_for_dma);
2089
2090 BTFIXUPSET_CALL(flush_cache_all, smp_flush_cache_all, BTFIXUPCALL_NORM);
2091 BTFIXUPSET_CALL(flush_cache_mm, smp_flush_cache_mm, BTFIXUPCALL_NORM);
2092 BTFIXUPSET_CALL(flush_cache_range, smp_flush_cache_range, BTFIXUPCALL_NORM);
2093 BTFIXUPSET_CALL(flush_cache_page, smp_flush_cache_page, BTFIXUPCALL_NORM);
2094 if (sparc_cpu_model != sun4d) {
2095 BTFIXUPSET_CALL(flush_tlb_all, smp_flush_tlb_all, BTFIXUPCALL_NORM);
2096 BTFIXUPSET_CALL(flush_tlb_mm, smp_flush_tlb_mm, BTFIXUPCALL_NORM);
2097 BTFIXUPSET_CALL(flush_tlb_range, smp_flush_tlb_range, BTFIXUPCALL_NORM);
2098 BTFIXUPSET_CALL(flush_tlb_page, smp_flush_tlb_page, BTFIXUPCALL_NORM);
2099 }
2100 BTFIXUPSET_CALL(__flush_page_to_ram, smp_flush_page_to_ram, BTFIXUPCALL_NORM);
2101 BTFIXUPSET_CALL(flush_sig_insns, smp_flush_sig_insns, BTFIXUPCALL_NORM);
2102 BTFIXUPSET_CALL(flush_page_for_dma, smp_flush_page_for_dma, BTFIXUPCALL_NORM);
2103 #endif
2104
2105 if (sparc_cpu_model == sun4d)
2106 ld_mmu_iounit();
2107 else
2108 ld_mmu_iommu();
2109 #ifdef CONFIG_SMP
2110 if (sparc_cpu_model == sun4d)
2111 sun4d_init_smp();
2112 else
2113 sun4m_init_smp();
2114 #endif
2115 }
2116