File: /usr/src/linux/arch/sparc64/mm/init.c
1 /* $Id: init.c,v 1.189 2001/09/02 23:27:18 kanoj Exp $
2 * arch/sparc64/mm/init.c
3 *
4 * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
6 */
7
8 #include <linux/config.h>
9 #include <linux/kernel.h>
10 #include <linux/sched.h>
11 #include <linux/string.h>
12 #include <linux/init.h>
13 #include <linux/bootmem.h>
14 #include <linux/mm.h>
15 #include <linux/slab.h>
16 #include <linux/blk.h>
17 #include <linux/swap.h>
18 #include <linux/swapctl.h>
19
20 #include <asm/head.h>
21 #include <asm/system.h>
22 #include <asm/page.h>
23 #include <asm/pgalloc.h>
24 #include <asm/pgtable.h>
25 #include <asm/oplib.h>
26 #include <asm/iommu.h>
27 #include <asm/io.h>
28 #include <asm/uaccess.h>
29 #include <asm/mmu_context.h>
30 #include <asm/dma.h>
31 #include <asm/starfire.h>
32 #include <asm/tlb.h>
33
34 mmu_gather_t mmu_gathers[NR_CPUS];
35
36 extern void device_scan(void);
37
38 struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS];
39
40 unsigned long *sparc64_valid_addr_bitmap;
41
42 /* Ugly, but necessary... -DaveM */
43 unsigned long phys_base;
44
45 enum ultra_tlb_layout tlb_type = spitfire;
46
47 /* get_new_mmu_context() uses "cache + 1". */
48 spinlock_t ctx_alloc_lock = SPIN_LOCK_UNLOCKED;
49 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
50 #define CTX_BMAP_SLOTS (1UL << (CTX_VERSION_SHIFT - 6))
51 unsigned long mmu_context_bmap[CTX_BMAP_SLOTS];
52
53 /* References to section boundaries */
54 extern char __init_begin, __init_end, _start, _end, etext, edata;
55
56 /* Initial ramdisk setup */
57 extern unsigned int sparc_ramdisk_image;
58 extern unsigned int sparc_ramdisk_size;
59
60 struct page *mem_map_zero;
61
62 int do_check_pgt_cache(int low, int high)
63 {
64 int freed = 0;
65
66 if (pgtable_cache_size > high) {
67 do {
68 #ifdef CONFIG_SMP
69 if (pgd_quicklist)
70 free_pgd_slow(get_pgd_fast()), freed++;
71 #endif
72 if (pte_quicklist[0])
73 free_pte_slow(pte_alloc_one_fast(NULL, 0)), freed++;
74 if (pte_quicklist[1])
75 free_pte_slow(pte_alloc_one_fast(NULL, 1 << (PAGE_SHIFT + 10))), freed++;
76 } while (pgtable_cache_size > low);
77 }
78 #ifndef CONFIG_SMP
79 if (pgd_cache_size > high / 4) {
80 struct page *page, *page2;
81 for (page2 = NULL, page = (struct page *)pgd_quicklist; page;) {
82 if ((unsigned long)page->pprev_hash == 3) {
83 if (page2)
84 page2->next_hash = page->next_hash;
85 else
86 (struct page *)pgd_quicklist = page->next_hash;
87 page->next_hash = NULL;
88 page->pprev_hash = NULL;
89 pgd_cache_size -= 2;
90 __free_page(page);
91 freed++;
92 if (page2)
93 page = page2->next_hash;
94 else
95 page = (struct page *)pgd_quicklist;
96 if (pgd_cache_size <= low / 4)
97 break;
98 continue;
99 }
100 page2 = page;
101 page = page->next_hash;
102 }
103 }
104 #endif
105 return freed;
106 }
107
108 extern void __update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
109
110 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
111 {
112 struct page *page = pte_page(pte);
113
114 if (VALID_PAGE(page) && page->mapping &&
115 test_bit(PG_dcache_dirty, &page->flags)) {
116 __flush_dcache_page(page->virtual,
117 (tlb_type == spitfire));
118 clear_bit(PG_dcache_dirty, &page->flags);
119 }
120 __update_mmu_cache(vma, address, pte);
121 }
122
123 /* In arch/sparc64/mm/ultra.S */
124 extern void __flush_icache_page(unsigned long);
125
126 void flush_icache_range(unsigned long start, unsigned long end)
127 {
128 /* Cheetah has coherent I-cache. */
129 if (tlb_type == spitfire) {
130 unsigned long kaddr;
131
132 for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE)
133 __flush_icache_page(__get_phys(kaddr));
134 }
135 }
136
137 void show_mem(void)
138 {
139 printk("Mem-info:\n");
140 show_free_areas();
141 printk("Free swap: %6dkB\n",
142 nr_swap_pages << (PAGE_SHIFT-10));
143 printk("%ld pages of RAM\n", num_physpages);
144 printk("%d free pages\n", nr_free_pages());
145 printk("%d pages in page table cache\n",pgtable_cache_size);
146 #ifndef CONFIG_SMP
147 printk("%d entries in page dir cache\n",pgd_cache_size);
148 #endif
149 show_buffers();
150 }
151
152 int mmu_info(char *buf)
153 {
154 if (tlb_type == cheetah)
155 return sprintf(buf, "MMU Type\t: Cheetah\n");
156 else if (tlb_type == spitfire)
157 return sprintf(buf, "MMU Type\t: Spitfire\n");
158 else
159 return sprintf(buf, "MMU Type\t: ???\n");
160 }
161
162 struct linux_prom_translation {
163 unsigned long virt;
164 unsigned long size;
165 unsigned long data;
166 };
167
168 extern unsigned long prom_boot_page;
169 extern void prom_remap(unsigned long physpage, unsigned long virtpage, int mmu_ihandle);
170 extern int prom_get_mmu_ihandle(void);
171 extern void register_prom_callbacks(void);
172
173 /* Exported for SMP bootup purposes. */
174 unsigned long kern_locked_tte_data;
175
176 void __init early_pgtable_allocfail(char *type)
177 {
178 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type);
179 prom_halt();
180 }
181
182 static void inherit_prom_mappings(void)
183 {
184 struct linux_prom_translation *trans;
185 unsigned long phys_page, tte_vaddr, tte_data;
186 void (*remap_func)(unsigned long, unsigned long, int);
187 pmd_t *pmdp, *pmd;
188 pte_t *ptep;
189 int node, n, i, tsz;
190 extern unsigned int obp_iaddr_patch[2], obp_daddr_patch[2];
191
192 node = prom_finddevice("/virtual-memory");
193 n = prom_getproplen(node, "translations");
194 if (n == 0 || n == -1) {
195 prom_printf("Couldn't get translation property\n");
196 prom_halt();
197 }
198 n += 5 * sizeof(struct linux_prom_translation);
199 for (tsz = 1; tsz < n; tsz <<= 1)
200 /* empty */;
201 trans = __alloc_bootmem(tsz, SMP_CACHE_BYTES, 0UL);
202 if (trans == NULL) {
203 prom_printf("inherit_prom_mappings: Cannot alloc translations.\n");
204 prom_halt();
205 }
206 memset(trans, 0, tsz);
207
208 if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
209 prom_printf("Couldn't get translation property\n");
210 prom_halt();
211 }
212 n = n / sizeof(*trans);
213
214 /*
215 * The obp translations are saved based on 8k pagesize, since obp can use
216 * a mixture of pagesizes. Misses to the 0xf0000000 - 0x100000000, ie obp
217 * range, are handled in entry.S and do not use the vpte scheme (see rant
218 * in inherit_locked_prom_mappings()).
219 */
220 #define OBP_PMD_SIZE 2048
221 #define BASE_PAGE_SIZE 8192
222 pmd = __alloc_bootmem(OBP_PMD_SIZE, OBP_PMD_SIZE, 0UL);
223 if (pmd == NULL)
224 early_pgtable_allocfail("pmd");
225 memset(pmd, 0, OBP_PMD_SIZE);
226 for (i = 0; i < n; i++) {
227 unsigned long vaddr;
228
229 if (trans[i].virt >= 0xf0000000 && trans[i].virt < 0x100000000) {
230 for (vaddr = trans[i].virt;
231 vaddr < trans[i].virt + trans[i].size;
232 vaddr += BASE_PAGE_SIZE) {
233 unsigned long val;
234
235 pmdp = pmd + ((vaddr >> 23) & 0x7ff);
236 if (pmd_none(*pmdp)) {
237 ptep = __alloc_bootmem(BASE_PAGE_SIZE,
238 BASE_PAGE_SIZE,
239 0UL);
240 if (ptep == NULL)
241 early_pgtable_allocfail("pte");
242 memset(ptep, 0, BASE_PAGE_SIZE);
243 pmd_set(pmdp, ptep);
244 }
245 ptep = (pte_t *)pmd_page(*pmdp) +
246 ((vaddr >> 13) & 0x3ff);
247
248 val = trans[i].data;
249
250 /* Clear diag TTE bits. */
251 if (tlb_type == spitfire)
252 val &= ~0x0003fe0000000000UL;
253
254 set_pte (ptep, __pte(val | _PAGE_MODIFIED));
255 trans[i].data += BASE_PAGE_SIZE;
256 }
257 }
258 }
259 phys_page = __pa(pmd);
260 obp_iaddr_patch[0] |= (phys_page >> 10);
261 obp_iaddr_patch[1] |= (phys_page & 0x3ff);
262 flushi((long)&obp_iaddr_patch[0]);
263 obp_daddr_patch[0] |= (phys_page >> 10);
264 obp_daddr_patch[1] |= (phys_page & 0x3ff);
265 flushi((long)&obp_daddr_patch[0]);
266
267 /* Now fixup OBP's idea about where we really are mapped. */
268 prom_printf("Remapping the kernel... ");
269
270 /* Spitfire Errata #32 workaround */
271 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
272 "flush %%g6"
273 : /* No outputs */
274 : "r" (0),
275 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
276
277 switch (tlb_type) {
278 default:
279 case spitfire:
280 phys_page = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
281 break;
282
283 case cheetah:
284 phys_page = cheetah_get_litlb_data(sparc64_highest_locked_tlbent());
285 break;
286 };
287
288 phys_page &= _PAGE_PADDR;
289 phys_page += ((unsigned long)&prom_boot_page -
290 (unsigned long)KERNBASE);
291
292 if (tlb_type == spitfire) {
293 /* Lock this into i/d tlb entry 59 */
294 __asm__ __volatile__(
295 "stxa %%g0, [%2] %3\n\t"
296 "stxa %0, [%1] %4\n\t"
297 "membar #Sync\n\t"
298 "flush %%g6\n\t"
299 "stxa %%g0, [%2] %5\n\t"
300 "stxa %0, [%1] %6\n\t"
301 "membar #Sync\n\t"
302 "flush %%g6"
303 : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
304 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
305 "r" (59 << 3), "r" (TLB_TAG_ACCESS),
306 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
307 "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
308 : "memory");
309 } else if (tlb_type == cheetah) {
310 /* Lock this into i/d tlb-0 entry 11 */
311 __asm__ __volatile__(
312 "stxa %%g0, [%2] %3\n\t"
313 "stxa %0, [%1] %4\n\t"
314 "membar #Sync\n\t"
315 "flush %%g6\n\t"
316 "stxa %%g0, [%2] %5\n\t"
317 "stxa %0, [%1] %6\n\t"
318 "membar #Sync\n\t"
319 "flush %%g6"
320 : : "r" (phys_page | _PAGE_VALID | _PAGE_SZ8K | _PAGE_CP |
321 _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W),
322 "r" ((0 << 16) | (11 << 3)), "r" (TLB_TAG_ACCESS),
323 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS),
324 "i" (ASI_IMMU), "i" (ASI_ITLB_DATA_ACCESS)
325 : "memory");
326 } else {
327 /* Implement me :-) */
328 BUG();
329 }
330
331 tte_vaddr = (unsigned long) KERNBASE;
332
333 /* Spitfire Errata #32 workaround */
334 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
335 "flush %%g6"
336 : /* No outputs */
337 : "r" (0),
338 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
339
340 if (tlb_type == spitfire)
341 tte_data = spitfire_get_dtlb_data(sparc64_highest_locked_tlbent());
342 else
343 tte_data = cheetah_get_ldtlb_data(sparc64_highest_locked_tlbent());
344
345 kern_locked_tte_data = tte_data;
346
347 remap_func = (void *) ((unsigned long) &prom_remap -
348 (unsigned long) &prom_boot_page);
349
350
351 /* Spitfire Errata #32 workaround */
352 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
353 "flush %%g6"
354 : /* No outputs */
355 : "r" (0),
356 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
357
358 remap_func((tlb_type == spitfire ?
359 (spitfire_get_dtlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR) :
360 (cheetah_get_litlb_data(sparc64_highest_locked_tlbent()) & _PAGE_PADDR)),
361 (unsigned long) KERNBASE,
362 prom_get_mmu_ihandle());
363
364 /* Flush out that temporary mapping. */
365 spitfire_flush_dtlb_nucleus_page(0x0);
366 spitfire_flush_itlb_nucleus_page(0x0);
367
368 /* Now lock us back into the TLBs via OBP. */
369 prom_dtlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
370 prom_itlb_load(sparc64_highest_locked_tlbent(), tte_data, tte_vaddr);
371
372 /* Re-read translations property. */
373 if ((n = prom_getproperty(node, "translations", (char *)trans, tsz)) == -1) {
374 prom_printf("Couldn't get translation property\n");
375 prom_halt();
376 }
377 n = n / sizeof(*trans);
378
379 for (i = 0; i < n; i++) {
380 unsigned long vaddr = trans[i].virt;
381 unsigned long size = trans[i].size;
382
383 if (vaddr < 0xf0000000UL) {
384 unsigned long avoid_start = (unsigned long) KERNBASE;
385 unsigned long avoid_end = avoid_start + (4 * 1024 * 1024);
386
387 if (vaddr < avoid_start) {
388 unsigned long top = vaddr + size;
389
390 if (top > avoid_start)
391 top = avoid_start;
392 prom_unmap(top - vaddr, vaddr);
393 }
394 if ((vaddr + size) > avoid_end) {
395 unsigned long bottom = vaddr;
396
397 if (bottom < avoid_end)
398 bottom = avoid_end;
399 prom_unmap((vaddr + size) - bottom, bottom);
400 }
401 }
402 }
403
404 prom_printf("done.\n");
405
406 register_prom_callbacks();
407 }
408
409 /* The OBP specifications for sun4u mark 0xfffffffc00000000 and
410 * upwards as reserved for use by the firmware (I wonder if this
411 * will be the same on Cheetah...). We use this virtual address
412 * range for the VPTE table mappings of the nucleus so we need
413 * to zap them when we enter the PROM. -DaveM
414 */
415 static void __flush_nucleus_vptes(void)
416 {
417 unsigned long prom_reserved_base = 0xfffffffc00000000UL;
418 int i;
419
420 /* Only DTLB must be checked for VPTE entries. */
421 if (tlb_type == spitfire) {
422 for (i = 0; i < 63; i++) {
423 unsigned long tag;
424
425 /* Spitfire Errata #32 workaround */
426 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
427 "flush %%g6"
428 : /* No outputs */
429 : "r" (0),
430 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
431
432 tag = spitfire_get_dtlb_tag(i);
433 if (((tag & ~(PAGE_MASK)) == 0) &&
434 ((tag & (PAGE_MASK)) >= prom_reserved_base)) {
435 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
436 "membar #Sync"
437 : /* no outputs */
438 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
439 spitfire_put_dtlb_data(i, 0x0UL);
440 }
441 }
442 } else if (tlb_type == cheetah) {
443 for (i = 0; i < 512; i++) {
444 unsigned long tag = cheetah_get_dtlb_tag(i);
445
446 if ((tag & ~PAGE_MASK) == 0 &&
447 (tag & PAGE_MASK) >= prom_reserved_base) {
448 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
449 "membar #Sync"
450 : /* no outputs */
451 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
452 cheetah_put_dtlb_data(i, 0x0UL);
453 }
454 }
455 } else {
456 /* Implement me :-) */
457 BUG();
458 }
459 }
460
461 static int prom_ditlb_set = 0;
462 struct prom_tlb_entry {
463 int tlb_ent;
464 unsigned long tlb_tag;
465 unsigned long tlb_data;
466 };
467 struct prom_tlb_entry prom_itlb[16], prom_dtlb[16];
468
469 void prom_world(int enter)
470 {
471 unsigned long pstate;
472 int i;
473
474 if (!enter)
475 set_fs(current->thread.current_ds);
476
477 if (!prom_ditlb_set)
478 return;
479
480 /* Make sure the following runs atomically. */
481 __asm__ __volatile__("flushw\n\t"
482 "rdpr %%pstate, %0\n\t"
483 "wrpr %0, %1, %%pstate"
484 : "=r" (pstate)
485 : "i" (PSTATE_IE));
486
487 if (enter) {
488 /* Kick out nucleus VPTEs. */
489 __flush_nucleus_vptes();
490
491 /* Install PROM world. */
492 for (i = 0; i < 16; i++) {
493 if (prom_dtlb[i].tlb_ent != -1) {
494 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
495 "membar #Sync"
496 : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
497 "i" (ASI_DMMU));
498 if (tlb_type == spitfire)
499 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
500 prom_dtlb[i].tlb_data);
501 else if (tlb_type == cheetah)
502 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
503 prom_dtlb[i].tlb_data);
504 }
505 if (prom_itlb[i].tlb_ent != -1) {
506 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
507 "membar #Sync"
508 : : "r" (prom_itlb[i].tlb_tag),
509 "r" (TLB_TAG_ACCESS),
510 "i" (ASI_IMMU));
511 if (tlb_type == spitfire)
512 spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
513 prom_itlb[i].tlb_data);
514 else if (tlb_type == cheetah)
515 cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
516 prom_itlb[i].tlb_data);
517 }
518 }
519 } else {
520 for (i = 0; i < 16; i++) {
521 if (prom_dtlb[i].tlb_ent != -1) {
522 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
523 "membar #Sync"
524 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
525 if (tlb_type == spitfire)
526 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
527 else
528 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent, 0x0UL);
529 }
530 if (prom_itlb[i].tlb_ent != -1) {
531 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
532 "membar #Sync"
533 : : "r" (TLB_TAG_ACCESS),
534 "i" (ASI_IMMU));
535 if (tlb_type == spitfire)
536 spitfire_put_itlb_data(prom_itlb[i].tlb_ent, 0x0UL);
537 else
538 cheetah_put_litlb_data(prom_itlb[i].tlb_ent, 0x0UL);
539 }
540 }
541 }
542 __asm__ __volatile__("wrpr %0, 0, %%pstate"
543 : : "r" (pstate));
544 }
545
546 void inherit_locked_prom_mappings(int save_p)
547 {
548 int i;
549 int dtlb_seen = 0;
550 int itlb_seen = 0;
551
552 /* Fucking losing PROM has more mappings in the TLB, but
553 * it (conveniently) fails to mention any of these in the
554 * translations property. The only ones that matter are
555 * the locked PROM tlb entries, so we impose the following
556 * irrecovable rule on the PROM, it is allowed 8 locked
557 * entries in the ITLB and 8 in the DTLB.
558 *
559 * Supposedly the upper 16GB of the address space is
560 * reserved for OBP, BUT I WISH THIS WAS DOCUMENTED
561 * SOMEWHERE!!!!!!!!!!!!!!!!! Furthermore the entire interface
562 * used between the client program and the firmware on sun5
563 * systems to coordinate mmu mappings is also COMPLETELY
564 * UNDOCUMENTED!!!!!! Thanks S(t)un!
565 */
566 if (save_p) {
567 for (i = 0; i < 16; i++) {
568 prom_itlb[i].tlb_ent = -1;
569 prom_dtlb[i].tlb_ent = -1;
570 }
571 }
572 if (tlb_type == spitfire) {
573 for (i = 0; i < SPITFIRE_HIGHEST_LOCKED_TLBENT; i++) {
574 unsigned long data;
575
576 /* Spitfire Errata #32 workaround */
577 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
578 "flush %%g6"
579 : /* No outputs */
580 : "r" (0),
581 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
582
583 data = spitfire_get_dtlb_data(i);
584 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
585 unsigned long tag;
586
587 /* Spitfire Errata #32 workaround */
588 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
589 "flush %%g6"
590 : /* No outputs */
591 : "r" (0),
592 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
593
594 tag = spitfire_get_dtlb_tag(i);
595 if (save_p) {
596 prom_dtlb[dtlb_seen].tlb_ent = i;
597 prom_dtlb[dtlb_seen].tlb_tag = tag;
598 prom_dtlb[dtlb_seen].tlb_data = data;
599 }
600 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
601 "membar #Sync"
602 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
603 spitfire_put_dtlb_data(i, 0x0UL);
604
605 dtlb_seen++;
606 if (dtlb_seen > 15)
607 break;
608 }
609 }
610
611 for (i = 0; i < SPITFIRE_HIGHEST_LOCKED_TLBENT; i++) {
612 unsigned long data;
613
614 /* Spitfire Errata #32 workaround */
615 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
616 "flush %%g6"
617 : /* No outputs */
618 : "r" (0),
619 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
620
621 data = spitfire_get_itlb_data(i);
622 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
623 unsigned long tag;
624
625 /* Spitfire Errata #32 workaround */
626 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
627 "flush %%g6"
628 : /* No outputs */
629 : "r" (0),
630 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
631
632 tag = spitfire_get_itlb_tag(i);
633 if (save_p) {
634 prom_itlb[itlb_seen].tlb_ent = i;
635 prom_itlb[itlb_seen].tlb_tag = tag;
636 prom_itlb[itlb_seen].tlb_data = data;
637 }
638 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
639 "membar #Sync"
640 : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
641 spitfire_put_itlb_data(i, 0x0UL);
642
643 itlb_seen++;
644 if (itlb_seen > 15)
645 break;
646 }
647 }
648 } else if (tlb_type == cheetah) {
649 for (i = 0; i < CHEETAH_HIGHEST_LOCKED_TLBENT; i++) {
650 unsigned long data;
651
652 data = cheetah_get_ldtlb_data(i);
653 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
654 unsigned long tag;
655
656 tag = cheetah_get_ldtlb_tag(i);
657 if (save_p) {
658 prom_dtlb[dtlb_seen].tlb_ent = i;
659 prom_dtlb[dtlb_seen].tlb_tag = tag;
660 prom_dtlb[dtlb_seen].tlb_data = data;
661 }
662 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
663 "membar #Sync"
664 : : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
665 cheetah_put_ldtlb_data(i, 0x0UL);
666
667 dtlb_seen++;
668 if (dtlb_seen > 15)
669 break;
670 }
671 }
672
673 for (i = 0; i < CHEETAH_HIGHEST_LOCKED_TLBENT; i++) {
674 unsigned long data;
675
676 data = cheetah_get_litlb_data(i);
677 if ((data & (_PAGE_L|_PAGE_VALID)) == (_PAGE_L|_PAGE_VALID)) {
678 unsigned long tag;
679
680 tag = cheetah_get_litlb_tag(i);
681 if (save_p) {
682 prom_itlb[itlb_seen].tlb_ent = i;
683 prom_itlb[itlb_seen].tlb_tag = tag;
684 prom_itlb[itlb_seen].tlb_data = data;
685 }
686 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
687 "membar #Sync"
688 : : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
689 cheetah_put_litlb_data(i, 0x0UL);
690
691 itlb_seen++;
692 if (itlb_seen > 15)
693 break;
694 }
695 }
696 } else {
697 /* Implement me :-) */
698 BUG();
699 }
700 if (save_p)
701 prom_ditlb_set = 1;
702 }
703
704 /* Give PROM back his world, done during reboots... */
705 void prom_reload_locked(void)
706 {
707 int i;
708
709 for (i = 0; i < 16; i++) {
710 if (prom_dtlb[i].tlb_ent != -1) {
711 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
712 "membar #Sync"
713 : : "r" (prom_dtlb[i].tlb_tag), "r" (TLB_TAG_ACCESS),
714 "i" (ASI_DMMU));
715 if (tlb_type == spitfire)
716 spitfire_put_dtlb_data(prom_dtlb[i].tlb_ent,
717 prom_dtlb[i].tlb_data);
718 else if (tlb_type == cheetah)
719 cheetah_put_ldtlb_data(prom_dtlb[i].tlb_ent,
720 prom_dtlb[i].tlb_data);
721 }
722
723 if (prom_itlb[i].tlb_ent != -1) {
724 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
725 "membar #Sync"
726 : : "r" (prom_itlb[i].tlb_tag),
727 "r" (TLB_TAG_ACCESS),
728 "i" (ASI_IMMU));
729 if (tlb_type == spitfire)
730 spitfire_put_itlb_data(prom_itlb[i].tlb_ent,
731 prom_itlb[i].tlb_data);
732 else
733 cheetah_put_litlb_data(prom_itlb[i].tlb_ent,
734 prom_itlb[i].tlb_data);
735 }
736 }
737 }
738
739 void __flush_dcache_range(unsigned long start, unsigned long end)
740 {
741 unsigned long va;
742
743 if (tlb_type == spitfire) {
744 int n = 0;
745
746 for (va = start; va < end; va += 32) {
747 spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
748 if (++n >= 512)
749 break;
750 }
751 } else {
752 start = __pa(start);
753 end = __pa(end);
754 for (va = start; va < end; va += 32)
755 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
756 "membar #Sync"
757 : /* no outputs */
758 : "r" (va),
759 "i" (ASI_DCACHE_INVALIDATE));
760 }
761 }
762
763 void __flush_cache_all(void)
764 {
765 /* Cheetah should be fine here too. */
766 if (tlb_type == spitfire) {
767 unsigned long va;
768
769 flushw_all();
770 for (va = 0; va < (PAGE_SIZE << 1); va += 32)
771 spitfire_put_icache_tag(va, 0x0);
772 __asm__ __volatile__("flush %g6");
773 }
774 }
775
776 /* If not locked, zap it. */
777 void __flush_tlb_all(void)
778 {
779 unsigned long pstate;
780 int i;
781
782 __asm__ __volatile__("flushw\n\t"
783 "rdpr %%pstate, %0\n\t"
784 "wrpr %0, %1, %%pstate"
785 : "=r" (pstate)
786 : "i" (PSTATE_IE));
787 if (tlb_type == spitfire) {
788 for (i = 0; i < 64; i++) {
789 /* Spitfire Errata #32 workaround */
790 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
791 "flush %%g6"
792 : /* No outputs */
793 : "r" (0),
794 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
795
796 if (!(spitfire_get_dtlb_data(i) & _PAGE_L)) {
797 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
798 "membar #Sync"
799 : /* no outputs */
800 : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
801 spitfire_put_dtlb_data(i, 0x0UL);
802 }
803
804 /* Spitfire Errata #32 workaround */
805 __asm__ __volatile__("stxa %0, [%1] %2\n\t"
806 "flush %%g6"
807 : /* No outputs */
808 : "r" (0),
809 "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
810
811 if (!(spitfire_get_itlb_data(i) & _PAGE_L)) {
812 __asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
813 "membar #Sync"
814 : /* no outputs */
815 : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
816 spitfire_put_itlb_data(i, 0x0UL);
817 }
818 }
819 } else if (tlb_type == cheetah) {
820 cheetah_flush_dtlb_all();
821 cheetah_flush_itlb_all();
822 }
823 __asm__ __volatile__("wrpr %0, 0, %%pstate"
824 : : "r" (pstate));
825 }
826
827 /* Caller does TLB context flushing on local CPU if necessary.
828 * The caller also ensures that CTX_VALID(mm->context) is false.
829 *
830 * We must be careful about boundary cases so that we never
831 * let the user have CTX 0 (nucleus) or we ever use a CTX
832 * version of zero (and thus NO_CONTEXT would not be caught
833 * by version mis-match tests in mmu_context.h).
834 */
835 void get_new_mmu_context(struct mm_struct *mm)
836 {
837 unsigned long ctx, new_ctx;
838
839 spin_lock(&ctx_alloc_lock);
840 ctx = CTX_HWBITS(tlb_context_cache + 1);
841 new_ctx = find_next_zero_bit(mmu_context_bmap, 1UL << CTX_VERSION_SHIFT, ctx);
842 if (new_ctx >= (1UL << CTX_VERSION_SHIFT)) {
843 new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
844 if (new_ctx >= ctx) {
845 int i;
846 new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
847 CTX_FIRST_VERSION;
848 if (new_ctx == 1)
849 new_ctx = CTX_FIRST_VERSION;
850
851 /* Don't call memset, for 16 entries that's just
852 * plain silly...
853 */
854 mmu_context_bmap[0] = 3;
855 mmu_context_bmap[1] = 0;
856 mmu_context_bmap[2] = 0;
857 mmu_context_bmap[3] = 0;
858 for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
859 mmu_context_bmap[i + 0] = 0;
860 mmu_context_bmap[i + 1] = 0;
861 mmu_context_bmap[i + 2] = 0;
862 mmu_context_bmap[i + 3] = 0;
863 }
864 goto out;
865 }
866 }
867 mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
868 new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
869 out:
870 tlb_context_cache = new_ctx;
871 spin_unlock(&ctx_alloc_lock);
872
873 mm->context = new_ctx;
874 }
875
876 #ifndef CONFIG_SMP
877 struct pgtable_cache_struct pgt_quicklists;
878 #endif
879
880 /* OK, we have to color these pages. The page tables are accessed
881 * by non-Dcache enabled mapping in the VPTE area by the dtlb_backend.S
882 * code, as well as by PAGE_OFFSET range direct-mapped addresses by
883 * other parts of the kernel. By coloring, we make sure that the tlbmiss
884 * fast handlers do not get data from old/garbage dcache lines that
885 * correspond to an old/stale virtual address (user/kernel) that
886 * previously mapped the pagetable page while accessing vpte range
887 * addresses. The idea is that if the vpte color and PAGE_OFFSET range
888 * color is the same, then when the kernel initializes the pagetable
889 * using the later address range, accesses with the first address
890 * range will not see the newly initialized data rather than the
891 * garbage.
892 */
893
894 pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
895 {
896 struct page *page = alloc_pages(GFP_KERNEL, 1);
897 unsigned long color = ((address >> (PAGE_SHIFT + 10)) & 1UL);
898
899 if (page) {
900 unsigned long *to_free;
901 unsigned long paddr;
902 pte_t *pte;
903
904 set_page_count((page + 1), 1);
905 paddr = (unsigned long) page_address(page);
906 memset((char *)paddr, 0, (PAGE_SIZE << 1));
907
908 if (!color) {
909 pte = (pte_t *) paddr;
910 to_free = (unsigned long *) (paddr + PAGE_SIZE);
911 } else {
912 pte = (pte_t *) (paddr + PAGE_SIZE);
913 to_free = (unsigned long *) paddr;
914 }
915
916 /* Now free the other one up, adjust cache size. */
917 *to_free = (unsigned long) pte_quicklist[color ^ 0x1];
918 pte_quicklist[color ^ 0x1] = to_free;
919 pgtable_cache_size++;
920
921 return pte;
922 }
923 return NULL;
924 }
925
926 void sparc_ultra_dump_itlb(void)
927 {
928 int slot;
929
930 if (tlb_type == spitfire) {
931 printk ("Contents of itlb: ");
932 for (slot = 0; slot < 14; slot++) printk (" ");
933 printk ("%2x:%016lx,%016lx\n",
934 0,
935 spitfire_get_itlb_tag(0), spitfire_get_itlb_data(0));
936 for (slot = 1; slot < 64; slot+=3) {
937 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
938 slot,
939 spitfire_get_itlb_tag(slot), spitfire_get_itlb_data(slot),
940 slot+1,
941 spitfire_get_itlb_tag(slot+1), spitfire_get_itlb_data(slot+1),
942 slot+2,
943 spitfire_get_itlb_tag(slot+2), spitfire_get_itlb_data(slot+2));
944 }
945 } else if (tlb_type == cheetah) {
946 printk ("Contents of itlb0:\n");
947 for (slot = 0; slot < 16; slot+=2) {
948 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
949 slot,
950 cheetah_get_litlb_tag(slot), cheetah_get_litlb_data(slot),
951 slot+1,
952 cheetah_get_litlb_tag(slot+1), cheetah_get_litlb_data(slot+1));
953 }
954 printk ("Contents of itlb2:\n");
955 for (slot = 0; slot < 128; slot+=2) {
956 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
957 slot,
958 cheetah_get_itlb_tag(slot), cheetah_get_itlb_data(slot),
959 slot+1,
960 cheetah_get_itlb_tag(slot+1), cheetah_get_itlb_data(slot+1));
961 }
962 }
963 }
964
965 void sparc_ultra_dump_dtlb(void)
966 {
967 int slot;
968
969 if (tlb_type == spitfire) {
970 printk ("Contents of dtlb: ");
971 for (slot = 0; slot < 14; slot++) printk (" ");
972 printk ("%2x:%016lx,%016lx\n", 0,
973 spitfire_get_dtlb_tag(0), spitfire_get_dtlb_data(0));
974 for (slot = 1; slot < 64; slot+=3) {
975 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx %2x:%016lx,%016lx\n",
976 slot,
977 spitfire_get_dtlb_tag(slot), spitfire_get_dtlb_data(slot),
978 slot+1,
979 spitfire_get_dtlb_tag(slot+1), spitfire_get_dtlb_data(slot+1),
980 slot+2,
981 spitfire_get_dtlb_tag(slot+2), spitfire_get_dtlb_data(slot+2));
982 }
983 } else if (tlb_type == cheetah) {
984 printk ("Contents of dtlb0:\n");
985 for (slot = 0; slot < 16; slot+=2) {
986 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
987 slot,
988 cheetah_get_ldtlb_tag(slot), cheetah_get_ldtlb_data(slot),
989 slot+1,
990 cheetah_get_ldtlb_tag(slot+1), cheetah_get_ldtlb_data(slot+1));
991 }
992 printk ("Contents of dtlb2:\n");
993 for (slot = 0; slot < 512; slot+=2) {
994 printk ("%2x:%016lx,%016lx %2x:%016lx,%016lx\n",
995 slot,
996 cheetah_get_dtlb_tag(slot), cheetah_get_dtlb_data(slot),
997 slot+1,
998 cheetah_get_dtlb_tag(slot+1), cheetah_get_dtlb_data(slot+1));
999 }
1000 }
1001 }
1002
1003 extern unsigned long cmdline_memory_size;
1004
1005 unsigned long __init bootmem_init(unsigned long *pages_avail)
1006 {
1007 unsigned long bootmap_size, start_pfn, end_pfn;
1008 unsigned long end_of_phys_memory = 0UL;
1009 unsigned long bootmap_pfn, bytes_avail, size;
1010 int i;
1011
1012 bytes_avail = 0UL;
1013 for (i = 0; sp_banks[i].num_bytes != 0; i++) {
1014 end_of_phys_memory = sp_banks[i].base_addr +
1015 sp_banks[i].num_bytes;
1016 bytes_avail += sp_banks[i].num_bytes;
1017 if (cmdline_memory_size) {
1018 if (bytes_avail > cmdline_memory_size) {
1019 unsigned long slack = bytes_avail - cmdline_memory_size;
1020
1021 bytes_avail -= slack;
1022 end_of_phys_memory -= slack;
1023
1024 sp_banks[i].num_bytes -= slack;
1025 if (sp_banks[i].num_bytes == 0) {
1026 sp_banks[i].base_addr = 0xdeadbeef;
1027 } else {
1028 sp_banks[i+1].num_bytes = 0;
1029 sp_banks[i+1].base_addr = 0xdeadbeef;
1030 }
1031 break;
1032 }
1033 }
1034 }
1035
1036 *pages_avail = bytes_avail >> PAGE_SHIFT;
1037
1038 /* Start with page aligned address of last symbol in kernel
1039 * image. The kernel is hard mapped below PAGE_OFFSET in a
1040 * 4MB locked TLB translation.
1041 */
1042 start_pfn = PAGE_ALIGN((unsigned long) &_end) -
1043 ((unsigned long) KERNBASE);
1044
1045 /* Adjust up to the physical address where the kernel begins. */
1046 start_pfn += phys_base;
1047
1048 /* Now shift down to get the real physical page frame number. */
1049 start_pfn >>= PAGE_SHIFT;
1050
1051 bootmap_pfn = start_pfn;
1052
1053 end_pfn = end_of_phys_memory >> PAGE_SHIFT;
1054
1055 #ifdef CONFIG_BLK_DEV_INITRD
1056 /* Now have to check initial ramdisk, so that bootmap does not overwrite it */
1057 if (sparc_ramdisk_image) {
1058 if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE)
1059 sparc_ramdisk_image -= KERNBASE;
1060 initrd_start = sparc_ramdisk_image + phys_base;
1061 initrd_end = initrd_start + sparc_ramdisk_size;
1062 if (initrd_end > end_of_phys_memory) {
1063 printk(KERN_CRIT "initrd extends beyond end of memory "
1064 "(0x%016lx > 0x%016lx)\ndisabling initrd\n",
1065 initrd_end, end_of_phys_memory);
1066 initrd_start = 0;
1067 }
1068 if (initrd_start) {
1069 if (initrd_start >= (start_pfn << PAGE_SHIFT) &&
1070 initrd_start < (start_pfn << PAGE_SHIFT) + 2 * PAGE_SIZE)
1071 bootmap_pfn = PAGE_ALIGN (initrd_end) >> PAGE_SHIFT;
1072 }
1073 }
1074 #endif
1075 /* Initialize the boot-time allocator. */
1076 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_pfn, phys_base>>PAGE_SHIFT, end_pfn);
1077
1078 /* Now register the available physical memory with the
1079 * allocator.
1080 */
1081 for (i = 0; sp_banks[i].num_bytes != 0; i++)
1082 free_bootmem(sp_banks[i].base_addr,
1083 sp_banks[i].num_bytes);
1084
1085 #ifdef CONFIG_BLK_DEV_INITRD
1086 if (initrd_start) {
1087 size = initrd_end - initrd_start;
1088
1089 /* Resert the initrd image area. */
1090 reserve_bootmem(initrd_start, size);
1091 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1092
1093 initrd_start += PAGE_OFFSET;
1094 initrd_end += PAGE_OFFSET;
1095 }
1096 #endif
1097 /* Reserve the kernel text/data/bss. */
1098 size = (start_pfn << PAGE_SHIFT) - phys_base;
1099 reserve_bootmem(phys_base, size);
1100 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1101
1102 /* Reserve the bootmem map. We do not account for it
1103 * in pages_avail because we will release that memory
1104 * in free_all_bootmem.
1105 */
1106 size = bootmap_size;
1107 reserve_bootmem((bootmap_pfn << PAGE_SHIFT), size);
1108 *pages_avail -= PAGE_ALIGN(size) >> PAGE_SHIFT;
1109
1110 return end_pfn;
1111 }
1112
1113 /* paging_init() sets up the page tables */
1114
1115 extern void sun_serial_setup(void);
1116 extern void cheetah_ecache_flush_init(void);
1117
1118 static unsigned long last_valid_pfn;
1119
1120 void __init paging_init(void)
1121 {
1122 extern pmd_t swapper_pmd_dir[1024];
1123 extern unsigned int sparc64_vpte_patchme1[1];
1124 extern unsigned int sparc64_vpte_patchme2[1];
1125 unsigned long alias_base = phys_base + PAGE_OFFSET;
1126 unsigned long second_alias_page = 0;
1127 unsigned long pt, flags, end_pfn, pages_avail;
1128 unsigned long shift = alias_base - ((unsigned long)KERNBASE);
1129 unsigned long real_end;
1130
1131 set_bit(0, mmu_context_bmap);
1132
1133 real_end = (unsigned long)&_end;
1134 #ifdef CONFIG_BLK_DEV_INITRD
1135 if (sparc_ramdisk_image)
1136 real_end = (PAGE_ALIGN(real_end) + PAGE_ALIGN(sparc_ramdisk_size));
1137 #endif
1138
1139 /* We assume physical memory starts at some 4mb multiple,
1140 * if this were not true we wouldn't boot up to this point
1141 * anyways.
1142 */
1143 pt = phys_base | _PAGE_VALID | _PAGE_SZ4MB;
1144 pt |= _PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W;
1145 __save_and_cli(flags);
1146 if (tlb_type == spitfire) {
1147 __asm__ __volatile__(
1148 " stxa %1, [%0] %3\n"
1149 " stxa %2, [%5] %4\n"
1150 " membar #Sync\n"
1151 " flush %%g6\n"
1152 " nop\n"
1153 " nop\n"
1154 " nop\n"
1155 : /* No outputs */
1156 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
1157 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (61 << 3)
1158 : "memory");
1159 if (real_end >= KERNBASE + 0x340000) {
1160 second_alias_page = alias_base + 0x400000;
1161 __asm__ __volatile__(
1162 " stxa %1, [%0] %3\n"
1163 " stxa %2, [%5] %4\n"
1164 " membar #Sync\n"
1165 " flush %%g6\n"
1166 " nop\n"
1167 " nop\n"
1168 " nop\n"
1169 : /* No outputs */
1170 : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
1171 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" (60 << 3)
1172 : "memory");
1173 }
1174 } else if (tlb_type == cheetah) {
1175 __asm__ __volatile__(
1176 " stxa %1, [%0] %3\n"
1177 " stxa %2, [%5] %4\n"
1178 " membar #Sync\n"
1179 " flush %%g6\n"
1180 " nop\n"
1181 " nop\n"
1182 " nop\n"
1183 : /* No outputs */
1184 : "r" (TLB_TAG_ACCESS), "r" (alias_base), "r" (pt),
1185 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (13<<3))
1186 : "memory");
1187 if (real_end >= KERNBASE + 0x340000) {
1188 second_alias_page = alias_base + 0x400000;
1189 __asm__ __volatile__(
1190 " stxa %1, [%0] %3\n"
1191 " stxa %2, [%5] %4\n"
1192 " membar #Sync\n"
1193 " flush %%g6\n"
1194 " nop\n"
1195 " nop\n"
1196 " nop\n"
1197 : /* No outputs */
1198 : "r" (TLB_TAG_ACCESS), "r" (second_alias_page), "r" (pt + 0x400000),
1199 "i" (ASI_DMMU), "i" (ASI_DTLB_DATA_ACCESS), "r" ((0<<16) | (12<<3))
1200 : "memory");
1201 }
1202 }
1203 __restore_flags(flags);
1204
1205 /* Now set kernel pgd to upper alias so physical page computations
1206 * work.
1207 */
1208 init_mm.pgd += ((shift) / (sizeof(pgd_t)));
1209
1210 memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));
1211
1212 /* Now can init the kernel/bad page tables. */
1213 pgd_set(&swapper_pg_dir[0], swapper_pmd_dir + (shift / sizeof(pgd_t)));
1214
1215 sparc64_vpte_patchme1[0] |= (pgd_val(init_mm.pgd[0]) >> 10);
1216 sparc64_vpte_patchme2[0] |= (pgd_val(init_mm.pgd[0]) & 0x3ff);
1217 flushi((long)&sparc64_vpte_patchme1[0]);
1218
1219 /* Setup bootmem... */
1220 pages_avail = 0;
1221 last_valid_pfn = end_pfn = bootmem_init(&pages_avail);
1222
1223 #ifdef CONFIG_SUN_SERIAL
1224 /* This does not logically belong here, but we need to
1225 * call it at the moment we are able to use the bootmem
1226 * allocator.
1227 */
1228 sun_serial_setup();
1229 #endif
1230
1231 /* Inherit non-locked OBP mappings. */
1232 inherit_prom_mappings();
1233
1234 /* Ok, we can use our TLB miss and window trap handlers safely.
1235 * We need to do a quick peek here to see if we are on StarFire
1236 * or not, so setup_tba can setup the IRQ globals correctly (it
1237 * needs to get the hard smp processor id correctly).
1238 */
1239 {
1240 extern void setup_tba(int);
1241 setup_tba(this_is_starfire);
1242 }
1243
1244 inherit_locked_prom_mappings(1);
1245
1246 /* We only created DTLB mapping of this stuff. */
1247 spitfire_flush_dtlb_nucleus_page(alias_base);
1248 if (second_alias_page)
1249 spitfire_flush_dtlb_nucleus_page(second_alias_page);
1250
1251 flush_tlb_all();
1252
1253 {
1254 unsigned long zones_size[MAX_NR_ZONES];
1255 unsigned long zholes_size[MAX_NR_ZONES];
1256 unsigned long npages;
1257 int znum;
1258
1259 for (znum = 0; znum < MAX_NR_ZONES; znum++)
1260 zones_size[znum] = zholes_size[znum] = 0;
1261
1262 npages = end_pfn - (phys_base >> PAGE_SHIFT);
1263 zones_size[ZONE_DMA] = npages;
1264 zholes_size[ZONE_DMA] = npages - pages_avail;
1265
1266 free_area_init_node(0, NULL, NULL, zones_size,
1267 phys_base, zholes_size);
1268 }
1269
1270 device_scan();
1271 }
1272
1273 /* Ok, it seems that the prom can allocate some more memory chunks
1274 * as a side effect of some prom calls we perform during the
1275 * boot sequence. My most likely theory is that it is from the
1276 * prom_set_traptable() call, and OBP is allocating a scratchpad
1277 * for saving client program register state etc.
1278 */
1279 void __init sort_memlist(struct linux_mlist_p1275 *thislist)
1280 {
1281 int swapi = 0;
1282 int i, mitr;
1283 unsigned long tmpaddr, tmpsize;
1284 unsigned long lowest;
1285
1286 for (i = 0; thislist[i].theres_more != 0; i++) {
1287 lowest = thislist[i].start_adr;
1288 for (mitr = i+1; thislist[mitr-1].theres_more != 0; mitr++)
1289 if (thislist[mitr].start_adr < lowest) {
1290 lowest = thislist[mitr].start_adr;
1291 swapi = mitr;
1292 }
1293 if (lowest == thislist[i].start_adr)
1294 continue;
1295 tmpaddr = thislist[swapi].start_adr;
1296 tmpsize = thislist[swapi].num_bytes;
1297 for (mitr = swapi; mitr > i; mitr--) {
1298 thislist[mitr].start_adr = thislist[mitr-1].start_adr;
1299 thislist[mitr].num_bytes = thislist[mitr-1].num_bytes;
1300 }
1301 thislist[i].start_adr = tmpaddr;
1302 thislist[i].num_bytes = tmpsize;
1303 }
1304 }
1305
1306 void __init rescan_sp_banks(void)
1307 {
1308 struct linux_prom64_registers memlist[64];
1309 struct linux_mlist_p1275 avail[64], *mlist;
1310 unsigned long bytes, base_paddr;
1311 int num_regs, node = prom_finddevice("/memory");
1312 int i;
1313
1314 num_regs = prom_getproperty(node, "available",
1315 (char *) memlist, sizeof(memlist));
1316 num_regs = (num_regs / sizeof(struct linux_prom64_registers));
1317 for (i = 0; i < num_regs; i++) {
1318 avail[i].start_adr = memlist[i].phys_addr;
1319 avail[i].num_bytes = memlist[i].reg_size;
1320 avail[i].theres_more = &avail[i + 1];
1321 }
1322 avail[i - 1].theres_more = NULL;
1323 sort_memlist(avail);
1324
1325 mlist = &avail[0];
1326 i = 0;
1327 bytes = mlist->num_bytes;
1328 base_paddr = mlist->start_adr;
1329
1330 sp_banks[0].base_addr = base_paddr;
1331 sp_banks[0].num_bytes = bytes;
1332
1333 while (mlist->theres_more != NULL){
1334 i++;
1335 mlist = mlist->theres_more;
1336 bytes = mlist->num_bytes;
1337 if (i >= SPARC_PHYS_BANKS-1) {
1338 printk ("The machine has more banks than "
1339 "this kernel can support\n"
1340 "Increase the SPARC_PHYS_BANKS "
1341 "setting (currently %d)\n",
1342 SPARC_PHYS_BANKS);
1343 i = SPARC_PHYS_BANKS-1;
1344 break;
1345 }
1346
1347 sp_banks[i].base_addr = mlist->start_adr;
1348 sp_banks[i].num_bytes = mlist->num_bytes;
1349 }
1350
1351 i++;
1352 sp_banks[i].base_addr = 0xdeadbeefbeefdeadUL;
1353 sp_banks[i].num_bytes = 0;
1354
1355 for (i = 0; sp_banks[i].num_bytes != 0; i++)
1356 sp_banks[i].num_bytes &= PAGE_MASK;
1357 }
1358
1359 static void __init taint_real_pages(void)
1360 {
1361 struct sparc_phys_banks saved_sp_banks[SPARC_PHYS_BANKS];
1362 int i;
1363
1364 for (i = 0; i < SPARC_PHYS_BANKS; i++) {
1365 saved_sp_banks[i].base_addr =
1366 sp_banks[i].base_addr;
1367 saved_sp_banks[i].num_bytes =
1368 sp_banks[i].num_bytes;
1369 }
1370
1371 rescan_sp_banks();
1372
1373 /* Find changes discovered in the sp_bank rescan and
1374 * reserve the lost portions in the bootmem maps.
1375 */
1376 for (i = 0; saved_sp_banks[i].num_bytes; i++) {
1377 unsigned long old_start, old_end;
1378
1379 old_start = saved_sp_banks[i].base_addr;
1380 old_end = old_start +
1381 saved_sp_banks[i].num_bytes;
1382 while (old_start < old_end) {
1383 int n;
1384
1385 for (n = 0; sp_banks[n].num_bytes; n++) {
1386 unsigned long new_start, new_end;
1387
1388 new_start = sp_banks[n].base_addr;
1389 new_end = new_start + sp_banks[n].num_bytes;
1390
1391 if (new_start <= old_start &&
1392 new_end >= (old_start + PAGE_SIZE)) {
1393 set_bit (old_start >> 22,
1394 sparc64_valid_addr_bitmap);
1395 goto do_next_page;
1396 }
1397 }
1398 reserve_bootmem(old_start, PAGE_SIZE);
1399
1400 do_next_page:
1401 old_start += PAGE_SIZE;
1402 }
1403 }
1404 }
1405
1406 void __init mem_init(void)
1407 {
1408 unsigned long codepages, datapages, initpages;
1409 unsigned long addr, last;
1410 int i;
1411
1412 i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
1413 i += 1;
1414 sparc64_valid_addr_bitmap = (unsigned long *)
1415 __alloc_bootmem(i << 3, SMP_CACHE_BYTES, 0UL);
1416 if (sparc64_valid_addr_bitmap == NULL) {
1417 prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
1418 prom_halt();
1419 }
1420 memset(sparc64_valid_addr_bitmap, 0, i << 3);
1421
1422 addr = PAGE_OFFSET + phys_base;
1423 last = PAGE_ALIGN((unsigned long)&_end) -
1424 ((unsigned long) KERNBASE);
1425 last += PAGE_OFFSET + phys_base;
1426 while (addr < last) {
1427 set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
1428 addr += PAGE_SIZE;
1429 }
1430
1431 taint_real_pages();
1432
1433 max_mapnr = last_valid_pfn - (phys_base >> PAGE_SHIFT);
1434 high_memory = __va(last_valid_pfn << PAGE_SHIFT);
1435
1436 num_physpages = free_all_bootmem() - 1;
1437
1438 /*
1439 * Set up the zero page, mark it reserved, so that page count
1440 * is not manipulated when freeing the page from user ptes.
1441 */
1442 mem_map_zero = _alloc_pages(GFP_KERNEL, 0);
1443 if (mem_map_zero == NULL) {
1444 prom_printf("paging_init: Cannot alloc zero page.\n");
1445 prom_halt();
1446 }
1447 SetPageReserved(mem_map_zero);
1448 clear_page(page_address(mem_map_zero));
1449
1450 codepages = (((unsigned long) &etext) - ((unsigned long)&_start));
1451 codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
1452 datapages = (((unsigned long) &edata) - ((unsigned long)&etext));
1453 datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
1454 initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin));
1455 initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
1456
1457 #ifndef CONFIG_SMP
1458 {
1459 /* Put empty_pg_dir on pgd_quicklist */
1460 extern pgd_t empty_pg_dir[1024];
1461 unsigned long addr = (unsigned long)empty_pg_dir;
1462 unsigned long alias_base = phys_base + PAGE_OFFSET -
1463 (long)(KERNBASE);
1464
1465 memset(empty_pg_dir, 0, sizeof(empty_pg_dir));
1466 addr += alias_base;
1467 free_pgd_fast((pgd_t *)addr);
1468 num_physpages++;
1469 }
1470 #endif
1471
1472 printk("Memory: %uk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
1473 nr_free_pages() << (PAGE_SHIFT-10),
1474 codepages << (PAGE_SHIFT-10),
1475 datapages << (PAGE_SHIFT-10),
1476 initpages << (PAGE_SHIFT-10),
1477 PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
1478
1479 if (tlb_type == cheetah)
1480 cheetah_ecache_flush_init();
1481 }
1482
1483 void free_initmem (void)
1484 {
1485 unsigned long addr, initend;
1486
1487 /*
1488 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
1489 */
1490 addr = PAGE_ALIGN((unsigned long)(&__init_begin));
1491 initend = (unsigned long)(&__init_end) & PAGE_MASK;
1492 for (; addr < initend; addr += PAGE_SIZE) {
1493 unsigned long page;
1494 struct page *p;
1495
1496 page = (addr +
1497 ((unsigned long) __va(phys_base)) -
1498 ((unsigned long) KERNBASE));
1499 p = virt_to_page(page);
1500
1501 ClearPageReserved(p);
1502 set_page_count(p, 1);
1503 __free_page(p);
1504 num_physpages++;
1505 }
1506 }
1507
1508 #ifdef CONFIG_BLK_DEV_INITRD
1509 void free_initrd_mem(unsigned long start, unsigned long end)
1510 {
1511 if (start < end)
1512 printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
1513 for (; start < end; start += PAGE_SIZE) {
1514 struct page *p = virt_to_page(start);
1515
1516 ClearPageReserved(p);
1517 set_page_count(p, 1);
1518 __free_page(p);
1519 num_physpages++;
1520 }
1521 }
1522 #endif
1523
1524 void si_meminfo(struct sysinfo *val)
1525 {
1526 val->totalram = num_physpages;
1527 val->sharedram = 0;
1528 val->freeram = nr_free_pages();
1529 val->bufferram = atomic_read(&buffermem_pages);
1530
1531 /* These are always zero on Sparc64. */
1532 val->totalhigh = 0;
1533 val->freehigh = 0;
1534
1535 val->mem_unit = PAGE_SIZE;
1536 }
1537