File: /usr/src/linux/arch/sparc/mm/sun4c.c
1 /* $Id: sun4c.c,v 1.207 2001/07/17 16:17:33 anton Exp $
2 * sun4c.c: Doing in software what should be done in hardware.
3 *
4 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
5 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
6 * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au)
7 * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org)
8 * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
9 */
10
11 #define NR_TASK_BUCKETS 512
12
13 #include <linux/config.h>
14 #include <linux/kernel.h>
15 #include <linux/mm.h>
16 #include <linux/init.h>
17 #include <linux/bootmem.h>
18 #include <linux/highmem.h>
19
20 #include <asm/scatterlist.h>
21 #include <asm/page.h>
22 #include <asm/pgalloc.h>
23 #include <asm/pgtable.h>
24 #include <asm/vaddrs.h>
25 #include <asm/idprom.h>
26 #include <asm/machines.h>
27 #include <asm/memreg.h>
28 #include <asm/processor.h>
29 #include <asm/auxio.h>
30 #include <asm/io.h>
31 #include <asm/oplib.h>
32 #include <asm/openprom.h>
33 #include <asm/mmu_context.h>
34 #include <asm/sun4paddr.h>
35 #include <asm/highmem.h>
36
37 /* Because of our dynamic kernel TLB miss strategy, and how
38 * our DVMA mapping allocation works, you _MUST_:
39 *
40 * 1) Disable interrupts _and_ not touch any dynamic kernel
41 * memory while messing with kernel MMU state. By
42 * dynamic memory I mean any object which is not in
43 * the kernel image itself or a task_struct (both of
44 * which are locked into the MMU).
45 * 2) Disable interrupts while messing with user MMU state.
46 */
47
48 extern int num_segmaps, num_contexts;
49
50 extern unsigned long page_kernel;
51
52 #ifdef CONFIG_SUN4
53 #define SUN4C_VAC_SIZE sun4c_vacinfo.num_bytes
54 #else
55 /* That's it, we prom_halt() on sun4c if the cache size is something other than 65536.
56 * So let's save some cycles and just use that everywhere except for that bootup
57 * sanity check.
58 */
59 #define SUN4C_VAC_SIZE 65536
60 #endif
61
62 #define SUN4C_KERNEL_BUCKETS 32
63
64 #ifndef MAX
65 #define MAX(a,b) ((a)<(b)?(b):(a))
66 #endif
67 #ifndef MIN
68 #define MIN(a,b) ((a)<(b)?(a):(b))
69 #endif
70
71 /* Flushing the cache. */
72 struct sun4c_vac_props sun4c_vacinfo;
73 unsigned long sun4c_kernel_faults;
74
75 /* Invalidate every sun4c cache line tag. */
76 void sun4c_flush_all(void)
77 {
78 unsigned long begin, end;
79
80 if (sun4c_vacinfo.on)
81 panic("SUN4C: AIEEE, trying to invalidate vac while"
82 " it is on.");
83
84 /* Clear 'valid' bit in all cache line tags */
85 begin = AC_CACHETAGS;
86 end = (AC_CACHETAGS + SUN4C_VAC_SIZE);
87 while (begin < end) {
88 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : :
89 "r" (begin), "i" (ASI_CONTROL));
90 begin += sun4c_vacinfo.linesize;
91 }
92 }
93
94 static __inline__ void sun4c_flush_context_hw(void)
95 {
96 unsigned long end = SUN4C_VAC_SIZE;
97
98 __asm__ __volatile__(
99 "1: addcc %0, -4096, %0\n\t"
100 " bne 1b\n\t"
101 " sta %%g0, [%0] %2"
102 : "=&r" (end)
103 : "0" (end), "i" (ASI_HWFLUSHCONTEXT)
104 : "cc");
105 }
106
107 /* Must be called minimally with IRQs disabled. */
108 static void sun4c_flush_segment_hw(unsigned long addr)
109 {
110 if (sun4c_get_segmap(addr) != invalid_segment) {
111 unsigned long vac_size = SUN4C_VAC_SIZE;
112
113 __asm__ __volatile__(
114 "1: addcc %0, -4096, %0\n\t"
115 " bne 1b\n\t"
116 " sta %%g0, [%2 + %0] %3"
117 : "=&r" (vac_size)
118 : "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG)
119 : "cc");
120 }
121 }
122
123 /* Must be called minimally with interrupts disabled. */
124 static __inline__ void sun4c_flush_page_hw(unsigned long addr)
125 {
126 addr &= PAGE_MASK;
127 if ((int)sun4c_get_pte(addr) < 0)
128 __asm__ __volatile__("sta %%g0, [%0] %1"
129 : : "r" (addr), "i" (ASI_HWFLUSHPAGE));
130 }
131
132 /* Don't inline the software version as it eats too many cache lines if expanded. */
133 static void sun4c_flush_context_sw(void)
134 {
135 unsigned long nbytes = SUN4C_VAC_SIZE;
136 unsigned long lsize = sun4c_vacinfo.linesize;
137
138 __asm__ __volatile__("
139 add %2, %2, %%g1
140 add %2, %%g1, %%g2
141 add %2, %%g2, %%g3
142 add %2, %%g3, %%g4
143 add %2, %%g4, %%g5
144 add %2, %%g5, %%o4
145 add %2, %%o4, %%o5
146 1: subcc %0, %%o5, %0
147 sta %%g0, [%0] %3
148 sta %%g0, [%0 + %2] %3
149 sta %%g0, [%0 + %%g1] %3
150 sta %%g0, [%0 + %%g2] %3
151 sta %%g0, [%0 + %%g3] %3
152 sta %%g0, [%0 + %%g4] %3
153 sta %%g0, [%0 + %%g5] %3
154 bg 1b
155 sta %%g0, [%1 + %%o4] %3
156 " : "=&r" (nbytes)
157 : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX)
158 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
159 }
160
161 /* Don't inline the software version as it eats too many cache lines if expanded. */
162 static void sun4c_flush_segment_sw(unsigned long addr)
163 {
164 if (sun4c_get_segmap(addr) != invalid_segment) {
165 unsigned long nbytes = SUN4C_VAC_SIZE;
166 unsigned long lsize = sun4c_vacinfo.linesize;
167
168 __asm__ __volatile__("
169 add %2, %2, %%g1
170 add %2, %%g1, %%g2
171 add %2, %%g2, %%g3
172 add %2, %%g3, %%g4
173 add %2, %%g4, %%g5
174 add %2, %%g5, %%o4
175 add %2, %%o4, %%o5
176 1: subcc %1, %%o5, %1
177 sta %%g0, [%0] %6
178 sta %%g0, [%0 + %2] %6
179 sta %%g0, [%0 + %%g1] %6
180 sta %%g0, [%0 + %%g2] %6
181 sta %%g0, [%0 + %%g3] %6
182 sta %%g0, [%0 + %%g4] %6
183 sta %%g0, [%0 + %%g5] %6
184 sta %%g0, [%0 + %%o4] %6
185 bg 1b
186 add %0, %%o5, %0
187 " : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize)
188 : "0" (addr), "1" (nbytes), "2" (lsize),
189 "i" (ASI_FLUSHSEG)
190 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
191 }
192 }
193
194 /* Bolix one page from the virtual cache. */
195 static void sun4c_flush_page(unsigned long addr)
196 {
197 addr &= PAGE_MASK;
198
199 if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) !=
200 _SUN4C_PAGE_VALID)
201 return;
202
203 if (sun4c_vacinfo.do_hwflushes) {
204 __asm__ __volatile__("sta %%g0, [%0] %1;nop;nop;nop;\n\t" : :
205 "r" (addr), "i" (ASI_HWFLUSHPAGE));
206 } else {
207 unsigned long left = PAGE_SIZE;
208 unsigned long lsize = sun4c_vacinfo.linesize;
209
210 __asm__ __volatile__("add %2, %2, %%g1\n\t"
211 "add %2, %%g1, %%g2\n\t"
212 "add %2, %%g2, %%g3\n\t"
213 "add %2, %%g3, %%g4\n\t"
214 "add %2, %%g4, %%g5\n\t"
215 "add %2, %%g5, %%o4\n\t"
216 "add %2, %%o4, %%o5\n"
217 "1:\n\t"
218 "subcc %1, %%o5, %1\n\t"
219 "sta %%g0, [%0] %6\n\t"
220 "sta %%g0, [%0 + %2] %6\n\t"
221 "sta %%g0, [%0 + %%g1] %6\n\t"
222 "sta %%g0, [%0 + %%g2] %6\n\t"
223 "sta %%g0, [%0 + %%g3] %6\n\t"
224 "sta %%g0, [%0 + %%g4] %6\n\t"
225 "sta %%g0, [%0 + %%g5] %6\n\t"
226 "sta %%g0, [%0 + %%o4] %6\n\t"
227 "bg 1b\n\t"
228 " add %0, %%o5, %0\n\t"
229 : "=&r" (addr), "=&r" (left), "=&r" (lsize)
230 : "0" (addr), "1" (left), "2" (lsize),
231 "i" (ASI_FLUSHPG)
232 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
233 }
234 }
235
236 /* Don't inline the software version as it eats too many cache lines if expanded. */
237 static void sun4c_flush_page_sw(unsigned long addr)
238 {
239 addr &= PAGE_MASK;
240 if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) ==
241 _SUN4C_PAGE_VALID) {
242 unsigned long left = PAGE_SIZE;
243 unsigned long lsize = sun4c_vacinfo.linesize;
244
245 __asm__ __volatile__("
246 add %2, %2, %%g1
247 add %2, %%g1, %%g2
248 add %2, %%g2, %%g3
249 add %2, %%g3, %%g4
250 add %2, %%g4, %%g5
251 add %2, %%g5, %%o4
252 add %2, %%o4, %%o5
253 1: subcc %1, %%o5, %1
254 sta %%g0, [%0] %6
255 sta %%g0, [%0 + %2] %6
256 sta %%g0, [%0 + %%g1] %6
257 sta %%g0, [%0 + %%g2] %6
258 sta %%g0, [%0 + %%g3] %6
259 sta %%g0, [%0 + %%g4] %6
260 sta %%g0, [%0 + %%g5] %6
261 sta %%g0, [%0 + %%o4] %6
262 bg 1b
263 add %0, %%o5, %0
264 " : "=&r" (addr), "=&r" (left), "=&r" (lsize)
265 : "0" (addr), "1" (left), "2" (lsize),
266 "i" (ASI_FLUSHPG)
267 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc");
268 }
269 }
270
271 /* The sun4c's do have an on chip store buffer. And the way you
272 * clear them out isn't so obvious. The only way I can think of
273 * to accomplish this is to read the current context register,
274 * store the same value there, then read an external hardware
275 * register.
276 */
277 void sun4c_complete_all_stores(void)
278 {
279 volatile int _unused;
280
281 _unused = sun4c_get_context();
282 sun4c_set_context(_unused);
283 #ifdef CONFIG_SUN_AUXIO
284 _unused = *AUXREG;
285 #endif
286 }
287
288 /* Bootup utility functions. */
289 static inline void sun4c_init_clean_segmap(unsigned char pseg)
290 {
291 unsigned long vaddr;
292
293 sun4c_put_segmap(0, pseg);
294 for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE)
295 sun4c_put_pte(vaddr, 0);
296 sun4c_put_segmap(0, invalid_segment);
297 }
298
299 static inline void sun4c_init_clean_mmu(unsigned long kernel_end)
300 {
301 unsigned long vaddr;
302 unsigned char savectx, ctx;
303
304 savectx = sun4c_get_context();
305 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
306 for (ctx = 0; ctx < num_contexts; ctx++) {
307 sun4c_set_context(ctx);
308 for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE)
309 sun4c_put_segmap(vaddr, invalid_segment);
310 for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE)
311 sun4c_put_segmap(vaddr, invalid_segment);
312 for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE)
313 sun4c_put_segmap(vaddr, invalid_segment);
314 for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE)
315 sun4c_put_segmap(vaddr, invalid_segment);
316 }
317 sun4c_set_context(savectx);
318 }
319
320 void __init sun4c_probe_vac(void)
321 {
322 sun4c_disable_vac();
323
324 if (ARCH_SUN4) {
325 switch (idprom->id_machtype) {
326
327 case (SM_SUN4|SM_4_110):
328 sun4c_vacinfo.type = NONE;
329 sun4c_vacinfo.num_bytes = 0;
330 sun4c_vacinfo.linesize = 0;
331 sun4c_vacinfo.do_hwflushes = 0;
332 prom_printf("No VAC. Get some bucks and buy a real computer.");
333 prom_halt();
334 break;
335
336 case (SM_SUN4|SM_4_260):
337 sun4c_vacinfo.type = WRITE_BACK;
338 sun4c_vacinfo.num_bytes = 128 * 1024;
339 sun4c_vacinfo.linesize = 16;
340 sun4c_vacinfo.do_hwflushes = 0;
341 break;
342
343 case (SM_SUN4|SM_4_330):
344 sun4c_vacinfo.type = WRITE_THROUGH;
345 sun4c_vacinfo.num_bytes = 128 * 1024;
346 sun4c_vacinfo.linesize = 16;
347 sun4c_vacinfo.do_hwflushes = 0;
348 break;
349
350 case (SM_SUN4|SM_4_470):
351 sun4c_vacinfo.type = WRITE_BACK;
352 sun4c_vacinfo.num_bytes = 128 * 1024;
353 sun4c_vacinfo.linesize = 32;
354 sun4c_vacinfo.do_hwflushes = 0;
355 break;
356
357 default:
358 prom_printf("Cannot initialize VAC - weird sun4 model idprom->id_machtype = %d", idprom->id_machtype);
359 prom_halt();
360 };
361 } else {
362 sun4c_vacinfo.type = WRITE_THROUGH;
363
364 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
365 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
366 /* PROM on SS1 lacks this info, to be super safe we
367 * hard code it here since this arch is cast in stone.
368 */
369 sun4c_vacinfo.num_bytes = 65536;
370 sun4c_vacinfo.linesize = 16;
371 } else {
372 sun4c_vacinfo.num_bytes =
373 prom_getintdefault(prom_root_node, "vac-size", 65536);
374 sun4c_vacinfo.linesize =
375 prom_getintdefault(prom_root_node, "vac-linesize", 16);
376 }
377 sun4c_vacinfo.do_hwflushes =
378 prom_getintdefault(prom_root_node, "vac-hwflush", 0);
379
380 if (sun4c_vacinfo.do_hwflushes == 0)
381 sun4c_vacinfo.do_hwflushes =
382 prom_getintdefault(prom_root_node, "vac_hwflush", 0);
383
384 if (sun4c_vacinfo.num_bytes != 65536) {
385 prom_printf("WEIRD Sun4C VAC cache size, tell davem");
386 prom_halt();
387 }
388 }
389
390 sun4c_vacinfo.num_lines =
391 (sun4c_vacinfo.num_bytes / sun4c_vacinfo.linesize);
392 switch (sun4c_vacinfo.linesize) {
393 case 16:
394 sun4c_vacinfo.log2lsize = 4;
395 break;
396 case 32:
397 sun4c_vacinfo.log2lsize = 5;
398 break;
399 default:
400 prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n",
401 sun4c_vacinfo.linesize);
402 prom_halt();
403 };
404
405 sun4c_flush_all();
406 sun4c_enable_vac();
407 }
408
409 /* Patch instructions for the low level kernel fault handler. */
410 extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff;
411 extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff;
412 extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff;
413 extern unsigned long num_context_patch1, num_context_patch1_16;
414 extern unsigned long num_context_patch2, num_context_patch2_16;
415 extern unsigned long vac_linesize_patch, vac_linesize_patch_32;
416 extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on;
417 extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on;
418
419 #define PATCH_INSN(src, dst) do { \
420 daddr = &(dst); \
421 iaddr = &(src); \
422 *daddr = *iaddr; \
423 } while (0);
424
425 static void patch_kernel_fault_handler(void)
426 {
427 unsigned long *iaddr, *daddr;
428
429 switch (num_segmaps) {
430 case 128:
431 /* Default, nothing to do. */
432 break;
433 case 256:
434 PATCH_INSN(invalid_segment_patch1_ff,
435 invalid_segment_patch1);
436 PATCH_INSN(invalid_segment_patch2_ff,
437 invalid_segment_patch2);
438 break;
439 case 512:
440 PATCH_INSN(invalid_segment_patch1_1ff,
441 invalid_segment_patch1);
442 PATCH_INSN(invalid_segment_patch2_1ff,
443 invalid_segment_patch2);
444 break;
445 default:
446 prom_printf("Unhandled number of segmaps: %d\n",
447 num_segmaps);
448 prom_halt();
449 };
450 switch (num_contexts) {
451 case 8:
452 /* Default, nothing to do. */
453 break;
454 case 16:
455 PATCH_INSN(num_context_patch1_16,
456 num_context_patch1);
457 #if 0
458 PATCH_INSN(num_context_patch2_16,
459 num_context_patch2);
460 #endif
461 break;
462 default:
463 prom_printf("Unhandled number of contexts: %d\n",
464 num_contexts);
465 prom_halt();
466 };
467
468 if (sun4c_vacinfo.do_hwflushes != 0) {
469 PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1);
470 PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2);
471 } else {
472 switch (sun4c_vacinfo.linesize) {
473 case 16:
474 /* Default, nothing to do. */
475 break;
476 case 32:
477 PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch);
478 break;
479 default:
480 prom_printf("Impossible VAC linesize %d, halting...\n",
481 sun4c_vacinfo.linesize);
482 prom_halt();
483 };
484 }
485 }
486
487 static void __init sun4c_probe_mmu(void)
488 {
489 if (ARCH_SUN4) {
490 switch (idprom->id_machtype) {
491 case (SM_SUN4|SM_4_110):
492 prom_printf("No support for 4100 yet\n");
493 prom_halt();
494 num_segmaps = 256;
495 num_contexts = 8;
496 break;
497
498 case (SM_SUN4|SM_4_260):
499 /* should be 512 segmaps. when it get fixed */
500 num_segmaps = 256;
501 num_contexts = 16;
502 break;
503
504 case (SM_SUN4|SM_4_330):
505 num_segmaps = 256;
506 num_contexts = 16;
507 break;
508
509 case (SM_SUN4|SM_4_470):
510 /* should be 1024 segmaps. when it get fixed */
511 num_segmaps = 256;
512 num_contexts = 64;
513 break;
514 default:
515 prom_printf("Invalid SUN4 model\n");
516 prom_halt();
517 };
518 } else {
519 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) ||
520 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) {
521 /* Hardcode these just to be safe, PROM on SS1 does
522 * not have this info available in the root node.
523 */
524 num_segmaps = 128;
525 num_contexts = 8;
526 } else {
527 num_segmaps =
528 prom_getintdefault(prom_root_node, "mmu-npmg", 128);
529 num_contexts =
530 prom_getintdefault(prom_root_node, "mmu-nctx", 0x8);
531 }
532 }
533 patch_kernel_fault_handler();
534 }
535
536 volatile unsigned long *sun4c_memerr_reg = 0;
537
538 void __init sun4c_probe_memerr_reg(void)
539 {
540 int node;
541 struct linux_prom_registers regs[1];
542
543 if (ARCH_SUN4) {
544 sun4c_memerr_reg = ioremap(sun4_memreg_physaddr, PAGE_SIZE);
545 } else {
546 node = prom_getchild(prom_root_node);
547 node = prom_searchsiblings(prom_root_node, "memory-error");
548 if (!node)
549 return;
550 prom_getproperty(node, "reg", (char *)regs, sizeof(regs));
551 /* hmm I think regs[0].which_io is zero here anyways */
552 sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size);
553 }
554 }
555
556 static inline void sun4c_init_ss2_cache_bug(void)
557 {
558 extern unsigned long start;
559
560 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) ||
561 (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) ||
562 (idprom->id_machtype == (SM_SUN4 | SM_4_330)) ||
563 (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) {
564 /* Whee.. */
565 printk("SS2 cache bug detected, uncaching trap table page\n");
566 sun4c_flush_page((unsigned int) &start);
567 sun4c_put_pte(((unsigned long) &start),
568 (sun4c_get_pte((unsigned long) &start) | _SUN4C_PAGE_NOCACHE));
569 }
570 }
571
572 /* Addr is always aligned on a page boundry for us already. */
573 static void sun4c_map_dma_area(unsigned long va, u32 addr, int len)
574 {
575 unsigned long page, end;
576
577 end = PAGE_ALIGN((addr + len));
578 while (addr < end) {
579 page = va;
580 sun4c_flush_page(page);
581 page -= PAGE_OFFSET;
582 page >>= PAGE_SHIFT;
583 page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY |
584 _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV);
585 sun4c_put_pte(addr, page);
586 addr += PAGE_SIZE;
587 va += PAGE_SIZE;
588 }
589 }
590
591 static unsigned long sun4c_translate_dvma(unsigned long busa)
592 {
593 /* Fortunately for us, bus_addr == uncached_virt in sun4c. */
594 unsigned long pte = sun4c_get_pte(busa);
595 return (pte << PAGE_SHIFT) + PAGE_OFFSET;
596 }
597
598 static void sun4c_unmap_dma_area(unsigned long busa, int len)
599 {
600 /* Fortunately for us, bus_addr == uncached_virt in sun4c. */
601 /* XXX Implement this */
602 }
603
604 /* TLB management. */
605
606 /* Don't change this struct without changing entry.S. This is used
607 * in the in-window kernel fault handler, and you don't want to mess
608 * with that. (See sun4c_fault in entry.S).
609 */
610 struct sun4c_mmu_entry {
611 struct sun4c_mmu_entry *next;
612 struct sun4c_mmu_entry *prev;
613 unsigned long vaddr;
614 unsigned char pseg;
615 unsigned char locked;
616
617 /* For user mappings only, and completely hidden from kernel
618 * TLB miss code.
619 */
620 unsigned char ctx;
621 struct sun4c_mmu_entry *lru_next;
622 struct sun4c_mmu_entry *lru_prev;
623 };
624
625 static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS];
626
627 static void __init sun4c_init_mmu_entry_pool(void)
628 {
629 int i;
630
631 for (i=0; i < SUN4C_MAX_SEGMAPS; i++) {
632 mmu_entry_pool[i].pseg = i;
633 mmu_entry_pool[i].next = 0;
634 mmu_entry_pool[i].prev = 0;
635 mmu_entry_pool[i].vaddr = 0;
636 mmu_entry_pool[i].locked = 0;
637 mmu_entry_pool[i].ctx = 0;
638 mmu_entry_pool[i].lru_next = 0;
639 mmu_entry_pool[i].lru_prev = 0;
640 }
641 mmu_entry_pool[invalid_segment].locked = 1;
642 }
643
644 static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on,
645 unsigned long bits_off)
646 {
647 unsigned long start, end;
648
649 end = vaddr + SUN4C_REAL_PGDIR_SIZE;
650 for (start = vaddr; start < end; start += PAGE_SIZE)
651 if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID)
652 sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) &
653 ~bits_off);
654 }
655
656 static inline void sun4c_init_map_kernelprom(unsigned long kernel_end)
657 {
658 unsigned long vaddr;
659 unsigned char pseg, ctx;
660 #ifdef CONFIG_SUN4
661 /* sun4/110 and 260 have no kadb. */
662 if ((idprom->id_machtype != (SM_SUN4 | SM_4_260)) &&
663 (idprom->id_machtype != (SM_SUN4 | SM_4_110))) {
664 #endif
665 for (vaddr = KADB_DEBUGGER_BEGVM;
666 vaddr < LINUX_OPPROM_ENDVM;
667 vaddr += SUN4C_REAL_PGDIR_SIZE) {
668 pseg = sun4c_get_segmap(vaddr);
669 if (pseg != invalid_segment) {
670 mmu_entry_pool[pseg].locked = 1;
671 for (ctx = 0; ctx < num_contexts; ctx++)
672 prom_putsegment(ctx, vaddr, pseg);
673 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0);
674 }
675 }
676 #ifdef CONFIG_SUN4
677 }
678 #endif
679 for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) {
680 pseg = sun4c_get_segmap(vaddr);
681 mmu_entry_pool[pseg].locked = 1;
682 for (ctx = 0; ctx < num_contexts; ctx++)
683 prom_putsegment(ctx, vaddr, pseg);
684 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE);
685 }
686 }
687
688 static void __init sun4c_init_lock_area(unsigned long start, unsigned long end)
689 {
690 int i, ctx;
691
692 while (start < end) {
693 for (i = 0; i < invalid_segment; i++)
694 if (!mmu_entry_pool[i].locked)
695 break;
696 mmu_entry_pool[i].locked = 1;
697 sun4c_init_clean_segmap(i);
698 for (ctx = 0; ctx < num_contexts; ctx++)
699 prom_putsegment(ctx, start, mmu_entry_pool[i].pseg);
700 start += SUN4C_REAL_PGDIR_SIZE;
701 }
702 }
703
704 /* Don't change this struct without changing entry.S. This is used
705 * in the in-window kernel fault handler, and you don't want to mess
706 * with that. (See sun4c_fault in entry.S).
707 */
708 struct sun4c_mmu_ring {
709 struct sun4c_mmu_entry ringhd;
710 int num_entries;
711 };
712
713 static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */
714 static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */
715 static struct sun4c_mmu_ring sun4c_ulru_ring; /* LRU user entries */
716 struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */
717 struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */
718
719 static inline void sun4c_init_rings(void)
720 {
721 int i;
722
723 for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) {
724 sun4c_context_ring[i].ringhd.next =
725 sun4c_context_ring[i].ringhd.prev =
726 &sun4c_context_ring[i].ringhd;
727 sun4c_context_ring[i].num_entries = 0;
728 }
729 sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev =
730 &sun4c_ufree_ring.ringhd;
731 sun4c_ufree_ring.num_entries = 0;
732 sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev =
733 &sun4c_ulru_ring.ringhd;
734 sun4c_ulru_ring.num_entries = 0;
735 sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev =
736 &sun4c_kernel_ring.ringhd;
737 sun4c_kernel_ring.num_entries = 0;
738 sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev =
739 &sun4c_kfree_ring.ringhd;
740 sun4c_kfree_ring.num_entries = 0;
741 }
742
743 static void add_ring(struct sun4c_mmu_ring *ring,
744 struct sun4c_mmu_entry *entry)
745 {
746 struct sun4c_mmu_entry *head = &ring->ringhd;
747
748 entry->prev = head;
749 (entry->next = head->next)->prev = entry;
750 head->next = entry;
751 ring->num_entries++;
752 }
753
754 static __inline__ void add_lru(struct sun4c_mmu_entry *entry)
755 {
756 struct sun4c_mmu_ring *ring = &sun4c_ulru_ring;
757 struct sun4c_mmu_entry *head = &ring->ringhd;
758
759 entry->lru_next = head;
760 (entry->lru_prev = head->lru_prev)->lru_next = entry;
761 head->lru_prev = entry;
762 }
763
764 static void add_ring_ordered(struct sun4c_mmu_ring *ring,
765 struct sun4c_mmu_entry *entry)
766 {
767 struct sun4c_mmu_entry *head = &ring->ringhd;
768 unsigned long addr = entry->vaddr;
769
770 while ((head->next != &ring->ringhd) && (head->next->vaddr < addr))
771 head = head->next;
772
773 entry->prev = head;
774 (entry->next = head->next)->prev = entry;
775 head->next = entry;
776 ring->num_entries++;
777
778 add_lru(entry);
779 }
780
781 static __inline__ void remove_ring(struct sun4c_mmu_ring *ring,
782 struct sun4c_mmu_entry *entry)
783 {
784 struct sun4c_mmu_entry *next = entry->next;
785
786 (next->prev = entry->prev)->next = next;
787 ring->num_entries--;
788 }
789
790 static void remove_lru(struct sun4c_mmu_entry *entry)
791 {
792 struct sun4c_mmu_entry *next = entry->lru_next;
793
794 (next->lru_prev = entry->lru_prev)->lru_next = next;
795 }
796
797 static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry)
798 {
799 remove_ring(sun4c_context_ring+ctx, entry);
800 remove_lru(entry);
801 add_ring(&sun4c_ufree_ring, entry);
802 }
803
804 static void free_kernel_entry(struct sun4c_mmu_entry *entry,
805 struct sun4c_mmu_ring *ring)
806 {
807 remove_ring(ring, entry);
808 add_ring(&sun4c_kfree_ring, entry);
809 }
810
811 static void __init sun4c_init_fill_kernel_ring(int howmany)
812 {
813 int i;
814
815 while (howmany) {
816 for (i = 0; i < invalid_segment; i++)
817 if (!mmu_entry_pool[i].locked)
818 break;
819 mmu_entry_pool[i].locked = 1;
820 sun4c_init_clean_segmap(i);
821 add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]);
822 howmany--;
823 }
824 }
825
826 static void __init sun4c_init_fill_user_ring(void)
827 {
828 int i;
829
830 for (i = 0; i < invalid_segment; i++) {
831 if (mmu_entry_pool[i].locked)
832 continue;
833 sun4c_init_clean_segmap(i);
834 add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]);
835 }
836 }
837
838 static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry)
839 {
840 int savectx, ctx;
841
842 savectx = sun4c_get_context();
843 for (ctx = 0; ctx < num_contexts; ctx++) {
844 sun4c_set_context(ctx);
845 sun4c_put_segmap(kentry->vaddr, invalid_segment);
846 }
847 sun4c_set_context(savectx);
848 }
849
850 static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry)
851 {
852 int savectx, ctx;
853
854 savectx = sun4c_get_context();
855 for (ctx = 0; ctx < num_contexts; ctx++) {
856 sun4c_set_context(ctx);
857 sun4c_put_segmap(kentry->vaddr, kentry->pseg);
858 }
859 sun4c_set_context(savectx);
860 }
861
862 #define sun4c_user_unmap(__entry) \
863 sun4c_put_segmap((__entry)->vaddr, invalid_segment)
864
865 static void sun4c_demap_context_hw(struct sun4c_mmu_ring *crp, unsigned char ctx)
866 {
867 struct sun4c_mmu_entry *head = &crp->ringhd;
868 unsigned long flags;
869
870 save_and_cli(flags);
871 if (head->next != head) {
872 struct sun4c_mmu_entry *entry = head->next;
873 int savectx = sun4c_get_context();
874
875 flush_user_windows();
876 sun4c_set_context(ctx);
877 sun4c_flush_context_hw();
878 do {
879 struct sun4c_mmu_entry *next = entry->next;
880
881 sun4c_user_unmap(entry);
882 free_user_entry(ctx, entry);
883
884 entry = next;
885 } while (entry != head);
886 sun4c_set_context(savectx);
887 }
888 restore_flags(flags);
889 }
890
891 static void sun4c_demap_context_sw(struct sun4c_mmu_ring *crp, unsigned char ctx)
892 {
893 struct sun4c_mmu_entry *head = &crp->ringhd;
894 unsigned long flags;
895
896 save_and_cli(flags);
897 if (head->next != head) {
898 struct sun4c_mmu_entry *entry = head->next;
899 int savectx = sun4c_get_context();
900
901 flush_user_windows();
902 sun4c_set_context(ctx);
903 sun4c_flush_context_sw();
904 do {
905 struct sun4c_mmu_entry *next = entry->next;
906
907 sun4c_user_unmap(entry);
908 free_user_entry(ctx, entry);
909
910 entry = next;
911 } while (entry != head);
912 sun4c_set_context(savectx);
913 }
914 restore_flags(flags);
915 }
916
917 static int sun4c_user_taken_entries = 0; /* This is how much we have. */
918 static int max_user_taken_entries = 0; /* This limits us and prevents deadlock. */
919
920 static struct sun4c_mmu_entry *sun4c_kernel_strategy(void)
921 {
922 struct sun4c_mmu_entry *this_entry;
923
924 /* If some are free, return first one. */
925 if (sun4c_kfree_ring.num_entries) {
926 this_entry = sun4c_kfree_ring.ringhd.next;
927 return this_entry;
928 }
929
930 /* Else free one up. */
931 this_entry = sun4c_kernel_ring.ringhd.prev;
932 if (sun4c_vacinfo.do_hwflushes)
933 sun4c_flush_segment_hw(this_entry->vaddr);
934 else
935 sun4c_flush_segment_sw(this_entry->vaddr);
936 sun4c_kernel_unmap(this_entry);
937 free_kernel_entry(this_entry, &sun4c_kernel_ring);
938 this_entry = sun4c_kfree_ring.ringhd.next;
939
940 return this_entry;
941 }
942
943 /* Using this method to free up mmu entries eliminates a lot of
944 * potential races since we have a kernel that incurs tlb
945 * replacement faults. There may be performance penalties.
946 *
947 * NOTE: Must be called with interrupts disabled.
948 */
949 static struct sun4c_mmu_entry *sun4c_user_strategy(void)
950 {
951 struct sun4c_mmu_entry *entry;
952 unsigned char ctx;
953 int savectx;
954
955 /* If some are free, return first one. */
956 if (sun4c_ufree_ring.num_entries) {
957 entry = sun4c_ufree_ring.ringhd.next;
958 goto unlink_out;
959 }
960
961 if (sun4c_user_taken_entries) {
962 entry = sun4c_kernel_strategy();
963 sun4c_user_taken_entries--;
964 goto kunlink_out;
965 }
966
967 /* Grab from the beginning of the LRU list. */
968 entry = sun4c_ulru_ring.ringhd.lru_next;
969 ctx = entry->ctx;
970
971 savectx = sun4c_get_context();
972 flush_user_windows();
973 sun4c_set_context(ctx);
974 if (sun4c_vacinfo.do_hwflushes)
975 sun4c_flush_segment_hw(entry->vaddr);
976 else
977 sun4c_flush_segment_sw(entry->vaddr);
978 sun4c_user_unmap(entry);
979 remove_ring(sun4c_context_ring + ctx, entry);
980 remove_lru(entry);
981 sun4c_set_context(savectx);
982
983 return entry;
984
985 unlink_out:
986 remove_ring(&sun4c_ufree_ring, entry);
987 return entry;
988 kunlink_out:
989 remove_ring(&sun4c_kfree_ring, entry);
990 return entry;
991 }
992
993 /* NOTE: Must be called with interrupts disabled. */
994 void sun4c_grow_kernel_ring(void)
995 {
996 struct sun4c_mmu_entry *entry;
997
998 /* Prevent deadlock condition. */
999 if (sun4c_user_taken_entries >= max_user_taken_entries)
1000 return;
1001
1002 if (sun4c_ufree_ring.num_entries) {
1003 entry = sun4c_ufree_ring.ringhd.next;
1004 remove_ring(&sun4c_ufree_ring, entry);
1005 add_ring(&sun4c_kfree_ring, entry);
1006 sun4c_user_taken_entries++;
1007 }
1008 }
1009
1010 /* 2 page buckets for task struct and kernel stack allocation.
1011 *
1012 * TASK_STACK_BEGIN
1013 * bucket[0]
1014 * bucket[1]
1015 * [ ... ]
1016 * bucket[NR_TASK_BUCKETS-1]
1017 * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS)
1018 *
1019 * Each slot looks like:
1020 *
1021 * page 1 -- task struct + beginning of kernel stack
1022 * page 2 -- rest of kernel stack
1023 */
1024
1025 union task_union *sun4c_bucket[NR_TASK_BUCKETS];
1026
1027 static int sun4c_lowbucket_avail;
1028
1029 #define BUCKET_EMPTY ((union task_union *) 0)
1030 #define BUCKET_SHIFT (PAGE_SHIFT + 1) /* log2(sizeof(struct task_bucket)) */
1031 #define BUCKET_SIZE (1 << BUCKET_SHIFT)
1032 #define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT))
1033 #define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR)
1034 #define BUCKET_PTE(page) \
1035 ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL))
1036 #define BUCKET_PTE_PAGE(pte) \
1037 (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT))
1038
1039 static void get_locked_segment(unsigned long addr)
1040 {
1041 struct sun4c_mmu_entry *stolen;
1042 unsigned long flags;
1043
1044 save_and_cli(flags);
1045 addr &= SUN4C_REAL_PGDIR_MASK;
1046 stolen = sun4c_user_strategy();
1047 max_user_taken_entries--;
1048 stolen->vaddr = addr;
1049 flush_user_windows();
1050 sun4c_kernel_map(stolen);
1051 restore_flags(flags);
1052 }
1053
1054 static void free_locked_segment(unsigned long addr)
1055 {
1056 struct sun4c_mmu_entry *entry;
1057 unsigned long flags;
1058 unsigned char pseg;
1059
1060 save_and_cli(flags);
1061 addr &= SUN4C_REAL_PGDIR_MASK;
1062 pseg = sun4c_get_segmap(addr);
1063 entry = &mmu_entry_pool[pseg];
1064
1065 flush_user_windows();
1066 if (sun4c_vacinfo.do_hwflushes)
1067 sun4c_flush_segment_hw(addr);
1068 else
1069 sun4c_flush_segment_sw(addr);
1070 sun4c_kernel_unmap(entry);
1071 add_ring(&sun4c_ufree_ring, entry);
1072 max_user_taken_entries++;
1073 restore_flags(flags);
1074 }
1075
1076 static inline void garbage_collect(int entry)
1077 {
1078 int start, end;
1079
1080 /* 32 buckets per segment... */
1081 entry &= ~31;
1082 start = entry;
1083 for (end = (start + 32); start < end; start++)
1084 if (sun4c_bucket[start] != BUCKET_EMPTY)
1085 return;
1086
1087 /* Entire segment empty, release it. */
1088 free_locked_segment(BUCKET_ADDR(entry));
1089 }
1090
1091 #ifdef CONFIG_SUN4
1092 #define TASK_STRUCT_ORDER 0
1093 #else
1094 #define TASK_STRUCT_ORDER 1
1095 #endif
1096
1097 static struct task_struct *sun4c_alloc_task_struct(void)
1098 {
1099 unsigned long addr, pages;
1100 int entry;
1101
1102 pages = __get_free_pages(GFP_KERNEL, TASK_STRUCT_ORDER);
1103 if (!pages)
1104 return (struct task_struct *) 0;
1105
1106 for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++)
1107 if (sun4c_bucket[entry] == BUCKET_EMPTY)
1108 break;
1109 if (entry == NR_TASK_BUCKETS) {
1110 free_pages(pages, TASK_STRUCT_ORDER);
1111 return (struct task_struct *) 0;
1112 }
1113 if (entry >= sun4c_lowbucket_avail)
1114 sun4c_lowbucket_avail = entry + 1;
1115
1116 addr = BUCKET_ADDR(entry);
1117 sun4c_bucket[entry] = (union task_union *) addr;
1118 if(sun4c_get_segmap(addr) == invalid_segment)
1119 get_locked_segment(addr);
1120
1121 /* We are changing the virtual color of the page(s)
1122 * so we must flush the cache to guarentee consistancy.
1123 */
1124 if (sun4c_vacinfo.do_hwflushes) {
1125 sun4c_flush_page_hw(pages);
1126 #ifndef CONFIG_SUN4
1127 sun4c_flush_page_hw(pages + PAGE_SIZE);
1128 #endif
1129 } else {
1130 sun4c_flush_page_sw(pages);
1131 #ifndef CONFIG_SUN4
1132 sun4c_flush_page_sw(pages + PAGE_SIZE);
1133 #endif
1134 }
1135
1136 sun4c_put_pte(addr, BUCKET_PTE(pages));
1137 #ifndef CONFIG_SUN4
1138 sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE));
1139 #endif
1140 return (struct task_struct *) addr;
1141 }
1142
1143 static void sun4c_free_task_struct_hw(struct task_struct *tsk)
1144 {
1145 unsigned long tsaddr = (unsigned long) tsk;
1146 unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tsaddr));
1147 int entry = BUCKET_NUM(tsaddr);
1148
1149 if (atomic_dec_and_test(&(tsk)->thread.refcount)) {
1150 /* We are deleting a mapping, so the flush here is mandatory. */
1151 sun4c_flush_page_hw(tsaddr);
1152 #ifndef CONFIG_SUN4
1153 sun4c_flush_page_hw(tsaddr + PAGE_SIZE);
1154 #endif
1155 sun4c_put_pte(tsaddr, 0);
1156 #ifndef CONFIG_SUN4
1157 sun4c_put_pte(tsaddr + PAGE_SIZE, 0);
1158 #endif
1159 sun4c_bucket[entry] = BUCKET_EMPTY;
1160 if (entry < sun4c_lowbucket_avail)
1161 sun4c_lowbucket_avail = entry;
1162
1163 free_pages(pages, TASK_STRUCT_ORDER);
1164 garbage_collect(entry);
1165 }
1166 }
1167
1168 static void sun4c_free_task_struct_sw(struct task_struct *tsk)
1169 {
1170 unsigned long tsaddr = (unsigned long) tsk;
1171 unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tsaddr));
1172 int entry = BUCKET_NUM(tsaddr);
1173
1174 if (atomic_dec_and_test(&(tsk)->thread.refcount)) {
1175 /* We are deleting a mapping, so the flush here is mandatory. */
1176 sun4c_flush_page_sw(tsaddr);
1177 #ifndef CONFIG_SUN4
1178 sun4c_flush_page_sw(tsaddr + PAGE_SIZE);
1179 #endif
1180 sun4c_put_pte(tsaddr, 0);
1181 #ifndef CONFIG_SUN4
1182 sun4c_put_pte(tsaddr + PAGE_SIZE, 0);
1183 #endif
1184 sun4c_bucket[entry] = BUCKET_EMPTY;
1185 if (entry < sun4c_lowbucket_avail)
1186 sun4c_lowbucket_avail = entry;
1187
1188 free_pages(pages, TASK_STRUCT_ORDER);
1189 garbage_collect(entry);
1190 }
1191 }
1192
1193 static void sun4c_get_task_struct(struct task_struct *tsk)
1194 {
1195 atomic_inc(&(tsk)->thread.refcount);
1196 }
1197
1198 static void __init sun4c_init_buckets(void)
1199 {
1200 int entry;
1201
1202 if (sizeof(union task_union) != (PAGE_SIZE << TASK_STRUCT_ORDER)) {
1203 prom_printf("task union not %d page(s)!\n", 1 << TASK_STRUCT_ORDER);
1204 }
1205 for (entry = 0; entry < NR_TASK_BUCKETS; entry++)
1206 sun4c_bucket[entry] = BUCKET_EMPTY;
1207 sun4c_lowbucket_avail = 0;
1208 }
1209
1210 static unsigned long sun4c_iobuffer_start;
1211 static unsigned long sun4c_iobuffer_end;
1212 static unsigned long sun4c_iobuffer_high;
1213 static unsigned long *sun4c_iobuffer_map;
1214 static int iobuffer_map_size;
1215
1216 /*
1217 * Alias our pages so they do not cause a trap.
1218 * Also one page may be aliased into several I/O areas and we may
1219 * finish these I/O separately.
1220 */
1221 static char *sun4c_lockarea(char *vaddr, unsigned long size)
1222 {
1223 unsigned long base, scan;
1224 unsigned long npages;
1225 unsigned long vpage;
1226 unsigned long pte;
1227 unsigned long apage;
1228 unsigned long high;
1229 unsigned long flags;
1230
1231 npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1232 size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1233
1234 scan = 0;
1235 save_and_cli(flags);
1236 for (;;) {
1237 scan = find_next_zero_bit(sun4c_iobuffer_map,
1238 iobuffer_map_size, scan);
1239 if ((base = scan) + npages > iobuffer_map_size) goto abend;
1240 for (;;) {
1241 if (scan >= base + npages) goto found;
1242 if (test_bit(scan, sun4c_iobuffer_map)) break;
1243 scan++;
1244 }
1245 }
1246
1247 found:
1248 high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start;
1249 high = SUN4C_REAL_PGDIR_ALIGN(high);
1250 while (high > sun4c_iobuffer_high) {
1251 get_locked_segment(sun4c_iobuffer_high);
1252 sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE;
1253 }
1254
1255 vpage = ((unsigned long) vaddr) & PAGE_MASK;
1256 for (scan = base; scan < base+npages; scan++) {
1257 pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT);
1258 pte |= pgprot_val(SUN4C_PAGE_KERNEL);
1259 pte |= _SUN4C_PAGE_NOCACHE;
1260 set_bit(scan, sun4c_iobuffer_map);
1261 apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start;
1262
1263 /* Flush original mapping so we see the right things later. */
1264 sun4c_flush_page(vpage);
1265
1266 sun4c_put_pte(apage, pte);
1267 vpage += PAGE_SIZE;
1268 }
1269 restore_flags(flags);
1270 return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start +
1271 (((unsigned long) vaddr) & ~PAGE_MASK));
1272
1273 abend:
1274 restore_flags(flags);
1275 printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size);
1276 panic("Out of iobuffer table");
1277 return 0;
1278 }
1279
1280 static void sun4c_unlockarea(char *vaddr, unsigned long size)
1281 {
1282 unsigned long vpage, npages;
1283 unsigned long flags;
1284 int scan, high;
1285
1286 vpage = (unsigned long)vaddr & PAGE_MASK;
1287 npages = (((unsigned long)vaddr & ~PAGE_MASK) +
1288 size + (PAGE_SIZE-1)) >> PAGE_SHIFT;
1289
1290 save_and_cli(flags);
1291 while (npages != 0) {
1292 --npages;
1293
1294 /* This mapping is marked non-cachable, no flush necessary. */
1295 sun4c_put_pte(vpage, 0);
1296 clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT,
1297 sun4c_iobuffer_map);
1298 vpage += PAGE_SIZE;
1299 }
1300
1301 /* garbage collect */
1302 scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT;
1303 while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5])
1304 scan -= 32;
1305 scan += 32;
1306 high = sun4c_iobuffer_start + (scan << PAGE_SHIFT);
1307 high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE;
1308 while (high < sun4c_iobuffer_high) {
1309 sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE;
1310 free_locked_segment(sun4c_iobuffer_high);
1311 }
1312 restore_flags(flags);
1313 }
1314
1315 /* Note the scsi code at init time passes to here buffers
1316 * which sit on the kernel stack, those are already locked
1317 * by implication and fool the page locking code above
1318 * if passed to by mistake.
1319 */
1320 static __u32 sun4c_get_scsi_one(char *bufptr, unsigned long len, struct sbus_bus *sbus)
1321 {
1322 unsigned long page;
1323
1324 page = ((unsigned long)bufptr) & PAGE_MASK;
1325 if (!VALID_PAGE(virt_to_page(page))) {
1326 sun4c_flush_page(page);
1327 return (__u32)bufptr; /* already locked */
1328 }
1329 return (__u32)sun4c_lockarea(bufptr, len);
1330 }
1331
1332 static void sun4c_get_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
1333 {
1334 while (sz >= 0) {
1335 sg[sz].dvma_address = (__u32)sun4c_lockarea(sg[sz].address, sg[sz].length);
1336 sg[sz].dvma_length = sg[sz].length;
1337 sz--;
1338 }
1339 }
1340
1341 static void sun4c_release_scsi_one(__u32 bufptr, unsigned long len, struct sbus_bus *sbus)
1342 {
1343 if (bufptr < sun4c_iobuffer_start)
1344 return; /* On kernel stack or similar, see above */
1345 sun4c_unlockarea((char *)bufptr, len);
1346 }
1347
1348 static void sun4c_release_scsi_sgl(struct scatterlist *sg, int sz, struct sbus_bus *sbus)
1349 {
1350 while (sz >= 0) {
1351 sun4c_unlockarea((char *)sg[sz].dvma_address, sg[sz].length);
1352 sz--;
1353 }
1354 }
1355
1356 #define TASK_ENTRY_SIZE BUCKET_SIZE /* see above */
1357 #define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
1358
1359 struct vm_area_struct sun4c_kstack_vma;
1360
1361 static void __init sun4c_init_lock_areas(void)
1362 {
1363 unsigned long sun4c_taskstack_start;
1364 unsigned long sun4c_taskstack_end;
1365 int bitmap_size;
1366
1367 sun4c_init_buckets();
1368 sun4c_taskstack_start = SUN4C_LOCK_VADDR;
1369 sun4c_taskstack_end = (sun4c_taskstack_start +
1370 (TASK_ENTRY_SIZE * NR_TASK_BUCKETS));
1371 if (sun4c_taskstack_end >= SUN4C_LOCK_END) {
1372 prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n");
1373 prom_halt();
1374 }
1375
1376 sun4c_iobuffer_start = sun4c_iobuffer_high =
1377 SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end);
1378 sun4c_iobuffer_end = SUN4C_LOCK_END;
1379 bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT;
1380 bitmap_size = (bitmap_size + 7) >> 3;
1381 bitmap_size = LONG_ALIGN(bitmap_size);
1382 iobuffer_map_size = bitmap_size << 3;
1383 sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL);
1384 memset((void *) sun4c_iobuffer_map, 0, bitmap_size);
1385
1386 sun4c_kstack_vma.vm_mm = &init_mm;
1387 sun4c_kstack_vma.vm_start = sun4c_taskstack_start;
1388 sun4c_kstack_vma.vm_end = sun4c_taskstack_end;
1389 sun4c_kstack_vma.vm_page_prot = PAGE_SHARED;
1390 sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC;
1391 insert_vm_struct(&init_mm, &sun4c_kstack_vma);
1392 }
1393
1394 /* Cache flushing on the sun4c. */
1395 static void sun4c_flush_cache_all(void)
1396 {
1397 unsigned long begin, end;
1398
1399 flush_user_windows();
1400 begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE);
1401 end = (begin + SUN4C_VAC_SIZE);
1402
1403 if (sun4c_vacinfo.linesize == 32) {
1404 while (begin < end) {
1405 __asm__ __volatile__("
1406 ld [%0 + 0x00], %%g0
1407 ld [%0 + 0x20], %%g0
1408 ld [%0 + 0x40], %%g0
1409 ld [%0 + 0x60], %%g0
1410 ld [%0 + 0x80], %%g0
1411 ld [%0 + 0xa0], %%g0
1412 ld [%0 + 0xc0], %%g0
1413 ld [%0 + 0xe0], %%g0
1414 ld [%0 + 0x100], %%g0
1415 ld [%0 + 0x120], %%g0
1416 ld [%0 + 0x140], %%g0
1417 ld [%0 + 0x160], %%g0
1418 ld [%0 + 0x180], %%g0
1419 ld [%0 + 0x1a0], %%g0
1420 ld [%0 + 0x1c0], %%g0
1421 ld [%0 + 0x1e0], %%g0
1422 " : : "r" (begin));
1423 begin += 512;
1424 }
1425 } else {
1426 while (begin < end) {
1427 __asm__ __volatile__("
1428 ld [%0 + 0x00], %%g0
1429 ld [%0 + 0x10], %%g0
1430 ld [%0 + 0x20], %%g0
1431 ld [%0 + 0x30], %%g0
1432 ld [%0 + 0x40], %%g0
1433 ld [%0 + 0x50], %%g0
1434 ld [%0 + 0x60], %%g0
1435 ld [%0 + 0x70], %%g0
1436 ld [%0 + 0x80], %%g0
1437 ld [%0 + 0x90], %%g0
1438 ld [%0 + 0xa0], %%g0
1439 ld [%0 + 0xb0], %%g0
1440 ld [%0 + 0xc0], %%g0
1441 ld [%0 + 0xd0], %%g0
1442 ld [%0 + 0xe0], %%g0
1443 ld [%0 + 0xf0], %%g0
1444 " : : "r" (begin));
1445 begin += 256;
1446 }
1447 }
1448 }
1449
1450 static void sun4c_flush_cache_mm_hw(struct mm_struct *mm)
1451 {
1452 int new_ctx = mm->context;
1453
1454 if (new_ctx != NO_CONTEXT) {
1455 flush_user_windows();
1456 if (sun4c_context_ring[new_ctx].num_entries) {
1457 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1458 unsigned long flags;
1459
1460 save_and_cli(flags);
1461 if (head->next != head) {
1462 struct sun4c_mmu_entry *entry = head->next;
1463 int savectx = sun4c_get_context();
1464
1465 sun4c_set_context(new_ctx);
1466 sun4c_flush_context_hw();
1467 do {
1468 struct sun4c_mmu_entry *next = entry->next;
1469
1470 sun4c_user_unmap(entry);
1471 free_user_entry(new_ctx, entry);
1472
1473 entry = next;
1474 } while (entry != head);
1475 sun4c_set_context(savectx);
1476 }
1477 restore_flags(flags);
1478 }
1479 }
1480 }
1481
1482 static void sun4c_flush_cache_range_hw(struct mm_struct *mm, unsigned long start, unsigned long end)
1483 {
1484 int new_ctx = mm->context;
1485
1486 if (new_ctx != NO_CONTEXT) {
1487 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1488 struct sun4c_mmu_entry *entry;
1489 unsigned long flags;
1490
1491 flush_user_windows();
1492
1493 save_and_cli(flags);
1494
1495 /* All user segmap chains are ordered on entry->vaddr. */
1496 for (entry = head->next;
1497 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1498 entry = entry->next)
1499 ;
1500
1501 /* Tracing various job mixtures showed that this conditional
1502 * only passes ~35% of the time for most worse case situations,
1503 * therefore we avoid all of this gross overhead ~65% of the time.
1504 */
1505 if ((entry != head) && (entry->vaddr < end)) {
1506 int octx = sun4c_get_context();
1507
1508 sun4c_set_context(new_ctx);
1509
1510 /* At this point, always, (start >= entry->vaddr) and
1511 * (entry->vaddr < end), once the latter condition
1512 * ceases to hold, or we hit the end of the list, we
1513 * exit the loop. The ordering of all user allocated
1514 * segmaps makes this all work out so beautifully.
1515 */
1516 do {
1517 struct sun4c_mmu_entry *next = entry->next;
1518 unsigned long realend;
1519
1520 /* "realstart" is always >= entry->vaddr */
1521 realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
1522 if (end < realend)
1523 realend = end;
1524 if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
1525 unsigned long page = entry->vaddr;
1526 while (page < realend) {
1527 sun4c_flush_page_hw(page);
1528 page += PAGE_SIZE;
1529 }
1530 } else {
1531 sun4c_flush_segment_hw(entry->vaddr);
1532 sun4c_user_unmap(entry);
1533 free_user_entry(new_ctx, entry);
1534 }
1535 entry = next;
1536 } while ((entry != head) && (entry->vaddr < end));
1537 sun4c_set_context(octx);
1538 }
1539 restore_flags(flags);
1540 }
1541 }
1542
1543 static void sun4c_flush_cache_page_hw(struct vm_area_struct *vma, unsigned long page)
1544 {
1545 struct mm_struct *mm = vma->vm_mm;
1546 int new_ctx = mm->context;
1547
1548 /* Sun4c has no separate I/D caches so cannot optimize for non
1549 * text page flushes.
1550 */
1551 if (new_ctx != NO_CONTEXT) {
1552 int octx = sun4c_get_context();
1553 unsigned long flags;
1554
1555 flush_user_windows();
1556 save_and_cli(flags);
1557 sun4c_set_context(new_ctx);
1558 sun4c_flush_page_hw(page);
1559 sun4c_set_context(octx);
1560 restore_flags(flags);
1561 }
1562 }
1563
1564 static void sun4c_flush_page_to_ram_hw(unsigned long page)
1565 {
1566 unsigned long flags;
1567
1568 save_and_cli(flags);
1569 sun4c_flush_page_hw(page);
1570 restore_flags(flags);
1571 }
1572
1573 static void sun4c_flush_cache_mm_sw(struct mm_struct *mm)
1574 {
1575 int new_ctx = mm->context;
1576
1577 if (new_ctx != NO_CONTEXT) {
1578 flush_user_windows();
1579
1580 if (sun4c_context_ring[new_ctx].num_entries) {
1581 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1582 unsigned long flags;
1583
1584 save_and_cli(flags);
1585 if (head->next != head) {
1586 struct sun4c_mmu_entry *entry = head->next;
1587 int savectx = sun4c_get_context();
1588
1589 sun4c_set_context(new_ctx);
1590 sun4c_flush_context_sw();
1591 do {
1592 struct sun4c_mmu_entry *next = entry->next;
1593
1594 sun4c_user_unmap(entry);
1595 free_user_entry(new_ctx, entry);
1596
1597 entry = next;
1598 } while (entry != head);
1599 sun4c_set_context(savectx);
1600 }
1601 restore_flags(flags);
1602 }
1603 }
1604 }
1605
1606 static void sun4c_flush_cache_range_sw(struct mm_struct *mm, unsigned long start, unsigned long end)
1607 {
1608 int new_ctx = mm->context;
1609
1610 if (new_ctx != NO_CONTEXT) {
1611 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1612 struct sun4c_mmu_entry *entry;
1613 unsigned long flags;
1614
1615 flush_user_windows();
1616
1617 save_and_cli(flags);
1618 /* All user segmap chains are ordered on entry->vaddr. */
1619 for (entry = head->next;
1620 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1621 entry = entry->next)
1622 ;
1623
1624 /* Tracing various job mixtures showed that this conditional
1625 * only passes ~35% of the time for most worse case situations,
1626 * therefore we avoid all of this gross overhead ~65% of the time.
1627 */
1628 if ((entry != head) && (entry->vaddr < end)) {
1629 int octx = sun4c_get_context();
1630 sun4c_set_context(new_ctx);
1631
1632 /* At this point, always, (start >= entry->vaddr) and
1633 * (entry->vaddr < end), once the latter condition
1634 * ceases to hold, or we hit the end of the list, we
1635 * exit the loop. The ordering of all user allocated
1636 * segmaps makes this all work out so beautifully.
1637 */
1638 do {
1639 struct sun4c_mmu_entry *next = entry->next;
1640 unsigned long realend;
1641
1642 /* "realstart" is always >= entry->vaddr */
1643 realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE;
1644 if (end < realend)
1645 realend = end;
1646 if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) {
1647 unsigned long page = entry->vaddr;
1648 while (page < realend) {
1649 sun4c_flush_page_sw(page);
1650 page += PAGE_SIZE;
1651 }
1652 } else {
1653 sun4c_flush_segment_sw(entry->vaddr);
1654 sun4c_user_unmap(entry);
1655 free_user_entry(new_ctx, entry);
1656 }
1657 entry = next;
1658 } while ((entry != head) && (entry->vaddr < end));
1659 sun4c_set_context(octx);
1660 }
1661 restore_flags(flags);
1662 }
1663 }
1664
1665 static void sun4c_flush_cache_page_sw(struct vm_area_struct *vma, unsigned long page)
1666 {
1667 struct mm_struct *mm = vma->vm_mm;
1668 int new_ctx = mm->context;
1669
1670 /* Sun4c has no separate I/D caches so cannot optimize for non
1671 * text page flushes.
1672 */
1673 if (new_ctx != NO_CONTEXT) {
1674 int octx = sun4c_get_context();
1675 unsigned long flags;
1676
1677 flush_user_windows();
1678 save_and_cli(flags);
1679 sun4c_set_context(new_ctx);
1680 sun4c_flush_page_sw(page);
1681 sun4c_set_context(octx);
1682 restore_flags(flags);
1683 }
1684 }
1685
1686 static void sun4c_flush_page_to_ram_sw(unsigned long page)
1687 {
1688 unsigned long flags;
1689
1690 save_and_cli(flags);
1691 sun4c_flush_page_sw(page);
1692 restore_flags(flags);
1693 }
1694
1695 /* Sun4c cache is unified, both instructions and data live there, so
1696 * no need to flush the on-stack instructions for new signal handlers.
1697 */
1698 static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
1699 {
1700 }
1701
1702 /* TLB flushing on the sun4c. These routines count on the cache
1703 * flushing code to flush the user register windows so that we need
1704 * not do so when we get here.
1705 */
1706
1707 static void sun4c_flush_tlb_all(void)
1708 {
1709 struct sun4c_mmu_entry *this_entry, *next_entry;
1710 unsigned long flags;
1711 int savectx, ctx;
1712
1713 save_and_cli(flags);
1714 this_entry = sun4c_kernel_ring.ringhd.next;
1715 savectx = sun4c_get_context();
1716 flush_user_windows();
1717 while (sun4c_kernel_ring.num_entries) {
1718 next_entry = this_entry->next;
1719 if (sun4c_vacinfo.do_hwflushes)
1720 sun4c_flush_segment_hw(this_entry->vaddr);
1721 else
1722 sun4c_flush_segment_sw(this_entry->vaddr);
1723 for (ctx = 0; ctx < num_contexts; ctx++) {
1724 sun4c_set_context(ctx);
1725 sun4c_put_segmap(this_entry->vaddr, invalid_segment);
1726 }
1727 free_kernel_entry(this_entry, &sun4c_kernel_ring);
1728 this_entry = next_entry;
1729 }
1730 sun4c_set_context(savectx);
1731 restore_flags(flags);
1732 }
1733
1734 static void sun4c_flush_tlb_mm_hw(struct mm_struct *mm)
1735 {
1736 int new_ctx = mm->context;
1737
1738 if (new_ctx != NO_CONTEXT) {
1739 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1740 unsigned long flags;
1741
1742 save_and_cli(flags);
1743 if (head->next != head) {
1744 struct sun4c_mmu_entry *entry = head->next;
1745 int savectx = sun4c_get_context();
1746
1747 sun4c_set_context(new_ctx);
1748 sun4c_flush_context_hw();
1749 do {
1750 struct sun4c_mmu_entry *next = entry->next;
1751
1752 sun4c_user_unmap(entry);
1753 free_user_entry(new_ctx, entry);
1754
1755 entry = next;
1756 } while (entry != head);
1757 sun4c_set_context(savectx);
1758 }
1759 restore_flags(flags);
1760 }
1761 }
1762
1763 static void sun4c_flush_tlb_range_hw(struct mm_struct *mm, unsigned long start, unsigned long end)
1764 {
1765 int new_ctx = mm->context;
1766
1767 if (new_ctx != NO_CONTEXT) {
1768 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1769 struct sun4c_mmu_entry *entry;
1770 unsigned long flags;
1771
1772 save_and_cli(flags);
1773 /* See commentary in sun4c_flush_cache_range_*(). */
1774 for (entry = head->next;
1775 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1776 entry = entry->next)
1777 ;
1778
1779 if ((entry != head) && (entry->vaddr < end)) {
1780 int octx = sun4c_get_context();
1781
1782 sun4c_set_context(new_ctx);
1783 do {
1784 struct sun4c_mmu_entry *next = entry->next;
1785
1786 sun4c_flush_segment_hw(entry->vaddr);
1787 sun4c_user_unmap(entry);
1788 free_user_entry(new_ctx, entry);
1789
1790 entry = next;
1791 } while ((entry != head) && (entry->vaddr < end));
1792 sun4c_set_context(octx);
1793 }
1794 restore_flags(flags);
1795 }
1796 }
1797
1798 static void sun4c_flush_tlb_page_hw(struct vm_area_struct *vma, unsigned long page)
1799 {
1800 struct mm_struct *mm = vma->vm_mm;
1801 int new_ctx = mm->context;
1802
1803 if (new_ctx != NO_CONTEXT) {
1804 int savectx = sun4c_get_context();
1805 unsigned long flags;
1806
1807 save_and_cli(flags);
1808 sun4c_set_context(new_ctx);
1809 page &= PAGE_MASK;
1810 sun4c_flush_page_hw(page);
1811 sun4c_put_pte(page, 0);
1812 sun4c_set_context(savectx);
1813 restore_flags(flags);
1814 }
1815 }
1816
1817 static void sun4c_flush_tlb_mm_sw(struct mm_struct *mm)
1818 {
1819 int new_ctx = mm->context;
1820
1821 if (new_ctx != NO_CONTEXT) {
1822 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1823 unsigned long flags;
1824
1825 save_and_cli(flags);
1826 if (head->next != head) {
1827 struct sun4c_mmu_entry *entry = head->next;
1828 int savectx = sun4c_get_context();
1829
1830 sun4c_set_context(new_ctx);
1831 sun4c_flush_context_sw();
1832 do {
1833 struct sun4c_mmu_entry *next = entry->next;
1834
1835 sun4c_user_unmap(entry);
1836 free_user_entry(new_ctx, entry);
1837
1838 entry = next;
1839 } while (entry != head);
1840 sun4c_set_context(savectx);
1841 }
1842 restore_flags(flags);
1843 }
1844 }
1845
1846 static void sun4c_flush_tlb_range_sw(struct mm_struct *mm, unsigned long start, unsigned long end)
1847 {
1848 int new_ctx = mm->context;
1849
1850 if (new_ctx != NO_CONTEXT) {
1851 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd;
1852 struct sun4c_mmu_entry *entry;
1853 unsigned long flags;
1854
1855 save_and_cli(flags);
1856 /* See commentary in sun4c_flush_cache_range_*(). */
1857 for (entry = head->next;
1858 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start);
1859 entry = entry->next)
1860 ;
1861
1862 if ((entry != head) && (entry->vaddr < end)) {
1863 int octx = sun4c_get_context();
1864
1865 sun4c_set_context(new_ctx);
1866 do {
1867 struct sun4c_mmu_entry *next = entry->next;
1868
1869 sun4c_flush_segment_sw(entry->vaddr);
1870 sun4c_user_unmap(entry);
1871 free_user_entry(new_ctx, entry);
1872
1873 entry = next;
1874 } while ((entry != head) && (entry->vaddr < end));
1875 sun4c_set_context(octx);
1876 }
1877 restore_flags(flags);
1878 }
1879 }
1880
1881 static void sun4c_flush_tlb_page_sw(struct vm_area_struct *vma, unsigned long page)
1882 {
1883 struct mm_struct *mm = vma->vm_mm;
1884 int new_ctx = mm->context;
1885
1886 if (new_ctx != NO_CONTEXT) {
1887 int savectx = sun4c_get_context();
1888 unsigned long flags;
1889
1890 save_and_cli(flags);
1891 sun4c_set_context(new_ctx);
1892 page &= PAGE_MASK;
1893 sun4c_flush_page_sw(page);
1894 sun4c_put_pte(page, 0);
1895 sun4c_set_context(savectx);
1896 restore_flags(flags);
1897 }
1898 }
1899
1900 static void sun4c_set_pte(pte_t *ptep, pte_t pte)
1901 {
1902 *ptep = pte;
1903 }
1904
1905 static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp)
1906 {
1907 }
1908
1909 static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep)
1910 {
1911 }
1912
1913 void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr,
1914 int bus_type, int rdonly)
1915 {
1916 unsigned long page_entry;
1917
1918 page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK);
1919 page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT));
1920 if (rdonly)
1921 page_entry &= ~_SUN4C_WRITEABLE;
1922 sun4c_put_pte(virt_addr, page_entry);
1923 }
1924
1925 void sun4c_unmapioaddr(unsigned long virt_addr)
1926 {
1927 sun4c_put_pte(virt_addr, 0);
1928 }
1929
1930 static void sun4c_alloc_context_hw(struct mm_struct *old_mm, struct mm_struct *mm)
1931 {
1932 struct ctx_list *ctxp;
1933
1934 ctxp = ctx_free.next;
1935 if (ctxp != &ctx_free) {
1936 remove_from_ctx_list(ctxp);
1937 add_to_used_ctxlist(ctxp);
1938 mm->context = ctxp->ctx_number;
1939 ctxp->ctx_mm = mm;
1940 return;
1941 }
1942 ctxp = ctx_used.next;
1943 if (ctxp->ctx_mm == old_mm)
1944 ctxp = ctxp->next;
1945 remove_from_ctx_list(ctxp);
1946 add_to_used_ctxlist(ctxp);
1947 ctxp->ctx_mm->context = NO_CONTEXT;
1948 ctxp->ctx_mm = mm;
1949 mm->context = ctxp->ctx_number;
1950 sun4c_demap_context_hw(&sun4c_context_ring[ctxp->ctx_number],
1951 ctxp->ctx_number);
1952 }
1953
1954 /* Switch the current MM context. */
1955 static void sun4c_switch_mm_hw(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu)
1956 {
1957 struct ctx_list *ctx;
1958 int dirty = 0;
1959
1960 if (mm->context == NO_CONTEXT) {
1961 dirty = 1;
1962 sun4c_alloc_context_hw(old_mm, mm);
1963 } else {
1964 /* Update the LRU ring of contexts. */
1965 ctx = ctx_list_pool + mm->context;
1966 remove_from_ctx_list(ctx);
1967 add_to_used_ctxlist(ctx);
1968 }
1969 if (dirty || old_mm != mm)
1970 sun4c_set_context(mm->context);
1971 }
1972
1973 static void sun4c_destroy_context_hw(struct mm_struct *mm)
1974 {
1975 struct ctx_list *ctx_old;
1976
1977 if (mm->context != NO_CONTEXT) {
1978 sun4c_demap_context_hw(&sun4c_context_ring[mm->context], mm->context);
1979 ctx_old = ctx_list_pool + mm->context;
1980 remove_from_ctx_list(ctx_old);
1981 add_to_free_ctxlist(ctx_old);
1982 mm->context = NO_CONTEXT;
1983 }
1984 }
1985
1986 static void sun4c_alloc_context_sw(struct mm_struct *old_mm, struct mm_struct *mm)
1987 {
1988 struct ctx_list *ctxp;
1989
1990 ctxp = ctx_free.next;
1991 if (ctxp != &ctx_free) {
1992 remove_from_ctx_list(ctxp);
1993 add_to_used_ctxlist(ctxp);
1994 mm->context = ctxp->ctx_number;
1995 ctxp->ctx_mm = mm;
1996 return;
1997 }
1998 ctxp = ctx_used.next;
1999 if(ctxp->ctx_mm == old_mm)
2000 ctxp = ctxp->next;
2001 remove_from_ctx_list(ctxp);
2002 add_to_used_ctxlist(ctxp);
2003 ctxp->ctx_mm->context = NO_CONTEXT;
2004 ctxp->ctx_mm = mm;
2005 mm->context = ctxp->ctx_number;
2006 sun4c_demap_context_sw(&sun4c_context_ring[ctxp->ctx_number],
2007 ctxp->ctx_number);
2008 }
2009
2010 /* Switch the current MM context. */
2011 static void sun4c_switch_mm_sw(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu)
2012 {
2013 struct ctx_list *ctx;
2014 int dirty = 0;
2015
2016 if (mm->context == NO_CONTEXT) {
2017 dirty = 1;
2018 sun4c_alloc_context_sw(old_mm, mm);
2019 } else {
2020 /* Update the LRU ring of contexts. */
2021 ctx = ctx_list_pool + mm->context;
2022 remove_from_ctx_list(ctx);
2023 add_to_used_ctxlist(ctx);
2024 }
2025
2026 if (dirty || old_mm != mm)
2027 sun4c_set_context(mm->context);
2028 }
2029
2030 static void sun4c_destroy_context_sw(struct mm_struct *mm)
2031 {
2032 struct ctx_list *ctx_old;
2033
2034 if (mm->context != NO_CONTEXT) {
2035 sun4c_demap_context_sw(&sun4c_context_ring[mm->context], mm->context);
2036 ctx_old = ctx_list_pool + mm->context;
2037 remove_from_ctx_list(ctx_old);
2038 add_to_free_ctxlist(ctx_old);
2039 mm->context = NO_CONTEXT;
2040 }
2041 }
2042
2043 static int sun4c_mmu_info(char *buf)
2044 {
2045 int used_user_entries, i;
2046 int len;
2047
2048 used_user_entries = 0;
2049 for (i = 0; i < num_contexts; i++)
2050 used_user_entries += sun4c_context_ring[i].num_entries;
2051
2052 len = sprintf(buf,
2053 "vacsize\t\t: %d bytes\n"
2054 "vachwflush\t: %s\n"
2055 "vaclinesize\t: %d bytes\n"
2056 "mmuctxs\t\t: %d\n"
2057 "mmupsegs\t: %d\n"
2058 "kernelpsegs\t: %d\n"
2059 "kfreepsegs\t: %d\n"
2060 "usedpsegs\t: %d\n"
2061 "ufreepsegs\t: %d\n"
2062 "user_taken\t: %d\n"
2063 "max_taken\t: %d\n",
2064 sun4c_vacinfo.num_bytes,
2065 (sun4c_vacinfo.do_hwflushes ? "yes" : "no"),
2066 sun4c_vacinfo.linesize,
2067 num_contexts,
2068 (invalid_segment + 1),
2069 sun4c_kernel_ring.num_entries,
2070 sun4c_kfree_ring.num_entries,
2071 used_user_entries,
2072 sun4c_ufree_ring.num_entries,
2073 sun4c_user_taken_entries,
2074 max_user_taken_entries);
2075
2076 return len;
2077 }
2078
2079 /* Nothing below here should touch the mmu hardware nor the mmu_entry
2080 * data structures.
2081 */
2082
2083 /* First the functions which the mid-level code uses to directly
2084 * manipulate the software page tables. Some defines since we are
2085 * emulating the i386 page directory layout.
2086 */
2087 #define PGD_PRESENT 0x001
2088 #define PGD_RW 0x002
2089 #define PGD_USER 0x004
2090 #define PGD_ACCESSED 0x020
2091 #define PGD_DIRTY 0x040
2092 #define PGD_TABLE (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY)
2093
2094 static int sun4c_pte_present(pte_t pte)
2095 {
2096 return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0);
2097 }
2098 static void sun4c_pte_clear(pte_t *ptep) { *ptep = __pte(0); }
2099
2100 static int sun4c_pmd_none(pmd_t pmd) { return !pmd_val(pmd); }
2101 static int sun4c_pmd_bad(pmd_t pmd)
2102 {
2103 return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) ||
2104 (!VALID_PAGE(virt_to_page(pmd_val(pmd)))));
2105 }
2106
2107 static int sun4c_pmd_present(pmd_t pmd)
2108 {
2109 return ((pmd_val(pmd) & PGD_PRESENT) != 0);
2110 }
2111 static void sun4c_pmd_clear(pmd_t *pmdp) { *pmdp = __pmd(0); }
2112
2113 static int sun4c_pgd_none(pgd_t pgd) { return 0; }
2114 static int sun4c_pgd_bad(pgd_t pgd) { return 0; }
2115 static int sun4c_pgd_present(pgd_t pgd) { return 1; }
2116 static void sun4c_pgd_clear(pgd_t * pgdp) { }
2117
2118 /*
2119 * The following only work if pte_present() is true.
2120 * Undefined behaviour if not..
2121 */
2122 static pte_t sun4c_pte_mkwrite(pte_t pte)
2123 {
2124 pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE);
2125 if (pte_val(pte) & _SUN4C_PAGE_MODIFIED)
2126 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
2127 return pte;
2128 }
2129
2130 static pte_t sun4c_pte_mkdirty(pte_t pte)
2131 {
2132 pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED);
2133 if (pte_val(pte) & _SUN4C_PAGE_WRITE)
2134 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE);
2135 return pte;
2136 }
2137
2138 static pte_t sun4c_pte_mkyoung(pte_t pte)
2139 {
2140 pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED);
2141 if (pte_val(pte) & _SUN4C_PAGE_READ)
2142 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ);
2143 return pte;
2144 }
2145
2146 /*
2147 * Conversion functions: convert a page and protection to a page entry,
2148 * and a page entry and page directory to the page they refer to.
2149 */
2150 static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot)
2151 {
2152 return __pte((page - mem_map) | pgprot_val(pgprot));
2153 }
2154
2155 static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot)
2156 {
2157 return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot));
2158 }
2159
2160 static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space)
2161 {
2162 return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot));
2163 }
2164
2165 static struct page *sun4c_pte_page(pte_t pte)
2166 {
2167 return (mem_map + (unsigned long)(pte_val(pte) & SUN4C_PFN_MASK));
2168 }
2169
2170 static inline unsigned long sun4c_pmd_page(pmd_t pmd)
2171 {
2172 return (pmd_val(pmd) & PAGE_MASK);
2173 }
2174
2175 static unsigned long sun4c_pgd_page(pgd_t pgd)
2176 {
2177 return 0;
2178 }
2179
2180 /* to find an entry in a page-table-directory */
2181 static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address)
2182 {
2183 return mm->pgd + (address >> SUN4C_PGDIR_SHIFT);
2184 }
2185
2186 /* Find an entry in the second-level page table.. */
2187 static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address)
2188 {
2189 return (pmd_t *) dir;
2190 }
2191
2192 /* Find an entry in the third-level page table.. */
2193 pte_t *sun4c_pte_offset(pmd_t * dir, unsigned long address)
2194 {
2195 return (pte_t *) sun4c_pmd_page(*dir) + ((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1));
2196 }
2197
2198 static void sun4c_free_pte_slow(pte_t *pte)
2199 {
2200 free_page((unsigned long)pte);
2201 }
2202
2203 static void sun4c_free_pgd_slow(pgd_t *pgd)
2204 {
2205 free_page((unsigned long)pgd);
2206 }
2207
2208 static pgd_t *sun4c_get_pgd_fast(void)
2209 {
2210 unsigned long *ret;
2211
2212 if ((ret = pgd_quicklist) != NULL) {
2213 pgd_quicklist = (unsigned long *)(*ret);
2214 ret[0] = ret[1];
2215 pgtable_cache_size--;
2216 } else {
2217 pgd_t *init;
2218
2219 ret = (unsigned long *)__get_free_page(GFP_KERNEL);
2220 memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t));
2221 init = sun4c_pgd_offset(&init_mm, 0);
2222 memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
2223 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
2224 }
2225 return (pgd_t *)ret;
2226 }
2227
2228 static void sun4c_free_pgd_fast(pgd_t *pgd)
2229 {
2230 *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
2231 pgd_quicklist = (unsigned long *) pgd;
2232 pgtable_cache_size++;
2233 }
2234
2235 static pte_t *sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address)
2236 {
2237 pte_t *pte = (pte_t *)__get_free_page(GFP_KERNEL);
2238 if (pte)
2239 memset(pte, 0, PAGE_SIZE);
2240 return pte;
2241 }
2242
2243 pte_t *sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
2244 {
2245 unsigned long *ret;
2246
2247 if ((ret = (unsigned long *)pte_quicklist) != NULL) {
2248 pte_quicklist = (unsigned long *)(*ret);
2249 ret[0] = ret[1];
2250 pgtable_cache_size--;
2251 }
2252 return (pte_t *)ret;
2253 }
2254
2255 static __inline__ void sun4c_free_pte_fast(pte_t *pte)
2256 {
2257 *(unsigned long *)pte = (unsigned long) pte_quicklist;
2258 pte_quicklist = (unsigned long *) pte;
2259 pgtable_cache_size++;
2260 }
2261
2262 /*
2263 * allocating and freeing a pmd is trivial: the 1-entry pmd is
2264 * inside the pgd, so has no extra memory associated with it.
2265 */
2266 static pmd_t *sun4c_pmd_alloc_one_fast(struct mm_struct *mm, unsigned long address)
2267 {
2268 BUG();
2269 return NULL;
2270 }
2271
2272 static void sun4c_free_pmd_fast(pmd_t * pmd)
2273 {
2274 }
2275
2276 static int sun4c_check_pgt_cache(int low, int high)
2277 {
2278 int freed = 0;
2279 if (pgtable_cache_size > high) {
2280 do {
2281 if (pgd_quicklist)
2282 sun4c_free_pgd_slow(sun4c_get_pgd_fast()), freed++;
2283 if (pte_quicklist)
2284 sun4c_free_pte_slow(sun4c_pte_alloc_one_fast(NULL, 0)), freed++;
2285 } while (pgtable_cache_size > low);
2286 }
2287 return freed;
2288 }
2289
2290 /* An experiment, turn off by default for now... -DaveM */
2291 #define SUN4C_PRELOAD_PSEG
2292
2293 void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t pte)
2294 {
2295 unsigned long flags;
2296 int pseg;
2297
2298 save_and_cli(flags);
2299 address &= PAGE_MASK;
2300 if ((pseg = sun4c_get_segmap(address)) == invalid_segment) {
2301 struct sun4c_mmu_entry *entry = sun4c_user_strategy();
2302 struct mm_struct *mm = vma->vm_mm;
2303 unsigned long start, end;
2304
2305 entry->vaddr = start = (address & SUN4C_REAL_PGDIR_MASK);
2306 entry->ctx = mm->context;
2307 add_ring_ordered(sun4c_context_ring + mm->context, entry);
2308 sun4c_put_segmap(entry->vaddr, entry->pseg);
2309 end = start + SUN4C_REAL_PGDIR_SIZE;
2310 while (start < end) {
2311 #ifdef SUN4C_PRELOAD_PSEG
2312 pgd_t *pgdp = sun4c_pgd_offset(mm, start);
2313 pte_t *ptep;
2314
2315 if (!pgdp)
2316 goto no_mapping;
2317 ptep = sun4c_pte_offset((pmd_t *) pgdp, start);
2318 if (!ptep || !(pte_val(*ptep) & _SUN4C_PAGE_PRESENT))
2319 goto no_mapping;
2320 sun4c_put_pte(start, pte_val(*ptep));
2321 goto next;
2322
2323 no_mapping:
2324 #endif
2325 sun4c_put_pte(start, 0);
2326 #ifdef SUN4C_PRELOAD_PSEG
2327 next:
2328 #endif
2329 start += PAGE_SIZE;
2330 }
2331 #ifndef SUN4C_PRELOAD_PSEG
2332 sun4c_put_pte(address, pte_val(pte));
2333 #endif
2334 restore_flags(flags);
2335 return;
2336 } else {
2337 struct sun4c_mmu_entry *entry = &mmu_entry_pool[pseg];
2338
2339 remove_lru(entry);
2340 add_lru(entry);
2341 }
2342
2343 sun4c_put_pte(address, pte_val(pte));
2344 restore_flags(flags);
2345 }
2346
2347 extern void sparc_context_init(int);
2348 extern unsigned long end;
2349 extern unsigned long bootmem_init(unsigned long *pages_avail);
2350 extern unsigned long last_valid_pfn;
2351 extern void sun_serial_setup(void);
2352
2353 void __init sun4c_paging_init(void)
2354 {
2355 int i, cnt;
2356 unsigned long kernel_end, vaddr;
2357 extern struct resource sparc_iomap;
2358 unsigned long end_pfn, pages_avail;
2359
2360 kernel_end = (unsigned long) &end;
2361 kernel_end += (SUN4C_REAL_PGDIR_SIZE * 4);
2362 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end);
2363
2364 pages_avail = 0;
2365 last_valid_pfn = bootmem_init(&pages_avail);
2366 end_pfn = last_valid_pfn;
2367
2368 /* This does not logically belong here, but we need to
2369 * call it at the moment we are able to use the bootmem
2370 * allocator.
2371 */
2372 sun_serial_setup();
2373
2374 sun4c_probe_mmu();
2375 invalid_segment = (num_segmaps - 1);
2376 sun4c_init_mmu_entry_pool();
2377 sun4c_init_rings();
2378 sun4c_init_map_kernelprom(kernel_end);
2379 sun4c_init_clean_mmu(kernel_end);
2380 sun4c_init_fill_kernel_ring(SUN4C_KERNEL_BUCKETS);
2381 sun4c_init_lock_area(sparc_iomap.start, IOBASE_END);
2382 sun4c_init_lock_area(DVMA_VADDR, DVMA_END);
2383 sun4c_init_lock_areas();
2384 sun4c_init_fill_user_ring();
2385
2386 sun4c_set_context(0);
2387 memset(swapper_pg_dir, 0, PAGE_SIZE);
2388 memset(pg0, 0, PAGE_SIZE);
2389 memset(pg1, 0, PAGE_SIZE);
2390 memset(pg2, 0, PAGE_SIZE);
2391 memset(pg3, 0, PAGE_SIZE);
2392
2393 /* Save work later. */
2394 vaddr = VMALLOC_START;
2395 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg0);
2396 vaddr += SUN4C_PGDIR_SIZE;
2397 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg1);
2398 vaddr += SUN4C_PGDIR_SIZE;
2399 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg2);
2400 vaddr += SUN4C_PGDIR_SIZE;
2401 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg3);
2402 sun4c_init_ss2_cache_bug();
2403 sparc_context_init(num_contexts);
2404
2405 {
2406 unsigned long zones_size[MAX_NR_ZONES];
2407 unsigned long zholes_size[MAX_NR_ZONES];
2408 unsigned long npages;
2409 int znum;
2410
2411 for (znum = 0; znum < MAX_NR_ZONES; znum++)
2412 zones_size[znum] = zholes_size[znum] = 0;
2413
2414 npages = max_low_pfn - (phys_base >> PAGE_SHIFT);
2415
2416 zones_size[ZONE_DMA] = npages;
2417 zholes_size[ZONE_DMA] = npages - pages_avail;
2418
2419 npages = highend_pfn - max_low_pfn;
2420 zones_size[ZONE_HIGHMEM] = npages;
2421 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages();
2422
2423 free_area_init_node(0, NULL, NULL, zones_size,
2424 phys_base, zholes_size);
2425 }
2426
2427 cnt = 0;
2428 for (i = 0; i < num_segmaps; i++)
2429 if (mmu_entry_pool[i].locked)
2430 cnt++;
2431
2432 max_user_taken_entries = num_segmaps - cnt - 40 - 1;
2433
2434 printk("SUN4C: %d mmu entries for the kernel\n", cnt);
2435 }
2436
2437 /* Load up routines and constants for sun4c mmu */
2438 void __init ld_mmu_sun4c(void)
2439 {
2440 extern void ___xchg32_sun4c(void);
2441
2442 printk("Loading sun4c MMU routines\n");
2443
2444 /* First the constants */
2445 BTFIXUPSET_SIMM13(pmd_shift, SUN4C_PMD_SHIFT);
2446 BTFIXUPSET_SETHI(pmd_size, SUN4C_PMD_SIZE);
2447 BTFIXUPSET_SETHI(pmd_mask, SUN4C_PMD_MASK);
2448 BTFIXUPSET_SIMM13(pgdir_shift, SUN4C_PGDIR_SHIFT);
2449 BTFIXUPSET_SETHI(pgdir_size, SUN4C_PGDIR_SIZE);
2450 BTFIXUPSET_SETHI(pgdir_mask, SUN4C_PGDIR_MASK);
2451
2452 BTFIXUPSET_SIMM13(ptrs_per_pte, SUN4C_PTRS_PER_PTE);
2453 BTFIXUPSET_SIMM13(ptrs_per_pmd, SUN4C_PTRS_PER_PMD);
2454 BTFIXUPSET_SIMM13(ptrs_per_pgd, SUN4C_PTRS_PER_PGD);
2455 BTFIXUPSET_SIMM13(user_ptrs_per_pgd, KERNBASE / SUN4C_PGDIR_SIZE);
2456
2457 BTFIXUPSET_INT(page_none, pgprot_val(SUN4C_PAGE_NONE));
2458 BTFIXUPSET_INT(page_shared, pgprot_val(SUN4C_PAGE_SHARED));
2459 BTFIXUPSET_INT(page_copy, pgprot_val(SUN4C_PAGE_COPY));
2460 BTFIXUPSET_INT(page_readonly, pgprot_val(SUN4C_PAGE_READONLY));
2461 BTFIXUPSET_INT(page_kernel, pgprot_val(SUN4C_PAGE_KERNEL));
2462 page_kernel = pgprot_val(SUN4C_PAGE_KERNEL);
2463 pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE |
2464 _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE;
2465
2466 /* Functions */
2467 #ifndef CONFIG_SMP
2468 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4c, BTFIXUPCALL_NORM);
2469 #endif
2470 BTFIXUPSET_CALL(do_check_pgt_cache, sun4c_check_pgt_cache, BTFIXUPCALL_NORM);
2471
2472 BTFIXUPSET_CALL(flush_cache_all, sun4c_flush_cache_all, BTFIXUPCALL_NORM);
2473
2474 if (sun4c_vacinfo.do_hwflushes) {
2475 BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm_hw, BTFIXUPCALL_NORM);
2476 BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range_hw, BTFIXUPCALL_NORM);
2477 BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page_hw, BTFIXUPCALL_NORM);
2478 BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram_hw, BTFIXUPCALL_NORM);
2479 BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm_hw, BTFIXUPCALL_NORM);
2480 BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range_hw, BTFIXUPCALL_NORM);
2481 BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page_hw, BTFIXUPCALL_NORM);
2482 BTFIXUPSET_CALL(free_task_struct, sun4c_free_task_struct_hw, BTFIXUPCALL_NORM);
2483 BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm_hw, BTFIXUPCALL_NORM);
2484 BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context_hw, BTFIXUPCALL_NORM);
2485 } else {
2486 BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm_sw, BTFIXUPCALL_NORM);
2487 BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range_sw, BTFIXUPCALL_NORM);
2488 BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page_sw, BTFIXUPCALL_NORM);
2489 BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram_sw, BTFIXUPCALL_NORM);
2490 BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm_sw, BTFIXUPCALL_NORM);
2491 BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range_sw, BTFIXUPCALL_NORM);
2492 BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page_sw, BTFIXUPCALL_NORM);
2493 BTFIXUPSET_CALL(free_task_struct, sun4c_free_task_struct_sw, BTFIXUPCALL_NORM);
2494 BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm_sw, BTFIXUPCALL_NORM);
2495 BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context_sw, BTFIXUPCALL_NORM);
2496 }
2497
2498 BTFIXUPSET_CALL(flush_tlb_all, sun4c_flush_tlb_all, BTFIXUPCALL_NORM);
2499
2500 BTFIXUPSET_CALL(flush_sig_insns, sun4c_flush_sig_insns, BTFIXUPCALL_NOP);
2501
2502 BTFIXUPSET_CALL(set_pte, sun4c_set_pte, BTFIXUPCALL_STO1O0);
2503
2504 BTFIXUPSET_CALL(pte_page, sun4c_pte_page, BTFIXUPCALL_NORM);
2505 #if PAGE_SHIFT <= 12
2506 BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_ANDNINT(PAGE_SIZE - 1));
2507 #else
2508 BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_NORM);
2509 #endif
2510
2511 BTFIXUPSET_CALL(pte_present, sun4c_pte_present, BTFIXUPCALL_NORM);
2512 BTFIXUPSET_CALL(pte_clear, sun4c_pte_clear, BTFIXUPCALL_STG0O0);
2513
2514 BTFIXUPSET_CALL(pmd_bad, sun4c_pmd_bad, BTFIXUPCALL_NORM);
2515 BTFIXUPSET_CALL(pmd_present, sun4c_pmd_present, BTFIXUPCALL_NORM);
2516 BTFIXUPSET_CALL(pmd_clear, sun4c_pmd_clear, BTFIXUPCALL_STG0O0);
2517
2518 BTFIXUPSET_CALL(pgd_none, sun4c_pgd_none, BTFIXUPCALL_RETINT(0));
2519 BTFIXUPSET_CALL(pgd_bad, sun4c_pgd_bad, BTFIXUPCALL_RETINT(0));
2520 BTFIXUPSET_CALL(pgd_present, sun4c_pgd_present, BTFIXUPCALL_RETINT(1));
2521 BTFIXUPSET_CALL(pgd_clear, sun4c_pgd_clear, BTFIXUPCALL_NOP);
2522
2523 BTFIXUPSET_CALL(mk_pte, sun4c_mk_pte, BTFIXUPCALL_NORM);
2524 BTFIXUPSET_CALL(mk_pte_phys, sun4c_mk_pte_phys, BTFIXUPCALL_NORM);
2525 BTFIXUPSET_CALL(mk_pte_io, sun4c_mk_pte_io, BTFIXUPCALL_NORM);
2526
2527 BTFIXUPSET_INT(pte_modify_mask, _SUN4C_PAGE_CHG_MASK);
2528 BTFIXUPSET_CALL(pmd_offset, sun4c_pmd_offset, BTFIXUPCALL_NORM);
2529 BTFIXUPSET_CALL(pte_offset, sun4c_pte_offset, BTFIXUPCALL_NORM);
2530 BTFIXUPSET_CALL(free_pte_fast, sun4c_free_pte_fast, BTFIXUPCALL_NORM);
2531 BTFIXUPSET_CALL(pte_alloc_one, sun4c_pte_alloc_one, BTFIXUPCALL_NORM);
2532 BTFIXUPSET_CALL(pte_alloc_one_fast, sun4c_pte_alloc_one_fast, BTFIXUPCALL_NORM);
2533 BTFIXUPSET_CALL(free_pmd_fast, sun4c_free_pmd_fast, BTFIXUPCALL_NOP);
2534 BTFIXUPSET_CALL(pmd_alloc_one_fast, sun4c_pmd_alloc_one_fast, BTFIXUPCALL_RETO0);
2535 BTFIXUPSET_CALL(free_pgd_fast, sun4c_free_pgd_fast, BTFIXUPCALL_NORM);
2536 BTFIXUPSET_CALL(get_pgd_fast, sun4c_get_pgd_fast, BTFIXUPCALL_NORM);
2537
2538 BTFIXUPSET_HALF(pte_writei, _SUN4C_PAGE_WRITE);
2539 BTFIXUPSET_HALF(pte_dirtyi, _SUN4C_PAGE_MODIFIED);
2540 BTFIXUPSET_HALF(pte_youngi, _SUN4C_PAGE_ACCESSED);
2541 BTFIXUPSET_HALF(pte_wrprotecti, _SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE);
2542 BTFIXUPSET_HALF(pte_mkcleani, _SUN4C_PAGE_MODIFIED|_SUN4C_PAGE_SILENT_WRITE);
2543 BTFIXUPSET_HALF(pte_mkoldi, _SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_SILENT_READ);
2544 BTFIXUPSET_CALL(pte_mkwrite, sun4c_pte_mkwrite, BTFIXUPCALL_NORM);
2545 BTFIXUPSET_CALL(pte_mkdirty, sun4c_pte_mkdirty, BTFIXUPCALL_NORM);
2546 BTFIXUPSET_CALL(pte_mkyoung, sun4c_pte_mkyoung, BTFIXUPCALL_NORM);
2547 BTFIXUPSET_CALL(update_mmu_cache, sun4c_update_mmu_cache, BTFIXUPCALL_NORM);
2548
2549 BTFIXUPSET_CALL(mmu_lockarea, sun4c_lockarea, BTFIXUPCALL_NORM);
2550 BTFIXUPSET_CALL(mmu_unlockarea, sun4c_unlockarea, BTFIXUPCALL_NORM);
2551
2552 BTFIXUPSET_CALL(mmu_get_scsi_one, sun4c_get_scsi_one, BTFIXUPCALL_NORM);
2553 BTFIXUPSET_CALL(mmu_get_scsi_sgl, sun4c_get_scsi_sgl, BTFIXUPCALL_NORM);
2554 BTFIXUPSET_CALL(mmu_release_scsi_one, sun4c_release_scsi_one, BTFIXUPCALL_NORM);
2555 BTFIXUPSET_CALL(mmu_release_scsi_sgl, sun4c_release_scsi_sgl, BTFIXUPCALL_NORM);
2556
2557 BTFIXUPSET_CALL(mmu_map_dma_area, sun4c_map_dma_area, BTFIXUPCALL_NORM);
2558 BTFIXUPSET_CALL(mmu_unmap_dma_area, sun4c_unmap_dma_area, BTFIXUPCALL_NORM);
2559 BTFIXUPSET_CALL(mmu_translate_dvma, sun4c_translate_dvma, BTFIXUPCALL_NORM);
2560
2561 /* Task struct and kernel stack allocating/freeing. */
2562 BTFIXUPSET_CALL(alloc_task_struct, sun4c_alloc_task_struct, BTFIXUPCALL_NORM);
2563 BTFIXUPSET_CALL(get_task_struct, sun4c_get_task_struct, BTFIXUPCALL_NORM);
2564
2565 BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM);
2566
2567 /* These should _never_ get called with two level tables. */
2568 BTFIXUPSET_CALL(pgd_set, sun4c_pgd_set, BTFIXUPCALL_NOP);
2569 BTFIXUPSET_CALL(pgd_page, sun4c_pgd_page, BTFIXUPCALL_RETO0);
2570 BTFIXUPSET_CALL(pmd_set, sun4c_pmd_set, BTFIXUPCALL_NOP);
2571 }
2572