File: /usr/src/linux/arch/mips/mm/r4xx0.c
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * r4xx0.c: R4000 processor variant specific MMU/Cache routines.
7 *
8 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
9 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
10 *
11 * To do:
12 *
13 * - this code is a overbloated pig
14 * - many of the bug workarounds are not efficient at all, but at
15 * least they are functional ...
16 */
17 #include <linux/init.h>
18 #include <linux/kernel.h>
19 #include <linux/sched.h>
20 #include <linux/mm.h>
21
22 #include <asm/bootinfo.h>
23 #include <asm/cpu.h>
24 #include <asm/bcache.h>
25 #include <asm/io.h>
26 #include <asm/page.h>
27 #include <asm/pgtable.h>
28 #include <asm/system.h>
29 #include <asm/mmu_context.h>
30
31 /* CP0 hazard avoidance. */
32 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
33 "nop; nop; nop; nop; nop; nop;\n\t" \
34 ".set reorder\n\t")
35
36 /* Primary cache parameters. */
37 static int icache_size, dcache_size; /* Size in bytes */
38 static int ic_lsize, dc_lsize; /* LineSize in bytes */
39
40 /* Secondary cache (if present) parameters. */
41 static unsigned int scache_size, sc_lsize; /* Again, in bytes */
42
43 #include <asm/cacheops.h>
44 #include <asm/r4kcache.h>
45
46 #undef DEBUG_CACHE
47
48 /*
49 * Dummy cache handling routines for machines without boardcaches
50 */
51 static void no_sc_noop(void) {}
52
53 static struct bcache_ops no_sc_ops = {
54 (void *)no_sc_noop, (void *)no_sc_noop,
55 (void *)no_sc_noop, (void *)no_sc_noop
56 };
57
58 struct bcache_ops *bcops = &no_sc_ops;
59
60 /*
61 * On processors with QED R4600 style two set assosicative cache
62 * this is the bit which selects the way in the cache for the
63 * indexed cachops.
64 */
65 #define icache_waybit (icache_size >> 1)
66 #define dcache_waybit (dcache_size >> 1)
67
68 /*
69 * Zero an entire page. Basically a simple unrolled loop should do the
70 * job but we want more performance by saving memory bus bandwidth. We
71 * have five flavours of the routine available for:
72 *
73 * - 16byte cachelines and no second level cache
74 * - 32byte cachelines second level cache
75 * - a version which handles the buggy R4600 v1.x
76 * - a version which handles the buggy R4600 v2.0
77 * - Finally a last version without fancy cache games for the SC and MC
78 * versions of R4000 and R4400.
79 */
80
81 static void r4k_clear_page_d16(void * page)
82 {
83 __asm__ __volatile__(
84 ".set\tnoreorder\n\t"
85 ".set\tnoat\n\t"
86 ".set\tmips3\n\t"
87 "daddiu\t$1,%0,%2\n"
88 "1:\tcache\t%3,(%0)\n\t"
89 "sd\t$0,(%0)\n\t"
90 "sd\t$0,8(%0)\n\t"
91 "cache\t%3,16(%0)\n\t"
92 "sd\t$0,16(%0)\n\t"
93 "sd\t$0,24(%0)\n\t"
94 "daddiu\t%0,64\n\t"
95 "cache\t%3,-32(%0)\n\t"
96 "sd\t$0,-32(%0)\n\t"
97 "sd\t$0,-24(%0)\n\t"
98 "cache\t%3,-16(%0)\n\t"
99 "sd\t$0,-16(%0)\n\t"
100 "bne\t$1,%0,1b\n\t"
101 "sd\t$0,-8(%0)\n\t"
102 ".set\tmips0\n\t"
103 ".set\tat\n\t"
104 ".set\treorder"
105 :"=r" (page)
106 :"0" (page),
107 "I" (PAGE_SIZE),
108 "i" (Create_Dirty_Excl_D)
109 :"$1","memory");
110 }
111
112 static void r4k_clear_page_d32(void * page)
113 {
114 __asm__ __volatile__(
115 ".set\tnoreorder\n\t"
116 ".set\tnoat\n\t"
117 ".set\tmips3\n\t"
118 "daddiu\t$1,%0,%2\n"
119 "1:\tcache\t%3,(%0)\n\t"
120 "sd\t$0,(%0)\n\t"
121 "sd\t$0,8(%0)\n\t"
122 "sd\t$0,16(%0)\n\t"
123 "sd\t$0,24(%0)\n\t"
124 "daddiu\t%0,64\n\t"
125 "cache\t%3,-32(%0)\n\t"
126 "sd\t$0,-32(%0)\n\t"
127 "sd\t$0,-24(%0)\n\t"
128 "sd\t$0,-16(%0)\n\t"
129 "bne\t$1,%0,1b\n\t"
130 "sd\t$0,-8(%0)\n\t"
131 ".set\tmips0\n\t"
132 ".set\tat\n\t"
133 ".set\treorder"
134 :"=r" (page)
135 :"0" (page),
136 "I" (PAGE_SIZE),
137 "i" (Create_Dirty_Excl_D)
138 :"$1","memory");
139 }
140
141
142 /*
143 * This flavour of r4k_clear_page is for the R4600 V1.x. Cite from the
144 * IDT R4600 V1.7 errata:
145 *
146 * 18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D,
147 * Hit_Invalidate_D and Create_Dirty_Excl_D should only be
148 * executed if there is no other dcache activity. If the dcache is
149 * accessed for another instruction immeidately preceding when these
150 * cache instructions are executing, it is possible that the dcache
151 * tag match outputs used by these cache instructions will be
152 * incorrect. These cache instructions should be preceded by at least
153 * four instructions that are not any kind of load or store
154 * instruction.
155 *
156 * This is not allowed: lw
157 * nop
158 * nop
159 * nop
160 * cache Hit_Writeback_Invalidate_D
161 *
162 * This is allowed: lw
163 * nop
164 * nop
165 * nop
166 * nop
167 * cache Hit_Writeback_Invalidate_D
168 */
169 static void r4k_clear_page_r4600_v1(void * page)
170 {
171 __asm__ __volatile__(
172 ".set\tnoreorder\n\t"
173 ".set\tnoat\n\t"
174 ".set\tmips3\n\t"
175 "daddiu\t$1,%0,%2\n"
176 "1:\tnop\n\t"
177 "nop\n\t"
178 "nop\n\t"
179 "nop\n\t"
180 "cache\t%3,(%0)\n\t"
181 "sd\t$0,(%0)\n\t"
182 "sd\t$0,8(%0)\n\t"
183 "sd\t$0,16(%0)\n\t"
184 "sd\t$0,24(%0)\n\t"
185 "daddiu\t%0,64\n\t"
186 "nop\n\t"
187 "nop\n\t"
188 "nop\n\t"
189 "cache\t%3,-32(%0)\n\t"
190 "sd\t$0,-32(%0)\n\t"
191 "sd\t$0,-24(%0)\n\t"
192 "sd\t$0,-16(%0)\n\t"
193 "bne\t$1,%0,1b\n\t"
194 "sd\t$0,-8(%0)\n\t"
195 ".set\tmips0\n\t"
196 ".set\tat\n\t"
197 ".set\treorder"
198 :"=r" (page)
199 :"0" (page),
200 "I" (PAGE_SIZE),
201 "i" (Create_Dirty_Excl_D)
202 :"$1","memory");
203 }
204
205 /*
206 * And this one is for the R4600 V2.0
207 */
208 static void r4k_clear_page_r4600_v2(void * page)
209 {
210 unsigned int flags;
211
212 __save_and_cli(flags);
213 *(volatile unsigned int *)KSEG1;
214 __asm__ __volatile__(
215 ".set\tnoreorder\n\t"
216 ".set\tnoat\n\t"
217 ".set\tmips3\n\t"
218 "daddiu\t$1,%0,%2\n"
219 "1:\tcache\t%3,(%0)\n\t"
220 "sd\t$0,(%0)\n\t"
221 "sd\t$0,8(%0)\n\t"
222 "sd\t$0,16(%0)\n\t"
223 "sd\t$0,24(%0)\n\t"
224 "daddiu\t%0,64\n\t"
225 "cache\t%3,-32(%0)\n\t"
226 "sd\t$0,-32(%0)\n\t"
227 "sd\t$0,-24(%0)\n\t"
228 "sd\t$0,-16(%0)\n\t"
229 "bne\t$1,%0,1b\n\t"
230 "sd\t$0,-8(%0)\n\t"
231 ".set\tmips0\n\t"
232 ".set\tat\n\t"
233 ".set\treorder"
234 :"=r" (page)
235 :"0" (page),
236 "I" (PAGE_SIZE),
237 "i" (Create_Dirty_Excl_D)
238 :"$1","memory");
239 __restore_flags(flags);
240 }
241
242 /*
243 * The next 4 versions are optimized for all possible scache configurations
244 * of the SC / MC versions of R4000 and R4400 ...
245 *
246 * Todo: For even better performance we should have a routine optimized for
247 * every legal combination of dcache / scache linesize. When I (Ralf) tried
248 * this the kernel crashed shortly after mounting the root filesystem. CPU
249 * bug? Weirdo cache instruction semantics?
250 */
251 static void r4k_clear_page_s16(void * page)
252 {
253 __asm__ __volatile__(
254 ".set\tnoreorder\n\t"
255 ".set\tnoat\n\t"
256 ".set\tmips3\n\t"
257 "daddiu\t$1,%0,%2\n"
258 "1:\tcache\t%3,(%0)\n\t"
259 "sd\t$0,(%0)\n\t"
260 "sd\t$0,8(%0)\n\t"
261 "cache\t%3,16(%0)\n\t"
262 "sd\t$0,16(%0)\n\t"
263 "sd\t$0,24(%0)\n\t"
264 "daddiu\t%0,64\n\t"
265 "cache\t%3,-32(%0)\n\t"
266 "sd\t$0,-32(%0)\n\t"
267 "sd\t$0,-24(%0)\n\t"
268 "cache\t%3,-16(%0)\n\t"
269 "sd\t$0,-16(%0)\n\t"
270 "bne\t$1,%0,1b\n\t"
271 "sd\t$0,-8(%0)\n\t"
272 ".set\tmips0\n\t"
273 ".set\tat\n\t"
274 ".set\treorder"
275 :"=r" (page)
276 :"0" (page),
277 "I" (PAGE_SIZE),
278 "i" (Create_Dirty_Excl_SD)
279 :"$1","memory");
280 }
281
282 static void r4k_clear_page_s32(void * page)
283 {
284 __asm__ __volatile__(
285 ".set\tnoreorder\n\t"
286 ".set\tnoat\n\t"
287 ".set\tmips3\n\t"
288 "daddiu\t$1,%0,%2\n"
289 "1:\tcache\t%3,(%0)\n\t"
290 "sd\t$0,(%0)\n\t"
291 "sd\t$0,8(%0)\n\t"
292 "sd\t$0,16(%0)\n\t"
293 "sd\t$0,24(%0)\n\t"
294 "daddiu\t%0,64\n\t"
295 "cache\t%3,-32(%0)\n\t"
296 "sd\t$0,-32(%0)\n\t"
297 "sd\t$0,-24(%0)\n\t"
298 "sd\t$0,-16(%0)\n\t"
299 "bne\t$1,%0,1b\n\t"
300 "sd\t$0,-8(%0)\n\t"
301 ".set\tmips0\n\t"
302 ".set\tat\n\t"
303 ".set\treorder"
304 :"=r" (page)
305 :"0" (page),
306 "I" (PAGE_SIZE),
307 "i" (Create_Dirty_Excl_SD)
308 :"$1","memory");
309 }
310
311 static void r4k_clear_page_s64(void * page)
312 {
313 __asm__ __volatile__(
314 ".set\tnoreorder\n\t"
315 ".set\tnoat\n\t"
316 ".set\tmips3\n\t"
317 "daddiu\t$1,%0,%2\n"
318 "1:\tcache\t%3,(%0)\n\t"
319 "sd\t$0,(%0)\n\t"
320 "sd\t$0,8(%0)\n\t"
321 "sd\t$0,16(%0)\n\t"
322 "sd\t$0,24(%0)\n\t"
323 "daddiu\t%0,64\n\t"
324 "sd\t$0,-32(%0)\n\t"
325 "sd\t$0,-24(%0)\n\t"
326 "sd\t$0,-16(%0)\n\t"
327 "bne\t$1,%0,1b\n\t"
328 "sd\t$0,-8(%0)\n\t"
329 ".set\tmips0\n\t"
330 ".set\tat\n\t"
331 ".set\treorder"
332 :"=r" (page)
333 :"0" (page),
334 "I" (PAGE_SIZE),
335 "i" (Create_Dirty_Excl_SD)
336 :"$1","memory");
337 }
338
339 static void r4k_clear_page_s128(void * page)
340 {
341 __asm__ __volatile__(
342 ".set\tnoreorder\n\t"
343 ".set\tnoat\n\t"
344 ".set\tmips3\n\t"
345 "daddiu\t$1,%0,%2\n"
346 "1:\tcache\t%3,(%0)\n\t"
347 "sd\t$0,(%0)\n\t"
348 "sd\t$0,8(%0)\n\t"
349 "sd\t$0,16(%0)\n\t"
350 "sd\t$0,24(%0)\n\t"
351 "sd\t$0,32(%0)\n\t"
352 "sd\t$0,40(%0)\n\t"
353 "sd\t$0,48(%0)\n\t"
354 "sd\t$0,56(%0)\n\t"
355 "daddiu\t%0,128\n\t"
356 "sd\t$0,-64(%0)\n\t"
357 "sd\t$0,-56(%0)\n\t"
358 "sd\t$0,-48(%0)\n\t"
359 "sd\t$0,-40(%0)\n\t"
360 "sd\t$0,-32(%0)\n\t"
361 "sd\t$0,-24(%0)\n\t"
362 "sd\t$0,-16(%0)\n\t"
363 "bne\t$1,%0,1b\n\t"
364 "sd\t$0,-8(%0)\n\t"
365 ".set\tmips0\n\t"
366 ".set\tat\n\t"
367 ".set\treorder"
368 :"=r" (page)
369 :"0" (page),
370 "I" (PAGE_SIZE),
371 "i" (Create_Dirty_Excl_SD)
372 :"$1","memory");
373 }
374
375
376 /*
377 * This is still inefficient. We only can do better if we know the
378 * virtual address where the copy will be accessed.
379 */
380
381 static void r4k_copy_page_d16(void * to, void * from)
382 {
383 unsigned long dummy1, dummy2;
384 unsigned long reg1, reg2, reg3, reg4;
385
386 __asm__ __volatile__(
387 ".set\tnoreorder\n\t"
388 ".set\tnoat\n\t"
389 ".set\tmips3\n\t"
390 "daddiu\t$1,%0,%8\n"
391 "1:\tcache\t%9,(%0)\n\t"
392 "lw\t%2,(%1)\n\t"
393 "lw\t%3,4(%1)\n\t"
394 "lw\t%4,8(%1)\n\t"
395 "lw\t%5,12(%1)\n\t"
396 "sw\t%2,(%0)\n\t"
397 "sw\t%3,4(%0)\n\t"
398 "sw\t%4,8(%0)\n\t"
399 "sw\t%5,12(%0)\n\t"
400 "cache\t%9,16(%0)\n\t"
401 "lw\t%2,16(%1)\n\t"
402 "lw\t%3,20(%1)\n\t"
403 "lw\t%4,24(%1)\n\t"
404 "lw\t%5,28(%1)\n\t"
405 "sw\t%2,16(%0)\n\t"
406 "sw\t%3,20(%0)\n\t"
407 "sw\t%4,24(%0)\n\t"
408 "sw\t%5,28(%0)\n\t"
409 "cache\t%9,32(%0)\n\t"
410 "daddiu\t%0,64\n\t"
411 "daddiu\t%1,64\n\t"
412 "lw\t%2,-32(%1)\n\t"
413 "lw\t%3,-28(%1)\n\t"
414 "lw\t%4,-24(%1)\n\t"
415 "lw\t%5,-20(%1)\n\t"
416 "sw\t%2,-32(%0)\n\t"
417 "sw\t%3,-28(%0)\n\t"
418 "sw\t%4,-24(%0)\n\t"
419 "sw\t%5,-20(%0)\n\t"
420 "cache\t%9,-16(%0)\n\t"
421 "lw\t%2,-16(%1)\n\t"
422 "lw\t%3,-12(%1)\n\t"
423 "lw\t%4,-8(%1)\n\t"
424 "lw\t%5,-4(%1)\n\t"
425 "sw\t%2,-16(%0)\n\t"
426 "sw\t%3,-12(%0)\n\t"
427 "sw\t%4,-8(%0)\n\t"
428 "bne\t$1,%0,1b\n\t"
429 "sw\t%5,-4(%0)\n\t"
430 ".set\tmips0\n\t"
431 ".set\tat\n\t"
432 ".set\treorder"
433 :"=r" (dummy1), "=r" (dummy2),
434 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
435 :"0" (to), "1" (from),
436 "I" (PAGE_SIZE),
437 "i" (Create_Dirty_Excl_D));
438 }
439
440 static void r4k_copy_page_d32(void * to, void * from)
441 {
442 unsigned long dummy1, dummy2;
443 unsigned long reg1, reg2, reg3, reg4;
444
445 __asm__ __volatile__(
446 ".set\tnoreorder\n\t"
447 ".set\tnoat\n\t"
448 ".set\tmips3\n\t"
449 "daddiu\t$1,%0,%8\n"
450 "1:\tcache\t%9,(%0)\n\t"
451 "lw\t%2,(%1)\n\t"
452 "lw\t%3,4(%1)\n\t"
453 "lw\t%4,8(%1)\n\t"
454 "lw\t%5,12(%1)\n\t"
455 "sw\t%2,(%0)\n\t"
456 "sw\t%3,4(%0)\n\t"
457 "sw\t%4,8(%0)\n\t"
458 "sw\t%5,12(%0)\n\t"
459 "lw\t%2,16(%1)\n\t"
460 "lw\t%3,20(%1)\n\t"
461 "lw\t%4,24(%1)\n\t"
462 "lw\t%5,28(%1)\n\t"
463 "sw\t%2,16(%0)\n\t"
464 "sw\t%3,20(%0)\n\t"
465 "sw\t%4,24(%0)\n\t"
466 "sw\t%5,28(%0)\n\t"
467 "cache\t%9,32(%0)\n\t"
468 "daddiu\t%0,64\n\t"
469 "daddiu\t%1,64\n\t"
470 "lw\t%2,-32(%1)\n\t"
471 "lw\t%3,-28(%1)\n\t"
472 "lw\t%4,-24(%1)\n\t"
473 "lw\t%5,-20(%1)\n\t"
474 "sw\t%2,-32(%0)\n\t"
475 "sw\t%3,-28(%0)\n\t"
476 "sw\t%4,-24(%0)\n\t"
477 "sw\t%5,-20(%0)\n\t"
478 "lw\t%2,-16(%1)\n\t"
479 "lw\t%3,-12(%1)\n\t"
480 "lw\t%4,-8(%1)\n\t"
481 "lw\t%5,-4(%1)\n\t"
482 "sw\t%2,-16(%0)\n\t"
483 "sw\t%3,-12(%0)\n\t"
484 "sw\t%4,-8(%0)\n\t"
485 "bne\t$1,%0,1b\n\t"
486 "sw\t%5,-4(%0)\n\t"
487 ".set\tmips0\n\t"
488 ".set\tat\n\t"
489 ".set\treorder"
490 :"=r" (dummy1), "=r" (dummy2),
491 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
492 :"0" (to), "1" (from),
493 "I" (PAGE_SIZE),
494 "i" (Create_Dirty_Excl_D));
495 }
496
497 /*
498 * Again a special version for the R4600 V1.x
499 */
500 static void r4k_copy_page_r4600_v1(void * to, void * from)
501 {
502 unsigned long dummy1, dummy2;
503 unsigned long reg1, reg2, reg3, reg4;
504
505 __asm__ __volatile__(
506 ".set\tnoreorder\n\t"
507 ".set\tnoat\n\t"
508 ".set\tmips3\n\t"
509 "daddiu\t$1,%0,%8\n"
510 "1:\tnop\n\t"
511 "nop\n\t"
512 "nop\n\t"
513 "nop\n\t"
514 "\tcache\t%9,(%0)\n\t"
515 "lw\t%2,(%1)\n\t"
516 "lw\t%3,4(%1)\n\t"
517 "lw\t%4,8(%1)\n\t"
518 "lw\t%5,12(%1)\n\t"
519 "sw\t%2,(%0)\n\t"
520 "sw\t%3,4(%0)\n\t"
521 "sw\t%4,8(%0)\n\t"
522 "sw\t%5,12(%0)\n\t"
523 "lw\t%2,16(%1)\n\t"
524 "lw\t%3,20(%1)\n\t"
525 "lw\t%4,24(%1)\n\t"
526 "lw\t%5,28(%1)\n\t"
527 "sw\t%2,16(%0)\n\t"
528 "sw\t%3,20(%0)\n\t"
529 "sw\t%4,24(%0)\n\t"
530 "sw\t%5,28(%0)\n\t"
531 "nop\n\t"
532 "nop\n\t"
533 "nop\n\t"
534 "nop\n\t"
535 "cache\t%9,32(%0)\n\t"
536 "daddiu\t%0,64\n\t"
537 "daddiu\t%1,64\n\t"
538 "lw\t%2,-32(%1)\n\t"
539 "lw\t%3,-28(%1)\n\t"
540 "lw\t%4,-24(%1)\n\t"
541 "lw\t%5,-20(%1)\n\t"
542 "sw\t%2,-32(%0)\n\t"
543 "sw\t%3,-28(%0)\n\t"
544 "sw\t%4,-24(%0)\n\t"
545 "sw\t%5,-20(%0)\n\t"
546 "lw\t%2,-16(%1)\n\t"
547 "lw\t%3,-12(%1)\n\t"
548 "lw\t%4,-8(%1)\n\t"
549 "lw\t%5,-4(%1)\n\t"
550 "sw\t%2,-16(%0)\n\t"
551 "sw\t%3,-12(%0)\n\t"
552 "sw\t%4,-8(%0)\n\t"
553 "bne\t$1,%0,1b\n\t"
554 "sw\t%5,-4(%0)\n\t"
555 ".set\tmips0\n\t"
556 ".set\tat\n\t"
557 ".set\treorder"
558 :"=r" (dummy1), "=r" (dummy2),
559 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
560 :"0" (to), "1" (from),
561 "I" (PAGE_SIZE),
562 "i" (Create_Dirty_Excl_D));
563 }
564
565 static void r4k_copy_page_r4600_v2(void * to, void * from)
566 {
567 unsigned long dummy1, dummy2;
568 unsigned long reg1, reg2, reg3, reg4;
569 unsigned int flags;
570
571 __save_and_cli(flags);
572 __asm__ __volatile__(
573 ".set\tnoreorder\n\t"
574 ".set\tnoat\n\t"
575 ".set\tmips3\n\t"
576 "daddiu\t$1,%0,%8\n"
577 "1:\tnop\n\t"
578 "nop\n\t"
579 "nop\n\t"
580 "nop\n\t"
581 "\tcache\t%9,(%0)\n\t"
582 "lw\t%2,(%1)\n\t"
583 "lw\t%3,4(%1)\n\t"
584 "lw\t%4,8(%1)\n\t"
585 "lw\t%5,12(%1)\n\t"
586 "sw\t%2,(%0)\n\t"
587 "sw\t%3,4(%0)\n\t"
588 "sw\t%4,8(%0)\n\t"
589 "sw\t%5,12(%0)\n\t"
590 "lw\t%2,16(%1)\n\t"
591 "lw\t%3,20(%1)\n\t"
592 "lw\t%4,24(%1)\n\t"
593 "lw\t%5,28(%1)\n\t"
594 "sw\t%2,16(%0)\n\t"
595 "sw\t%3,20(%0)\n\t"
596 "sw\t%4,24(%0)\n\t"
597 "sw\t%5,28(%0)\n\t"
598 "nop\n\t"
599 "nop\n\t"
600 "nop\n\t"
601 "nop\n\t"
602 "cache\t%9,32(%0)\n\t"
603 "daddiu\t%0,64\n\t"
604 "daddiu\t%1,64\n\t"
605 "lw\t%2,-32(%1)\n\t"
606 "lw\t%3,-28(%1)\n\t"
607 "lw\t%4,-24(%1)\n\t"
608 "lw\t%5,-20(%1)\n\t"
609 "sw\t%2,-32(%0)\n\t"
610 "sw\t%3,-28(%0)\n\t"
611 "sw\t%4,-24(%0)\n\t"
612 "sw\t%5,-20(%0)\n\t"
613 "lw\t%2,-16(%1)\n\t"
614 "lw\t%3,-12(%1)\n\t"
615 "lw\t%4,-8(%1)\n\t"
616 "lw\t%5,-4(%1)\n\t"
617 "sw\t%2,-16(%0)\n\t"
618 "sw\t%3,-12(%0)\n\t"
619 "sw\t%4,-8(%0)\n\t"
620 "bne\t$1,%0,1b\n\t"
621 "sw\t%5,-4(%0)\n\t"
622 ".set\tmips0\n\t"
623 ".set\tat\n\t"
624 ".set\treorder"
625 :"=r" (dummy1), "=r" (dummy2),
626 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
627 :"0" (to), "1" (from),
628 "I" (PAGE_SIZE),
629 "i" (Create_Dirty_Excl_D));
630 __restore_flags(flags);
631 }
632
633 /*
634 * These are for R4000SC / R4400MC
635 */
636 static void r4k_copy_page_s16(void * to, void * from)
637 {
638 unsigned long dummy1, dummy2;
639 unsigned long reg1, reg2, reg3, reg4;
640
641 __asm__ __volatile__(
642 ".set\tnoreorder\n\t"
643 ".set\tnoat\n\t"
644 ".set\tmips3\n\t"
645 "daddiu\t$1,%0,%8\n"
646 "1:\tcache\t%9,(%0)\n\t"
647 "lw\t%2,(%1)\n\t"
648 "lw\t%3,4(%1)\n\t"
649 "lw\t%4,8(%1)\n\t"
650 "lw\t%5,12(%1)\n\t"
651 "sw\t%2,(%0)\n\t"
652 "sw\t%3,4(%0)\n\t"
653 "sw\t%4,8(%0)\n\t"
654 "sw\t%5,12(%0)\n\t"
655 "cache\t%9,16(%0)\n\t"
656 "lw\t%2,16(%1)\n\t"
657 "lw\t%3,20(%1)\n\t"
658 "lw\t%4,24(%1)\n\t"
659 "lw\t%5,28(%1)\n\t"
660 "sw\t%2,16(%0)\n\t"
661 "sw\t%3,20(%0)\n\t"
662 "sw\t%4,24(%0)\n\t"
663 "sw\t%5,28(%0)\n\t"
664 "cache\t%9,32(%0)\n\t"
665 "daddiu\t%0,64\n\t"
666 "daddiu\t%1,64\n\t"
667 "lw\t%2,-32(%1)\n\t"
668 "lw\t%3,-28(%1)\n\t"
669 "lw\t%4,-24(%1)\n\t"
670 "lw\t%5,-20(%1)\n\t"
671 "sw\t%2,-32(%0)\n\t"
672 "sw\t%3,-28(%0)\n\t"
673 "sw\t%4,-24(%0)\n\t"
674 "sw\t%5,-20(%0)\n\t"
675 "cache\t%9,-16(%0)\n\t"
676 "lw\t%2,-16(%1)\n\t"
677 "lw\t%3,-12(%1)\n\t"
678 "lw\t%4,-8(%1)\n\t"
679 "lw\t%5,-4(%1)\n\t"
680 "sw\t%2,-16(%0)\n\t"
681 "sw\t%3,-12(%0)\n\t"
682 "sw\t%4,-8(%0)\n\t"
683 "bne\t$1,%0,1b\n\t"
684 "sw\t%5,-4(%0)\n\t"
685 ".set\tmips0\n\t"
686 ".set\tat\n\t"
687 ".set\treorder"
688 :"=r" (dummy1), "=r" (dummy2),
689 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
690 :"0" (to), "1" (from),
691 "I" (PAGE_SIZE),
692 "i" (Create_Dirty_Excl_SD));
693 }
694
695 static void r4k_copy_page_s32(void * to, void * from)
696 {
697 unsigned long dummy1, dummy2;
698 unsigned long reg1, reg2, reg3, reg4;
699
700 __asm__ __volatile__(
701 ".set\tnoreorder\n\t"
702 ".set\tnoat\n\t"
703 ".set\tmips3\n\t"
704 "daddiu\t$1,%0,%8\n"
705 "1:\tcache\t%9,(%0)\n\t"
706 "lw\t%2,(%1)\n\t"
707 "lw\t%3,4(%1)\n\t"
708 "lw\t%4,8(%1)\n\t"
709 "lw\t%5,12(%1)\n\t"
710 "sw\t%2,(%0)\n\t"
711 "sw\t%3,4(%0)\n\t"
712 "sw\t%4,8(%0)\n\t"
713 "sw\t%5,12(%0)\n\t"
714 "lw\t%2,16(%1)\n\t"
715 "lw\t%3,20(%1)\n\t"
716 "lw\t%4,24(%1)\n\t"
717 "lw\t%5,28(%1)\n\t"
718 "sw\t%2,16(%0)\n\t"
719 "sw\t%3,20(%0)\n\t"
720 "sw\t%4,24(%0)\n\t"
721 "sw\t%5,28(%0)\n\t"
722 "cache\t%9,32(%0)\n\t"
723 "daddiu\t%0,64\n\t"
724 "daddiu\t%1,64\n\t"
725 "lw\t%2,-32(%1)\n\t"
726 "lw\t%3,-28(%1)\n\t"
727 "lw\t%4,-24(%1)\n\t"
728 "lw\t%5,-20(%1)\n\t"
729 "sw\t%2,-32(%0)\n\t"
730 "sw\t%3,-28(%0)\n\t"
731 "sw\t%4,-24(%0)\n\t"
732 "sw\t%5,-20(%0)\n\t"
733 "lw\t%2,-16(%1)\n\t"
734 "lw\t%3,-12(%1)\n\t"
735 "lw\t%4,-8(%1)\n\t"
736 "lw\t%5,-4(%1)\n\t"
737 "sw\t%2,-16(%0)\n\t"
738 "sw\t%3,-12(%0)\n\t"
739 "sw\t%4,-8(%0)\n\t"
740 "bne\t$1,%0,1b\n\t"
741 "sw\t%5,-4(%0)\n\t"
742 ".set\tmips0\n\t"
743 ".set\tat\n\t"
744 ".set\treorder"
745 :"=r" (dummy1), "=r" (dummy2),
746 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
747 :"0" (to), "1" (from),
748 "I" (PAGE_SIZE),
749 "i" (Create_Dirty_Excl_SD));
750 }
751
752 static void r4k_copy_page_s64(void * to, void * from)
753 {
754 unsigned long dummy1, dummy2;
755 unsigned long reg1, reg2, reg3, reg4;
756
757 __asm__ __volatile__(
758 ".set\tnoreorder\n\t"
759 ".set\tnoat\n\t"
760 ".set\tmips3\n\t"
761 "daddiu\t$1,%0,%8\n"
762 "1:\tcache\t%9,(%0)\n\t"
763 "lw\t%2,(%1)\n\t"
764 "lw\t%3,4(%1)\n\t"
765 "lw\t%4,8(%1)\n\t"
766 "lw\t%5,12(%1)\n\t"
767 "sw\t%2,(%0)\n\t"
768 "sw\t%3,4(%0)\n\t"
769 "sw\t%4,8(%0)\n\t"
770 "sw\t%5,12(%0)\n\t"
771 "lw\t%2,16(%1)\n\t"
772 "lw\t%3,20(%1)\n\t"
773 "lw\t%4,24(%1)\n\t"
774 "lw\t%5,28(%1)\n\t"
775 "sw\t%2,16(%0)\n\t"
776 "sw\t%3,20(%0)\n\t"
777 "sw\t%4,24(%0)\n\t"
778 "sw\t%5,28(%0)\n\t"
779 "daddiu\t%0,64\n\t"
780 "daddiu\t%1,64\n\t"
781 "lw\t%2,-32(%1)\n\t"
782 "lw\t%3,-28(%1)\n\t"
783 "lw\t%4,-24(%1)\n\t"
784 "lw\t%5,-20(%1)\n\t"
785 "sw\t%2,-32(%0)\n\t"
786 "sw\t%3,-28(%0)\n\t"
787 "sw\t%4,-24(%0)\n\t"
788 "sw\t%5,-20(%0)\n\t"
789 "lw\t%2,-16(%1)\n\t"
790 "lw\t%3,-12(%1)\n\t"
791 "lw\t%4,-8(%1)\n\t"
792 "lw\t%5,-4(%1)\n\t"
793 "sw\t%2,-16(%0)\n\t"
794 "sw\t%3,-12(%0)\n\t"
795 "sw\t%4,-8(%0)\n\t"
796 "bne\t$1,%0,1b\n\t"
797 "sw\t%5,-4(%0)\n\t"
798 ".set\tmips0\n\t"
799 ".set\tat\n\t"
800 ".set\treorder"
801 :"=r" (dummy1), "=r" (dummy2),
802 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
803 :"0" (to), "1" (from),
804 "I" (PAGE_SIZE),
805 "i" (Create_Dirty_Excl_SD));
806 }
807
808 static void r4k_copy_page_s128(void * to, void * from)
809 {
810 unsigned long dummy1, dummy2;
811 unsigned long reg1, reg2, reg3, reg4;
812
813 __asm__ __volatile__(
814 ".set\tnoreorder\n\t"
815 ".set\tnoat\n\t"
816 ".set\tmips3\n\t"
817 "daddiu\t$1,%0,%8\n"
818 "1:\tcache\t%9,(%0)\n\t"
819 "lw\t%2,(%1)\n\t"
820 "lw\t%3,4(%1)\n\t"
821 "lw\t%4,8(%1)\n\t"
822 "lw\t%5,12(%1)\n\t"
823 "sw\t%2,(%0)\n\t"
824 "sw\t%3,4(%0)\n\t"
825 "sw\t%4,8(%0)\n\t"
826 "sw\t%5,12(%0)\n\t"
827 "lw\t%2,16(%1)\n\t"
828 "lw\t%3,20(%1)\n\t"
829 "lw\t%4,24(%1)\n\t"
830 "lw\t%5,28(%1)\n\t"
831 "sw\t%2,16(%0)\n\t"
832 "sw\t%3,20(%0)\n\t"
833 "sw\t%4,24(%0)\n\t"
834 "sw\t%5,28(%0)\n\t"
835 "lw\t%2,32(%1)\n\t"
836 "lw\t%3,36(%1)\n\t"
837 "lw\t%4,40(%1)\n\t"
838 "lw\t%5,44(%1)\n\t"
839 "sw\t%2,32(%0)\n\t"
840 "sw\t%3,36(%0)\n\t"
841 "sw\t%4,40(%0)\n\t"
842 "sw\t%5,44(%0)\n\t"
843 "lw\t%2,48(%1)\n\t"
844 "lw\t%3,52(%1)\n\t"
845 "lw\t%4,56(%1)\n\t"
846 "lw\t%5,60(%1)\n\t"
847 "sw\t%2,48(%0)\n\t"
848 "sw\t%3,52(%0)\n\t"
849 "sw\t%4,56(%0)\n\t"
850 "sw\t%5,60(%0)\n\t"
851 "daddiu\t%0,128\n\t"
852 "daddiu\t%1,128\n\t"
853 "lw\t%2,-64(%1)\n\t"
854 "lw\t%3,-60(%1)\n\t"
855 "lw\t%4,-56(%1)\n\t"
856 "lw\t%5,-52(%1)\n\t"
857 "sw\t%2,-64(%0)\n\t"
858 "sw\t%3,-60(%0)\n\t"
859 "sw\t%4,-56(%0)\n\t"
860 "sw\t%5,-52(%0)\n\t"
861 "lw\t%2,-48(%1)\n\t"
862 "lw\t%3,-44(%1)\n\t"
863 "lw\t%4,-40(%1)\n\t"
864 "lw\t%5,-36(%1)\n\t"
865 "sw\t%2,-48(%0)\n\t"
866 "sw\t%3,-44(%0)\n\t"
867 "sw\t%4,-40(%0)\n\t"
868 "sw\t%5,-36(%0)\n\t"
869 "lw\t%2,-32(%1)\n\t"
870 "lw\t%3,-28(%1)\n\t"
871 "lw\t%4,-24(%1)\n\t"
872 "lw\t%5,-20(%1)\n\t"
873 "sw\t%2,-32(%0)\n\t"
874 "sw\t%3,-28(%0)\n\t"
875 "sw\t%4,-24(%0)\n\t"
876 "sw\t%5,-20(%0)\n\t"
877 "lw\t%2,-16(%1)\n\t"
878 "lw\t%3,-12(%1)\n\t"
879 "lw\t%4,-8(%1)\n\t"
880 "lw\t%5,-4(%1)\n\t"
881 "sw\t%2,-16(%0)\n\t"
882 "sw\t%3,-12(%0)\n\t"
883 "sw\t%4,-8(%0)\n\t"
884 "bne\t$1,%0,1b\n\t"
885 "sw\t%5,-4(%0)\n\t"
886 ".set\tmips0\n\t"
887 ".set\tat\n\t"
888 ".set\treorder"
889 :"=r" (dummy1), "=r" (dummy2),
890 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
891 :"0" (to), "1" (from),
892 "I" (PAGE_SIZE),
893 "i" (Create_Dirty_Excl_SD));
894 }
895
896
897 /*
898 * If you think for one second that this stuff coming up is a lot
899 * of bulky code eating too many kernel cache lines. Think _again_.
900 *
901 * Consider:
902 * 1) Taken branches have a 3 cycle penalty on R4k
903 * 2) The branch itself is a real dead cycle on even R4600/R5000.
904 * 3) Only one of the following variants of each type is even used by
905 * the kernel based upon the cache parameters we detect at boot time.
906 *
907 * QED.
908 */
909
910 static inline void r4k_flush_cache_all_s16d16i16(void)
911 {
912 unsigned long flags;
913
914 __save_and_cli(flags);
915 blast_dcache16(); blast_icache16(); blast_scache16();
916 __restore_flags(flags);
917 }
918
919 static inline void r4k_flush_cache_all_s32d16i16(void)
920 {
921 unsigned long flags;
922
923 __save_and_cli(flags);
924 blast_dcache16(); blast_icache16(); blast_scache32();
925 __restore_flags(flags);
926 }
927
928 static inline void r4k_flush_cache_all_s64d16i16(void)
929 {
930 unsigned long flags;
931
932 __save_and_cli(flags);
933 blast_dcache16(); blast_icache16(); blast_scache64();
934 __restore_flags(flags);
935 }
936
937 static inline void r4k_flush_cache_all_s128d16i16(void)
938 {
939 unsigned long flags;
940
941 __save_and_cli(flags);
942 blast_dcache16(); blast_icache16(); blast_scache128();
943 __restore_flags(flags);
944 }
945
946 static inline void r4k_flush_cache_all_s32d32i32(void)
947 {
948 unsigned long flags;
949
950 __save_and_cli(flags);
951 blast_dcache32(); blast_icache32(); blast_scache32();
952 __restore_flags(flags);
953 }
954
955 static inline void r4k_flush_cache_all_s64d32i32(void)
956 {
957 unsigned long flags;
958
959 __save_and_cli(flags);
960 blast_dcache32(); blast_icache32(); blast_scache64();
961 __restore_flags(flags);
962 }
963
964 static inline void r4k_flush_cache_all_s128d32i32(void)
965 {
966 unsigned long flags;
967
968 __save_and_cli(flags);
969 blast_dcache32(); blast_icache32(); blast_scache128();
970 __restore_flags(flags);
971 }
972
973 static inline void r4k_flush_cache_all_d16i16(void)
974 {
975 unsigned long flags;
976
977 __save_and_cli(flags);
978 blast_dcache16(); blast_icache16();
979 __restore_flags(flags);
980 }
981
982 static inline void r4k_flush_cache_all_d32i32(void)
983 {
984 unsigned long flags;
985
986 __save_and_cli(flags);
987 blast_dcache32(); blast_icache32();
988 __restore_flags(flags);
989 }
990
991 static void
992 r4k_flush_cache_range_s16d16i16(struct mm_struct *mm,
993 unsigned long start,
994 unsigned long end)
995 {
996 struct vm_area_struct *vma;
997 unsigned long flags;
998
999 if (mm->context == 0)
1000 return;
1001
1002 start &= PAGE_MASK;
1003 #ifdef DEBUG_CACHE
1004 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1005 #endif
1006 vma = find_vma(mm, start);
1007 if (vma) {
1008 if (mm->context != current->active_mm->context) {
1009 r4k_flush_cache_all_s16d16i16();
1010 } else {
1011 pgd_t *pgd;
1012 pmd_t *pmd;
1013 pte_t *pte;
1014
1015 __save_and_cli(flags);
1016 while(start < end) {
1017 pgd = pgd_offset(mm, start);
1018 pmd = pmd_offset(pgd, start);
1019 pte = pte_offset(pmd, start);
1020
1021 if(pte_val(*pte) & _PAGE_VALID)
1022 blast_scache16_page(start);
1023 start += PAGE_SIZE;
1024 }
1025 __restore_flags(flags);
1026 }
1027 }
1028 }
1029
1030 static void
1031 r4k_flush_cache_range_s32d16i16(struct mm_struct *mm,
1032 unsigned long start,
1033 unsigned long end)
1034 {
1035 struct vm_area_struct *vma;
1036 unsigned long flags;
1037
1038 if (mm->context == 0)
1039 return;
1040
1041 start &= PAGE_MASK;
1042 #ifdef DEBUG_CACHE
1043 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1044 #endif
1045 vma = find_vma(mm, start);
1046 if (vma) {
1047 if (mm->context != current->active_mm->context) {
1048 r4k_flush_cache_all_s32d16i16();
1049 } else {
1050 pgd_t *pgd;
1051 pmd_t *pmd;
1052 pte_t *pte;
1053
1054 __save_and_cli(flags);
1055 while(start < end) {
1056 pgd = pgd_offset(mm, start);
1057 pmd = pmd_offset(pgd, start);
1058 pte = pte_offset(pmd, start);
1059
1060 if(pte_val(*pte) & _PAGE_VALID)
1061 blast_scache32_page(start);
1062 start += PAGE_SIZE;
1063 }
1064 __restore_flags(flags);
1065 }
1066 }
1067 }
1068
1069 static void r4k_flush_cache_range_s64d16i16(struct mm_struct *mm,
1070 unsigned long start,
1071 unsigned long end)
1072 {
1073 struct vm_area_struct *vma;
1074 unsigned long flags;
1075
1076 if (mm->context == 0)
1077 return;
1078
1079 start &= PAGE_MASK;
1080 #ifdef DEBUG_CACHE
1081 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1082 #endif
1083 vma = find_vma(mm, start);
1084 if(vma) {
1085 if (mm->context != current->active_mm->context) {
1086 r4k_flush_cache_all_s64d16i16();
1087 } else {
1088 pgd_t *pgd;
1089 pmd_t *pmd;
1090 pte_t *pte;
1091
1092 __save_and_cli(flags);
1093 while(start < end) {
1094 pgd = pgd_offset(mm, start);
1095 pmd = pmd_offset(pgd, start);
1096 pte = pte_offset(pmd, start);
1097
1098 if(pte_val(*pte) & _PAGE_VALID)
1099 blast_scache64_page(start);
1100 start += PAGE_SIZE;
1101 }
1102 __restore_flags(flags);
1103 }
1104 }
1105 }
1106
1107 static void r4k_flush_cache_range_s128d16i16(struct mm_struct *mm,
1108 unsigned long start,
1109 unsigned long end)
1110 {
1111 struct vm_area_struct *vma;
1112 unsigned long flags;
1113
1114 if (mm->context == 0)
1115 return;
1116
1117 start &= PAGE_MASK;
1118 #ifdef DEBUG_CACHE
1119 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1120 #endif
1121 vma = find_vma(mm, start);
1122 if (vma) {
1123 if (mm->context != current->active_mm->context) {
1124 r4k_flush_cache_all_s128d16i16();
1125 } else {
1126 pgd_t *pgd;
1127 pmd_t *pmd;
1128 pte_t *pte;
1129
1130 __save_and_cli(flags);
1131 while(start < end) {
1132 pgd = pgd_offset(mm, start);
1133 pmd = pmd_offset(pgd, start);
1134 pte = pte_offset(pmd, start);
1135
1136 if(pte_val(*pte) & _PAGE_VALID)
1137 blast_scache128_page(start);
1138 start += PAGE_SIZE;
1139 }
1140 __restore_flags(flags);
1141 }
1142 }
1143 }
1144
1145 static void r4k_flush_cache_range_s32d32i32(struct mm_struct *mm,
1146 unsigned long start,
1147 unsigned long end)
1148 {
1149 struct vm_area_struct *vma;
1150 unsigned long flags;
1151
1152 if (mm->context == 0)
1153 return;
1154
1155 start &= PAGE_MASK;
1156 #ifdef DEBUG_CACHE
1157 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1158 #endif
1159 vma = find_vma(mm, start);
1160 if (vma) {
1161 if (mm->context != current->active_mm->context) {
1162 r4k_flush_cache_all_s32d32i32();
1163 } else {
1164 pgd_t *pgd;
1165 pmd_t *pmd;
1166 pte_t *pte;
1167
1168 __save_and_cli(flags);
1169 while(start < end) {
1170 pgd = pgd_offset(mm, start);
1171 pmd = pmd_offset(pgd, start);
1172 pte = pte_offset(pmd, start);
1173
1174 if(pte_val(*pte) & _PAGE_VALID)
1175 blast_scache32_page(start);
1176 start += PAGE_SIZE;
1177 }
1178 __restore_flags(flags);
1179 }
1180 }
1181 }
1182
1183 static void r4k_flush_cache_range_s64d32i32(struct mm_struct *mm,
1184 unsigned long start,
1185 unsigned long end)
1186 {
1187 struct vm_area_struct *vma;
1188 unsigned long flags;
1189
1190 if (mm->context == 0)
1191 return;
1192
1193 start &= PAGE_MASK;
1194 #ifdef DEBUG_CACHE
1195 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1196 #endif
1197 vma = find_vma(mm, start);
1198 if (vma) {
1199 if (mm->context != current->active_mm->context) {
1200 r4k_flush_cache_all_s64d32i32();
1201 } else {
1202 pgd_t *pgd;
1203 pmd_t *pmd;
1204 pte_t *pte;
1205
1206 __save_and_cli(flags);
1207 while(start < end) {
1208 pgd = pgd_offset(mm, start);
1209 pmd = pmd_offset(pgd, start);
1210 pte = pte_offset(pmd, start);
1211
1212 if(pte_val(*pte) & _PAGE_VALID)
1213 blast_scache64_page(start);
1214 start += PAGE_SIZE;
1215 }
1216 __restore_flags(flags);
1217 }
1218 }
1219 }
1220
1221 static void r4k_flush_cache_range_s128d32i32(struct mm_struct *mm,
1222 unsigned long start,
1223 unsigned long end)
1224 {
1225 struct vm_area_struct *vma;
1226 unsigned long flags;
1227
1228 if (mm->context == 0)
1229 return;
1230
1231 start &= PAGE_MASK;
1232 #ifdef DEBUG_CACHE
1233 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1234 #endif
1235 vma = find_vma(mm, start);
1236 if (vma) {
1237 if (mm->context != current->active_mm->context) {
1238 r4k_flush_cache_all_s128d32i32();
1239 } else {
1240 pgd_t *pgd;
1241 pmd_t *pmd;
1242 pte_t *pte;
1243
1244 __save_and_cli(flags);
1245 while(start < end) {
1246 pgd = pgd_offset(mm, start);
1247 pmd = pmd_offset(pgd, start);
1248 pte = pte_offset(pmd, start);
1249
1250 if(pte_val(*pte) & _PAGE_VALID)
1251 blast_scache128_page(start);
1252 start += PAGE_SIZE;
1253 }
1254 __restore_flags(flags);
1255 }
1256 }
1257 }
1258
1259 static void r4k_flush_cache_range_d16i16(struct mm_struct *mm,
1260 unsigned long start,
1261 unsigned long end)
1262 {
1263 if (mm->context != 0) {
1264 unsigned long flags;
1265
1266 #ifdef DEBUG_CACHE
1267 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1268 #endif
1269 __save_and_cli(flags);
1270 blast_dcache16(); blast_icache16();
1271 __restore_flags(flags);
1272 }
1273 }
1274
1275 static void r4k_flush_cache_range_d32i32(struct mm_struct *mm,
1276 unsigned long start,
1277 unsigned long end)
1278 {
1279 if (mm->context != 0) {
1280 unsigned long flags;
1281
1282 #ifdef DEBUG_CACHE
1283 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1284 #endif
1285 __save_and_cli(flags);
1286 blast_dcache32(); blast_icache32();
1287 __restore_flags(flags);
1288 }
1289 }
1290
1291 /*
1292 * On architectures like the Sparc, we could get rid of lines in
1293 * the cache created only by a certain context, but on the MIPS
1294 * (and actually certain Sparc's) we cannot.
1295 */
1296 static void r4k_flush_cache_mm_s16d16i16(struct mm_struct *mm)
1297 {
1298 if (mm->context != 0) {
1299 #ifdef DEBUG_CACHE
1300 printk("cmm[%d]", (int)mm->context);
1301 #endif
1302 r4k_flush_cache_all_s16d16i16();
1303 }
1304 }
1305
1306 static void r4k_flush_cache_mm_s32d16i16(struct mm_struct *mm)
1307 {
1308 if (mm->context != 0) {
1309 #ifdef DEBUG_CACHE
1310 printk("cmm[%d]", (int)mm->context);
1311 #endif
1312 r4k_flush_cache_all_s32d16i16();
1313 }
1314 }
1315
1316 static void r4k_flush_cache_mm_s64d16i16(struct mm_struct *mm)
1317 {
1318 if (mm->context != 0) {
1319 #ifdef DEBUG_CACHE
1320 printk("cmm[%d]", (int)mm->context);
1321 #endif
1322 r4k_flush_cache_all_s64d16i16();
1323 }
1324 }
1325
1326 static void r4k_flush_cache_mm_s128d16i16(struct mm_struct *mm)
1327 {
1328 if (mm->context != 0) {
1329 #ifdef DEBUG_CACHE
1330 printk("cmm[%d]", (int)mm->context);
1331 #endif
1332 r4k_flush_cache_all_s128d16i16();
1333 }
1334 }
1335
1336 static void r4k_flush_cache_mm_s32d32i32(struct mm_struct *mm)
1337 {
1338 if (mm->context != 0) {
1339 #ifdef DEBUG_CACHE
1340 printk("cmm[%d]", (int)mm->context);
1341 #endif
1342 r4k_flush_cache_all_s32d32i32();
1343 }
1344 }
1345
1346 static void r4k_flush_cache_mm_s64d32i32(struct mm_struct *mm)
1347 {
1348 if (mm->context != 0) {
1349 #ifdef DEBUG_CACHE
1350 printk("cmm[%d]", (int)mm->context);
1351 #endif
1352 r4k_flush_cache_all_s64d32i32();
1353 }
1354 }
1355
1356 static void r4k_flush_cache_mm_s128d32i32(struct mm_struct *mm)
1357 {
1358 if (mm->context != 0) {
1359 #ifdef DEBUG_CACHE
1360 printk("cmm[%d]", (int)mm->context);
1361 #endif
1362 r4k_flush_cache_all_s128d32i32();
1363 }
1364 }
1365
1366 static void r4k_flush_cache_mm_d16i16(struct mm_struct *mm)
1367 {
1368 if (mm->context != 0) {
1369 #ifdef DEBUG_CACHE
1370 printk("cmm[%d]", (int)mm->context);
1371 #endif
1372 r4k_flush_cache_all_d16i16();
1373 }
1374 }
1375
1376 static void r4k_flush_cache_mm_d32i32(struct mm_struct *mm)
1377 {
1378 if (mm->context != 0) {
1379 #ifdef DEBUG_CACHE
1380 printk("cmm[%d]", (int)mm->context);
1381 #endif
1382 r4k_flush_cache_all_d32i32();
1383 }
1384 }
1385
1386 static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma,
1387 unsigned long page)
1388 {
1389 struct mm_struct *mm = vma->vm_mm;
1390 unsigned long flags;
1391 pgd_t *pgdp;
1392 pmd_t *pmdp;
1393 pte_t *ptep;
1394
1395 /*
1396 * If ownes no valid ASID yet, cannot possibly have gotten
1397 * this page into the cache.
1398 */
1399 if (mm->context == 0)
1400 return;
1401
1402 #ifdef DEBUG_CACHE
1403 printk("cpage[%d,%08lx]", (int)mm->context, page);
1404 #endif
1405 __save_and_cli(flags);
1406 page &= PAGE_MASK;
1407 pgdp = pgd_offset(mm, page);
1408 pmdp = pmd_offset(pgdp, page);
1409 ptep = pte_offset(pmdp, page);
1410
1411 /*
1412 * If the page isn't marked valid, the page cannot possibly be
1413 * in the cache.
1414 */
1415 if (!(pte_val(*ptep) & _PAGE_VALID))
1416 goto out;
1417
1418 /*
1419 * Doing flushes for another ASID than the current one is
1420 * too difficult since stupid R4k caches do a TLB translation
1421 * for every cache flush operation. So we do indexed flushes
1422 * in that case, which doesn't overly flush the cache too much.
1423 */
1424 if (mm->context != current->active_mm->context) {
1425 /*
1426 * Do indexed flush, too much work to get the (possible)
1427 * tlb refills to work correctly.
1428 */
1429 page = (KSEG0 + (page & (scache_size - 1)));
1430 blast_dcache16_page_indexed(page);
1431 blast_scache16_page_indexed(page);
1432 } else
1433 blast_scache16_page(page);
1434 out:
1435 __restore_flags(flags);
1436 }
1437
1438 static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma,
1439 unsigned long page)
1440 {
1441 struct mm_struct *mm = vma->vm_mm;
1442 unsigned long flags;
1443 pgd_t *pgdp;
1444 pmd_t *pmdp;
1445 pte_t *ptep;
1446
1447 /*
1448 * If ownes no valid ASID yet, cannot possibly have gotten
1449 * this page into the cache.
1450 */
1451 if (mm->context == 0)
1452 return;
1453
1454 #ifdef DEBUG_CACHE
1455 printk("cpage[%d,%08lx]", (int)mm->context, page);
1456 #endif
1457 __save_and_cli(flags);
1458 page &= PAGE_MASK;
1459 pgdp = pgd_offset(mm, page);
1460 pmdp = pmd_offset(pgdp, page);
1461 ptep = pte_offset(pmdp, page);
1462
1463 /* If the page isn't marked valid, the page cannot possibly be
1464 * in the cache.
1465 */
1466 if (!(pte_val(*ptep) & _PAGE_VALID))
1467 goto out;
1468
1469 /*
1470 * Doing flushes for another ASID than the current one is
1471 * too difficult since stupid R4k caches do a TLB translation
1472 * for every cache flush operation. So we do indexed flushes
1473 * in that case, which doesn't overly flush the cache too much.
1474 */
1475 if (mm->context != current->active_mm->context) {
1476 /*
1477 * Do indexed flush, too much work to get the (possible)
1478 * tlb refills to work correctly.
1479 */
1480 page = (KSEG0 + (page & (scache_size - 1)));
1481 blast_dcache16_page_indexed(page);
1482 blast_scache32_page_indexed(page);
1483 } else
1484 blast_scache32_page(page);
1485 out:
1486 __restore_flags(flags);
1487 }
1488
1489 static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma,
1490 unsigned long page)
1491 {
1492 struct mm_struct *mm = vma->vm_mm;
1493 unsigned long flags;
1494 pgd_t *pgdp;
1495 pmd_t *pmdp;
1496 pte_t *ptep;
1497
1498 /*
1499 * If ownes no valid ASID yet, cannot possibly have gotten
1500 * this page into the cache.
1501 */
1502 if (mm->context == 0)
1503 return;
1504
1505 #ifdef DEBUG_CACHE
1506 printk("cpage[%d,%08lx]", (int)mm->context, page);
1507 #endif
1508 __save_and_cli(flags);
1509 page &= PAGE_MASK;
1510 pgdp = pgd_offset(mm, page);
1511 pmdp = pmd_offset(pgdp, page);
1512 ptep = pte_offset(pmdp, page);
1513
1514 /* If the page isn't marked valid, the page cannot possibly be
1515 * in the cache.
1516 */
1517 if (!(pte_val(*ptep) & _PAGE_VALID))
1518 goto out;
1519
1520 /*
1521 * Doing flushes for another ASID than the current one is
1522 * too difficult since stupid R4k caches do a TLB translation
1523 * for every cache flush operation. So we do indexed flushes
1524 * in that case, which doesn't overly flush the cache too much.
1525 */
1526 if (mm->context != current->active_mm->context) {
1527 /*
1528 * Do indexed flush, too much work to get the (possible)
1529 * tlb refills to work correctly.
1530 */
1531 page = (KSEG0 + (page & (scache_size - 1)));
1532 blast_dcache16_page_indexed(page);
1533 blast_scache64_page_indexed(page);
1534 } else
1535 blast_scache64_page(page);
1536 out:
1537 __restore_flags(flags);
1538 }
1539
1540 static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma,
1541 unsigned long page)
1542 {
1543 struct mm_struct *mm = vma->vm_mm;
1544 unsigned long flags;
1545 pgd_t *pgdp;
1546 pmd_t *pmdp;
1547 pte_t *ptep;
1548
1549 /*
1550 * If ownes no valid ASID yet, cannot possibly have gotten
1551 * this page into the cache.
1552 */
1553 if (mm->context == 0)
1554 return;
1555
1556 #ifdef DEBUG_CACHE
1557 printk("cpage[%d,%08lx]", (int)mm->context, page);
1558 #endif
1559 __save_and_cli(flags);
1560 page &= PAGE_MASK;
1561 pgdp = pgd_offset(mm, page);
1562 pmdp = pmd_offset(pgdp, page);
1563 ptep = pte_offset(pmdp, page);
1564
1565 /*
1566 * If the page isn't marked valid, the page cannot possibly be
1567 * in the cache.
1568 */
1569 if (!(pte_val(*ptep) & _PAGE_VALID))
1570 goto out;
1571
1572 /*
1573 * Doing flushes for another ASID than the current one is
1574 * too difficult since stupid R4k caches do a TLB translation
1575 * for every cache flush operation. So we do indexed flushes
1576 * in that case, which doesn't overly flush the cache too much.
1577 */
1578 if (mm->context != current->active_mm->context) {
1579 /*
1580 * Do indexed flush, too much work to get the (possible)
1581 * tlb refills to work correctly.
1582 */
1583 page = (KSEG0 + (page & (scache_size - 1)));
1584 blast_dcache16_page_indexed(page);
1585 blast_scache128_page_indexed(page);
1586 } else
1587 blast_scache128_page(page);
1588 out:
1589 __restore_flags(flags);
1590 }
1591
1592 static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma,
1593 unsigned long page)
1594 {
1595 struct mm_struct *mm = vma->vm_mm;
1596 unsigned long flags;
1597 pgd_t *pgdp;
1598 pmd_t *pmdp;
1599 pte_t *ptep;
1600
1601 /*
1602 * If ownes no valid ASID yet, cannot possibly have gotten
1603 * this page into the cache.
1604 */
1605 if (mm->context == 0)
1606 return;
1607
1608 #ifdef DEBUG_CACHE
1609 printk("cpage[%d,%08lx]", (int)mm->context, page);
1610 #endif
1611 __save_and_cli(flags);
1612 page &= PAGE_MASK;
1613 pgdp = pgd_offset(mm, page);
1614 pmdp = pmd_offset(pgdp, page);
1615 ptep = pte_offset(pmdp, page);
1616
1617 /*
1618 * If the page isn't marked valid, the page cannot possibly be
1619 * in the cache.
1620 */
1621 if (!(pte_val(*ptep) & _PAGE_VALID))
1622 goto out;
1623
1624 /*
1625 * Doing flushes for another ASID than the current one is
1626 * too difficult since stupid R4k caches do a TLB translation
1627 * for every cache flush operation. So we do indexed flushes
1628 * in that case, which doesn't overly flush the cache too much.
1629 */
1630 if (mm->context != current->active_mm->context) {
1631 /*
1632 * Do indexed flush, too much work to get the (possible)
1633 * tlb refills to work correctly.
1634 */
1635 page = (KSEG0 + (page & (scache_size - 1)));
1636 blast_dcache32_page_indexed(page);
1637 blast_scache32_page_indexed(page);
1638 } else
1639 blast_scache32_page(page);
1640 out:
1641 __restore_flags(flags);
1642 }
1643
1644 static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma,
1645 unsigned long page)
1646 {
1647 struct mm_struct *mm = vma->vm_mm;
1648 unsigned long flags;
1649 pgd_t *pgdp;
1650 pmd_t *pmdp;
1651 pte_t *ptep;
1652
1653 /*
1654 * If ownes no valid ASID yet, cannot possibly have gotten
1655 * this page into the cache.
1656 */
1657 if (mm->context == 0)
1658 return;
1659
1660 #ifdef DEBUG_CACHE
1661 printk("cpage[%d,%08lx]", (int)mm->context, page);
1662 #endif
1663 __save_and_cli(flags);
1664 page &= PAGE_MASK;
1665 pgdp = pgd_offset(mm, page);
1666 pmdp = pmd_offset(pgdp, page);
1667 ptep = pte_offset(pmdp, page);
1668
1669 /*
1670 * If the page isn't marked valid, the page cannot possibly be
1671 * in the cache.
1672 */
1673 if (!(pte_val(*ptep) & _PAGE_VALID))
1674 goto out;
1675
1676 /*
1677 * Doing flushes for another ASID than the current one is
1678 * too difficult since stupid R4k caches do a TLB translation
1679 * for every cache flush operation. So we do indexed flushes
1680 * in that case, which doesn't overly flush the cache too much.
1681 */
1682 if (mm->context != current->active_mm->context) {
1683 /*
1684 * Do indexed flush, too much work to get the (possible)
1685 * tlb refills to work correctly.
1686 */
1687 page = (KSEG0 + (page & (scache_size - 1)));
1688 blast_dcache32_page_indexed(page);
1689 blast_scache64_page_indexed(page);
1690 } else
1691 blast_scache64_page(page);
1692 out:
1693 __restore_flags(flags);
1694 }
1695
1696 static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma,
1697 unsigned long page)
1698 {
1699 struct mm_struct *mm = vma->vm_mm;
1700 unsigned long flags;
1701 pgd_t *pgdp;
1702 pmd_t *pmdp;
1703 pte_t *ptep;
1704
1705 /*
1706 * If ownes no valid ASID yet, cannot possibly have gotten
1707 * this page into the cache.
1708 */
1709 if (mm->context == 0)
1710 return;
1711
1712 #ifdef DEBUG_CACHE
1713 printk("cpage[%d,%08lx]", (int)mm->context, page);
1714 #endif
1715 __save_and_cli(flags);
1716 page &= PAGE_MASK;
1717 pgdp = pgd_offset(mm, page);
1718 pmdp = pmd_offset(pgdp, page);
1719 ptep = pte_offset(pmdp, page);
1720
1721 /*
1722 * If the page isn't marked valid, the page cannot possibly be
1723 * in the cache.
1724 */
1725 if (!(pte_val(*ptep) & _PAGE_VALID))
1726 goto out;
1727
1728 /*
1729 * Doing flushes for another ASID than the current one is
1730 * too difficult since stupid R4k caches do a TLB translation
1731 * for every cache flush operation. So we do indexed flushes
1732 * in that case, which doesn't overly flush the cache too much.
1733 */
1734 if (mm->context != current->active_mm->context) {
1735 /* Do indexed flush, too much work to get the (possible)
1736 * tlb refills to work correctly.
1737 */
1738 page = (KSEG0 + (page & (scache_size - 1)));
1739 blast_dcache32_page_indexed(page);
1740 blast_scache128_page_indexed(page);
1741 } else
1742 blast_scache128_page(page);
1743 out:
1744 __restore_flags(flags);
1745 }
1746
1747 static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma,
1748 unsigned long page)
1749 {
1750 struct mm_struct *mm = vma->vm_mm;
1751 unsigned long flags;
1752 pgd_t *pgdp;
1753 pmd_t *pmdp;
1754 pte_t *ptep;
1755
1756 /*
1757 * If ownes no valid ASID yet, cannot possibly have gotten
1758 * this page into the cache.
1759 */
1760 if (mm->context == 0)
1761 return;
1762
1763 #ifdef DEBUG_CACHE
1764 printk("cpage[%d,%08lx]", (int)mm->context, page);
1765 #endif
1766 __save_and_cli(flags);
1767 page &= PAGE_MASK;
1768 pgdp = pgd_offset(mm, page);
1769 pmdp = pmd_offset(pgdp, page);
1770 ptep = pte_offset(pmdp, page);
1771
1772 /*
1773 * If the page isn't marked valid, the page cannot possibly be
1774 * in the cache.
1775 */
1776 if (!(pte_val(*ptep) & _PAGE_VALID))
1777 goto out;
1778
1779 /*
1780 * Doing flushes for another ASID than the current one is
1781 * too difficult since stupid R4k caches do a TLB translation
1782 * for every cache flush operation. So we do indexed flushes
1783 * in that case, which doesn't overly flush the cache too much.
1784 */
1785 if (mm == current->active_mm) {
1786 blast_dcache16_page(page);
1787 } else {
1788 /* Do indexed flush, too much work to get the (possible)
1789 * tlb refills to work correctly.
1790 */
1791 page = (KSEG0 + (page & (dcache_size - 1)));
1792 blast_dcache16_page_indexed(page);
1793 }
1794 out:
1795 __restore_flags(flags);
1796 }
1797
1798 static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma,
1799 unsigned long page)
1800 {
1801 struct mm_struct *mm = vma->vm_mm;
1802 unsigned long flags;
1803 pgd_t *pgdp;
1804 pmd_t *pmdp;
1805 pte_t *ptep;
1806
1807 /*
1808 * If ownes no valid ASID yet, cannot possibly have gotten
1809 * this page into the cache.
1810 */
1811 if (mm->context == 0)
1812 return;
1813
1814 #ifdef DEBUG_CACHE
1815 printk("cpage[%d,%08lx]", (int)mm->context, page);
1816 #endif
1817 __save_and_cli(flags);
1818 page &= PAGE_MASK;
1819 pgdp = pgd_offset(mm, page);
1820 pmdp = pmd_offset(pgdp, page);
1821 ptep = pte_offset(pmdp, page);
1822
1823 /*
1824 * If the page isn't marked valid, the page cannot possibly be
1825 * in the cache.
1826 */
1827 if (!(pte_val(*ptep) & _PAGE_PRESENT))
1828 goto out;
1829
1830 /*
1831 * Doing flushes for another ASID than the current one is
1832 * too difficult since stupid R4k caches do a TLB translation
1833 * for every cache flush operation. So we do indexed flushes
1834 * in that case, which doesn't overly flush the cache too much.
1835 */
1836 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1837 blast_dcache32_page(page);
1838 } else {
1839 /*
1840 * Do indexed flush, too much work to get the (possible)
1841 * tlb refills to work correctly.
1842 */
1843 page = (KSEG0 + (page & (dcache_size - 1)));
1844 blast_dcache32_page_indexed(page);
1845 }
1846 out:
1847 __restore_flags(flags);
1848 }
1849
1850 static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma,
1851 unsigned long page)
1852 {
1853 struct mm_struct *mm = vma->vm_mm;
1854 unsigned long flags;
1855 pgd_t *pgdp;
1856 pmd_t *pmdp;
1857 pte_t *ptep;
1858
1859 /*
1860 * If ownes no valid ASID yet, cannot possibly have gotten
1861 * this page into the cache.
1862 */
1863 if (mm->context == 0)
1864 return;
1865
1866 #ifdef DEBUG_CACHE
1867 printk("cpage[%d,%08lx]", (int)mm->context, page);
1868 #endif
1869 __save_and_cli(flags);
1870 page &= PAGE_MASK;
1871 pgdp = pgd_offset(mm, page);
1872 pmdp = pmd_offset(pgdp, page);
1873 ptep = pte_offset(pmdp, page);
1874
1875 /*
1876 * If the page isn't marked valid, the page cannot possibly be
1877 * in the cache.
1878 */
1879 if (!(pte_val(*ptep) & _PAGE_PRESENT))
1880 goto out;
1881
1882 /*
1883 * Doing flushes for another ASID than the current one is
1884 * too difficult since stupid R4k caches do a TLB translation
1885 * for every cache flush operation. So we do indexed flushes
1886 * in that case, which doesn't overly flush the cache too much.
1887 */
1888 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1889 blast_dcache32_page(page);
1890 } else {
1891 /* Do indexed flush, too much work to get the (possible)
1892 * tlb refills to work correctly.
1893 */
1894 page = (KSEG0 + (page & (dcache_size - 1)));
1895 blast_dcache32_page_indexed(page);
1896 blast_dcache32_page_indexed(page ^ dcache_waybit);
1897 }
1898 out:
1899 __restore_flags(flags);
1900 }
1901
1902 /* If the addresses passed to these routines are valid, they are
1903 * either:
1904 *
1905 * 1) In KSEG0, so we can do a direct flush of the page.
1906 * 2) In KSEG2, and since every process can translate those
1907 * addresses all the time in kernel mode we can do a direct
1908 * flush.
1909 * 3) In KSEG1, no flush necessary.
1910 */
1911 static void r4k_flush_page_to_ram_s16(struct page *page)
1912 {
1913 blast_scache16_page((unsigned long)page_address(page));
1914 }
1915
1916 static void r4k_flush_page_to_ram_s32(struct page *page)
1917 {
1918 blast_scache32_page((unsigned long)page_address(page));
1919 }
1920
1921 static void r4k_flush_page_to_ram_s64(struct page *page)
1922 {
1923 blast_scache64_page((unsigned long)page_address(page));
1924 }
1925
1926 static void r4k_flush_page_to_ram_s128(struct page *page)
1927 {
1928 blast_scache128_page((unsigned long)page_address(page));
1929 }
1930
1931 static void r4k_flush_page_to_ram_d16(struct page *page)
1932 {
1933 blast_dcache16_page((unsigned long)page_address(page));
1934 }
1935
1936 static void r4k_flush_page_to_ram_d32(struct page *page)
1937 {
1938 blast_dcache32_page((unsigned long)page_address(page));
1939 }
1940
1941 static void r4k_flush_page_to_ram_d32_r4600(struct page *page)
1942 {
1943 unsigned long flags;
1944
1945 __save_and_cli(flags); /* For R4600 v1.7 bug. */
1946 blast_dcache32_page((unsigned long)page_address(page));
1947 __restore_flags(flags);
1948 }
1949
1950 static void
1951 r4k_flush_icache_page_s(struct vm_area_struct *vma, struct page *page)
1952 {
1953 /*
1954 * We did an scache flush therefore PI is already clean.
1955 */
1956 }
1957
1958 static void
1959 r4k_flush_icache_range(unsigned long start, unsigned long end)
1960 {
1961 flush_cache_all();
1962 }
1963
1964 /*
1965 * Ok, this seriously sucks. We use them to flush a user page but don't
1966 * know the virtual address, so we have to blast away the whole icache
1967 * which is significantly more expensive than the real thing.
1968 */
1969 static void
1970 r4k_flush_icache_page_p(struct vm_area_struct *vma, struct page *page)
1971 {
1972 if (!(vma->vm_flags & VM_EXEC))
1973 return;
1974
1975 flush_cache_all();
1976 }
1977
1978 /*
1979 * Writeback and invalidate the primary cache dcache before DMA.
1980 *
1981 * R4600 v2.0 bug: "The CACHE instructions Hit_Writeback_Inv_D,
1982 * Hit_Writeback_D, Hit_Invalidate_D and Create_Dirty_Exclusive_D will only
1983 * operate correctly if the internal data cache refill buffer is empty. These
1984 * CACHE instructions should be separated from any potential data cache miss
1985 * by a load instruction to an uncached address to empty the response buffer."
1986 * (Revision 2.0 device errata from IDT available on http://www.idt.com/
1987 * in .pdf format.)
1988 */
1989 static void
1990 r4k_dma_cache_wback_inv_pc(unsigned long addr, unsigned long size)
1991 {
1992 unsigned long end, a;
1993 unsigned int flags;
1994
1995 if (size >= dcache_size) {
1996 flush_cache_all();
1997 } else {
1998 /* Workaround for R4600 bug. See comment above. */
1999 __save_and_cli(flags);
2000 *(volatile unsigned long *)KSEG1;
2001
2002 a = addr & ~(dc_lsize - 1);
2003 end = (addr + size) & ~(dc_lsize - 1);
2004 while (1) {
2005 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
2006 if (a == end) break;
2007 a += dc_lsize;
2008 }
2009 __restore_flags(flags);
2010 }
2011 bc_wback_inv(addr, size);
2012 }
2013
2014 static void
2015 r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size)
2016 {
2017 unsigned long end, a;
2018
2019 if (size >= scache_size) {
2020 flush_cache_all();
2021 return;
2022 }
2023
2024 a = addr & ~(sc_lsize - 1);
2025 end = (addr + size) & ~(sc_lsize - 1);
2026 while (1) {
2027 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
2028 if (a == end) break;
2029 a += sc_lsize;
2030 }
2031 }
2032
2033 static void
2034 r4k_dma_cache_inv_pc(unsigned long addr, unsigned long size)
2035 {
2036 unsigned long end, a;
2037 unsigned int flags;
2038
2039 if (size >= dcache_size) {
2040 flush_cache_all();
2041 } else {
2042 /* Workaround for R4600 bug. See comment above. */
2043 __save_and_cli(flags);
2044 *(volatile unsigned long *)KSEG1;
2045
2046 a = addr & ~(dc_lsize - 1);
2047 end = (addr + size) & ~(dc_lsize - 1);
2048 while (1) {
2049 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
2050 if (a == end) break;
2051 a += dc_lsize;
2052 }
2053 __restore_flags(flags);
2054 }
2055
2056 bc_inv(addr, size);
2057 }
2058
2059 static void
2060 r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
2061 {
2062 unsigned long end, a;
2063
2064 if (size >= scache_size) {
2065 flush_cache_all();
2066 return;
2067 }
2068
2069 a = addr & ~(sc_lsize - 1);
2070 end = (addr + size) & ~(sc_lsize - 1);
2071 while (1) {
2072 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
2073 if (a == end) break;
2074 a += sc_lsize;
2075 }
2076 }
2077
2078 static void
2079 r4k_dma_cache_wback(unsigned long addr, unsigned long size)
2080 {
2081 panic("r4k_dma_cache called - should not happen.\n");
2082 }
2083
2084 /*
2085 * While we're protected against bad userland addresses we don't care
2086 * very much about what happens in that case. Usually a segmentation
2087 * fault will dump the process later on anyway ...
2088 */
2089 static void r4k_flush_cache_sigtramp(unsigned long addr)
2090 {
2091 __asm__ __volatile__("nop;nop;nop;nop"); /* R4600 V1.7 */
2092 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
2093 protected_flush_icache_line(addr & ~(ic_lsize - 1));
2094 }
2095
2096 static void r4600v20k_flush_cache_sigtramp(unsigned long addr)
2097 {
2098 unsigned int flags;
2099
2100 __save_and_cli(flags);
2101
2102 /* Clear internal cache refill buffer */
2103 *(volatile unsigned int *)KSEG1;
2104
2105 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
2106 protected_flush_icache_line(addr & ~(ic_lsize - 1));
2107
2108 __restore_flags(flags);
2109 }
2110
2111 #undef DEBUG_TLB
2112 #undef DEBUG_TLBUPDATE
2113
2114 void flush_tlb_all(void)
2115 {
2116 unsigned long flags;
2117 unsigned long old_ctx;
2118 int entry;
2119
2120 #ifdef DEBUG_TLB
2121 printk("[tlball]");
2122 #endif
2123
2124 __save_and_cli(flags);
2125 /* Save old context and create impossible VPN2 value */
2126 old_ctx = (get_entryhi() & 0xff);
2127 set_entryhi(KSEG0);
2128 set_entrylo0(0);
2129 set_entrylo1(0);
2130 BARRIER;
2131
2132 entry = get_wired();
2133
2134 /* Blast 'em all away. */
2135 while(entry < mips_cpu.tlbsize) {
2136 set_index(entry);
2137 BARRIER;
2138 tlb_write_indexed();
2139 BARRIER;
2140 entry++;
2141 }
2142 BARRIER;
2143 set_entryhi(old_ctx);
2144 __restore_flags(flags);
2145 }
2146
2147 void flush_tlb_mm(struct mm_struct *mm)
2148 {
2149 if (mm->context != 0) {
2150 unsigned long flags;
2151
2152 #ifdef DEBUG_TLB
2153 printk("[tlbmm<%d>]", mm->context);
2154 #endif
2155 __save_and_cli(flags);
2156 get_new_mmu_context(mm, asid_cache);
2157 if (mm == current->active_mm)
2158 set_entryhi(mm->context & 0xff);
2159 __restore_flags(flags);
2160 }
2161 }
2162
2163 void flush_tlb_range(struct mm_struct *mm, unsigned long start,
2164 unsigned long end)
2165 {
2166 if(mm->context != 0) {
2167 unsigned long flags;
2168 int size;
2169
2170 #ifdef DEBUG_TLB
2171 printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & 0xff),
2172 start, end);
2173 #endif
2174 __save_and_cli(flags);
2175 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
2176 size = (size + 1) >> 1;
2177 if(size <= mips_cpu.tlbsize/2) {
2178 int oldpid = (get_entryhi() & 0xff);
2179 int newpid = (mm->context & 0xff);
2180
2181 start &= (PAGE_MASK << 1);
2182 end += ((PAGE_SIZE << 1) - 1);
2183 end &= (PAGE_MASK << 1);
2184 while(start < end) {
2185 int idx;
2186
2187 set_entryhi(start | newpid);
2188 start += (PAGE_SIZE << 1);
2189 BARRIER;
2190 tlb_probe();
2191 BARRIER;
2192 idx = get_index();
2193 set_entrylo0(0);
2194 set_entrylo1(0);
2195 set_entryhi(KSEG0);
2196 BARRIER;
2197 if(idx < 0)
2198 continue;
2199 tlb_write_indexed();
2200 BARRIER;
2201 }
2202 set_entryhi(oldpid);
2203 } else {
2204 get_new_mmu_context(mm, asid_cache);
2205 if (mm == current->active_mm)
2206 set_entryhi(mm->context & 0xff);
2207 }
2208 __restore_flags(flags);
2209 }
2210 }
2211
2212 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
2213 {
2214 if (vma->vm_mm->context != 0) {
2215 unsigned long flags;
2216 int oldpid, newpid, idx;
2217
2218 #ifdef DEBUG_TLB
2219 printk("[tlbpage<%d,%08lx>]", vma->vm_mm->context, page);
2220 #endif
2221 newpid = (vma->vm_mm->context & 0xff);
2222 page &= (PAGE_MASK << 1);
2223 __save_and_cli(flags);
2224 oldpid = (get_entryhi() & 0xff);
2225 set_entryhi(page | newpid);
2226 BARRIER;
2227 tlb_probe();
2228 BARRIER;
2229 idx = get_index();
2230 set_entrylo0(0);
2231 set_entrylo1(0);
2232 set_entryhi(KSEG0);
2233 if(idx < 0)
2234 goto finish;
2235 BARRIER;
2236 tlb_write_indexed();
2237
2238 finish:
2239 BARRIER;
2240 set_entryhi(oldpid);
2241 __restore_flags(flags);
2242 }
2243 }
2244
2245 void pgd_init(unsigned long page)
2246 {
2247 unsigned long *p = (unsigned long *) page;
2248 int i;
2249
2250 for(i = 0; i < USER_PTRS_PER_PGD; i+=8) {
2251 p[i + 0] = (unsigned long) invalid_pte_table;
2252 p[i + 1] = (unsigned long) invalid_pte_table;
2253 p[i + 2] = (unsigned long) invalid_pte_table;
2254 p[i + 3] = (unsigned long) invalid_pte_table;
2255 p[i + 4] = (unsigned long) invalid_pte_table;
2256 p[i + 5] = (unsigned long) invalid_pte_table;
2257 p[i + 6] = (unsigned long) invalid_pte_table;
2258 p[i + 7] = (unsigned long) invalid_pte_table;
2259 }
2260 }
2261
2262 /* We will need multiple versions of update_mmu_cache(), one that just
2263 * updates the TLB with the new pte(s), and another which also checks
2264 * for the R4k "end of page" hardware bug and does the needy.
2265 */
2266 void update_mmu_cache(struct vm_area_struct * vma,
2267 unsigned long address, pte_t pte)
2268 {
2269 unsigned long flags;
2270 pgd_t *pgdp;
2271 pmd_t *pmdp;
2272 pte_t *ptep;
2273 int idx, pid;
2274
2275 /*
2276 * Handle debugger faulting in for debugee.
2277 */
2278 if (current->active_mm != vma->vm_mm)
2279 return;
2280
2281 pid = get_entryhi() & 0xff;
2282
2283 #ifdef DEBUG_TLB
2284 if((pid != (vma->vm_mm->context & 0xff)) || (vma->vm_mm->context == 0)) {
2285 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%d tlbpid=%d\n",
2286 (int) (vma->vm_mm->context & 0xff), pid);
2287 }
2288 #endif
2289
2290 __save_and_cli(flags);
2291 address &= (PAGE_MASK << 1);
2292 set_entryhi(address | (pid));
2293 pgdp = pgd_offset(vma->vm_mm, address);
2294 BARRIER;
2295 tlb_probe();
2296 BARRIER;
2297 pmdp = pmd_offset(pgdp, address);
2298 idx = get_index();
2299 ptep = pte_offset(pmdp, address);
2300 BARRIER;
2301 set_entrylo0(pte_val(*ptep++) >> 6);
2302 set_entrylo1(pte_val(*ptep) >> 6);
2303 set_entryhi(address | (pid));
2304 BARRIER;
2305 if(idx < 0) {
2306 tlb_write_random();
2307 } else {
2308 tlb_write_indexed();
2309 }
2310 BARRIER;
2311 set_entryhi(pid);
2312 BARRIER;
2313 __restore_flags(flags);
2314 }
2315
2316 #if 0
2317 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
2318 unsigned long address, pte_t pte)
2319 {
2320 unsigned long flags;
2321 pgd_t *pgdp;
2322 pmd_t *pmdp;
2323 pte_t *ptep;
2324 int idx;
2325
2326 __save_and_cli(flags);
2327 address &= (PAGE_MASK << 1);
2328 set_entryhi(address | (get_entryhi() & 0xff));
2329 pgdp = pgd_offset(vma->vm_mm, address);
2330 tlb_probe();
2331 pmdp = pmd_offset(pgdp, address);
2332 idx = get_index();
2333 ptep = pte_offset(pmdp, address);
2334 set_entrylo0(pte_val(*ptep++) >> 6);
2335 set_entrylo1(pte_val(*ptep) >> 6);
2336 BARRIER;
2337 if(idx < 0)
2338 tlb_write_random();
2339 else
2340 tlb_write_indexed();
2341 BARRIER;
2342 __restore_flags(flags);
2343 }
2344 #endif
2345
2346 void show_regs(struct pt_regs * regs)
2347 {
2348 /* Saved main processor registers. */
2349 printk("$0 : %08lx %08lx %08lx %08lx\n",
2350 0UL, regs->regs[1], regs->regs[2], regs->regs[3]);
2351 printk("$4 : %08lx %08lx %08lx %08lx\n",
2352 regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7]);
2353 printk("$8 : %08lx %08lx %08lx %08lx\n",
2354 regs->regs[8], regs->regs[9], regs->regs[10], regs->regs[11]);
2355 printk("$12: %08lx %08lx %08lx %08lx\n",
2356 regs->regs[12], regs->regs[13], regs->regs[14], regs->regs[15]);
2357 printk("$16: %08lx %08lx %08lx %08lx\n",
2358 regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]);
2359 printk("$20: %08lx %08lx %08lx %08lx\n",
2360 regs->regs[20], regs->regs[21], regs->regs[22], regs->regs[23]);
2361 printk("$24: %08lx %08lx\n",
2362 regs->regs[24], regs->regs[25]);
2363 printk("$28: %08lx %08lx %08lx %08lx\n",
2364 regs->regs[28], regs->regs[29], regs->regs[30], regs->regs[31]);
2365
2366 /* Saved cp0 registers. */
2367 printk("epc : %08lx\nStatus: %08lx\nCause : %08lx\n",
2368 regs->cp0_epc, regs->cp0_status, regs->cp0_cause);
2369 }
2370
2371 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
2372 unsigned long entryhi, unsigned long pagemask)
2373 {
2374 unsigned long flags;
2375 unsigned long wired;
2376 unsigned long old_pagemask;
2377 unsigned long old_ctx;
2378
2379 __save_and_cli(flags);
2380 /* Save old context and create impossible VPN2 value */
2381 old_ctx = (get_entryhi() & 0xff);
2382 old_pagemask = get_pagemask();
2383 wired = get_wired();
2384 set_wired (wired + 1);
2385 set_index (wired);
2386 BARRIER;
2387 set_pagemask (pagemask);
2388 set_entryhi(entryhi);
2389 set_entrylo0(entrylo0);
2390 set_entrylo1(entrylo1);
2391 BARRIER;
2392 tlb_write_indexed();
2393 BARRIER;
2394
2395 set_entryhi(old_ctx);
2396 BARRIER;
2397 set_pagemask (old_pagemask);
2398 flush_tlb_all();
2399 __restore_flags(flags);
2400 }
2401
2402 /* Detect and size the various r4k caches. */
2403 static void __init probe_icache(unsigned long config)
2404 {
2405 switch (mips_cpu.cputype) {
2406 case CPU_VR41XX:
2407 icache_size = 1 << (10 + ((config >> 9) & 7));
2408 break;
2409 default:
2410 icache_size = 1 << (12 + ((config >> 9) & 7));
2411 break;
2412 }
2413 ic_lsize = 16 << ((config >> 5) & 1);
2414
2415 printk("Primary instruction cache %dkb, linesize %d bytes.\n",
2416 icache_size >> 10, ic_lsize);
2417 }
2418
2419 static void __init probe_dcache(unsigned long config)
2420 {
2421 switch (mips_cpu.cputype) {
2422 case CPU_VR41XX:
2423 dcache_size = 1 << (10 + ((config >> 6) & 7));
2424 break;
2425 default:
2426 dcache_size = 1 << (12 + ((config >> 6) & 7));
2427 break;
2428 }
2429 dc_lsize = 16 << ((config >> 4) & 1);
2430
2431 printk("Primary data cache %dkb, linesize %d bytes.\n",
2432 dcache_size >> 10, dc_lsize);
2433 }
2434
2435
2436 /* If you even _breathe_ on this function, look at the gcc output
2437 * and make sure it does not pop things on and off the stack for
2438 * the cache sizing loop that executes in KSEG1 space or else
2439 * you will crash and burn badly. You have been warned.
2440 */
2441 static int __init probe_scache(unsigned long config)
2442 {
2443 extern unsigned long stext;
2444 unsigned long flags, addr, begin, end, pow2;
2445 int tmp;
2446
2447 tmp = ((config >> 17) & 1);
2448 if(tmp)
2449 return 0;
2450 tmp = ((config >> 22) & 3);
2451 switch(tmp) {
2452 case 0:
2453 sc_lsize = 16;
2454 break;
2455 case 1:
2456 sc_lsize = 32;
2457 break;
2458 case 2:
2459 sc_lsize = 64;
2460 break;
2461 case 3:
2462 sc_lsize = 128;
2463 break;
2464 }
2465
2466 begin = (unsigned long) &stext;
2467 begin &= ~((4 * 1024 * 1024) - 1);
2468 end = begin + (4 * 1024 * 1024);
2469
2470 /* This is such a bitch, you'd think they would make it
2471 * easy to do this. Away you daemons of stupidity!
2472 */
2473 __save_and_cli(flags);
2474
2475 /* Fill each size-multiple cache line with a valid tag. */
2476 pow2 = (64 * 1024);
2477 for(addr = begin; addr < end; addr = (begin + pow2)) {
2478 unsigned long *p = (unsigned long *) addr;
2479 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
2480 pow2 <<= 1;
2481 }
2482
2483 /* Load first line with zero (therefore invalid) tag. */
2484 set_taglo(0);
2485 set_taghi(0);
2486 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
2487 __asm__ __volatile__("\n\t.set noreorder\n\t"
2488 ".set mips3\n\t"
2489 "cache 8, (%0)\n\t"
2490 ".set mips0\n\t"
2491 ".set reorder\n\t" : : "r" (begin));
2492 __asm__ __volatile__("\n\t.set noreorder\n\t"
2493 ".set mips3\n\t"
2494 "cache 9, (%0)\n\t"
2495 ".set mips0\n\t"
2496 ".set reorder\n\t" : : "r" (begin));
2497 __asm__ __volatile__("\n\t.set noreorder\n\t"
2498 ".set mips3\n\t"
2499 "cache 11, (%0)\n\t"
2500 ".set mips0\n\t"
2501 ".set reorder\n\t" : : "r" (begin));
2502
2503 /* Now search for the wrap around point. */
2504 pow2 = (128 * 1024);
2505 tmp = 0;
2506 for(addr = (begin + (128 * 1024)); addr < (end); addr = (begin + pow2)) {
2507 __asm__ __volatile__("\n\t.set noreorder\n\t"
2508 ".set mips3\n\t"
2509 "cache 7, (%0)\n\t"
2510 ".set mips0\n\t"
2511 ".set reorder\n\t" : : "r" (addr));
2512 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
2513 if(!get_taglo())
2514 break;
2515 pow2 <<= 1;
2516 }
2517 __restore_flags(flags);
2518 addr -= begin;
2519 printk("Secondary cache sized at %dK linesize %d bytes.\n",
2520 (int) (addr >> 10), sc_lsize);
2521 scache_size = addr;
2522 return 1;
2523 }
2524
2525 static void __init setup_noscache_funcs(void)
2526 {
2527 unsigned int prid;
2528
2529 switch(dc_lsize) {
2530 case 16:
2531 _clear_page = r4k_clear_page_d16;
2532 _copy_page = r4k_copy_page_d16;
2533 _flush_cache_all = r4k_flush_cache_all_d16i16;
2534 _flush_cache_mm = r4k_flush_cache_mm_d16i16;
2535 _flush_cache_range = r4k_flush_cache_range_d16i16;
2536 _flush_cache_page = r4k_flush_cache_page_d16i16;
2537 _flush_page_to_ram = r4k_flush_page_to_ram_d16;
2538 break;
2539 case 32:
2540 prid = read_32bit_cp0_register(CP0_PRID) & 0xfff0;
2541 if (prid == 0x2010) { /* R4600 V1.7 */
2542 _clear_page = r4k_clear_page_r4600_v1;
2543 _copy_page = r4k_copy_page_r4600_v1;
2544 _flush_page_to_ram = r4k_flush_page_to_ram_d32_r4600;
2545 } else if (prid == 0x2020) { /* R4600 V2.0 */
2546 _clear_page = r4k_clear_page_r4600_v2;
2547 _copy_page = r4k_copy_page_r4600_v2;
2548 _flush_page_to_ram = r4k_flush_page_to_ram_d32;
2549 } else {
2550 _clear_page = r4k_clear_page_d32;
2551 _copy_page = r4k_copy_page_d32;
2552 _flush_page_to_ram = r4k_flush_page_to_ram_d32;
2553 }
2554 _flush_cache_all = r4k_flush_cache_all_d32i32;
2555 _flush_cache_mm = r4k_flush_cache_mm_d32i32;
2556 _flush_cache_range = r4k_flush_cache_range_d32i32;
2557 _flush_cache_page = r4k_flush_cache_page_d32i32;
2558 break;
2559 }
2560 ___flush_cache_all = _flush_cache_all;
2561
2562 _flush_icache_page = r4k_flush_icache_page_p;
2563
2564 _dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc;
2565 _dma_cache_wback = r4k_dma_cache_wback;
2566 _dma_cache_inv = r4k_dma_cache_inv_pc;
2567 }
2568
2569 static void __init setup_scache_funcs(void)
2570 {
2571 switch(sc_lsize) {
2572 case 16:
2573 switch(dc_lsize) {
2574 case 16:
2575 _flush_cache_all = r4k_flush_cache_all_s16d16i16;
2576 _flush_cache_mm = r4k_flush_cache_mm_s16d16i16;
2577 _flush_cache_range = r4k_flush_cache_range_s16d16i16;
2578 _flush_cache_page = r4k_flush_cache_page_s16d16i16;
2579 break;
2580 case 32:
2581 panic("Invalid cache configuration detected");
2582 };
2583 _flush_page_to_ram = r4k_flush_page_to_ram_s16;
2584 _clear_page = r4k_clear_page_s16;
2585 _copy_page = r4k_copy_page_s16;
2586 break;
2587 case 32:
2588 switch(dc_lsize) {
2589 case 16:
2590 _flush_cache_all = r4k_flush_cache_all_s32d16i16;
2591 _flush_cache_mm = r4k_flush_cache_mm_s32d16i16;
2592 _flush_cache_range = r4k_flush_cache_range_s32d16i16;
2593 _flush_cache_page = r4k_flush_cache_page_s32d16i16;
2594 break;
2595 case 32:
2596 _flush_cache_all = r4k_flush_cache_all_s32d32i32;
2597 _flush_cache_mm = r4k_flush_cache_mm_s32d32i32;
2598 _flush_cache_range = r4k_flush_cache_range_s32d32i32;
2599 _flush_cache_page = r4k_flush_cache_page_s32d32i32;
2600 break;
2601 };
2602 _flush_page_to_ram = r4k_flush_page_to_ram_s32;
2603 _clear_page = r4k_clear_page_s32;
2604 _copy_page = r4k_copy_page_s32;
2605 break;
2606 case 64:
2607 switch(dc_lsize) {
2608 case 16:
2609 _flush_cache_all = r4k_flush_cache_all_s64d16i16;
2610 _flush_cache_mm = r4k_flush_cache_mm_s64d16i16;
2611 _flush_cache_range = r4k_flush_cache_range_s64d16i16;
2612 _flush_cache_page = r4k_flush_cache_page_s64d16i16;
2613 break;
2614 case 32:
2615 _flush_cache_all = r4k_flush_cache_all_s64d32i32;
2616 _flush_cache_mm = r4k_flush_cache_mm_s64d32i32;
2617 _flush_cache_range = r4k_flush_cache_range_s64d32i32;
2618 _flush_cache_page = r4k_flush_cache_page_s64d32i32;
2619 break;
2620 };
2621 _flush_page_to_ram = r4k_flush_page_to_ram_s64;
2622 _clear_page = r4k_clear_page_s64;
2623 _copy_page = r4k_copy_page_s64;
2624 break;
2625 case 128:
2626 switch(dc_lsize) {
2627 case 16:
2628 _flush_cache_all = r4k_flush_cache_all_s128d16i16;
2629 _flush_cache_mm = r4k_flush_cache_mm_s128d16i16;
2630 _flush_cache_range = r4k_flush_cache_range_s128d16i16;
2631 _flush_cache_page = r4k_flush_cache_page_s128d16i16;
2632 break;
2633 case 32:
2634 _flush_cache_all = r4k_flush_cache_all_s128d32i32;
2635 _flush_cache_mm = r4k_flush_cache_mm_s128d32i32;
2636 _flush_cache_range = r4k_flush_cache_range_s128d32i32;
2637 _flush_cache_page = r4k_flush_cache_page_s128d32i32;
2638 break;
2639 };
2640 _flush_page_to_ram = r4k_flush_page_to_ram_s128;
2641 _clear_page = r4k_clear_page_s128;
2642 _copy_page = r4k_copy_page_s128;
2643 break;
2644 }
2645 ___flush_cache_all = _flush_cache_all;
2646 _flush_icache_page = r4k_flush_icache_page_s;
2647 _dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc;
2648 _dma_cache_wback = r4k_dma_cache_wback;
2649 _dma_cache_inv = r4k_dma_cache_inv_sc;
2650 }
2651
2652 typedef int (*probe_func_t)(unsigned long);
2653
2654 static inline void __init setup_scache(unsigned int config)
2655 {
2656 probe_func_t probe_scache_kseg1;
2657 int sc_present = 0;
2658
2659 /* Maybe the cpu knows about a l2 cache? */
2660 probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache));
2661 sc_present = probe_scache_kseg1(config);
2662
2663 if (sc_present) {
2664 setup_scache_funcs();
2665 return;
2666 }
2667
2668 setup_noscache_funcs();
2669 }
2670
2671 void __init ld_mmu_r4xx0(void)
2672 {
2673 unsigned long config = read_32bit_cp0_register(CP0_CONFIG);
2674
2675 printk("CPU revision is: %08x\n", read_32bit_cp0_register(CP0_PRID));
2676
2677 #ifdef CONFIG_MIPS_UNCACHED
2678 change_cp0_config(CONF_CM_CMASK, CONF_CM_UNCACHED);
2679 #else
2680 change_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_NONCOHERENT);
2681 #endif
2682
2683 probe_icache(config);
2684 probe_dcache(config);
2685 setup_scache(config);
2686
2687 switch(mips_cpu.cputype) {
2688 case CPU_R4600: /* QED style two way caches? */
2689 case CPU_R4700:
2690 case CPU_R5000:
2691 case CPU_NEVADA:
2692 _flush_cache_page = r4k_flush_cache_page_d32i32_r4600;
2693 }
2694
2695 _flush_cache_sigtramp = r4k_flush_cache_sigtramp;
2696 _flush_icache_range = r4k_flush_icache_range; /* Ouch */
2697 if ((read_32bit_cp0_register(CP0_PRID) & 0xfff0) == 0x2020) {
2698 _flush_cache_sigtramp = r4600v20k_flush_cache_sigtramp;
2699 }
2700
2701 __flush_cache_all();
2702 write_32bit_cp0_register(CP0_WIRED, 0);
2703
2704 /*
2705 * You should never change this register:
2706 * - On R4600 1.7 the tlbp never hits for pages smaller than
2707 * the value in the c0_pagemask register.
2708 * - The entire mm handling assumes the c0_pagemask register to
2709 * be set for 4kb pages.
2710 */
2711 set_pagemask(PM_4K);
2712 flush_tlb_all();
2713 }
2714