File: /usr/src/linux/arch/mips64/mm/r4xx0.c
1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * r4xx0.c: R4000 processor variant specific MMU/Cache routines.
7 *
8 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com)
9 * Copyright (C) 1997, 1998, 1999, 2000, 2001 Ralf Baechle (ralf@gnu.org)
10 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
11 */
12 #include <linux/init.h>
13 #include <linux/kernel.h>
14 #include <linux/sched.h>
15 #include <linux/mm.h>
16
17 #include <asm/bcache.h>
18 #include <asm/io.h>
19 #include <asm/page.h>
20 #include <asm/pgtable.h>
21 #include <asm/system.h>
22 #include <asm/bootinfo.h>
23 #include <asm/sgialib.h>
24 #include <asm/mmu_context.h>
25
26 /* CP0 hazard avoidance. */
27 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
28 "nop; nop; nop; nop; nop; nop;\n\t" \
29 ".set reorder\n\t")
30
31 /* Primary cache parameters. */
32 static int icache_size, dcache_size; /* Size in bytes */
33 static int ic_lsize, dc_lsize; /* LineSize in bytes */
34
35 /* Secondary cache (if present) parameters. */
36 static unsigned int scache_size, sc_lsize; /* Again, in bytes */
37
38 #include <asm/r4kcacheops.h>
39 #include <asm/r4kcache.h>
40
41 #undef DEBUG_CACHE
42
43 /*
44 * Dummy cache handling routines for machines without boardcaches
45 */
46 static void no_sc_noop(void) {}
47
48 static struct bcache_ops no_sc_ops = {
49 (void *)no_sc_noop, (void *)no_sc_noop,
50 (void *)no_sc_noop, (void *)no_sc_noop
51 };
52
53 struct bcache_ops *bcops = &no_sc_ops;
54
55 /*
56 * On processors with QED R4600 style two set assosicative cache
57 * this is the bit which selects the way in the cache for the
58 * indexed cachops.
59 */
60 #define icache_waybit (icache_size >> 1)
61 #define dcache_waybit (dcache_size >> 1)
62
63 /*
64 * Zero an entire page. Basically a simple unrolled loop should do the
65 * job but we want more performance by saving memory bus bandwidth. We
66 * have five flavours of the routine available for:
67 *
68 * - 16byte cachelines and no second level cache
69 * - 32byte cachelines second level cache
70 * - a version which handles the buggy R4600 v1.x
71 * - a version which handles the buggy R4600 v2.0
72 * - Finally a last version without fancy cache games for the SC and MC
73 * versions of R4000 and R4400.
74 */
75
76 static void r4k_clear_page_d16(void * page)
77 {
78 __asm__ __volatile__(
79 ".set\tnoreorder\n\t"
80 ".set\tnoat\n\t"
81 "daddiu\t$1,%0,%2\n"
82 "1:\tcache\t%3,(%0)\n\t"
83 "sd\t$0,(%0)\n\t"
84 "sd\t$0,8(%0)\n\t"
85 "cache\t%3,16(%0)\n\t"
86 "sd\t$0,16(%0)\n\t"
87 "sd\t$0,24(%0)\n\t"
88 "daddiu\t%0,64\n\t"
89 "cache\t%3,-32(%0)\n\t"
90 "sd\t$0,-32(%0)\n\t"
91 "sd\t$0,-24(%0)\n\t"
92 "cache\t%3,-16(%0)\n\t"
93 "sd\t$0,-16(%0)\n\t"
94 "bne\t$1,%0,1b\n\t"
95 "sd\t$0,-8(%0)\n\t"
96 ".set\tat\n\t"
97 ".set\treorder"
98 :"=r" (page)
99 :"0" (page), "I" (PAGE_SIZE), "i" (Create_Dirty_Excl_D)
100 :"$1", "memory");
101 }
102
103 static void r4k_clear_page_d32(void * page)
104 {
105 __asm__ __volatile__(
106 ".set\tnoreorder\n\t"
107 ".set\tnoat\n\t"
108 "daddiu\t$1,%0,%2\n"
109 "1:\tcache\t%3,(%0)\n\t"
110 "sd\t$0,(%0)\n\t"
111 "sd\t$0,8(%0)\n\t"
112 "sd\t$0,16(%0)\n\t"
113 "sd\t$0,24(%0)\n\t"
114 "daddiu\t%0,64\n\t"
115 "cache\t%3,-32(%0)\n\t"
116 "sd\t$0,-32(%0)\n\t"
117 "sd\t$0,-24(%0)\n\t"
118 "sd\t$0,-16(%0)\n\t"
119 "bne\t$1,%0,1b\n\t"
120 "sd\t$0,-8(%0)\n\t"
121 ".set\tat\n\t"
122 ".set\treorder"
123 :"=r" (page)
124 :"0" (page), "I" (PAGE_SIZE), "i" (Create_Dirty_Excl_D)
125 :"$1", "memory");
126 }
127
128
129 /*
130 * This flavour of r4k_clear_page is for the R4600 V1.x. Cite from the
131 * IDT R4600 V1.7 errata:
132 *
133 * 18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D,
134 * Hit_Invalidate_D and Create_Dirty_Excl_D should only be
135 * executed if there is no other dcache activity. If the dcache is
136 * accessed for another instruction immeidately preceding when these
137 * cache instructions are executing, it is possible that the dcache
138 * tag match outputs used by these cache instructions will be
139 * incorrect. These cache instructions should be preceded by at least
140 * four instructions that are not any kind of load or store
141 * instruction.
142 *
143 * This is not allowed: lw
144 * nop
145 * nop
146 * nop
147 * cache Hit_Writeback_Invalidate_D
148 *
149 * This is allowed: lw
150 * nop
151 * nop
152 * nop
153 * nop
154 * cache Hit_Writeback_Invalidate_D
155 */
156 static void r4k_clear_page_r4600_v1(void * page)
157 {
158 __asm__ __volatile__(
159 ".set\tnoreorder\n\t"
160 ".set\tnoat\n\t"
161 "daddiu\t$1,%0,%2\n"
162 "1:\tnop\n\t"
163 "nop\n\t"
164 "nop\n\t"
165 "nop\n\t"
166 "cache\t%3,(%0)\n\t"
167 "sd\t$0,(%0)\n\t"
168 "sd\t$0,8(%0)\n\t"
169 "sd\t$0,16(%0)\n\t"
170 "sd\t$0,24(%0)\n\t"
171 "daddiu\t%0,64\n\t"
172 "nop\n\t"
173 "nop\n\t"
174 "nop\n\t"
175 "cache\t%3,-32(%0)\n\t"
176 "sd\t$0,-32(%0)\n\t"
177 "sd\t$0,-24(%0)\n\t"
178 "sd\t$0,-16(%0)\n\t"
179 "bne\t$1,%0,1b\n\t"
180 "sd\t$0,-8(%0)\n\t"
181 ".set\tat\n\t"
182 ".set\treorder"
183 :"=r" (page)
184 :"0" (page),
185 "I" (PAGE_SIZE),
186 "i" (Create_Dirty_Excl_D)
187 :"$1", "memory");
188 }
189
190 /*
191 * And this one is for the R4600 V2.0
192 */
193 static void r4k_clear_page_r4600_v2(void * page)
194 {
195 unsigned int flags;
196
197 __save_and_cli(flags);
198 *(volatile unsigned int *)KSEG1;
199 __asm__ __volatile__(
200 ".set\tnoreorder\n\t"
201 ".set\tnoat\n\t"
202 "daddiu\t$1,%0,%2\n"
203 "1:\tcache\t%3,(%0)\n\t"
204 "sd\t$0,(%0)\n\t"
205 "sd\t$0,8(%0)\n\t"
206 "sd\t$0,16(%0)\n\t"
207 "sd\t$0,24(%0)\n\t"
208 "daddiu\t%0,64\n\t"
209 "cache\t%3,-32(%0)\n\t"
210 "sd\t$0,-32(%0)\n\t"
211 "sd\t$0,-24(%0)\n\t"
212 "sd\t$0,-16(%0)\n\t"
213 "bne\t$1,%0,1b\n\t"
214 "sd\t$0,-8(%0)\n\t"
215 ".set\tat\n\t"
216 ".set\treorder"
217 :"=r" (page)
218 :"0" (page), "I" (PAGE_SIZE), "i" (Create_Dirty_Excl_D)
219 :"$1", "memory");
220 __restore_flags(flags);
221 }
222
223 /*
224 * The next 4 versions are optimized for all possible scache configurations
225 * of the SC / MC versions of R4000 and R4400 ...
226 *
227 * Todo: For even better performance we should have a routine optimized for
228 * every legal combination of dcache / scache linesize. When I (Ralf) tried
229 * this the kernel crashed shortly after mounting the root filesystem. CPU
230 * bug? Weirdo cache instruction semantics?
231 */
232 static void r4k_clear_page_s16(void * page)
233 {
234 __asm__ __volatile__(
235 ".set\tnoreorder\n\t"
236 ".set\tnoat\n\t"
237 "daddiu\t$1,%0,%2\n"
238 "1:\tcache\t%3,(%0)\n\t"
239 "sd\t$0,(%0)\n\t"
240 "sd\t$0,8(%0)\n\t"
241 "cache\t%3,16(%0)\n\t"
242 "sd\t$0,16(%0)\n\t"
243 "sd\t$0,24(%0)\n\t"
244 "daddiu\t%0,64\n\t"
245 "cache\t%3,-32(%0)\n\t"
246 "sd\t$0,-32(%0)\n\t"
247 "sd\t$0,-24(%0)\n\t"
248 "cache\t%3,-16(%0)\n\t"
249 "sd\t$0,-16(%0)\n\t"
250 "bne\t$1,%0,1b\n\t"
251 "sd\t$0,-8(%0)\n\t"
252 ".set\tat\n\t"
253 ".set\treorder"
254 :"=r" (page)
255 :"0" (page), "I" (PAGE_SIZE), "i" (Create_Dirty_Excl_SD)
256 :"$1","memory");
257 }
258
259 static void r4k_clear_page_s32(void * page)
260 {
261 __asm__ __volatile__(
262 ".set\tnoreorder\n\t"
263 ".set\tnoat\n\t"
264 "daddiu\t$1,%0,%2\n"
265 "1:\tcache\t%3,(%0)\n\t"
266 "sd\t$0,(%0)\n\t"
267 "sd\t$0,8(%0)\n\t"
268 "sd\t$0,16(%0)\n\t"
269 "sd\t$0,24(%0)\n\t"
270 "daddiu\t%0,64\n\t"
271 "cache\t%3,-32(%0)\n\t"
272 "sd\t$0,-32(%0)\n\t"
273 "sd\t$0,-24(%0)\n\t"
274 "sd\t$0,-16(%0)\n\t"
275 "bne\t$1,%0,1b\n\t"
276 "sd\t$0,-8(%0)\n\t"
277 ".set\tat\n\t"
278 ".set\treorder"
279 :"=r" (page)
280 :"0" (page), "I" (PAGE_SIZE), "i" (Create_Dirty_Excl_SD)
281 :"$1","memory");
282 }
283
284 static void r4k_clear_page_s64(void * page)
285 {
286 __asm__ __volatile__(
287 ".set\tnoreorder\n\t"
288 ".set\tnoat\n\t"
289 "daddiu\t$1,%0,%2\n"
290 "1:\tcache\t%3,(%0)\n\t"
291 "sd\t$0,(%0)\n\t"
292 "sd\t$0,8(%0)\n\t"
293 "sd\t$0,16(%0)\n\t"
294 "sd\t$0,24(%0)\n\t"
295 "daddiu\t%0,64\n\t"
296 "sd\t$0,-32(%0)\n\t"
297 "sd\t$0,-24(%0)\n\t"
298 "sd\t$0,-16(%0)\n\t"
299 "bne\t$1,%0,1b\n\t"
300 "sd\t$0,-8(%0)\n\t"
301 ".set\tat\n\t"
302 ".set\treorder"
303 :"=r" (page)
304 :"0" (page),
305 "I" (PAGE_SIZE),
306 "i" (Create_Dirty_Excl_SD)
307 :"$1","memory");
308 }
309
310 static void r4k_clear_page_s128(void * page)
311 {
312 __asm__ __volatile__(
313 ".set\tnoreorder\n\t"
314 ".set\tnoat\n\t"
315 "daddiu\t$1,%0,%2\n"
316 "1:\tcache\t%3,(%0)\n\t"
317 "sd\t$0,(%0)\n\t"
318 "sd\t$0,8(%0)\n\t"
319 "sd\t$0,16(%0)\n\t"
320 "sd\t$0,24(%0)\n\t"
321 "sd\t$0,32(%0)\n\t"
322 "sd\t$0,40(%0)\n\t"
323 "sd\t$0,48(%0)\n\t"
324 "sd\t$0,56(%0)\n\t"
325 "daddiu\t%0,128\n\t"
326 "sd\t$0,-64(%0)\n\t"
327 "sd\t$0,-56(%0)\n\t"
328 "sd\t$0,-48(%0)\n\t"
329 "sd\t$0,-40(%0)\n\t"
330 "sd\t$0,-32(%0)\n\t"
331 "sd\t$0,-24(%0)\n\t"
332 "sd\t$0,-16(%0)\n\t"
333 "bne\t$1,%0,1b\n\t"
334 "sd\t$0,-8(%0)\n\t"
335 ".set\tat\n\t"
336 ".set\treorder"
337 :"=r" (page)
338 :"0" (page),
339 "I" (PAGE_SIZE),
340 "i" (Create_Dirty_Excl_SD)
341 :"$1", "memory");
342 }
343
344
345 /*
346 * This is still inefficient. We only can do better if we know the
347 * virtual address where the copy will be accessed.
348 */
349
350 static void r4k_copy_page_d16(void * to, void * from)
351 {
352 unsigned long dummy1, dummy2, reg1, reg2;
353
354 __asm__ __volatile__(
355 ".set\tnoreorder\n\t"
356 ".set\tnoat\n\t"
357 "daddiu\t$1,%0,%6\n"
358 "1:\tcache\t%7,(%0)\n\t"
359 "ld\t%2,(%1)\n\t"
360 "ld\t%3,8(%1)\n\t"
361 "sd\t%2,(%0)\n\t"
362 "sd\t%3,8(%0)\n\t"
363 "cache\t%7,16(%0)\n\t"
364 "ld\t%2,16(%1)\n\t"
365 "ld\t%3,24(%1)\n\t"
366 "sd\t%2,16(%0)\n\t"
367 "sd\t%3,24(%0)\n\t"
368 "cache\t%7,32(%0)\n\t"
369 "daddiu\t%0,64\n\t"
370 "daddiu\t%1,64\n\t"
371 "ld\t%2,-32(%1)\n\t"
372 "ld\t%3,-24(%1)\n\t"
373 "sd\t%2,-32(%0)\n\t"
374 "sd\t%3,-24(%0)\n\t"
375 "cache\t%7,-16(%0)\n\t"
376 "ld\t%2,-16(%1)\n\t"
377 "ld\t%3,-8(%1)\n\t"
378 "sd\t%2,-16(%0)\n\t"
379 "bne\t$1,%0,1b\n\t"
380 " sd\t%3,-8(%0)\n\t"
381 ".set\tat\n\t"
382 ".set\treorder"
383 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
384 :"0" (to), "1" (from), "I" (PAGE_SIZE),
385 "i" (Create_Dirty_Excl_D));
386 }
387
388 static void r4k_copy_page_d32(void * to, void * from)
389 {
390 unsigned long dummy1, dummy2, reg1, reg2;
391
392 __asm__ __volatile__(
393 ".set\tnoreorder\n\t"
394 ".set\tnoat\n\t"
395 "daddiu\t$1,%0,%6\n"
396 "1:\tcache\t%7,(%0)\n\t"
397 "ld\t%2,(%1)\n\t"
398 "ld\t%3,8(%1)\n\t"
399 "sd\t%2,(%0)\n\t"
400 "sd\t%3,8(%0)\n\t"
401 "ld\t%2,16(%1)\n\t"
402 "ld\t%3,24(%1)\n\t"
403 "sd\t%2,16(%0)\n\t"
404 "sd\t%3,24(%0)\n\t"
405 "cache\t%7,32(%0)\n\t"
406 "daddiu\t%0,64\n\t"
407 "daddiu\t%1,64\n\t"
408 "ld\t%2,-32(%1)\n\t"
409 "ld\t%3,-24(%1)\n\t"
410 "sd\t%2,-32(%0)\n\t"
411 "sd\t%3,-24(%0)\n\t"
412 "ld\t%2,-16(%1)\n\t"
413 "ld\t%3,-8(%1)\n\t"
414 "sd\t%2,-16(%0)\n\t"
415 "bne\t$1,%0,1b\n\t"
416 " sd\t%3,-8(%0)\n\t"
417 ".set\tat\n\t"
418 ".set\treorder"
419 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
420 :"0" (to), "1" (from), "I" (PAGE_SIZE),
421 "i" (Create_Dirty_Excl_D));
422 }
423
424 /*
425 * Again a special version for the R4600 V1.x
426 */
427 static void r4k_copy_page_r4600_v1(void * to, void * from)
428 {
429 unsigned long dummy1, dummy2, reg1, reg2;
430
431 __asm__ __volatile__(
432 ".set\tnoreorder\n\t"
433 ".set\tnoat\n\t"
434 "daddiu\t$1,%0,%6\n"
435 "1:\tnop\n\t"
436 "nop\n\t"
437 "nop\n\t"
438 "nop\n\t"
439 "\tcache\t%7,(%0)\n\t"
440 "ld\t%2,(%1)\n\t"
441 "ld\t%3,8(%1)\n\t"
442 "sd\t%2,(%0)\n\t"
443 "sd\t%3,8(%0)\n\t"
444 "ld\t%2,16(%1)\n\t"
445 "ld\t%3,24(%1)\n\t"
446 "sd\t%2,16(%0)\n\t"
447 "sd\t%3,24(%0)\n\t"
448 "nop\n\t"
449 "nop\n\t"
450 "nop\n\t"
451 "nop\n\t"
452 "cache\t%7,32(%0)\n\t"
453 "daddiu\t%0,64\n\t"
454 "daddiu\t%1,64\n\t"
455 "ld\t%2,-32(%1)\n\t"
456 "ld\t%3,-24(%1)\n\t"
457 "sd\t%2,-32(%0)\n\t"
458 "sd\t%3,-24(%0)\n\t"
459 "ld\t%2,-16(%1)\n\t"
460 "ld\t%3,-8(%1)\n\t"
461 "sd\t%2,-16(%0)\n\t"
462 "bne\t$1,%0,1b\n\t"
463 " sd\t%3,-8(%0)\n\t"
464 ".set\tat\n\t"
465 ".set\treorder"
466 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
467 :"0" (to), "1" (from), "I" (PAGE_SIZE),
468 "i" (Create_Dirty_Excl_D));
469 }
470
471 static void r4k_copy_page_r4600_v2(void * to, void * from)
472 {
473 unsigned long dummy1, dummy2, reg1, reg2;
474 unsigned int flags;
475
476 __save_and_cli(flags);
477 __asm__ __volatile__(
478 ".set\tnoreorder\n\t"
479 ".set\tnoat\n\t"
480 "daddiu\t$1,%0,%6\n"
481 "1:\tnop\n\t"
482 "nop\n\t"
483 "nop\n\t"
484 "nop\n\t"
485 "\tcache\t%7,(%0)\n\t"
486 "ld\t%2,(%1)\n\t"
487 "ld\t%3,8(%1)\n\t"
488 "sd\t%2,(%0)\n\t"
489 "sd\t%3,8(%0)\n\t"
490 "ld\t%2,16(%1)\n\t"
491 "ld\t%3,24(%1)\n\t"
492 "sd\t%2,16(%0)\n\t"
493 "sd\t%3,24(%0)\n\t"
494 "nop\n\t"
495 "nop\n\t"
496 "nop\n\t"
497 "nop\n\t"
498 "cache\t%7,32(%0)\n\t"
499 "daddiu\t%0,64\n\t"
500 "daddiu\t%1,64\n\t"
501 "ld\t%2,-32(%1)\n\t"
502 "ld\t%3,-24(%1)\n\t"
503 "sd\t%2,-32(%0)\n\t"
504 "sd\t%3,-24(%0)\n\t"
505 "ld\t%2,-16(%1)\n\t"
506 "ld\t%3,-8(%1)\n\t"
507 "sd\t%2,-16(%0)\n\t"
508 "bne\t$1,%0,1b\n\t"
509 " sd\t%3,-8(%0)\n\t"
510 ".set\tat\n\t"
511 ".set\treorder"
512 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
513 :"0" (to), "1" (from), "I" (PAGE_SIZE),
514 "i" (Create_Dirty_Excl_D));
515 __restore_flags(flags);
516 }
517
518 /*
519 * These are for R4000SC / R4400MC
520 */
521 static void r4k_copy_page_s16(void * to, void * from)
522 {
523 unsigned long dummy1, dummy2, reg1, reg2;
524
525 __asm__ __volatile__(
526 ".set\tnoreorder\n\t"
527 ".set\tnoat\n\t"
528 "daddiu\t$1,%0,%6\n"
529 "1:\tcache\t%7,(%0)\n\t"
530 "ld\t%2,(%1)\n\t"
531 "ld\t%3,8(%1)\n\t"
532 "sd\t%2,(%0)\n\t"
533 "sd\t%3,8(%0)\n\t"
534 "cache\t%7,16(%0)\n\t"
535 "ld\t%2,16(%1)\n\t"
536 "ld\t%3,24(%1)\n\t"
537 "sd\t%2,16(%0)\n\t"
538 "sd\t%3,24(%0)\n\t"
539 "cache\t%7,32(%0)\n\t"
540 "daddiu\t%0,64\n\t"
541 "daddiu\t%1,64\n\t"
542 "ld\t%2,-32(%1)\n\t"
543 "ld\t%3,-24(%1)\n\t"
544 "sd\t%2,-32(%0)\n\t"
545 "sd\t%3,-24(%0)\n\t"
546 "cache\t%7,-16(%0)\n\t"
547 "ld\t%2,-16(%1)\n\t"
548 "ld\t%3,-8(%1)\n\t"
549 "sd\t%2,-16(%0)\n\t"
550 "bne\t$1,%0,1b\n\t"
551 " sd\t%3,-8(%0)\n\t"
552 ".set\tat\n\t"
553 ".set\treorder"
554 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
555 :"0" (to), "1" (from), "I" (PAGE_SIZE),
556 "i" (Create_Dirty_Excl_SD));
557 }
558
559 static void r4k_copy_page_s32(void * to, void * from)
560 {
561 unsigned long dummy1, dummy2, reg1, reg2;
562
563 __asm__ __volatile__(
564 ".set\tnoreorder\n\t"
565 ".set\tnoat\n\t"
566 "daddiu\t$1,%0,%6\n"
567 "1:\tcache\t%7,(%0)\n\t"
568 "ld\t%2,(%1)\n\t"
569 "ld\t%3,8(%1)\n\t"
570 "sd\t%2,(%0)\n\t"
571 "sd\t%3,8(%0)\n\t"
572 "ld\t%2,16(%1)\n\t"
573 "ld\t%3,24(%1)\n\t"
574 "sd\t%2,16(%0)\n\t"
575 "sd\t%3,24(%0)\n\t"
576 "cache\t%7,32(%0)\n\t"
577 "daddiu\t%0,64\n\t"
578 "daddiu\t%1,64\n\t"
579 "ld\t%2,-32(%1)\n\t"
580 "ld\t%3,-24(%1)\n\t"
581 "sd\t%2,-32(%0)\n\t"
582 "sd\t%3,-24(%0)\n\t"
583 "ld\t%2,-16(%1)\n\t"
584 "ld\t%3,-8(%1)\n\t"
585 "sd\t%2,-16(%0)\n\t"
586 "bne\t$1,%0,1b\n\t"
587 " sd\t%3,-8(%0)\n\t"
588 ".set\tat\n\t"
589 ".set\treorder"
590 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
591 :"0" (to), "1" (from), "I" (PAGE_SIZE),
592 "i" (Create_Dirty_Excl_SD));
593 }
594
595 static void r4k_copy_page_s64(void * to, void * from)
596 {
597 unsigned long dummy1, dummy2, reg1, reg2;
598
599 __asm__ __volatile__(
600 ".set\tnoreorder\n\t"
601 ".set\tnoat\n\t"
602 "daddiu\t$1,%0,%6\n"
603 "1:\tcache\t%7,(%0)\n\t"
604 "ld\t%2,(%1)\n\t"
605 "ld\t%3,8(%1)\n\t"
606 "sd\t%2,(%0)\n\t"
607 "sd\t%3,8(%0)\n\t"
608 "ld\t%2,16(%1)\n\t"
609 "ld\t%3,24(%1)\n\t"
610 "sd\t%2,16(%0)\n\t"
611 "sd\t%3,24(%0)\n\t"
612 "daddiu\t%0,64\n\t"
613 "daddiu\t%1,64\n\t"
614 "ld\t%2,-32(%1)\n\t"
615 "ld\t%3,-24(%1)\n\t"
616 "sd\t%2,-32(%0)\n\t"
617 "sd\t%3,-24(%0)\n\t"
618 "ld\t%2,-16(%1)\n\t"
619 "ld\t%3,-8(%1)\n\t"
620 "sd\t%2,-16(%0)\n\t"
621 "bne\t$1,%0,1b\n\t"
622 " sd\t%3,-8(%0)\n\t"
623 ".set\tat\n\t"
624 ".set\treorder"
625 :"=r" (dummy1), "=r" (dummy2), "=&r" (reg1), "=&r" (reg2)
626 :"0" (to), "1" (from), "I" (PAGE_SIZE),
627 "i" (Create_Dirty_Excl_SD));
628 }
629
630 static void r4k_copy_page_s128(void * to, void * from)
631 {
632 unsigned long dummy1, dummy2;
633 unsigned long reg1, reg2, reg3, reg4;
634
635 __asm__ __volatile__(
636 ".set\tnoreorder\n\t"
637 ".set\tnoat\n\t"
638 "daddiu\t$1,%0,%8\n"
639 "1:\tcache\t%9,(%0)\n\t"
640 "ld\t%2,(%1)\n\t"
641 "ld\t%3,8(%1)\n\t"
642 "ld\t%4,16(%1)\n\t"
643 "ld\t%5,24(%1)\n\t"
644 "sd\t%2,(%0)\n\t"
645 "sd\t%3,8(%0)\n\t"
646 "sd\t%4,16(%0)\n\t"
647 "sd\t%5,24(%0)\n\t"
648 "ld\t%2,32(%1)\n\t"
649 "ld\t%3,40(%1)\n\t"
650 "ld\t%4,48(%1)\n\t"
651 "ld\t%5,56(%1)\n\t"
652 "sd\t%2,32(%0)\n\t"
653 "sd\t%3,40(%0)\n\t"
654 "sd\t%4,48(%0)\n\t"
655 "sd\t%5,56(%0)\n\t"
656 "daddiu\t%0,128\n\t"
657 "daddiu\t%1,128\n\t"
658 "ld\t%2,-64(%1)\n\t"
659 "ld\t%3,-56(%1)\n\t"
660 "ld\t%4,-48(%1)\n\t"
661 "ld\t%5,-40(%1)\n\t"
662 "sd\t%2,-64(%0)\n\t"
663 "sd\t%3,-56(%0)\n\t"
664 "sd\t%4,-48(%0)\n\t"
665 "sd\t%5,-40(%0)\n\t"
666 "ld\t%2,-32(%1)\n\t"
667 "ld\t%3,-24(%1)\n\t"
668 "ld\t%4,-16(%1)\n\t"
669 "ld\t%5,-8(%1)\n\t"
670 "sd\t%2,-32(%0)\n\t"
671 "sd\t%3,-24(%0)\n\t"
672 "sd\t%4,-16(%0)\n\t"
673 "bne\t$1,%0,1b\n\t"
674 " sd\t%5,-8(%0)\n\t"
675 ".set\tat\n\t"
676 ".set\treorder"
677 :"=r" (dummy1), "=r" (dummy2),
678 "=&r" (reg1), "=&r" (reg2), "=&r" (reg3), "=&r" (reg4)
679 :"0" (to), "1" (from),
680 "I" (PAGE_SIZE),
681 "i" (Create_Dirty_Excl_SD));
682 }
683
684
685 /*
686 * If you think for one second that this stuff coming up is a lot
687 * of bulky code eating too many kernel cache lines. Think _again_.
688 *
689 * Consider:
690 * 1) Taken branches have a 3 cycle penalty on R4k
691 * 2) The branch itself is a real dead cycle on even R4600/R5000.
692 * 3) Only one of the following variants of each type is even used by
693 * the kernel based upon the cache parameters we detect at boot time.
694 *
695 * QED.
696 */
697
698 static inline void r4k_flush_cache_all_s16d16i16(void)
699 {
700 unsigned long flags;
701
702 __save_and_cli(flags);
703 blast_dcache16(); blast_icache16(); blast_scache16();
704 __restore_flags(flags);
705 }
706
707 static inline void r4k_flush_cache_all_s32d16i16(void)
708 {
709 unsigned long flags;
710
711 __save_and_cli(flags);
712 blast_dcache16(); blast_icache16(); blast_scache32();
713 __restore_flags(flags);
714 }
715
716 static inline void r4k_flush_cache_all_s64d16i16(void)
717 {
718 unsigned long flags;
719
720 __save_and_cli(flags);
721 blast_dcache16(); blast_icache16(); blast_scache64();
722 __restore_flags(flags);
723 }
724
725 static inline void r4k_flush_cache_all_s128d16i16(void)
726 {
727 unsigned long flags;
728
729 __save_and_cli(flags);
730 blast_dcache16(); blast_icache16(); blast_scache128();
731 __restore_flags(flags);
732 }
733
734 static inline void r4k_flush_cache_all_s32d32i32(void)
735 {
736 unsigned long flags;
737
738 __save_and_cli(flags);
739 blast_dcache32(); blast_icache32(); blast_scache32();
740 __restore_flags(flags);
741 }
742
743 static inline void r4k_flush_cache_all_s64d32i32(void)
744 {
745 unsigned long flags;
746
747 __save_and_cli(flags);
748 blast_dcache32(); blast_icache32(); blast_scache64();
749 __restore_flags(flags);
750 }
751
752 static inline void r4k_flush_cache_all_s128d32i32(void)
753 {
754 unsigned long flags;
755
756 __save_and_cli(flags);
757 blast_dcache32(); blast_icache32(); blast_scache128();
758 __restore_flags(flags);
759 }
760
761 static inline void r4k_flush_cache_all_d16i16(void)
762 {
763 unsigned long flags;
764
765 __save_and_cli(flags);
766 blast_dcache16(); blast_icache16();
767 __restore_flags(flags);
768 }
769
770 static inline void r4k_flush_cache_all_d32i32(void)
771 {
772 unsigned long flags;
773
774 __save_and_cli(flags);
775 blast_dcache32(); blast_icache32();
776 __restore_flags(flags);
777 }
778
779 static void r4k_flush_cache_range_s16d16i16(struct mm_struct *mm,
780 unsigned long start,
781 unsigned long end)
782 {
783 struct vm_area_struct *vma;
784 unsigned long flags;
785
786 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
787 return;
788
789 start &= PAGE_MASK;
790 #ifdef DEBUG_CACHE
791 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
792 #endif
793 vma = find_vma(mm, start);
794 if(vma) {
795 if (CPU_CONTEXT(smp_processor_id(), mm) !=
796 CPU_CONTEXT(smp_processor_id(), current->mm)) {
797 r4k_flush_cache_all_s16d16i16();
798 } else {
799 pgd_t *pgd;
800 pmd_t *pmd;
801 pte_t *pte;
802
803 __save_and_cli(flags);
804 while(start < end) {
805 pgd = pgd_offset(mm, start);
806 pmd = pmd_offset(pgd, start);
807 pte = pte_offset(pmd, start);
808
809 if(pte_val(*pte) & _PAGE_VALID)
810 blast_scache16_page(start);
811 start += PAGE_SIZE;
812 }
813 __restore_flags(flags);
814 }
815 }
816 }
817
818 static void r4k_flush_cache_range_s32d16i16(struct mm_struct *mm,
819 unsigned long start,
820 unsigned long end)
821 {
822 struct vm_area_struct *vma;
823 unsigned long flags;
824
825 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
826 return;
827
828 start &= PAGE_MASK;
829 #ifdef DEBUG_CACHE
830 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
831 #endif
832 vma = find_vma(mm, start);
833 if(vma) {
834 if (CPU_CONTEXT(smp_processor_id(), mm) !=
835 CPU_CONTEXT(smp_processor_id(), current->mm)) {
836 r4k_flush_cache_all_s32d16i16();
837 } else {
838 pgd_t *pgd;
839 pmd_t *pmd;
840 pte_t *pte;
841
842 __save_and_cli(flags);
843 while(start < end) {
844 pgd = pgd_offset(mm, start);
845 pmd = pmd_offset(pgd, start);
846 pte = pte_offset(pmd, start);
847
848 if(pte_val(*pte) & _PAGE_VALID)
849 blast_scache32_page(start);
850 start += PAGE_SIZE;
851 }
852 __restore_flags(flags);
853 }
854 }
855 }
856
857 static void r4k_flush_cache_range_s64d16i16(struct mm_struct *mm,
858 unsigned long start,
859 unsigned long end)
860 {
861 struct vm_area_struct *vma;
862 unsigned long flags;
863
864 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
865 return;
866
867 start &= PAGE_MASK;
868 #ifdef DEBUG_CACHE
869 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
870 #endif
871 vma = find_vma(mm, start);
872 if(vma) {
873 if (CPU_CONTEXT(smp_processor_id(), mm) !=
874 CPU_CONTEXT(smp_processor_id(), current->mm)) {
875 r4k_flush_cache_all_s64d16i16();
876 } else {
877 pgd_t *pgd;
878 pmd_t *pmd;
879 pte_t *pte;
880
881 __save_and_cli(flags);
882 while(start < end) {
883 pgd = pgd_offset(mm, start);
884 pmd = pmd_offset(pgd, start);
885 pte = pte_offset(pmd, start);
886
887 if(pte_val(*pte) & _PAGE_VALID)
888 blast_scache64_page(start);
889 start += PAGE_SIZE;
890 }
891 __restore_flags(flags);
892 }
893 }
894 }
895
896 static void r4k_flush_cache_range_s128d16i16(struct mm_struct *mm,
897 unsigned long start,
898 unsigned long end)
899 {
900 struct vm_area_struct *vma;
901 unsigned long flags;
902
903 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
904 return;
905
906 start &= PAGE_MASK;
907 #ifdef DEBUG_CACHE
908 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
909 #endif
910 vma = find_vma(mm, start);
911 if(vma) {
912 if (CPU_CONTEXT(smp_processor_id(), mm) !=
913 CPU_CONTEXT(smp_processor_id(), current->mm)) {
914 r4k_flush_cache_all_s128d16i16();
915 } else {
916 pgd_t *pgd;
917 pmd_t *pmd;
918 pte_t *pte;
919
920 __save_and_cli(flags);
921 while(start < end) {
922 pgd = pgd_offset(mm, start);
923 pmd = pmd_offset(pgd, start);
924 pte = pte_offset(pmd, start);
925
926 if(pte_val(*pte) & _PAGE_VALID)
927 blast_scache128_page(start);
928 start += PAGE_SIZE;
929 }
930 __restore_flags(flags);
931 }
932 }
933 }
934
935 static void r4k_flush_cache_range_s32d32i32(struct mm_struct *mm,
936 unsigned long start,
937 unsigned long end)
938 {
939 struct vm_area_struct *vma;
940 unsigned long flags;
941
942 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
943 return;
944
945 start &= PAGE_MASK;
946 #ifdef DEBUG_CACHE
947 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
948 #endif
949 vma = find_vma(mm, start);
950 if(vma) {
951 if (CPU_CONTEXT(smp_processor_id(), mm) !=
952 CPU_CONTEXT(smp_processor_id(), current->mm)) {
953 r4k_flush_cache_all_s32d32i32();
954 } else {
955 pgd_t *pgd;
956 pmd_t *pmd;
957 pte_t *pte;
958
959 __save_and_cli(flags);
960 while(start < end) {
961 pgd = pgd_offset(mm, start);
962 pmd = pmd_offset(pgd, start);
963 pte = pte_offset(pmd, start);
964
965 if(pte_val(*pte) & _PAGE_VALID)
966 blast_scache32_page(start);
967 start += PAGE_SIZE;
968 }
969 __restore_flags(flags);
970 }
971 }
972 }
973
974 static void r4k_flush_cache_range_s64d32i32(struct mm_struct *mm,
975 unsigned long start,
976 unsigned long end)
977 {
978 struct vm_area_struct *vma;
979 unsigned long flags;
980
981 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
982 return;
983
984 start &= PAGE_MASK;
985 #ifdef DEBUG_CACHE
986 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
987 #endif
988 vma = find_vma(mm, start);
989 if(vma) {
990 if (CPU_CONTEXT(smp_processor_id(), mm) !=
991 CPU_CONTEXT(smp_processor_id(), current->mm)) {
992 r4k_flush_cache_all_s64d32i32();
993 } else {
994 pgd_t *pgd;
995 pmd_t *pmd;
996 pte_t *pte;
997
998 __save_and_cli(flags);
999 while(start < end) {
1000 pgd = pgd_offset(mm, start);
1001 pmd = pmd_offset(pgd, start);
1002 pte = pte_offset(pmd, start);
1003
1004 if(pte_val(*pte) & _PAGE_VALID)
1005 blast_scache64_page(start);
1006 start += PAGE_SIZE;
1007 }
1008 __restore_flags(flags);
1009 }
1010 }
1011 }
1012
1013 static void r4k_flush_cache_range_s128d32i32(struct mm_struct *mm,
1014 unsigned long start,
1015 unsigned long end)
1016 {
1017 struct vm_area_struct *vma;
1018 unsigned long flags;
1019
1020 if (CPU_CONTEXT(smp_processor_id(), mm) != 0)
1021 return;
1022
1023 start &= PAGE_MASK;
1024 #ifdef DEBUG_CACHE
1025 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1026 #endif
1027 vma = find_vma(mm, start);
1028 if(vma) {
1029 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1030 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1031 r4k_flush_cache_all_s128d32i32();
1032 } else {
1033 pgd_t *pgd;
1034 pmd_t *pmd;
1035 pte_t *pte;
1036
1037 __save_and_cli(flags);
1038 while(start < end) {
1039 pgd = pgd_offset(mm, start);
1040 pmd = pmd_offset(pgd, start);
1041 pte = pte_offset(pmd, start);
1042
1043 if(pte_val(*pte) & _PAGE_VALID)
1044 blast_scache128_page(start);
1045 start += PAGE_SIZE;
1046 }
1047 __restore_flags(flags);
1048 }
1049 }
1050 }
1051
1052 static void r4k_flush_cache_range_d16i16(struct mm_struct *mm,
1053 unsigned long start,
1054 unsigned long end)
1055 {
1056 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1057 unsigned long flags;
1058
1059 #ifdef DEBUG_CACHE
1060 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1061 #endif
1062 __save_and_cli(flags);
1063 blast_dcache16(); blast_icache16();
1064 __restore_flags(flags);
1065 }
1066 }
1067
1068 static void r4k_flush_cache_range_d32i32(struct mm_struct *mm,
1069 unsigned long start,
1070 unsigned long end)
1071 {
1072 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1073 unsigned long flags;
1074
1075 #ifdef DEBUG_CACHE
1076 printk("crange[%d,%08lx,%08lx]", (int)mm->context, start, end);
1077 #endif
1078 __save_and_cli(flags);
1079 blast_dcache32(); blast_icache32();
1080 __restore_flags(flags);
1081 }
1082 }
1083
1084 /*
1085 * On architectures like the Sparc, we could get rid of lines in
1086 * the cache created only by a certain context, but on the MIPS
1087 * (and actually certain Sparc's) we cannot.
1088 */
1089 static void r4k_flush_cache_mm_s16d16i16(struct mm_struct *mm)
1090 {
1091 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1092 #ifdef DEBUG_CACHE
1093 printk("cmm[%d]", (int)mm->context);
1094 #endif
1095 r4k_flush_cache_all_s16d16i16();
1096 }
1097 }
1098
1099 static void r4k_flush_cache_mm_s32d16i16(struct mm_struct *mm)
1100 {
1101 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1102 #ifdef DEBUG_CACHE
1103 printk("cmm[%d]", (int)mm->context);
1104 #endif
1105 r4k_flush_cache_all_s32d16i16();
1106 }
1107 }
1108
1109 static void r4k_flush_cache_mm_s64d16i16(struct mm_struct *mm)
1110 {
1111 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1112 #ifdef DEBUG_CACHE
1113 printk("cmm[%d]", (int)mm->context);
1114 #endif
1115 r4k_flush_cache_all_s64d16i16();
1116 }
1117 }
1118
1119 static void r4k_flush_cache_mm_s128d16i16(struct mm_struct *mm)
1120 {
1121 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1122 #ifdef DEBUG_CACHE
1123 printk("cmm[%d]", (int)mm->context);
1124 #endif
1125 r4k_flush_cache_all_s128d16i16();
1126 }
1127 }
1128
1129 static void r4k_flush_cache_mm_s32d32i32(struct mm_struct *mm)
1130 {
1131 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1132 #ifdef DEBUG_CACHE
1133 printk("cmm[%d]", (int)mm->context);
1134 #endif
1135 r4k_flush_cache_all_s32d32i32();
1136 }
1137 }
1138
1139 static void r4k_flush_cache_mm_s64d32i32(struct mm_struct *mm)
1140 {
1141 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1142 #ifdef DEBUG_CACHE
1143 printk("cmm[%d]", (int)mm->context);
1144 #endif
1145 r4k_flush_cache_all_s64d32i32();
1146 }
1147 }
1148
1149 static void r4k_flush_cache_mm_s128d32i32(struct mm_struct *mm)
1150 {
1151 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1152 #ifdef DEBUG_CACHE
1153 printk("cmm[%d]", (int)mm->context);
1154 #endif
1155 r4k_flush_cache_all_s128d32i32();
1156 }
1157 }
1158
1159 static void r4k_flush_cache_mm_d16i16(struct mm_struct *mm)
1160 {
1161 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1162 #ifdef DEBUG_CACHE
1163 printk("cmm[%d]", (int)mm->context);
1164 #endif
1165 r4k_flush_cache_all_d16i16();
1166 }
1167 }
1168
1169 static void r4k_flush_cache_mm_d32i32(struct mm_struct *mm)
1170 {
1171 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1172 #ifdef DEBUG_CACHE
1173 printk("cmm[%d]", (int)mm->context);
1174 #endif
1175 r4k_flush_cache_all_d32i32();
1176 }
1177 }
1178
1179 static void r4k_flush_cache_page_s16d16i16(struct vm_area_struct *vma,
1180 unsigned long page)
1181 {
1182 struct mm_struct *mm = vma->vm_mm;
1183 unsigned long flags;
1184 pgd_t *pgdp;
1185 pmd_t *pmdp;
1186 pte_t *ptep;
1187
1188 /*
1189 * If ownes no valid ASID yet, cannot possibly have gotten
1190 * this page into the cache.
1191 */
1192 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1193 return;
1194
1195 #ifdef DEBUG_CACHE
1196 printk("cpage[%d,%08lx]", (int)mm->context, page);
1197 #endif
1198 __save_and_cli(flags);
1199 page &= PAGE_MASK;
1200 pgdp = pgd_offset(mm, page);
1201 pmdp = pmd_offset(pgdp, page);
1202 ptep = pte_offset(pmdp, page);
1203
1204 /*
1205 * If the page isn't marked valid, the page cannot possibly be
1206 * in the cache.
1207 */
1208 if(!(pte_val(*ptep) & _PAGE_VALID))
1209 goto out;
1210
1211 /* Doing flushes for another ASID than the current one is
1212 * too difficult since stupid R4k caches do a TLB translation
1213 * for every cache flush operation. So we do indexed flushes
1214 * in that case, which doesn't overly flush the cache too much.
1215 */
1216 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1217 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1218 /* Do indexed flush, too much work to get the (possible)
1219 * tlb refills to work correctly.
1220 */
1221 page = (KSEG0 + (page & (scache_size - 1)));
1222 blast_dcache16_page_indexed(page);
1223 blast_scache16_page_indexed(page);
1224 } else
1225 blast_scache16_page(page);
1226 out:
1227 __restore_flags(flags);
1228 }
1229
1230 static void r4k_flush_cache_page_s32d16i16(struct vm_area_struct *vma,
1231 unsigned long page)
1232 {
1233 struct mm_struct *mm = vma->vm_mm;
1234 unsigned long flags;
1235 pgd_t *pgdp;
1236 pmd_t *pmdp;
1237 pte_t *ptep;
1238
1239 /*
1240 * If ownes no valid ASID yet, cannot possibly have gotten
1241 * this page into the cache.
1242 */
1243 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1244 return;
1245
1246 #ifdef DEBUG_CACHE
1247 printk("cpage[%d,%08lx]", (int)mm->context, page);
1248 #endif
1249 __save_and_cli(flags);
1250 page &= PAGE_MASK;
1251 pgdp = pgd_offset(mm, page);
1252 pmdp = pmd_offset(pgdp, page);
1253 ptep = pte_offset(pmdp, page);
1254
1255 /* If the page isn't marked valid, the page cannot possibly be
1256 * in the cache.
1257 */
1258 if(!(pte_val(*ptep) & _PAGE_VALID))
1259 goto out;
1260
1261 /* Doing flushes for another ASID than the current one is
1262 * too difficult since stupid R4k caches do a TLB translation
1263 * for every cache flush operation. So we do indexed flushes
1264 * in that case, which doesn't overly flush the cache too much.
1265 */
1266 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1267 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1268 /* Do indexed flush, too much work to get the (possible)
1269 * tlb refills to work correctly.
1270 */
1271 page = (KSEG0 + (page & (scache_size - 1)));
1272 blast_dcache16_page_indexed(page);
1273 blast_scache32_page_indexed(page);
1274 } else
1275 blast_scache32_page(page);
1276 out:
1277 __restore_flags(flags);
1278 }
1279
1280 static void r4k_flush_cache_page_s64d16i16(struct vm_area_struct *vma,
1281 unsigned long page)
1282 {
1283 struct mm_struct *mm = vma->vm_mm;
1284 unsigned long flags;
1285 pgd_t *pgdp;
1286 pmd_t *pmdp;
1287 pte_t *ptep;
1288
1289 /*
1290 * If ownes no valid ASID yet, cannot possibly have gotten
1291 * this page into the cache.
1292 */
1293 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1294 return;
1295
1296 #ifdef DEBUG_CACHE
1297 printk("cpage[%d,%08lx]", (int)mm->context, page);
1298 #endif
1299 __save_and_cli(flags);
1300 page &= PAGE_MASK;
1301 pgdp = pgd_offset(mm, page);
1302 pmdp = pmd_offset(pgdp, page);
1303 ptep = pte_offset(pmdp, page);
1304
1305 /* If the page isn't marked valid, the page cannot possibly be
1306 * in the cache.
1307 */
1308 if(!(pte_val(*ptep) & _PAGE_VALID))
1309 goto out;
1310
1311 /*
1312 * Doing flushes for another ASID than the current one is
1313 * too difficult since stupid R4k caches do a TLB translation
1314 * for every cache flush operation. So we do indexed flushes
1315 * in that case, which doesn't overly flush the cache too much.
1316 */
1317 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1318 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1319 /* Do indexed flush, too much work to get the (possible)
1320 * tlb refills to work correctly.
1321 */
1322 page = (KSEG0 + (page & (scache_size - 1)));
1323 blast_dcache16_page_indexed(page);
1324 blast_scache64_page_indexed(page);
1325 } else
1326 blast_scache64_page(page);
1327 out:
1328 __restore_flags(flags);
1329 }
1330
1331 static void r4k_flush_cache_page_s128d16i16(struct vm_area_struct *vma,
1332 unsigned long page)
1333 {
1334 struct mm_struct *mm = vma->vm_mm;
1335 unsigned long flags;
1336 pgd_t *pgdp;
1337 pmd_t *pmdp;
1338 pte_t *ptep;
1339
1340 /*
1341 * If ownes no valid ASID yet, cannot possibly have gotten
1342 * this page into the cache.
1343 */
1344 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1345 return;
1346
1347 #ifdef DEBUG_CACHE
1348 printk("cpage[%d,%08lx]", (int)mm->context, page);
1349 #endif
1350 __save_and_cli(flags);
1351 page &= PAGE_MASK;
1352 pgdp = pgd_offset(mm, page);
1353 pmdp = pmd_offset(pgdp, page);
1354 ptep = pte_offset(pmdp, page);
1355
1356 /*
1357 * If the page isn't marked valid, the page cannot possibly be
1358 * in the cache.
1359 */
1360 if(!(pte_val(*ptep) & _PAGE_VALID))
1361 goto out;
1362
1363 /* Doing flushes for another ASID than the current one is
1364 * too difficult since stupid R4k caches do a TLB translation
1365 * for every cache flush operation. So we do indexed flushes
1366 * in that case, which doesn't overly flush the cache too much.
1367 */
1368 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1369 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1370 /*
1371 * Do indexed flush, too much work to get the (possible)
1372 * tlb refills to work correctly.
1373 */
1374 page = (KSEG0 + (page & (scache_size - 1)));
1375 blast_dcache16_page_indexed(page);
1376 blast_scache128_page_indexed(page);
1377 } else
1378 blast_scache128_page(page);
1379 out:
1380 __restore_flags(flags);
1381 }
1382
1383 static void r4k_flush_cache_page_s32d32i32(struct vm_area_struct *vma,
1384 unsigned long page)
1385 {
1386 struct mm_struct *mm = vma->vm_mm;
1387 unsigned long flags;
1388 pgd_t *pgdp;
1389 pmd_t *pmdp;
1390 pte_t *ptep;
1391
1392 /*
1393 * If ownes no valid ASID yet, cannot possibly have gotten
1394 * this page into the cache.
1395 */
1396 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1397 return;
1398
1399 #ifdef DEBUG_CACHE
1400 printk("cpage[%d,%08lx]", (int)mm->context, page);
1401 #endif
1402 __save_and_cli(flags);
1403 page &= PAGE_MASK;
1404 pgdp = pgd_offset(mm, page);
1405 pmdp = pmd_offset(pgdp, page);
1406 ptep = pte_offset(pmdp, page);
1407
1408 /*
1409 * If the page isn't marked valid, the page cannot possibly be
1410 * in the cache.
1411 */
1412 if(!(pte_val(*ptep) & _PAGE_VALID))
1413 goto out;
1414
1415 /*
1416 * Doing flushes for another ASID than the current one is
1417 * too difficult since stupid R4k caches do a TLB translation
1418 * for every cache flush operation. So we do indexed flushes
1419 * in that case, which doesn't overly flush the cache too much.
1420 */
1421 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1422 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1423 /*
1424 * Do indexed flush, too much work to get the (possible)
1425 * tlb refills to work correctly.
1426 */
1427 page = (KSEG0 + (page & (scache_size - 1)));
1428 blast_dcache32_page_indexed(page);
1429 blast_scache32_page_indexed(page);
1430 } else
1431 blast_scache32_page(page);
1432 out:
1433 __restore_flags(flags);
1434 }
1435
1436 static void r4k_flush_cache_page_s64d32i32(struct vm_area_struct *vma,
1437 unsigned long page)
1438 {
1439 struct mm_struct *mm = vma->vm_mm;
1440 unsigned long flags;
1441 pgd_t *pgdp;
1442 pmd_t *pmdp;
1443 pte_t *ptep;
1444
1445 /*
1446 * If ownes no valid ASID yet, cannot possibly have gotten
1447 * this page into the cache.
1448 */
1449 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1450 return;
1451
1452 #ifdef DEBUG_CACHE
1453 printk("cpage[%d,%08lx]", (int)mm->context, page);
1454 #endif
1455 __save_and_cli(flags);
1456 page &= PAGE_MASK;
1457 pgdp = pgd_offset(mm, page);
1458 pmdp = pmd_offset(pgdp, page);
1459 ptep = pte_offset(pmdp, page);
1460
1461 /*
1462 * If the page isn't marked valid, the page cannot possibly be
1463 * in the cache.
1464 */
1465 if(!(pte_val(*ptep) & _PAGE_VALID))
1466 goto out;
1467
1468 /*
1469 * Doing flushes for another ASID than the current one is
1470 * too difficult since stupid R4k caches do a TLB translation
1471 * for every cache flush operation. So we do indexed flushes
1472 * in that case, which doesn't overly flush the cache too much.
1473 */
1474 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1475 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1476 /*
1477 * Do indexed flush, too much work to get the (possible)
1478 * tlb refills to work correctly.
1479 */
1480 page = (KSEG0 + (page & (scache_size - 1)));
1481 blast_dcache32_page_indexed(page);
1482 blast_scache64_page_indexed(page);
1483 } else
1484 blast_scache64_page(page);
1485 out:
1486 __restore_flags(flags);
1487 }
1488
1489 static void r4k_flush_cache_page_s128d32i32(struct vm_area_struct *vma,
1490 unsigned long page)
1491 {
1492 struct mm_struct *mm = vma->vm_mm;
1493 unsigned long flags;
1494 pgd_t *pgdp;
1495 pmd_t *pmdp;
1496 pte_t *ptep;
1497
1498 /*
1499 * If ownes no valid ASID yet, cannot possibly have gotten
1500 * this page into the cache.
1501 */
1502 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1503 return;
1504
1505 #ifdef DEBUG_CACHE
1506 printk("cpage[%d,%08lx]", (int)mm->context, page);
1507 #endif
1508 __save_and_cli(flags);
1509 page &= PAGE_MASK;
1510 pgdp = pgd_offset(mm, page);
1511 pmdp = pmd_offset(pgdp, page);
1512 ptep = pte_offset(pmdp, page);
1513
1514 /* If the page isn't marked valid, the page cannot possibly be
1515 * in the cache.
1516 */
1517 if(!(pte_val(*ptep) & _PAGE_VALID))
1518 goto out;
1519
1520 /*
1521 * Doing flushes for another ASID than the current one is
1522 * too difficult since stupid R4k caches do a TLB translation
1523 * for every cache flush operation. So we do indexed flushes
1524 * in that case, which doesn't overly flush the cache too much.
1525 */
1526 if (CPU_CONTEXT(smp_processor_id(), mm) !=
1527 CPU_CONTEXT(smp_processor_id(), current->mm)) {
1528 /* Do indexed flush, too much work to get the (possible)
1529 * tlb refills to work correctly.
1530 */
1531 page = (KSEG0 + (page & (scache_size - 1)));
1532 blast_dcache32_page_indexed(page);
1533 blast_scache128_page_indexed(page);
1534 } else
1535 blast_scache128_page(page);
1536 out:
1537 __restore_flags(flags);
1538 }
1539
1540 static void r4k_flush_cache_page_d16i16(struct vm_area_struct *vma,
1541 unsigned long page)
1542 {
1543 struct mm_struct *mm = vma->vm_mm;
1544 unsigned long flags;
1545 pgd_t *pgdp;
1546 pmd_t *pmdp;
1547 pte_t *ptep;
1548
1549 /*
1550 * If ownes no valid ASID yet, cannot possibly have gotten
1551 * this page into the cache.
1552 */
1553 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1554 return;
1555
1556 #ifdef DEBUG_CACHE
1557 printk("cpage[%d,%08lx]", (int)mm->context, page);
1558 #endif
1559 __save_and_cli(flags);
1560 page &= PAGE_MASK;
1561 pgdp = pgd_offset(mm, page);
1562 pmdp = pmd_offset(pgdp, page);
1563 ptep = pte_offset(pmdp, page);
1564
1565 /* If the page isn't marked valid, the page cannot possibly be
1566 * in the cache.
1567 */
1568 if(!(pte_val(*ptep) & _PAGE_VALID))
1569 goto out;
1570
1571 /*
1572 * Doing flushes for another ASID than the current one is
1573 * too difficult since stupid R4k caches do a TLB translation
1574 * for every cache flush operation. So we do indexed flushes
1575 * in that case, which doesn't overly flush the cache too much.
1576 */
1577 if(mm == current->mm) {
1578 blast_dcache16_page(page);
1579 } else {
1580 /* Do indexed flush, too much work to get the (possible)
1581 * tlb refills to work correctly.
1582 */
1583 page = (KSEG0 + (page & (dcache_size - 1)));
1584 blast_dcache16_page_indexed(page);
1585 }
1586 out:
1587 __restore_flags(flags);
1588 }
1589
1590 static void r4k_flush_cache_page_d32i32(struct vm_area_struct *vma,
1591 unsigned long page)
1592 {
1593 struct mm_struct *mm = vma->vm_mm;
1594 unsigned long flags;
1595 pgd_t *pgdp;
1596 pmd_t *pmdp;
1597 pte_t *ptep;
1598
1599 /*
1600 * If ownes no valid ASID yet, cannot possibly have gotten
1601 * this page into the cache.
1602 */
1603 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1604 return;
1605
1606 #ifdef DEBUG_CACHE
1607 printk("cpage[%d,%08lx]", (int)mm->context, page);
1608 #endif
1609 __save_and_cli(flags);
1610 page &= PAGE_MASK;
1611 pgdp = pgd_offset(mm, page);
1612 pmdp = pmd_offset(pgdp, page);
1613 ptep = pte_offset(pmdp, page);
1614
1615 /*
1616 * If the page isn't marked valid, the page cannot possibly be
1617 * in the cache.
1618 */
1619 if(!(pte_val(*ptep) & _PAGE_PRESENT))
1620 goto out;
1621
1622 /*
1623 * Doing flushes for another ASID than the current one is
1624 * too difficult since stupid R4k caches do a TLB translation
1625 * for every cache flush operation. So we do indexed flushes
1626 * in that case, which doesn't overly flush the cache too much.
1627 */
1628 if((mm == current->mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1629 blast_dcache32_page(page);
1630 } else {
1631 /*
1632 * Do indexed flush, too much work to get the (possible)
1633 * tlb refills to work correctly.
1634 */
1635 page = (KSEG0 + (page & (dcache_size - 1)));
1636 blast_dcache32_page_indexed(page);
1637 }
1638 out:
1639 __restore_flags(flags);
1640 }
1641
1642 static void r4k_flush_cache_page_d32i32_r4600(struct vm_area_struct *vma,
1643 unsigned long page)
1644 {
1645 struct mm_struct *mm = vma->vm_mm;
1646 unsigned long flags;
1647 pgd_t *pgdp;
1648 pmd_t *pmdp;
1649 pte_t *ptep;
1650
1651 /*
1652 * If ownes no valid ASID yet, cannot possibly have gotten
1653 * this page into the cache.
1654 */
1655 if (CPU_CONTEXT(smp_processor_id(), mm) == 0)
1656 return;
1657
1658 #ifdef DEBUG_CACHE
1659 printk("cpage[%d,%08lx]", (int)mm->context, page);
1660 #endif
1661 __save_and_cli(flags);
1662 page &= PAGE_MASK;
1663 pgdp = pgd_offset(mm, page);
1664 pmdp = pmd_offset(pgdp, page);
1665 ptep = pte_offset(pmdp, page);
1666
1667 /*
1668 * If the page isn't marked valid, the page cannot possibly be
1669 * in the cache.
1670 */
1671 if(!(pte_val(*ptep) & _PAGE_PRESENT))
1672 goto out;
1673
1674 /*
1675 * Doing flushes for another ASID than the current one is
1676 * too difficult since stupid R4k caches do a TLB translation
1677 * for every cache flush operation. So we do indexed flushes
1678 * in that case, which doesn't overly flush the cache too much.
1679 */
1680 if((mm == current->mm) && (pte_val(*ptep) & _PAGE_VALID)) {
1681 blast_dcache32_page(page);
1682 } else {
1683 /* Do indexed flush, too much work to get the (possible)
1684 * tlb refills to work correctly.
1685 */
1686 page = (KSEG0 + (page & (dcache_size - 1)));
1687 blast_dcache32_page_indexed(page);
1688 blast_dcache32_page_indexed(page ^ dcache_waybit);
1689 }
1690 out:
1691 __restore_flags(flags);
1692 }
1693
1694 static void r4k_flush_page_to_ram_s16(struct page *page)
1695 {
1696 blast_scache16_page((unsigned long)page_address(page));
1697 }
1698
1699 static void r4k_flush_page_to_ram_s32(struct page *page)
1700 {
1701 blast_scache32_page((unsigned long)page_address(page));
1702 }
1703
1704 static void r4k_flush_page_to_ram_s64(struct page *page)
1705 {
1706 blast_scache64_page((unsigned long)page_address(page));
1707 }
1708
1709 static void r4k_flush_page_to_ram_s128(struct page *page)
1710 {
1711 blast_scache128_page((unsigned long)page_address(page));
1712 }
1713
1714 static void r4k_flush_page_to_ram_d16(struct page *page)
1715 {
1716 unsigned long flags;
1717
1718 __save_and_cli(flags);
1719 blast_dcache16_page((unsigned long)page_address(page));
1720 __restore_flags(flags);
1721 }
1722
1723 static void r4k_flush_page_to_ram_d32(struct page *page)
1724 {
1725 unsigned long flags;
1726
1727 __save_and_cli(flags);
1728 blast_dcache32_page((unsigned long)page_address(page));
1729 __restore_flags(flags);
1730 }
1731
1732 /*
1733 * Writeback and invalidate the primary cache dcache before DMA.
1734 *
1735 * R4600 v2.0 bug: "The CACHE instructions Hit_Writeback_Inv_D,
1736 * Hit_Writeback_D, Hit_Invalidate_D and Create_Dirty_Exclusive_D will only
1737 * operate correctly if the internal data cache refill buffer is empty. These
1738 * CACHE instructions should be separated from any potential data cache miss
1739 * by a load instruction to an uncached address to empty the response buffer."
1740 * (Revision 2.0 device errata from IDT available on http://www.idt.com/
1741 * in .pdf format.)
1742 */
1743 static void r4k_dma_cache_wback_inv_pc(unsigned long addr, unsigned long size)
1744 {
1745 unsigned long end, a;
1746 unsigned int flags;
1747
1748 if (size >= (unsigned long)dcache_size) {
1749 flush_cache_l1();
1750 } else {
1751 /* Workaround for R4600 bug. See comment above. */
1752 __save_and_cli(flags);
1753 *(volatile unsigned long *)KSEG1;
1754
1755 a = addr & ~((unsigned long)dc_lsize - 1);
1756 end = (addr + size) & ~((unsigned long)dc_lsize - 1);
1757 while (1) {
1758 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
1759 if (a == end) break;
1760 a += dc_lsize;
1761 }
1762 __restore_flags(flags);
1763 }
1764 bc_wback_inv(addr, size);
1765 }
1766
1767 static void r4k_dma_cache_wback_inv_sc(unsigned long addr, unsigned long size)
1768 {
1769 unsigned long end, a;
1770
1771 if (size >= (unsigned long)scache_size) {
1772 flush_cache_l1();
1773 return;
1774 }
1775
1776 a = addr & ~((unsigned long)sc_lsize - 1);
1777 end = (addr + size) & ~((unsigned long)sc_lsize - 1);
1778 while (1) {
1779 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
1780 if (a == end) break;
1781 a += sc_lsize;
1782 }
1783 }
1784
1785 static void r4k_dma_cache_inv_pc(unsigned long addr, unsigned long size)
1786 {
1787 unsigned long end, a;
1788 unsigned int flags;
1789
1790 if (size >= (unsigned long)dcache_size) {
1791 flush_cache_l1();
1792 } else {
1793 /* Workaround for R4600 bug. See comment above. */
1794 __save_and_cli(flags);
1795 *(volatile unsigned long *)KSEG1;
1796
1797 a = addr & ~((unsigned long)dc_lsize - 1);
1798 end = (addr + size) & ~((unsigned long)dc_lsize - 1);
1799 while (1) {
1800 flush_dcache_line(a); /* Hit_Writeback_Inv_D */
1801 if (a == end) break;
1802 a += dc_lsize;
1803 }
1804 __restore_flags(flags);
1805 }
1806
1807 bc_inv(addr, size);
1808 }
1809
1810 static void r4k_dma_cache_inv_sc(unsigned long addr, unsigned long size)
1811 {
1812 unsigned long end, a;
1813
1814 if (size >= (unsigned long)scache_size) {
1815 flush_cache_l1();
1816 return;
1817 }
1818
1819 a = addr & ~((unsigned long)sc_lsize - 1);
1820 end = (addr + size) & ~((unsigned long)sc_lsize - 1);
1821 while (1) {
1822 flush_scache_line(a); /* Hit_Writeback_Inv_SD */
1823 if (a == end) break;
1824 a += sc_lsize;
1825 }
1826 }
1827
1828 static void r4k_dma_cache_wback(unsigned long addr, unsigned long size)
1829 {
1830 panic("r4k_dma_cache called - should not happen.\n");
1831 }
1832
1833 /*
1834 * While we're protected against bad userland addresses we don't care
1835 * very much about what happens in that case. Usually a segmentation
1836 * fault will dump the process later on anyway ...
1837 */
1838 static void r4k_flush_cache_sigtramp(unsigned long addr)
1839 {
1840 __asm__ __volatile__("nop;nop;nop;nop"); /* R4600 V1.7 */
1841
1842 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
1843 protected_flush_icache_line(addr & ~(ic_lsize - 1));
1844 }
1845
1846 static void r4600v20k_flush_cache_sigtramp(unsigned long addr)
1847 {
1848 unsigned int flags;
1849
1850 __save_and_cli(flags);
1851
1852 /* Clear internal cache refill buffer */
1853 *(volatile unsigned int *)KSEG1;
1854
1855 protected_writeback_dcache_line(addr & ~(dc_lsize - 1));
1856 protected_flush_icache_line(addr & ~(ic_lsize - 1));
1857
1858 __restore_flags(flags);
1859 }
1860
1861 #undef DEBUG_TLB
1862
1863 #define NTLB_ENTRIES 48 /* Fixed on all R4XX0 variants... */
1864
1865 #define NTLB_ENTRIES_HALF 24 /* Fixed on all R4XX0 variants... */
1866
1867 static inline void r4k_flush_tlb_all(void)
1868 {
1869 unsigned long flags;
1870 unsigned long old_ctx;
1871 int entry;
1872
1873 #ifdef DEBUG_TLB
1874 printk("[tlball]");
1875 #endif
1876
1877 __save_and_cli(flags);
1878 /* Save old context and create impossible VPN2 value */
1879 old_ctx = (get_entryhi() & 0xff);
1880 set_entryhi(KSEG0);
1881 set_entrylo0(0);
1882 set_entrylo1(0);
1883 BARRIER;
1884
1885 entry = get_wired();
1886
1887 /* Blast 'em all away. */
1888 while(entry < NTLB_ENTRIES) {
1889 set_index(entry);
1890 BARRIER;
1891 tlb_write_indexed();
1892 BARRIER;
1893 entry++;
1894 }
1895 BARRIER;
1896 set_entryhi(old_ctx);
1897 __restore_flags(flags);
1898 }
1899
1900 static void r4k_flush_tlb_mm(struct mm_struct *mm)
1901 {
1902 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1903 unsigned long flags;
1904
1905 #ifdef DEBUG_TLB
1906 printk("[tlbmm<%d>]", mm->context);
1907 #endif
1908 __save_and_cli(flags);
1909 get_new_cpu_mmu_context(mm, smp_processor_id());
1910 if(mm == current->mm)
1911 set_entryhi(CPU_CONTEXT(smp_processor_id(), mm) & 0xff);
1912 __restore_flags(flags);
1913 }
1914 }
1915
1916 static void r4k_flush_tlb_range(struct mm_struct *mm, unsigned long start,
1917 unsigned long end)
1918 {
1919 if (CPU_CONTEXT(smp_processor_id(), mm) != 0) {
1920 unsigned long flags;
1921 int size;
1922
1923 #ifdef DEBUG_TLB
1924 printk("[tlbrange<%02x,%08lx,%08lx>]", (mm->context & 0xff),
1925 start, end);
1926 #endif
1927 __save_and_cli(flags);
1928 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
1929 size = (size + 1) >> 1;
1930 if(size <= NTLB_ENTRIES_HALF) {
1931 int oldpid = (get_entryhi() & 0xff);
1932 int newpid = (CPU_CONTEXT(smp_processor_id(), mm) & 0xff);
1933
1934 start &= (PAGE_MASK << 1);
1935 end += ((PAGE_SIZE << 1) - 1);
1936 end &= (PAGE_MASK << 1);
1937 while(start < end) {
1938 int idx;
1939
1940 set_entryhi(start | newpid);
1941 start += (PAGE_SIZE << 1);
1942 BARRIER;
1943 tlb_probe();
1944 BARRIER;
1945 idx = get_index();
1946 set_entrylo0(0);
1947 set_entrylo1(0);
1948 set_entryhi(KSEG0);
1949 BARRIER;
1950 if(idx < 0)
1951 continue;
1952 tlb_write_indexed();
1953 BARRIER;
1954 }
1955 set_entryhi(oldpid);
1956 } else {
1957 get_new_cpu_mmu_context(mm, smp_processor_id());
1958 if(mm == current->mm)
1959 set_entryhi(CPU_CONTEXT(smp_processor_id(),
1960 mm) & 0xff);
1961 }
1962 __restore_flags(flags);
1963 }
1964 }
1965
1966 static void r4k_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
1967 {
1968 if (CPU_CONTEXT(smp_processor_id(), vma->vm_mm) != 0) {
1969 unsigned long flags;
1970 int oldpid, newpid, idx;
1971
1972 #ifdef DEBUG_TLB
1973 printk("[tlbpage<%d,%08lx>]", vma->vm_mm->context, page);
1974 #endif
1975 newpid = (CPU_CONTEXT(smp_processor_id(), vma->vm_mm) & 0xff);
1976 page &= (PAGE_MASK << 1);
1977 __save_and_cli(flags);
1978 oldpid = (get_entryhi() & 0xff);
1979 set_entryhi(page | newpid);
1980 BARRIER;
1981 tlb_probe();
1982 BARRIER;
1983 idx = get_index();
1984 set_entrylo0(0);
1985 set_entrylo1(0);
1986 set_entryhi(KSEG0);
1987 if(idx < 0)
1988 goto finish;
1989 BARRIER;
1990 tlb_write_indexed();
1991
1992 finish:
1993 BARRIER;
1994 set_entryhi(oldpid);
1995 __restore_flags(flags);
1996 }
1997 }
1998
1999 static void r4k_flush_cache_l2(void)
2000 {
2001 }
2002
2003 /* We will need multiple versions of update_mmu_cache(), one that just
2004 * updates the TLB with the new pte(s), and another which also checks
2005 * for the R4k "end of page" hardware bug and does the needy.
2006 */
2007 static void r4k_update_mmu_cache(struct vm_area_struct * vma,
2008 unsigned long address, pte_t pte)
2009 {
2010 unsigned long flags;
2011 pgd_t *pgdp;
2012 pmd_t *pmdp;
2013 pte_t *ptep;
2014 int idx, pid;
2015
2016 /*
2017 * Handle debugger faulting in for debugee.
2018 */
2019 if (current->active_mm != vma->vm_mm)
2020 return;
2021
2022 __save_and_cli(flags);
2023 pid = (get_entryhi() & 0xff);
2024
2025 #ifdef DEBUG_TLB
2026 if((pid != (CPU_CONTEXT(smp_processor_id(), vma->vm_mm) & 0xff)) ||
2027 (CPU_CONTEXT(smp_processor_id(), vma->vm_mm) == 0)) {
2028 printk("update_mmu_cache: Wheee, bogus tlbpid mmpid=%d
2029 tlbpid=%d\n", (int) (CPU_CONTEXT(smp_processor_id(),
2030 vma->vm_mm) & 0xff), pid);
2031 }
2032 #endif
2033
2034 address &= (PAGE_MASK << 1);
2035 set_entryhi(address | (pid));
2036 pgdp = pgd_offset(vma->vm_mm, address);
2037 BARRIER;
2038 tlb_probe();
2039 BARRIER;
2040 pmdp = pmd_offset(pgdp, address);
2041 idx = get_index();
2042 ptep = pte_offset(pmdp, address);
2043 BARRIER;
2044 set_entrylo0(pte_val(*ptep++) >> 6);
2045 set_entrylo1(pte_val(*ptep) >> 6);
2046 set_entryhi(address | (pid));
2047 BARRIER;
2048 if(idx < 0) {
2049 tlb_write_random();
2050 } else {
2051 tlb_write_indexed();
2052 }
2053 BARRIER;
2054 set_entryhi(pid);
2055 BARRIER;
2056 __restore_flags(flags);
2057 }
2058
2059 #if 0
2060 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma,
2061 unsigned long address, pte_t pte)
2062 {
2063 unsigned long flags;
2064 pgd_t *pgdp;
2065 pmd_t *pmdp;
2066 pte_t *ptep;
2067 int idx;
2068
2069 __save_and_cli(flags);
2070 address &= (PAGE_MASK << 1);
2071 set_entryhi(address | (get_entryhi() & 0xff));
2072 pgdp = pgd_offset(vma->vm_mm, address);
2073 tlb_probe();
2074 pmdp = pmd_offset(pgdp, address);
2075 idx = get_index();
2076 ptep = pte_offset(pmdp, address);
2077 set_entrylo0(pte_val(*ptep++) >> 6);
2078 set_entrylo1(pte_val(*ptep) >> 6);
2079 BARRIER;
2080 if(idx < 0)
2081 tlb_write_random();
2082 else
2083 tlb_write_indexed();
2084 BARRIER;
2085 __restore_flags(flags);
2086 }
2087 #endif
2088
2089 static void r4k_show_regs(struct pt_regs *regs)
2090 {
2091 /* Saved main processor registers. */
2092 printk("$0 : %016lx %016lx %016lx %016lx\n",
2093 0UL, regs->regs[1], regs->regs[2], regs->regs[3]);
2094 printk("$4 : %016lx %016lx %016lx %016lx\n",
2095 regs->regs[4], regs->regs[5], regs->regs[6], regs->regs[7]);
2096 printk("$8 : %016lx %016lx %016lx %016lx\n",
2097 regs->regs[8], regs->regs[9], regs->regs[10], regs->regs[11]);
2098 printk("$12 : %016lx %016lx %016lx %016lx\n",
2099 regs->regs[12], regs->regs[13], regs->regs[14], regs->regs[15]);
2100 printk("$16 : %016lx %016lx %016lx %016lx\n",
2101 regs->regs[16], regs->regs[17], regs->regs[18], regs->regs[19]);
2102 printk("$20 : %016lx %016lx %016lx %016lx\n",
2103 regs->regs[20], regs->regs[21], regs->regs[22], regs->regs[23]);
2104 printk("$24 : %016lx %016lx\n",
2105 regs->regs[24], regs->regs[25]);
2106 printk("$28 : %016lx %016lx %016lx %016lx\n",
2107 regs->regs[28], regs->regs[29], regs->regs[30], regs->regs[31]);
2108 printk("Hi : %016lx\n", regs->hi);
2109 printk("Lo : %016lx\n", regs->lo);
2110
2111 /* Saved cp0 registers. */
2112 printk("epc : %016lx\nbadvaddr: %016lx\n",
2113 regs->cp0_epc, regs->cp0_badvaddr);
2114 printk("Status : %08x\nCause : %08x\n",
2115 (unsigned int) regs->cp0_status, (unsigned int) regs->cp0_cause);
2116 }
2117
2118 /* Detect and size the various r4k caches. */
2119 static void __init probe_icache(unsigned long config)
2120 {
2121 icache_size = 1 << (12 + ((config >> 9) & 7));
2122 ic_lsize = 16 << ((config >> 5) & 1);
2123
2124 printk("Primary instruction cache %dkb, linesize %d bytes)\n",
2125 icache_size >> 10, ic_lsize);
2126 }
2127
2128 static void __init probe_dcache(unsigned long config)
2129 {
2130 dcache_size = 1 << (12 + ((config >> 6) & 7));
2131 dc_lsize = 16 << ((config >> 4) & 1);
2132
2133 printk("Primary data cache %dkb, linesize %d bytes)\n",
2134 dcache_size >> 10, dc_lsize);
2135 }
2136
2137
2138 /* If you even _breathe_ on this function, look at the gcc output
2139 * and make sure it does not pop things on and off the stack for
2140 * the cache sizing loop that executes in KSEG1 space or else
2141 * you will crash and burn badly. You have been warned.
2142 */
2143 static int __init probe_scache(unsigned long config)
2144 {
2145 extern unsigned long stext;
2146 unsigned long flags, addr, begin, end, pow2;
2147 int tmp;
2148
2149 tmp = ((config >> 17) & 1);
2150 if(tmp)
2151 return 0;
2152 tmp = ((config >> 22) & 3);
2153 switch(tmp) {
2154 case 0:
2155 sc_lsize = 16;
2156 break;
2157 case 1:
2158 sc_lsize = 32;
2159 break;
2160 case 2:
2161 sc_lsize = 64;
2162 break;
2163 case 3:
2164 sc_lsize = 128;
2165 break;
2166 }
2167
2168 begin = (unsigned long) &stext;
2169 begin &= ~((4 * 1024 * 1024) - 1);
2170 end = begin + (4 * 1024 * 1024);
2171
2172 /* This is such a bitch, you'd think they would make it
2173 * easy to do this. Away you daemons of stupidity!
2174 */
2175 __save_and_cli(flags);
2176
2177 /* Fill each size-multiple cache line with a valid tag. */
2178 pow2 = (64 * 1024);
2179 for(addr = begin; addr < end; addr = (begin + pow2)) {
2180 unsigned long *p = (unsigned long *) addr;
2181 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */
2182 pow2 <<= 1;
2183 }
2184
2185 /* Load first line with zero (therefore invalid) tag. */
2186 set_taglo(0);
2187 set_taghi(0);
2188 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */
2189 __asm__ __volatile__("\n\t.set noreorder\n\t"
2190 "cache 8, (%0)\n\t"
2191 ".set reorder\n\t" : : "r" (begin));
2192 __asm__ __volatile__("\n\t.set noreorder\n\t"
2193 "cache 9, (%0)\n\t"
2194 ".set reorder\n\t" : : "r" (begin));
2195 __asm__ __volatile__("\n\t.set noreorder\n\t"
2196 "cache 11, (%0)\n\t"
2197 ".set reorder\n\t" : : "r" (begin));
2198
2199 /* Now search for the wrap around point. */
2200 pow2 = (128 * 1024);
2201 tmp = 0;
2202 for(addr = (begin + (128 * 1024)); addr < (end); addr = (begin + pow2)) {
2203 __asm__ __volatile__("\n\t.set noreorder\n\t"
2204 "cache 7, (%0)\n\t"
2205 ".set reorder\n\t" : : "r" (addr));
2206 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */
2207 if(!get_taglo())
2208 break;
2209 pow2 <<= 1;
2210 }
2211 __restore_flags(flags);
2212 addr -= begin;
2213 printk("Secondary cache sized at %dK linesize %d\n",
2214 (int) (addr >> 10), sc_lsize);
2215 scache_size = addr;
2216 return 1;
2217 }
2218
2219 static void __init setup_noscache_funcs(void)
2220 {
2221 unsigned int prid;
2222
2223 switch(dc_lsize) {
2224 case 16:
2225 _clear_page = r4k_clear_page_d16;
2226 _copy_page = r4k_copy_page_d16;
2227 _flush_cache_l1 = r4k_flush_cache_all_d16i16;
2228 _flush_cache_mm = r4k_flush_cache_mm_d16i16;
2229 _flush_cache_range = r4k_flush_cache_range_d16i16;
2230 _flush_cache_page = r4k_flush_cache_page_d16i16;
2231 break;
2232 case 32:
2233 prid = read_32bit_cp0_register(CP0_PRID) & 0xfff0;
2234 if (prid == 0x2010) { /* R4600 V1.7 */
2235 _clear_page = r4k_clear_page_r4600_v1;
2236 _copy_page = r4k_copy_page_r4600_v1;
2237 } else if (prid == 0x2020) { /* R4600 V2.0 */
2238 _clear_page = r4k_clear_page_r4600_v2;
2239 _copy_page = r4k_copy_page_r4600_v2;
2240 } else {
2241 _clear_page = r4k_clear_page_d32;
2242 _copy_page = r4k_copy_page_d32;
2243 }
2244 _flush_cache_l1 = r4k_flush_cache_all_d32i32;
2245 _flush_cache_mm = r4k_flush_cache_mm_d32i32;
2246 _flush_cache_range = r4k_flush_cache_range_d32i32;
2247 _flush_cache_page = r4k_flush_cache_page_d32i32;
2248 break;
2249 }
2250
2251 switch(ic_lsize) {
2252 case 16:
2253 _flush_page_to_ram = r4k_flush_page_to_ram_d16;
2254 break;
2255 case 32:
2256 _flush_page_to_ram = r4k_flush_page_to_ram_d32;
2257 break;
2258 }
2259
2260 _dma_cache_wback_inv = r4k_dma_cache_wback_inv_pc;
2261 _dma_cache_wback = r4k_dma_cache_wback;
2262 _dma_cache_inv = r4k_dma_cache_inv_pc;
2263 }
2264
2265 static void __init setup_scache_funcs(void)
2266 {
2267 switch(sc_lsize) {
2268 case 16:
2269 switch(dc_lsize) {
2270 case 16:
2271 _flush_cache_l1 = r4k_flush_cache_all_s16d16i16;
2272 _flush_cache_mm = r4k_flush_cache_mm_s16d16i16;
2273 _flush_cache_range = r4k_flush_cache_range_s16d16i16;
2274 _flush_cache_page = r4k_flush_cache_page_s16d16i16;
2275 break;
2276 case 32:
2277 panic("Invalid cache configuration detected");
2278 };
2279 _flush_page_to_ram = r4k_flush_page_to_ram_s16;
2280 _clear_page = r4k_clear_page_s16;
2281 _copy_page = r4k_copy_page_s16;
2282 break;
2283 case 32:
2284 switch(dc_lsize) {
2285 case 16:
2286 _flush_cache_l1 = r4k_flush_cache_all_s32d16i16;
2287 _flush_cache_mm = r4k_flush_cache_mm_s32d16i16;
2288 _flush_cache_range = r4k_flush_cache_range_s32d16i16;
2289 _flush_cache_page = r4k_flush_cache_page_s32d16i16;
2290 break;
2291 case 32:
2292 _flush_cache_l1 = r4k_flush_cache_all_s32d32i32;
2293 _flush_cache_mm = r4k_flush_cache_mm_s32d32i32;
2294 _flush_cache_range = r4k_flush_cache_range_s32d32i32;
2295 _flush_cache_page = r4k_flush_cache_page_s32d32i32;
2296 break;
2297 };
2298 _flush_page_to_ram = r4k_flush_page_to_ram_s32;
2299 _clear_page = r4k_clear_page_s32;
2300 _copy_page = r4k_copy_page_s32;
2301 break;
2302 case 64:
2303 switch(dc_lsize) {
2304 case 16:
2305 _flush_cache_l1 = r4k_flush_cache_all_s64d16i16;
2306 _flush_cache_mm = r4k_flush_cache_mm_s64d16i16;
2307 _flush_cache_range = r4k_flush_cache_range_s64d16i16;
2308 _flush_cache_page = r4k_flush_cache_page_s64d16i16;
2309 break;
2310 case 32:
2311 _flush_cache_l1 = r4k_flush_cache_all_s64d32i32;
2312 _flush_cache_mm = r4k_flush_cache_mm_s64d32i32;
2313 _flush_cache_range = r4k_flush_cache_range_s64d32i32;
2314 _flush_cache_page = r4k_flush_cache_page_s64d32i32;
2315 break;
2316 };
2317 _flush_page_to_ram = r4k_flush_page_to_ram_s64;
2318 _clear_page = r4k_clear_page_s64;
2319 _copy_page = r4k_copy_page_s64;
2320 break;
2321 case 128:
2322 switch(dc_lsize) {
2323 case 16:
2324 _flush_cache_l1 = r4k_flush_cache_all_s128d16i16;
2325 _flush_cache_mm = r4k_flush_cache_mm_s128d16i16;
2326 _flush_cache_range = r4k_flush_cache_range_s128d16i16;
2327 _flush_cache_page = r4k_flush_cache_page_s128d16i16;
2328 break;
2329 case 32:
2330 _flush_cache_l1 = r4k_flush_cache_all_s128d32i32;
2331 _flush_cache_mm = r4k_flush_cache_mm_s128d32i32;
2332 _flush_cache_range = r4k_flush_cache_range_s128d32i32;
2333 _flush_cache_page = r4k_flush_cache_page_s128d32i32;
2334 break;
2335 };
2336 _flush_page_to_ram = r4k_flush_page_to_ram_s128;
2337 _clear_page = r4k_clear_page_s128;
2338 _copy_page = r4k_copy_page_s128;
2339 break;
2340 }
2341 _dma_cache_wback_inv = r4k_dma_cache_wback_inv_sc;
2342 _dma_cache_wback = r4k_dma_cache_wback;
2343 _dma_cache_inv = r4k_dma_cache_inv_sc;
2344 }
2345
2346 typedef int (*probe_func_t)(unsigned long);
2347
2348 static inline void __init setup_scache(unsigned int config)
2349 {
2350 probe_func_t probe_scache_kseg1;
2351 int sc_present = 0;
2352
2353 /* Maybe the cpu knows about a l2 cache? */
2354 probe_scache_kseg1 = (probe_func_t) (KSEG1ADDR(&probe_scache));
2355 sc_present = probe_scache_kseg1(config);
2356
2357 if (sc_present) {
2358 setup_scache_funcs();
2359 return;
2360 }
2361
2362 setup_noscache_funcs();
2363 }
2364
2365 void __init ld_mmu_r4xx0(void)
2366 {
2367 unsigned long config = read_32bit_cp0_register(CP0_CONFIG);
2368
2369 printk("CPU revision is: %08x\n", read_32bit_cp0_register(CP0_PRID));
2370
2371 #ifdef CONFIG_MIPS_UNCACHED
2372 set_cp0_config(CONF_CM_CMASK, CONF_CM_UNCACHED);
2373 #else
2374 set_cp0_config(CONF_CM_CMASK, CONF_CM_CACHABLE_NONCOHERENT);
2375 #endif /* UNCACHED */
2376
2377 probe_icache(config);
2378 probe_dcache(config);
2379 setup_scache(config);
2380
2381 switch(mips_cputype) {
2382 case CPU_R4600: /* QED style two way caches? */
2383 case CPU_R4700:
2384 case CPU_R5000:
2385 case CPU_NEVADA:
2386 _flush_cache_page = r4k_flush_cache_page_d32i32_r4600;
2387 }
2388
2389 _flush_cache_sigtramp = r4k_flush_cache_sigtramp;
2390 if ((read_32bit_cp0_register(CP0_PRID) & 0xfff0) == 0x2020) {
2391 _flush_cache_sigtramp = r4600v20k_flush_cache_sigtramp;
2392 }
2393
2394 _flush_tlb_all = r4k_flush_tlb_all;
2395 _flush_tlb_mm = r4k_flush_tlb_mm;
2396 _flush_tlb_range = r4k_flush_tlb_range;
2397 _flush_tlb_page = r4k_flush_tlb_page;
2398 _flush_cache_l2 = r4k_flush_cache_l2;
2399
2400 update_mmu_cache = r4k_update_mmu_cache;
2401
2402 _show_regs = r4k_show_regs;
2403
2404 flush_cache_l1();
2405
2406 /*
2407 * You should never change this register:
2408 * - On R4600 1.7 the tlbp never hits for pages smaller than
2409 * the value in the c0_pagemask register.
2410 * - The entire mm handling assumes the c0_pagemask register to
2411 * be set for 4kb pages.
2412 */
2413 write_32bit_cp0_register(CP0_PAGEMASK, PM_4K);
2414 _flush_tlb_all();
2415 }
2416