File: /usr/src/linux/arch/arm/mm/mm-armv.c
1 /*
2 * linux/arch/arm/mm/mm-armv.c
3 *
4 * Copyright (C) 1998-2000 Russell King
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * Page table sludge for ARM v3 and v4 processor architectures.
11 */
12 #include <linux/sched.h>
13 #include <linux/mm.h>
14 #include <linux/init.h>
15 #include <linux/bootmem.h>
16
17 #include <asm/pgtable.h>
18 #include <asm/pgalloc.h>
19 #include <asm/page.h>
20 #include <asm/io.h>
21 #include <asm/setup.h>
22
23 #include <asm/mach/map.h>
24
25 /*
26 * These are useful for identifing cache coherency
27 * problems by allowing the cache or the cache and
28 * writebuffer to be turned off. (Note: the write
29 * buffer should not be on and the cache off).
30 */
31 static int __init nocache_setup(char *__unused)
32 {
33 cr_alignment &= ~4;
34 cr_no_alignment &= ~4;
35 flush_cache_all();
36 set_cr(cr_alignment);
37 return 1;
38 }
39
40 static int __init nowrite_setup(char *__unused)
41 {
42 cr_alignment &= ~(8|4);
43 cr_no_alignment &= ~(8|4);
44 flush_cache_all();
45 set_cr(cr_alignment);
46 return 1;
47 }
48
49 static int __init noalign_setup(char *__unused)
50 {
51 cr_alignment &= ~2;
52 cr_no_alignment &= ~2;
53 set_cr(cr_alignment);
54 return 1;
55 }
56
57 __setup("noalign", noalign_setup);
58 __setup("nocache", nocache_setup);
59 __setup("nowb", nowrite_setup);
60
61 #define FIRST_KERNEL_PGD_NR (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD)
62
63 #define clean_cache_area(start,size) \
64 cpu_cache_clean_invalidate_range((unsigned long)start, ((unsigned long)start) + size, 0);
65
66
67 /*
68 * need to get a 16k page for level 1
69 */
70 pgd_t *get_pgd_slow(struct mm_struct *mm)
71 {
72 pgd_t *new_pgd, *init_pgd;
73 pmd_t *new_pmd, *init_pmd;
74 pte_t *new_pte, *init_pte;
75
76 new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2);
77 if (!new_pgd)
78 goto no_pgd;
79
80 memzero(new_pgd, FIRST_KERNEL_PGD_NR * sizeof(pgd_t));
81
82 init_pgd = pgd_offset_k(0);
83
84 if (vectors_base() == 0) {
85 init_pmd = pmd_offset(init_pgd, 0);
86 init_pte = pte_offset(init_pmd, 0);
87
88 /*
89 * This lock is here just to satisfy pmd_alloc and pte_lock
90 */
91 spin_lock(&mm->page_table_lock);
92
93 /*
94 * On ARM, first page must always be allocated since it
95 * contains the machine vectors.
96 */
97 new_pmd = pmd_alloc(mm, new_pgd, 0);
98 if (!new_pmd)
99 goto no_pmd;
100
101 new_pte = pte_alloc(mm, new_pmd, 0);
102 if (!new_pte)
103 goto no_pte;
104
105 set_pte(new_pte, *init_pte);
106
107 spin_unlock(&mm->page_table_lock);
108 }
109
110 /*
111 * Copy over the kernel and IO PGD entries
112 */
113 memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR,
114 (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t));
115
116 /*
117 * FIXME: this should not be necessary
118 */
119 clean_cache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
120
121 return new_pgd;
122
123 no_pte:
124 spin_unlock(&mm->page_table_lock);
125 pmd_free(new_pmd);
126 free_pages((unsigned long)new_pgd, 2);
127 return NULL;
128
129 no_pmd:
130 spin_unlock(&mm->page_table_lock);
131 free_pages((unsigned long)new_pgd, 2);
132 return NULL;
133
134 no_pgd:
135 return NULL;
136 }
137
138 void free_pgd_slow(pgd_t *pgd)
139 {
140 pmd_t *pmd;
141 pte_t *pte;
142
143 if (!pgd)
144 return;
145
146 /* pgd is always present and good */
147 pmd = (pmd_t *)pgd;
148 if (pmd_none(*pmd))
149 goto free;
150 if (pmd_bad(*pmd)) {
151 pmd_ERROR(*pmd);
152 pmd_clear(pmd);
153 goto free;
154 }
155
156 pte = pte_offset(pmd, 0);
157 pmd_clear(pmd);
158 pte_free(pte);
159 pmd_free(pmd);
160 free:
161 free_pages((unsigned long) pgd, 2);
162 }
163
164 /*
165 * Create a SECTION PGD between VIRT and PHYS in domain
166 * DOMAIN with protection PROT
167 */
168 static inline void
169 alloc_init_section(unsigned long virt, unsigned long phys, int prot)
170 {
171 pmd_t pmd;
172
173 pmd_val(pmd) = phys | prot;
174
175 set_pmd(pmd_offset(pgd_offset_k(virt), virt), pmd);
176 }
177
178 /*
179 * Add a PAGE mapping between VIRT and PHYS in domain
180 * DOMAIN with protection PROT. Note that due to the
181 * way we map the PTEs, we must allocate two PTE_SIZE'd
182 * blocks - one for the Linux pte table, and one for
183 * the hardware pte table.
184 */
185 static inline void
186 alloc_init_page(unsigned long virt, unsigned long phys, int domain, int prot)
187 {
188 pmd_t *pmdp;
189 pte_t *ptep;
190
191 pmdp = pmd_offset(pgd_offset_k(virt), virt);
192
193 if (pmd_none(*pmdp)) {
194 pte_t *ptep = alloc_bootmem_low_pages(2 * PTRS_PER_PTE *
195 sizeof(pte_t));
196
197 ptep += PTRS_PER_PTE;
198
199 set_pmd(pmdp, __mk_pmd(ptep, PMD_TYPE_TABLE | PMD_DOMAIN(domain)));
200 }
201 ptep = pte_offset(pmdp, virt);
202
203 set_pte(ptep, mk_pte_phys(phys, __pgprot(prot)));
204 }
205
206 /*
207 * Clear any PGD mapping. On a two-level page table system,
208 * the clearance is done by the middle-level functions (pmd)
209 * rather than the top-level (pgd) functions.
210 */
211 static inline void clear_mapping(unsigned long virt)
212 {
213 pmd_clear(pmd_offset(pgd_offset_k(virt), virt));
214 }
215
216 /*
217 * Create the page directory entries and any necessary
218 * page tables for the mapping specified by `md'. We
219 * are able to cope here with varying sizes and address
220 * offsets, and we take full advantage of sections.
221 */
222 static void __init create_mapping(struct map_desc *md)
223 {
224 unsigned long virt, length;
225 int prot_sect, prot_pte;
226 long off;
227
228 prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
229 (md->prot_read ? L_PTE_USER : 0) |
230 (md->prot_write ? L_PTE_WRITE : 0) |
231 (md->cacheable ? L_PTE_CACHEABLE : 0) |
232 (md->bufferable ? L_PTE_BUFFERABLE : 0);
233
234 prot_sect = PMD_TYPE_SECT | PMD_DOMAIN(md->domain) |
235 (md->prot_read ? PMD_SECT_AP_READ : 0) |
236 (md->prot_write ? PMD_SECT_AP_WRITE : 0) |
237 (md->cacheable ? PMD_SECT_CACHEABLE : 0) |
238 (md->bufferable ? PMD_SECT_BUFFERABLE : 0);
239
240 virt = md->virtual;
241 off = md->physical - virt;
242 length = md->length;
243
244 while ((virt & 0xfffff || (virt + off) & 0xfffff) && length >= PAGE_SIZE) {
245 alloc_init_page(virt, virt + off, md->domain, prot_pte);
246
247 virt += PAGE_SIZE;
248 length -= PAGE_SIZE;
249 }
250
251 while (length >= PGDIR_SIZE) {
252 alloc_init_section(virt, virt + off, prot_sect);
253
254 virt += PGDIR_SIZE;
255 length -= PGDIR_SIZE;
256 }
257
258 while (length >= PAGE_SIZE) {
259 alloc_init_page(virt, virt + off, md->domain, prot_pte);
260
261 virt += PAGE_SIZE;
262 length -= PAGE_SIZE;
263 }
264 }
265
266 /*
267 * In order to soft-boot, we need to insert a 1:1 mapping in place of
268 * the user-mode pages. This will then ensure that we have predictable
269 * results when turning the mmu off
270 */
271 void setup_mm_for_reboot(char mode)
272 {
273 pgd_t *pgd;
274 pmd_t pmd;
275 int i;
276
277 if (current->mm && current->mm->pgd)
278 pgd = current->mm->pgd;
279 else
280 pgd = init_mm.pgd;
281
282 for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++) {
283 pmd_val(pmd) = (i << PGDIR_SHIFT) |
284 PMD_SECT_AP_WRITE | PMD_SECT_AP_READ |
285 PMD_TYPE_SECT;
286 set_pmd(pmd_offset(pgd + i, i << PGDIR_SHIFT), pmd);
287 }
288 }
289
290 /*
291 * Setup initial mappings. We use the page we allocated for zero page to hold
292 * the mappings, which will get overwritten by the vectors in traps_init().
293 * The mappings must be in virtual address order.
294 */
295 void __init memtable_init(struct meminfo *mi)
296 {
297 struct map_desc *init_maps, *p, *q;
298 unsigned long address = 0;
299 int i;
300
301 init_maps = p = alloc_bootmem_low_pages(PAGE_SIZE);
302
303 for (i = 0; i < mi->nr_banks; i++) {
304 if (mi->bank[i].size == 0)
305 continue;
306
307 p->physical = mi->bank[i].start;
308 p->virtual = __phys_to_virt(p->physical);
309 p->length = mi->bank[i].size;
310 p->domain = DOMAIN_KERNEL;
311 p->prot_read = 0;
312 p->prot_write = 1;
313 p->cacheable = 1;
314 p->bufferable = 1;
315
316 p ++;
317 }
318
319 #ifdef FLUSH_BASE
320 p->physical = FLUSH_BASE_PHYS;
321 p->virtual = FLUSH_BASE;
322 p->length = PGDIR_SIZE;
323 p->domain = DOMAIN_KERNEL;
324 p->prot_read = 1;
325 p->prot_write = 0;
326 p->cacheable = 1;
327 p->bufferable = 1;
328
329 p ++;
330 #endif
331
332 #ifdef FLUSH_BASE_MINICACHE
333 p->physical = FLUSH_BASE_PHYS + PGDIR_SIZE;
334 p->virtual = FLUSH_BASE_MINICACHE;
335 p->length = PGDIR_SIZE;
336 p->domain = DOMAIN_KERNEL;
337 p->prot_read = 1;
338 p->prot_write = 0;
339 p->cacheable = 1;
340 p->bufferable = 0;
341
342 p ++;
343 #endif
344
345 /*
346 * Go through the initial mappings, but clear out any
347 * pgdir entries that are not in the description.
348 */
349 q = init_maps;
350 do {
351 if (address < q->virtual || q == p) {
352 clear_mapping(address);
353 address += PGDIR_SIZE;
354 } else {
355 create_mapping(q);
356
357 address = q->virtual + q->length;
358 address = (address + PGDIR_SIZE - 1) & PGDIR_MASK;
359
360 q ++;
361 }
362 } while (address != 0);
363
364 /*
365 * Create a mapping for the machine vectors at virtual address 0
366 * or 0xffff0000. We should always try the high mapping.
367 */
368 init_maps->physical = virt_to_phys(init_maps);
369 init_maps->virtual = vectors_base();
370 init_maps->length = PAGE_SIZE;
371 init_maps->domain = DOMAIN_USER;
372 init_maps->prot_read = 0;
373 init_maps->prot_write = 0;
374 init_maps->cacheable = 1;
375 init_maps->bufferable = 0;
376
377 create_mapping(init_maps);
378
379 flush_cache_all();
380 }
381
382 /*
383 * Create the architecture specific mappings
384 */
385 void __init iotable_init(struct map_desc *io_desc)
386 {
387 int i;
388
389 for (i = 0; io_desc[i].last == 0; i++)
390 create_mapping(io_desc + i);
391 }
392
393 static inline void free_memmap(int node, unsigned long start, unsigned long end)
394 {
395 unsigned long pg, pgend;
396
397 start = __phys_to_virt(start);
398 end = __phys_to_virt(end);
399
400 pg = PAGE_ALIGN((unsigned long)(virt_to_page(start)));
401 pgend = ((unsigned long)(virt_to_page(end))) & PAGE_MASK;
402
403 start = __virt_to_phys(pg);
404 end = __virt_to_phys(pgend);
405
406 free_bootmem_node(NODE_DATA(node), start, end - start);
407 }
408
409 static inline void free_unused_memmap_node(int node, struct meminfo *mi)
410 {
411 unsigned long bank_start, prev_bank_end = 0;
412 unsigned int i;
413
414 /*
415 * [FIXME] This relies on each bank being in address order. This
416 * may not be the case, especially if the user has provided the
417 * information on the command line.
418 */
419 for (i = 0; i < mi->nr_banks; i++) {
420 if (mi->bank[i].size == 0 || mi->bank[i].node != node)
421 continue;
422
423 bank_start = mi->bank[i].start & PAGE_MASK;
424
425 /*
426 * If we had a previous bank, and there is a space
427 * between the current bank and the previous, free it.
428 */
429 if (prev_bank_end && prev_bank_end != bank_start)
430 free_memmap(node, prev_bank_end, bank_start);
431
432 prev_bank_end = PAGE_ALIGN(mi->bank[i].start +
433 mi->bank[i].size);
434 }
435 }
436
437 /*
438 * The mem_map array can get very big. Free
439 * the unused area of the memory map.
440 */
441 void __init create_memmap_holes(struct meminfo *mi)
442 {
443 int node;
444
445 for (node = 0; node < numnodes; node++)
446 free_unused_memmap_node(node, mi);
447 }
448
449 /*
450 * PTE table allocation cache.
451 *
452 * This is a move away from our custom 2K page allocator. We now use the
453 * slab cache to keep track of these objects.
454 *
455 * With this, it is questionable as to whether the PGT cache gains us
456 * anything. We may be better off dropping the PTE stuff from our PGT
457 * cache implementation.
458 */
459 kmem_cache_t *pte_cache;
460
461 /*
462 * The constructor gets called for each object within the cache when the
463 * cache page is created. Note that if slab tries to misalign the blocks,
464 * we BUG() loudly.
465 */
466 static void pte_cache_ctor(void *pte, kmem_cache_t *cache, unsigned long flags)
467 {
468 unsigned long block = (unsigned long)pte;
469
470 if (block & 2047)
471 BUG();
472
473 memzero(pte, 2 * PTRS_PER_PTE * sizeof(pte_t));
474 cpu_cache_clean_invalidate_range(block, block +
475 PTRS_PER_PTE * sizeof(pte_t), 0);
476 }
477
478 void __init pgtable_cache_init(void)
479 {
480 pte_cache = kmem_cache_create("pte-cache",
481 2 * PTRS_PER_PTE * sizeof(pte_t), 0, 0,
482 pte_cache_ctor, NULL);
483 if (!pte_cache)
484 BUG();
485 }
486