File: /usr/src/linux/include/asm-ia64/pgalloc.h
1 #ifndef _ASM_IA64_PGALLOC_H
2 #define _ASM_IA64_PGALLOC_H
3
4 /*
5 * This file contains the functions and defines necessary to allocate
6 * page tables.
7 *
8 * This hopefully works with any (fixed) ia-64 page-size, as defined
9 * in <asm/page.h> (currently 8192).
10 *
11 * Copyright (C) 1998-2001 Hewlett-Packard Co
12 * Copyright (C) 1998-2001 David Mosberger-Tang <davidm@hpl.hp.com>
13 * Copyright (C) 2000, Goutham Rao <goutham.rao@intel.com>
14 */
15
16 #include <linux/config.h>
17
18 #include <linux/mm.h>
19 #include <linux/threads.h>
20
21 #include <asm/mmu_context.h>
22 #include <asm/processor.h>
23
24 /*
25 * Very stupidly, we used to get new pgd's and pmd's, init their contents
26 * to point to the NULL versions of the next level page table, later on
27 * completely re-init them the same way, then free them up. This wasted
28 * a lot of work and caused unnecessary memory traffic. How broken...
29 * We fix this by caching them.
30 */
31 #define pgd_quicklist (local_cpu_data->pgd_quick)
32 #define pmd_quicklist (local_cpu_data->pmd_quick)
33 #define pte_quicklist (local_cpu_data->pte_quick)
34 #define pgtable_cache_size (local_cpu_data->pgtable_cache_sz)
35
36 static inline pgd_t*
37 pgd_alloc_one_fast (struct mm_struct *mm)
38 {
39 unsigned long *ret = pgd_quicklist;
40
41 if (__builtin_expect(ret != NULL, 1)) {
42 pgd_quicklist = (unsigned long *)(*ret);
43 ret[0] = 0;
44 --pgtable_cache_size;
45 } else
46 ret = NULL;
47 return (pgd_t *) ret;
48 }
49
50 static inline pgd_t*
51 pgd_alloc (struct mm_struct *mm)
52 {
53 /* the VM system never calls pgd_alloc_one_fast(), so we do it here. */
54 pgd_t *pgd = pgd_alloc_one_fast(mm);
55
56 if (__builtin_expect(pgd == NULL, 0)) {
57 pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
58 if (__builtin_expect(pgd != NULL, 1))
59 clear_page(pgd);
60 }
61 return pgd;
62 }
63
64 static inline void
65 pgd_free (pgd_t *pgd)
66 {
67 *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
68 pgd_quicklist = (unsigned long *) pgd;
69 ++pgtable_cache_size;
70 }
71
72 static inline void
73 pgd_populate (struct mm_struct *mm, pgd_t *pgd_entry, pmd_t *pmd)
74 {
75 pgd_val(*pgd_entry) = __pa(pmd);
76 }
77
78
79 static inline pmd_t*
80 pmd_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
81 {
82 unsigned long *ret = (unsigned long *)pmd_quicklist;
83
84 if (__builtin_expect(ret != NULL, 1)) {
85 pmd_quicklist = (unsigned long *)(*ret);
86 ret[0] = 0;
87 --pgtable_cache_size;
88 }
89 return (pmd_t *)ret;
90 }
91
92 static inline pmd_t*
93 pmd_alloc_one (struct mm_struct *mm, unsigned long addr)
94 {
95 pmd_t *pmd = (pmd_t *) __get_free_page(GFP_KERNEL);
96
97 if (__builtin_expect(pmd != NULL, 1))
98 clear_page(pmd);
99 return pmd;
100 }
101
102 static inline void
103 pmd_free (pmd_t *pmd)
104 {
105 *(unsigned long *)pmd = (unsigned long) pmd_quicklist;
106 pmd_quicklist = (unsigned long *) pmd;
107 ++pgtable_cache_size;
108 }
109
110 static inline void
111 pmd_populate (struct mm_struct *mm, pmd_t *pmd_entry, pte_t *pte)
112 {
113 pmd_val(*pmd_entry) = __pa(pte);
114 }
115
116 static inline pte_t*
117 pte_alloc_one_fast (struct mm_struct *mm, unsigned long addr)
118 {
119 unsigned long *ret = (unsigned long *)pte_quicklist;
120
121 if (__builtin_expect(ret != NULL, 1)) {
122 pte_quicklist = (unsigned long *)(*ret);
123 ret[0] = 0;
124 --pgtable_cache_size;
125 }
126 return (pte_t *)ret;
127 }
128
129
130 static inline pte_t*
131 pte_alloc_one (struct mm_struct *mm, unsigned long addr)
132 {
133 pte_t *pte = (pte_t *) __get_free_page(GFP_KERNEL);
134
135 if (__builtin_expect(pte != NULL, 1))
136 clear_page(pte);
137 return pte;
138 }
139
140 static inline void
141 pte_free (pte_t *pte)
142 {
143 *(unsigned long *)pte = (unsigned long) pte_quicklist;
144 pte_quicklist = (unsigned long *) pte;
145 ++pgtable_cache_size;
146 }
147
148 extern int do_check_pgt_cache (int, int);
149
150 /*
151 * Now for some TLB flushing routines. This is the kind of stuff that
152 * can be very expensive, so try to avoid them whenever possible.
153 */
154
155 /*
156 * Flush everything (kernel mapping may also have changed due to
157 * vmalloc/vfree).
158 */
159 extern void __flush_tlb_all (void);
160
161 #ifdef CONFIG_SMP
162 extern void smp_flush_tlb_all (void);
163 # define flush_tlb_all() smp_flush_tlb_all()
164 #else
165 # define flush_tlb_all() __flush_tlb_all()
166 #endif
167
168 /*
169 * Serialize usage of ptc.g:
170 */
171 extern spinlock_t ptcg_lock;
172
173 /*
174 * Flush a specified user mapping
175 */
176 static inline void
177 flush_tlb_mm (struct mm_struct *mm)
178 {
179 if (mm) {
180 mm->context = 0;
181 if (mm == current->active_mm) {
182 /* This is called, e.g., as a result of exec(). */
183 get_new_mmu_context(mm);
184 reload_context(mm);
185 }
186 }
187 }
188
189 extern void flush_tlb_range (struct mm_struct *mm, unsigned long start, unsigned long end);
190
191 /*
192 * Page-granular tlb flush.
193 */
194 static inline void
195 flush_tlb_page (struct vm_area_struct *vma, unsigned long addr)
196 {
197 #ifdef CONFIG_SMP
198 flush_tlb_range(vma->vm_mm, (addr & PAGE_MASK), (addr & PAGE_MASK) + PAGE_SIZE);
199 #else
200 if (vma->vm_mm == current->active_mm)
201 asm volatile ("ptc.l %0,%1" :: "r"(addr), "r"(PAGE_SHIFT << 2) : "memory");
202 #endif
203 }
204
205 /*
206 * Flush the TLB entries mapping the virtually mapped linear page
207 * table corresponding to address range [START-END).
208 */
209 static inline void
210 flush_tlb_pgtables (struct mm_struct *mm, unsigned long start, unsigned long end)
211 {
212 if (rgn_index(start) != rgn_index(end))
213 printk("flush_tlb_pgtables: can't flush across regions!!\n");
214 flush_tlb_range(mm, ia64_thash(start), ia64_thash(end));
215 }
216
217 /*
218 * Now for some cache flushing routines. This is the kind of stuff
219 * that can be very expensive, so try to avoid them whenever possible.
220 */
221
222 /* Caches aren't brain-dead on the IA-64. */
223 #define flush_cache_all() do { } while (0)
224 #define flush_cache_mm(mm) do { } while (0)
225 #define flush_cache_range(mm, start, end) do { } while (0)
226 #define flush_cache_page(vma, vmaddr) do { } while (0)
227 #define flush_page_to_ram(page) do { } while (0)
228
229 extern void flush_icache_range (unsigned long start, unsigned long end);
230
231 static inline void
232 flush_dcache_page (struct page *page)
233 {
234 clear_bit(PG_arch_1, &page->flags);
235 }
236
237 static inline void
238 clear_user_page (void *addr, unsigned long vaddr, struct page *page)
239 {
240 clear_page(addr);
241 flush_dcache_page(page);
242 }
243
244 static inline void
245 copy_user_page (void *to, void *from, unsigned long vaddr, struct page *page)
246 {
247 copy_page(to, from);
248 flush_dcache_page(page);
249 }
250
251 /*
252 * IA-64 doesn't have any external MMU info: the page tables contain all the necessary
253 * information. However, we use this macro to take care of any (delayed) i-cache flushing
254 * that may be necessary.
255 */
256 static inline void
257 update_mmu_cache (struct vm_area_struct *vma, unsigned long vaddr, pte_t pte)
258 {
259 unsigned long addr;
260 struct page *page;
261
262 if (!pte_exec(pte))
263 return; /* not an executable page... */
264
265 page = pte_page(pte);
266 /* don't use VADDR: it may not be mapped on this CPU (or may have just been flushed): */
267 addr = (unsigned long) page_address(page);
268
269 if (test_bit(PG_arch_1, &page->flags))
270 return; /* i-cache is already coherent with d-cache */
271
272 flush_icache_range(addr, addr + PAGE_SIZE);
273 set_bit(PG_arch_1, &page->flags); /* mark page as clean */
274 }
275
276 #endif /* _ASM_IA64_PGALLOC_H */
277