File: /usr/src/linux/include/asm-sparc64/pgalloc.h
1 /* $Id: pgalloc.h,v 1.21 2001/08/22 22:16:56 kanoj Exp $ */
2 #ifndef _SPARC64_PGALLOC_H
3 #define _SPARC64_PGALLOC_H
4
5 #include <linux/config.h>
6 #include <linux/kernel.h>
7 #include <linux/sched.h>
8
9 #include <asm/page.h>
10
11 /* Cache and TLB flush operations. */
12
13 /* These are the same regardless of whether this is an SMP kernel or not. */
14 #define flush_cache_mm(__mm) \
15 do { if ((__mm) == current->mm) flushw_user(); } while(0)
16 #define flush_cache_range(mm, start, end) \
17 flush_cache_mm(mm)
18 #define flush_cache_page(vma, page) \
19 flush_cache_mm((vma)->vm_mm)
20
21 /* This is unnecessary on the SpitFire since D-CACHE is write-through. */
22 #define flush_page_to_ram(page) do { } while (0)
23
24 /*
25 * On spitfire, the icache doesn't snoop local stores and we don't
26 * use block commit stores (which invalidate icache lines) during
27 * module load, so we need this.
28 */
29 extern void flush_icache_range(unsigned long start, unsigned long end);
30
31 extern void __flush_dcache_page(void *addr, int flush_icache);
32 #define flush_dcache_page(page) \
33 do { if ((page)->mapping && \
34 !((page)->mapping->i_mmap) && \
35 !((page)->mapping->i_mmap_shared)) \
36 set_bit(PG_dcache_dirty, &(page)->flags); \
37 else \
38 __flush_dcache_page((page)->virtual, \
39 ((tlb_type == spitfire) && \
40 (page)->mapping != NULL)); \
41 } while(0)
42
43 extern void __flush_dcache_range(unsigned long start, unsigned long end);
44
45 extern void __flush_cache_all(void);
46
47 extern void __flush_tlb_all(void);
48 extern void __flush_tlb_mm(unsigned long context, unsigned long r);
49 extern void __flush_tlb_range(unsigned long context, unsigned long start,
50 unsigned long r, unsigned long end,
51 unsigned long pgsz, unsigned long size);
52 extern void __flush_tlb_page(unsigned long context, unsigned long page, unsigned long r);
53
54 #ifndef CONFIG_SMP
55
56 #define flush_cache_all() __flush_cache_all()
57 #define flush_tlb_all() __flush_tlb_all()
58
59 #define flush_tlb_mm(__mm) \
60 do { if(CTX_VALID((__mm)->context)) \
61 __flush_tlb_mm(CTX_HWBITS((__mm)->context), SECONDARY_CONTEXT); \
62 } while(0)
63
64 #define flush_tlb_range(__mm, start, end) \
65 do { if(CTX_VALID((__mm)->context)) { \
66 unsigned long __start = (start)&PAGE_MASK; \
67 unsigned long __end = PAGE_ALIGN(end); \
68 __flush_tlb_range(CTX_HWBITS((__mm)->context), __start, \
69 SECONDARY_CONTEXT, __end, PAGE_SIZE, \
70 (__end - __start)); \
71 } \
72 } while(0)
73
74 #define flush_tlb_page(vma, page) \
75 do { struct mm_struct *__mm = (vma)->vm_mm; \
76 if(CTX_VALID(__mm->context)) \
77 __flush_tlb_page(CTX_HWBITS(__mm->context), (page)&PAGE_MASK, \
78 SECONDARY_CONTEXT); \
79 } while(0)
80
81 #else /* CONFIG_SMP */
82
83 extern void smp_flush_cache_all(void);
84 extern void smp_flush_tlb_all(void);
85 extern void smp_flush_tlb_mm(struct mm_struct *mm);
86 extern void smp_flush_tlb_range(struct mm_struct *mm, unsigned long start,
87 unsigned long end);
88 extern void smp_flush_tlb_page(struct mm_struct *mm, unsigned long page);
89
90 #define flush_cache_all() smp_flush_cache_all()
91 #define flush_tlb_all() smp_flush_tlb_all()
92 #define flush_tlb_mm(mm) smp_flush_tlb_mm(mm)
93 #define flush_tlb_range(mm, start, end) \
94 smp_flush_tlb_range(mm, start, end)
95 #define flush_tlb_page(vma, page) \
96 smp_flush_tlb_page((vma)->vm_mm, page)
97
98 #endif /* ! CONFIG_SMP */
99
100 #define VPTE_BASE_SPITFIRE 0xfffffffe00000000
101 #if 1
102 #define VPTE_BASE_CHEETAH VPTE_BASE_SPITFIRE
103 #else
104 #define VPTE_BASE_CHEETAH 0xffe0000000000000
105 #endif
106
107 extern __inline__ void flush_tlb_pgtables(struct mm_struct *mm, unsigned long start,
108 unsigned long end)
109 {
110 /* Note the signed type. */
111 long s = start, e = end, vpte_base;
112 if (s > e)
113 /* Nobody should call us with start below VM hole and end above.
114 See if it is really true. */
115 BUG();
116 #if 0
117 /* Currently free_pgtables guarantees this. */
118 s &= PMD_MASK;
119 e = (e + PMD_SIZE - 1) & PMD_MASK;
120 #endif
121 vpte_base = (tlb_type == spitfire ?
122 VPTE_BASE_SPITFIRE :
123 VPTE_BASE_CHEETAH);
124 flush_tlb_range(mm,
125 vpte_base + (s >> (PAGE_SHIFT - 3)),
126 vpte_base + (e >> (PAGE_SHIFT - 3)));
127 }
128 #undef VPTE_BASE_SPITFIRE
129 #undef VPTE_BASE_CHEETAH
130
131 /* Page table allocation/freeing. */
132 #ifdef CONFIG_SMP
133 /* Sliiiicck */
134 #define pgt_quicklists cpu_data[smp_processor_id()]
135 #else
136 extern struct pgtable_cache_struct {
137 unsigned long *pgd_cache;
138 unsigned long *pte_cache[2];
139 unsigned int pgcache_size;
140 unsigned int pgdcache_size;
141 } pgt_quicklists;
142 #endif
143 #define pgd_quicklist (pgt_quicklists.pgd_cache)
144 #define pmd_quicklist ((unsigned long *)0)
145 #define pte_quicklist (pgt_quicklists.pte_cache)
146 #define pgtable_cache_size (pgt_quicklists.pgcache_size)
147 #define pgd_cache_size (pgt_quicklists.pgdcache_size)
148
149 #ifndef CONFIG_SMP
150
151 extern __inline__ void free_pgd_fast(pgd_t *pgd)
152 {
153 struct page *page = virt_to_page(pgd);
154
155 if (!page->pprev_hash) {
156 (unsigned long *)page->next_hash = pgd_quicklist;
157 pgd_quicklist = (unsigned long *)page;
158 }
159 (unsigned long)page->pprev_hash |=
160 (((unsigned long)pgd & (PAGE_SIZE / 2)) ? 2 : 1);
161 pgd_cache_size++;
162 }
163
164 extern __inline__ pgd_t *get_pgd_fast(void)
165 {
166 struct page *ret;
167
168 if ((ret = (struct page *)pgd_quicklist) != NULL) {
169 unsigned long mask = (unsigned long)ret->pprev_hash;
170 unsigned long off = 0;
171
172 if (mask & 1)
173 mask &= ~1;
174 else {
175 off = PAGE_SIZE / 2;
176 mask &= ~2;
177 }
178 (unsigned long)ret->pprev_hash = mask;
179 if (!mask)
180 pgd_quicklist = (unsigned long *)ret->next_hash;
181 ret = (struct page *)(__page_address(ret) + off);
182 pgd_cache_size--;
183 } else {
184 struct page *page = alloc_page(GFP_KERNEL);
185
186 if (page) {
187 ret = (struct page *)page_address(page);
188 clear_page(ret);
189 (unsigned long)page->pprev_hash = 2;
190 (unsigned long *)page->next_hash = pgd_quicklist;
191 pgd_quicklist = (unsigned long *)page;
192 pgd_cache_size++;
193 }
194 }
195 return (pgd_t *)ret;
196 }
197
198 #else /* CONFIG_SMP */
199
200 extern __inline__ void free_pgd_fast(pgd_t *pgd)
201 {
202 *(unsigned long *)pgd = (unsigned long) pgd_quicklist;
203 pgd_quicklist = (unsigned long *) pgd;
204 pgtable_cache_size++;
205 }
206
207 extern __inline__ pgd_t *get_pgd_fast(void)
208 {
209 unsigned long *ret;
210
211 if((ret = pgd_quicklist) != NULL) {
212 pgd_quicklist = (unsigned long *)(*ret);
213 ret[0] = 0;
214 pgtable_cache_size--;
215 } else {
216 ret = (unsigned long *) __get_free_page(GFP_KERNEL);
217 if(ret)
218 memset(ret, 0, PAGE_SIZE);
219 }
220 return (pgd_t *)ret;
221 }
222
223 extern __inline__ void free_pgd_slow(pgd_t *pgd)
224 {
225 free_page((unsigned long)pgd);
226 }
227
228 #endif /* CONFIG_SMP */
229
230 #define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD)
231
232 extern __inline__ pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address)
233 {
234 pmd_t *pmd = (pmd_t *)__get_free_page(GFP_KERNEL);
235 if (pmd)
236 memset(pmd, 0, PAGE_SIZE);
237 return pmd;
238 }
239
240 extern __inline__ pmd_t *pmd_alloc_one_fast(struct mm_struct *mm, unsigned long address)
241 {
242 unsigned long *ret;
243 int color = 0;
244
245 if (pte_quicklist[color] == NULL)
246 color = 1;
247 if((ret = (unsigned long *)pte_quicklist[color]) != NULL) {
248 pte_quicklist[color] = (unsigned long *)(*ret);
249 ret[0] = 0;
250 pgtable_cache_size--;
251 }
252 return (pmd_t *)ret;
253 }
254
255 extern __inline__ void free_pmd_fast(pmd_t *pmd)
256 {
257 unsigned long color;
258
259 color = (((unsigned long)pmd >> PAGE_SHIFT) & 0x1UL);
260 *(unsigned long *)pmd = (unsigned long) pte_quicklist[color];
261 pte_quicklist[color] = (unsigned long *) pmd;
262 pgtable_cache_size++;
263 }
264
265 extern __inline__ void free_pmd_slow(pmd_t *pmd)
266 {
267 free_page((unsigned long)pmd);
268 }
269
270 #define pmd_populate(MM, PMD, PTE) pmd_set(PMD, PTE)
271
272 extern pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address);
273
274 extern __inline__ pte_t *pte_alloc_one_fast(struct mm_struct *mm, unsigned long address)
275 {
276 unsigned long color = (address >> (PAGE_SHIFT + 10)) & 0x1UL;
277 unsigned long *ret;
278
279 if((ret = (unsigned long *)pte_quicklist[color]) != NULL) {
280 pte_quicklist[color] = (unsigned long *)(*ret);
281 ret[0] = 0;
282 pgtable_cache_size--;
283 }
284 return (pte_t *)ret;
285 }
286
287 extern __inline__ void free_pte_fast(pte_t *pte)
288 {
289 unsigned long color = (((unsigned long)pte >> PAGE_SHIFT) & 0x1);
290 *(unsigned long *)pte = (unsigned long) pte_quicklist[color];
291 pte_quicklist[color] = (unsigned long *) pte;
292 pgtable_cache_size++;
293 }
294
295 extern __inline__ void free_pte_slow(pte_t *pte)
296 {
297 free_page((unsigned long)pte);
298 }
299
300 #define pte_free(pte) free_pte_fast(pte)
301 #define pmd_free(pmd) free_pmd_fast(pmd)
302 #define pgd_free(pgd) free_pgd_fast(pgd)
303 #define pgd_alloc(mm) get_pgd_fast()
304
305 extern int do_check_pgt_cache(int, int);
306
307 #endif /* _SPARC64_PGALLOC_H */
308