File: /usr/src/linux/arch/ia64/mm/tlb.c
1 /*
2 * TLB support routines.
3 *
4 * Copyright (C) 1998-2001 Hewlett-Packard Co
5 * Copyright (C) 1998-2001 David Mosberger-Tang <davidm@hpl.hp.com>
6 *
7 * 08/02/00 A. Mallick <asit.k.mallick@intel.com>
8 * Modified RID allocation for SMP
9 * Goutham Rao <goutham.rao@intel.com>
10 * IPI based ptc implementation and A-step IPI implementation.
11 */
12 #include <linux/config.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/sched.h>
16 #include <linux/smp.h>
17 #include <linux/mm.h>
18
19 #include <asm/mmu_context.h>
20 #include <asm/pgalloc.h>
21 #include <asm/pal.h>
22 #include <asm/delay.h>
23
24 #define SUPPORTED_PGBITS ( \
25 1 << _PAGE_SIZE_256M | \
26 1 << _PAGE_SIZE_64M | \
27 1 << _PAGE_SIZE_16M | \
28 1 << _PAGE_SIZE_4M | \
29 1 << _PAGE_SIZE_1M | \
30 1 << _PAGE_SIZE_256K | \
31 1 << _PAGE_SIZE_64K | \
32 1 << _PAGE_SIZE_16K | \
33 1 << _PAGE_SIZE_8K | \
34 1 << _PAGE_SIZE_4K )
35
36 struct ia64_ctx ia64_ctx = {
37 lock: SPIN_LOCK_UNLOCKED,
38 next: 1,
39 limit: (1 << 15) - 1, /* start out with the safe (architected) limit */
40 max_ctx: ~0U
41 };
42
43 /*
44 * Seralize usage of ptc.g
45 */
46 spinlock_t ptcg_lock = SPIN_LOCK_UNLOCKED; /* see <asm/pgtable.h> */
47
48 #if defined(CONFIG_SMP) && !defined(CONFIG_ITANIUM_PTCG)
49
50 #include <linux/irq.h>
51
52 unsigned long flush_end, flush_start, flush_nbits, flush_rid;
53 atomic_t flush_cpu_count;
54
55 /*
56 * flush_tlb_no_ptcg is called with ptcg_lock locked
57 */
58 static inline void
59 flush_tlb_no_ptcg (unsigned long start, unsigned long end, unsigned long nbits)
60 {
61 extern void smp_send_flush_tlb (void);
62 unsigned long saved_tpr = 0;
63 unsigned long flags;
64
65 /*
66 * Some times this is called with interrupts disabled and causes
67 * dead-lock; to avoid this we enable interrupt and raise the TPR
68 * to enable ONLY IPI.
69 */
70 __save_flags(flags);
71 if (!(flags & IA64_PSR_I)) {
72 saved_tpr = ia64_get_tpr();
73 ia64_srlz_d();
74 ia64_set_tpr(IA64_IPI_VECTOR - 16);
75 ia64_srlz_d();
76 local_irq_enable();
77 }
78
79 spin_lock(&ptcg_lock);
80 flush_rid = ia64_get_rr(start);
81 ia64_srlz_d();
82 flush_start = start;
83 flush_end = end;
84 flush_nbits = nbits;
85 atomic_set(&flush_cpu_count, smp_num_cpus - 1);
86 smp_send_flush_tlb();
87 /*
88 * Purge local TLB entries. ALAT invalidation is done in ia64_leave_kernel.
89 */
90 do {
91 asm volatile ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory");
92 start += (1UL << nbits);
93 } while (start < end);
94
95 ia64_srlz_i(); /* srlz.i implies srlz.d */
96
97 /*
98 * Wait for other CPUs to finish purging entries.
99 */
100 #if defined(CONFIG_ITANIUM_BSTEP_SPECIFIC)
101 {
102 extern void smp_resend_flush_tlb (void);
103 unsigned long start = ia64_get_itc();
104
105 while (atomic_read(&flush_cpu_count) > 0) {
106 if ((ia64_get_itc() - start) > 400000UL) {
107 smp_resend_flush_tlb();
108 start = ia64_get_itc();
109 }
110 }
111 }
112 #else
113 while (atomic_read(&flush_cpu_count)) {
114 /* Nothing */
115 }
116 #endif
117 if (!(flags & IA64_PSR_I)) {
118 local_irq_disable();
119 ia64_set_tpr(saved_tpr);
120 ia64_srlz_d();
121 }
122 }
123
124 #endif /* CONFIG_SMP && !CONFIG_ITANIUM_PTCG */
125
126 /*
127 * Acquire the ia64_ctx.lock before calling this function!
128 */
129 void
130 wrap_mmu_context (struct mm_struct *mm)
131 {
132 unsigned long tsk_context, max_ctx = ia64_ctx.max_ctx;
133 struct task_struct *tsk;
134
135 if (ia64_ctx.next > max_ctx)
136 ia64_ctx.next = 300; /* skip daemons */
137 ia64_ctx.limit = max_ctx + 1;
138
139 /*
140 * Scan all the task's mm->context and set proper safe range
141 */
142
143 read_lock(&tasklist_lock);
144 repeat:
145 for_each_task(tsk) {
146 if (!tsk->mm)
147 continue;
148 tsk_context = tsk->mm->context;
149 if (tsk_context == ia64_ctx.next) {
150 if (++ia64_ctx.next >= ia64_ctx.limit) {
151 /* empty range: reset the range limit and start over */
152 if (ia64_ctx.next > max_ctx)
153 ia64_ctx.next = 300;
154 ia64_ctx.limit = max_ctx + 1;
155 goto repeat;
156 }
157 }
158 if ((tsk_context > ia64_ctx.next) && (tsk_context < ia64_ctx.limit))
159 ia64_ctx.limit = tsk_context;
160 }
161 read_unlock(&tasklist_lock);
162 flush_tlb_all();
163 }
164
165 void
166 __flush_tlb_all (void)
167 {
168 unsigned long i, j, flags, count0, count1, stride0, stride1, addr;
169
170 addr = local_cpu_data->ptce_base;
171 count0 = local_cpu_data->ptce_count[0];
172 count1 = local_cpu_data->ptce_count[1];
173 stride0 = local_cpu_data->ptce_stride[0];
174 stride1 = local_cpu_data->ptce_stride[1];
175
176 local_irq_save(flags);
177 for (i = 0; i < count0; ++i) {
178 for (j = 0; j < count1; ++j) {
179 asm volatile ("ptc.e %0" :: "r"(addr));
180 addr += stride1;
181 }
182 addr += stride0;
183 }
184 local_irq_restore(flags);
185 ia64_insn_group_barrier();
186 ia64_srlz_i(); /* srlz.i implies srlz.d */
187 ia64_insn_group_barrier();
188 }
189
190 void
191 flush_tlb_range (struct mm_struct *mm, unsigned long start, unsigned long end)
192 {
193 unsigned long size = end - start;
194 unsigned long nbits;
195
196 if (mm != current->active_mm) {
197 /* this does happen, but perhaps it's not worth optimizing for? */
198 #ifdef CONFIG_SMP
199 flush_tlb_all();
200 #else
201 mm->context = 0;
202 #endif
203 return;
204 }
205
206 nbits = ia64_fls(size + 0xfff);
207 if (((1UL << nbits) & SUPPORTED_PGBITS) == 0) {
208 if (nbits > _PAGE_SIZE_256M)
209 nbits = _PAGE_SIZE_256M;
210 else
211 /*
212 * Some page sizes are not implemented in the
213 * IA-64 arch, so if we get asked to clear an
214 * unsupported page size, round up to the
215 * nearest page size. Note that we depend on
216 * the fact that if page size N is not
217 * implemented, 2*N _is_ implemented.
218 */
219 ++nbits;
220 if (((1UL << nbits) & SUPPORTED_PGBITS) == 0)
221 panic("flush_tlb_range: BUG: nbits=%lu\n", nbits);
222 }
223 start &= ~((1UL << nbits) - 1);
224
225 #if defined(CONFIG_SMP) && !defined(CONFIG_ITANIUM_PTCG)
226 flush_tlb_no_ptcg(start, end, nbits);
227 #else
228 spin_lock(&ptcg_lock);
229 do {
230 # ifdef CONFIG_SMP
231 /*
232 * Flush ALAT entries also.
233 */
234 asm volatile ("ptc.ga %0,%1;;srlz.i;;" :: "r"(start), "r"(nbits<<2) : "memory");
235 # else
236 asm volatile ("ptc.l %0,%1" :: "r"(start), "r"(nbits<<2) : "memory");
237 # endif
238 start += (1UL << nbits);
239 } while (start < end);
240 #endif /* CONFIG_SMP && !defined(CONFIG_ITANIUM_PTCG) */
241 spin_unlock(&ptcg_lock);
242 ia64_insn_group_barrier();
243 ia64_srlz_i(); /* srlz.i implies srlz.d */
244 ia64_insn_group_barrier();
245 }
246
247 void __init
248 ia64_tlb_init (void)
249 {
250 ia64_ptce_info_t ptce_info;
251
252 ia64_get_ptce(&ptce_info);
253 local_cpu_data->ptce_base = ptce_info.base;
254 local_cpu_data->ptce_count[0] = ptce_info.count[0];
255 local_cpu_data->ptce_count[1] = ptce_info.count[1];
256 local_cpu_data->ptce_stride[0] = ptce_info.stride[0];
257 local_cpu_data->ptce_stride[1] = ptce_info.stride[1];
258
259 __flush_tlb_all(); /* nuke left overs from bootstrapping... */
260 }
261