File: /usr/src/linux/arch/i386/kernel/i387.c
1 /*
2 * linux/arch/i386/kernel/i387.c
3 *
4 * Copyright (C) 1994 Linus Torvalds
5 *
6 * Pentium III FXSR, SSE support
7 * General FPU state handling cleanups
8 * Gareth Hughes <gareth@valinux.com>, May 2000
9 */
10
11 #include <linux/config.h>
12 #include <linux/sched.h>
13 #include <asm/processor.h>
14 #include <asm/i387.h>
15 #include <asm/math_emu.h>
16 #include <asm/sigcontext.h>
17 #include <asm/user.h>
18 #include <asm/ptrace.h>
19 #include <asm/uaccess.h>
20
21 #ifdef CONFIG_MATH_EMULATION
22 #define HAVE_HWFP (boot_cpu_data.hard_math)
23 #else
24 #define HAVE_HWFP 1
25 #endif
26
27 /*
28 * The _current_ task is using the FPU for the first time
29 * so initialize it and set the mxcsr to its default
30 * value at reset if we support XMM instructions and then
31 * remeber the current task has used the FPU.
32 */
33 void init_fpu(void)
34 {
35 __asm__("fninit");
36 if ( cpu_has_xmm )
37 load_mxcsr(0x1f80);
38
39 current->used_math = 1;
40 }
41
42 /*
43 * FPU lazy state save handling.
44 */
45
46 static inline void __save_init_fpu( struct task_struct *tsk )
47 {
48 if ( cpu_has_fxsr ) {
49 asm volatile( "fxsave %0 ; fnclex"
50 : "=m" (tsk->thread.i387.fxsave) );
51 } else {
52 asm volatile( "fnsave %0 ; fwait"
53 : "=m" (tsk->thread.i387.fsave) );
54 }
55 tsk->flags &= ~PF_USEDFPU;
56 }
57
58 void save_init_fpu( struct task_struct *tsk )
59 {
60 __save_init_fpu(tsk);
61 stts();
62 }
63
64 void kernel_fpu_begin(void)
65 {
66 struct task_struct *tsk = current;
67
68 if (tsk->flags & PF_USEDFPU) {
69 __save_init_fpu(tsk);
70 return;
71 }
72 clts();
73 }
74
75 void restore_fpu( struct task_struct *tsk )
76 {
77 if ( cpu_has_fxsr ) {
78 asm volatile( "fxrstor %0"
79 : : "m" (tsk->thread.i387.fxsave) );
80 } else {
81 asm volatile( "frstor %0"
82 : : "m" (tsk->thread.i387.fsave) );
83 }
84 }
85
86 /*
87 * FPU tag word conversions.
88 */
89
90 static inline unsigned short twd_i387_to_fxsr( unsigned short twd )
91 {
92 unsigned int tmp; /* to avoid 16 bit prefixes in the code */
93
94 /* Transform each pair of bits into 01 (valid) or 00 (empty) */
95 tmp = ~twd;
96 tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
97 /* and move the valid bits to the lower byte. */
98 tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
99 tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
100 tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
101 return tmp;
102 }
103
104 static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave )
105 {
106 struct _fpxreg *st = NULL;
107 unsigned long twd = (unsigned long) fxsave->twd;
108 unsigned long tag;
109 unsigned long ret = 0xffff0000;
110 int i;
111
112 #define FPREG_ADDR(f, n) ((char *)&(f)->st_space + (n) * 16);
113
114 for ( i = 0 ; i < 8 ; i++ ) {
115 if ( twd & 0x1 ) {
116 st = (struct _fpxreg *) FPREG_ADDR( fxsave, i );
117
118 switch ( st->exponent & 0x7fff ) {
119 case 0x7fff:
120 tag = 2; /* Special */
121 break;
122 case 0x0000:
123 if ( !st->significand[0] &&
124 !st->significand[1] &&
125 !st->significand[2] &&
126 !st->significand[3] ) {
127 tag = 1; /* Zero */
128 } else {
129 tag = 2; /* Special */
130 }
131 break;
132 default:
133 if ( st->significand[3] & 0x8000 ) {
134 tag = 0; /* Valid */
135 } else {
136 tag = 2; /* Special */
137 }
138 break;
139 }
140 } else {
141 tag = 3; /* Empty */
142 }
143 ret |= (tag << (2 * i));
144 twd = twd >> 1;
145 }
146 return ret;
147 }
148
149 /*
150 * FPU state interaction.
151 */
152
153 unsigned short get_fpu_cwd( struct task_struct *tsk )
154 {
155 if ( cpu_has_fxsr ) {
156 return tsk->thread.i387.fxsave.cwd;
157 } else {
158 return (unsigned short)tsk->thread.i387.fsave.cwd;
159 }
160 }
161
162 unsigned short get_fpu_swd( struct task_struct *tsk )
163 {
164 if ( cpu_has_fxsr ) {
165 return tsk->thread.i387.fxsave.swd;
166 } else {
167 return (unsigned short)tsk->thread.i387.fsave.swd;
168 }
169 }
170
171 unsigned short get_fpu_twd( struct task_struct *tsk )
172 {
173 if ( cpu_has_fxsr ) {
174 return tsk->thread.i387.fxsave.twd;
175 } else {
176 return (unsigned short)tsk->thread.i387.fsave.twd;
177 }
178 }
179
180 unsigned short get_fpu_mxcsr( struct task_struct *tsk )
181 {
182 if ( cpu_has_xmm ) {
183 return tsk->thread.i387.fxsave.mxcsr;
184 } else {
185 return 0x1f80;
186 }
187 }
188
189 void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd )
190 {
191 if ( cpu_has_fxsr ) {
192 tsk->thread.i387.fxsave.cwd = cwd;
193 } else {
194 tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000);
195 }
196 }
197
198 void set_fpu_swd( struct task_struct *tsk, unsigned short swd )
199 {
200 if ( cpu_has_fxsr ) {
201 tsk->thread.i387.fxsave.swd = swd;
202 } else {
203 tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000);
204 }
205 }
206
207 void set_fpu_twd( struct task_struct *tsk, unsigned short twd )
208 {
209 if ( cpu_has_fxsr ) {
210 tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd);
211 } else {
212 tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000);
213 }
214 }
215
216 void set_fpu_mxcsr( struct task_struct *tsk, unsigned short mxcsr )
217 {
218 if ( cpu_has_xmm ) {
219 tsk->thread.i387.fxsave.mxcsr = (mxcsr & 0xffbf);
220 }
221 }
222
223 /*
224 * FXSR floating point environment conversions.
225 */
226
227 static inline int convert_fxsr_to_user( struct _fpstate *buf,
228 struct i387_fxsave_struct *fxsave )
229 {
230 unsigned long env[7];
231 struct _fpreg *to;
232 struct _fpxreg *from;
233 int i;
234
235 env[0] = (unsigned long)fxsave->cwd | 0xffff0000;
236 env[1] = (unsigned long)fxsave->swd | 0xffff0000;
237 env[2] = twd_fxsr_to_i387(fxsave);
238 env[3] = fxsave->fip;
239 env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16);
240 env[5] = fxsave->foo;
241 env[6] = fxsave->fos;
242
243 if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) )
244 return 1;
245
246 to = &buf->_st[0];
247 from = (struct _fpxreg *) &fxsave->st_space[0];
248 for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
249 if ( __copy_to_user( to, from, sizeof(*to) ) )
250 return 1;
251 }
252 return 0;
253 }
254
255 static inline int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave,
256 struct _fpstate *buf )
257 {
258 unsigned long env[7];
259 struct _fpxreg *to;
260 struct _fpreg *from;
261 int i;
262
263 if ( __copy_from_user( env, buf, 7 * sizeof(long) ) )
264 return 1;
265
266 fxsave->cwd = (unsigned short)(env[0] & 0xffff);
267 fxsave->swd = (unsigned short)(env[1] & 0xffff);
268 fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff));
269 fxsave->fip = env[3];
270 fxsave->fop = (unsigned short)((env[4] & 0xffff0000) >> 16);
271 fxsave->fcs = (env[4] & 0xffff);
272 fxsave->foo = env[5];
273 fxsave->fos = env[6];
274
275 to = (struct _fpxreg *) &fxsave->st_space[0];
276 from = &buf->_st[0];
277 for ( i = 0 ; i < 8 ; i++, to++, from++ ) {
278 if ( __copy_from_user( to, from, sizeof(*from) ) )
279 return 1;
280 }
281 return 0;
282 }
283
284 /*
285 * Signal frame handlers.
286 */
287
288 static inline int save_i387_fsave( struct _fpstate *buf )
289 {
290 struct task_struct *tsk = current;
291
292 unlazy_fpu( tsk );
293 tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd;
294 if ( __copy_to_user( buf, &tsk->thread.i387.fsave,
295 sizeof(struct i387_fsave_struct) ) )
296 return -1;
297 return 1;
298 }
299
300 static inline int save_i387_fxsave( struct _fpstate *buf )
301 {
302 struct task_struct *tsk = current;
303 int err = 0;
304
305 unlazy_fpu( tsk );
306
307 if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) )
308 return -1;
309
310 err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status );
311 err |= __put_user( X86_FXSR_MAGIC, &buf->magic );
312 if ( err )
313 return -1;
314
315 if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave,
316 sizeof(struct i387_fxsave_struct) ) )
317 return -1;
318 return 1;
319 }
320
321 int save_i387( struct _fpstate *buf )
322 {
323 if ( !current->used_math )
324 return 0;
325
326 /* This will cause a "finit" to be triggered by the next
327 * attempted FPU operation by the 'current' process.
328 */
329 current->used_math = 0;
330
331 if ( HAVE_HWFP ) {
332 if ( cpu_has_fxsr ) {
333 return save_i387_fxsave( buf );
334 } else {
335 return save_i387_fsave( buf );
336 }
337 } else {
338 return save_i387_soft( ¤t->thread.i387.soft, buf );
339 }
340 }
341
342 static inline int restore_i387_fsave( struct _fpstate *buf )
343 {
344 struct task_struct *tsk = current;
345 clear_fpu( tsk );
346 return __copy_from_user( &tsk->thread.i387.fsave, buf,
347 sizeof(struct i387_fsave_struct) );
348 }
349
350 static inline int restore_i387_fxsave( struct _fpstate *buf )
351 {
352 struct task_struct *tsk = current;
353 clear_fpu( tsk );
354 if ( __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0],
355 sizeof(struct i387_fxsave_struct) ) )
356 return 1;
357 /* mxcsr bit 6 and 31-16 must be zero for security reasons */
358 tsk->thread.i387.fxsave.mxcsr &= 0xffbf;
359 return convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf );
360 }
361
362 int restore_i387( struct _fpstate *buf )
363 {
364 int err;
365
366 if ( HAVE_HWFP ) {
367 if ( cpu_has_fxsr ) {
368 err = restore_i387_fxsave( buf );
369 } else {
370 err = restore_i387_fsave( buf );
371 }
372 } else {
373 err = restore_i387_soft( ¤t->thread.i387.soft, buf );
374 }
375 current->used_math = 1;
376 return err;
377 }
378
379 /*
380 * ptrace request handlers.
381 */
382
383 static inline int get_fpregs_fsave( struct user_i387_struct *buf,
384 struct task_struct *tsk )
385 {
386 return __copy_to_user( buf, &tsk->thread.i387.fsave,
387 sizeof(struct user_i387_struct) );
388 }
389
390 static inline int get_fpregs_fxsave( struct user_i387_struct *buf,
391 struct task_struct *tsk )
392 {
393 return convert_fxsr_to_user( (struct _fpstate *)buf,
394 &tsk->thread.i387.fxsave );
395 }
396
397 int get_fpregs( struct user_i387_struct *buf, struct task_struct *tsk )
398 {
399 if ( HAVE_HWFP ) {
400 if ( cpu_has_fxsr ) {
401 return get_fpregs_fxsave( buf, tsk );
402 } else {
403 return get_fpregs_fsave( buf, tsk );
404 }
405 } else {
406 return save_i387_soft( &tsk->thread.i387.soft,
407 (struct _fpstate *)buf );
408 }
409 }
410
411 static inline int set_fpregs_fsave( struct task_struct *tsk,
412 struct user_i387_struct *buf )
413 {
414 return __copy_from_user( &tsk->thread.i387.fsave, buf,
415 sizeof(struct user_i387_struct) );
416 }
417
418 static inline int set_fpregs_fxsave( struct task_struct *tsk,
419 struct user_i387_struct *buf )
420 {
421 return convert_fxsr_from_user( &tsk->thread.i387.fxsave,
422 (struct _fpstate *)buf );
423 }
424
425 int set_fpregs( struct task_struct *tsk, struct user_i387_struct *buf )
426 {
427 if ( HAVE_HWFP ) {
428 if ( cpu_has_fxsr ) {
429 return set_fpregs_fxsave( tsk, buf );
430 } else {
431 return set_fpregs_fsave( tsk, buf );
432 }
433 } else {
434 return restore_i387_soft( &tsk->thread.i387.soft,
435 (struct _fpstate *)buf );
436 }
437 }
438
439 int get_fpxregs( struct user_fxsr_struct *buf, struct task_struct *tsk )
440 {
441 if ( cpu_has_fxsr ) {
442 if (__copy_to_user( (void *)buf, &tsk->thread.i387.fxsave,
443 sizeof(struct user_fxsr_struct) ))
444 return -EFAULT;
445 return 0;
446 } else {
447 return -EIO;
448 }
449 }
450
451 int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct *buf )
452 {
453 if ( cpu_has_fxsr ) {
454 __copy_from_user( &tsk->thread.i387.fxsave, (void *)buf,
455 sizeof(struct user_fxsr_struct) );
456 /* mxcsr bit 6 and 31-16 must be zero for security reasons */
457 tsk->thread.i387.fxsave.mxcsr &= 0xffbf;
458 return 0;
459 } else {
460 return -EIO;
461 }
462 }
463
464 /*
465 * FPU state for core dumps.
466 */
467
468 static inline void copy_fpu_fsave( struct task_struct *tsk,
469 struct user_i387_struct *fpu )
470 {
471 memcpy( fpu, &tsk->thread.i387.fsave,
472 sizeof(struct user_i387_struct) );
473 }
474
475 static inline void copy_fpu_fxsave( struct task_struct *tsk,
476 struct user_i387_struct *fpu )
477 {
478 unsigned short *to;
479 unsigned short *from;
480 int i;
481
482 memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) );
483
484 to = (unsigned short *)&fpu->st_space[0];
485 from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0];
486 for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) {
487 memcpy( to, from, 5 * sizeof(unsigned short) );
488 }
489 }
490
491 int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu )
492 {
493 int fpvalid;
494 struct task_struct *tsk = current;
495
496 fpvalid = tsk->used_math;
497 if ( fpvalid ) {
498 unlazy_fpu( tsk );
499 if ( cpu_has_fxsr ) {
500 copy_fpu_fxsave( tsk, fpu );
501 } else {
502 copy_fpu_fsave( tsk, fpu );
503 }
504 }
505
506 return fpvalid;
507 }
508
509 int dump_extended_fpu( struct pt_regs *regs, struct user_fxsr_struct *fpu )
510 {
511 int fpvalid;
512 struct task_struct *tsk = current;
513
514 fpvalid = tsk->used_math && cpu_has_fxsr;
515 if ( fpvalid ) {
516 unlazy_fpu( tsk );
517 memcpy( fpu, &tsk->thread.i387.fxsave,
518 sizeof(struct user_fxsr_struct) );
519 }
520
521 return fpvalid;
522 }
523