File: /usr/src/linux/arch/alpha/lib/csum_partial_copy.c
1 /*
2 * csum_partial_copy - do IP checksumming and copy
3 *
4 * (C) Copyright 1996 Linus Torvalds
5 * accellerated versions (and 21264 assembly versions ) contributed by
6 * Rick Gorton <rick.gorton@alpha-processor.com>
7 *
8 * Don't look at this too closely - you'll go mad. The things
9 * we do for performance..
10 */
11
12 #include <linux/types.h>
13 #include <linux/string.h>
14 #include <asm/uaccess.h>
15
16
17 #define ldq_u(x,y) \
18 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
19
20 #define stq_u(x,y) \
21 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
22
23 #define extql(x,y,z) \
24 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
25
26 #define extqh(x,y,z) \
27 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
28
29 #define mskql(x,y,z) \
30 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
31
32 #define mskqh(x,y,z) \
33 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
34
35 #define insql(x,y,z) \
36 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
37
38 #define insqh(x,y,z) \
39 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
40
41
42 #define __get_user_u(x,ptr) \
43 ({ \
44 long __guu_err; \
45 __asm__ __volatile__( \
46 "1: ldq_u %0,%2\n" \
47 "2:\n" \
48 ".section __ex_table,\"a\"\n" \
49 " .gprel32 1b\n" \
50 " lda %0,2b-1b(%1)\n" \
51 ".previous" \
52 : "=r"(x), "=r"(__guu_err) \
53 : "m"(__m(ptr)), "1"(0)); \
54 __guu_err; \
55 })
56
57 #define __put_user_u(x,ptr) \
58 ({ \
59 long __puu_err; \
60 __asm__ __volatile__( \
61 "1: stq_u %2,%1\n" \
62 "2:\n" \
63 ".section __ex_table,\"a\"\n" \
64 " .gprel32 1b" \
65 " lda $31,2b-1b(%0)\n" \
66 ".previous" \
67 : "=r"(__puu_err) \
68 : "m"(__m(addr)), "rJ"(x), "0"(0)); \
69 __puu_err; \
70 })
71
72
73 static inline unsigned short from64to16(unsigned long x)
74 {
75 /* Using extract instructions is a bit more efficient
76 than the original shift/bitmask version. */
77
78 union {
79 unsigned long ul;
80 unsigned int ui[2];
81 unsigned short us[4];
82 } in_v, tmp_v, out_v;
83
84 in_v.ul = x;
85 tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
86
87 /* Since the bits of tmp_v.sh[3] are going to always be zero,
88 we don't have to bother to add that in. */
89 out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
90 + (unsigned long) tmp_v.us[2];
91
92 /* Similarly, out_v.us[2] is always zero for the final add. */
93 return out_v.us[0] + out_v.us[1];
94 }
95
96
97
98 /*
99 * Ok. This isn't fun, but this is the EASY case.
100 */
101 static inline unsigned long
102 csum_partial_cfu_aligned(const unsigned long *src, unsigned long *dst,
103 long len, unsigned long checksum,
104 int *errp)
105 {
106 unsigned long carry = 0;
107 int err = 0;
108
109 while (len >= 0) {
110 unsigned long word;
111 err |= __get_user(word, src);
112 checksum += carry;
113 src++;
114 checksum += word;
115 len -= 8;
116 carry = checksum < word;
117 *dst = word;
118 dst++;
119 }
120 len += 8;
121 checksum += carry;
122 if (len) {
123 unsigned long word, tmp;
124 err |= __get_user(word, src);
125 tmp = *dst;
126 mskql(word, len, word);
127 checksum += word;
128 mskqh(tmp, len, tmp);
129 carry = checksum < word;
130 *dst = word | tmp;
131 checksum += carry;
132 }
133 if (err) *errp = err;
134 return checksum;
135 }
136
137 /*
138 * This is even less fun, but this is still reasonably
139 * easy.
140 */
141 static inline unsigned long
142 csum_partial_cfu_dest_aligned(const unsigned long *src, unsigned long *dst,
143 unsigned long soff,
144 long len, unsigned long checksum,
145 int *errp)
146 {
147 unsigned long first;
148 unsigned long word, carry;
149 unsigned long lastsrc = 7+len+(unsigned long)src;
150 int err = 0;
151
152 err |= __get_user_u(first,src);
153 carry = 0;
154 while (len >= 0) {
155 unsigned long second;
156
157 err |= __get_user_u(second, src+1);
158 extql(first, soff, word);
159 len -= 8;
160 src++;
161 extqh(second, soff, first);
162 checksum += carry;
163 word |= first;
164 first = second;
165 checksum += word;
166 *dst = word;
167 dst++;
168 carry = checksum < word;
169 }
170 len += 8;
171 checksum += carry;
172 if (len) {
173 unsigned long tmp;
174 unsigned long second;
175 err |= __get_user_u(second, lastsrc);
176 tmp = *dst;
177 extql(first, soff, word);
178 extqh(second, soff, first);
179 word |= first;
180 mskql(word, len, word);
181 checksum += word;
182 mskqh(tmp, len, tmp);
183 carry = checksum < word;
184 *dst = word | tmp;
185 checksum += carry;
186 }
187 if (err) *errp = err;
188 return checksum;
189 }
190
191 /*
192 * This is slightly less fun than the above..
193 */
194 static inline unsigned long
195 csum_partial_cfu_src_aligned(const unsigned long *src, unsigned long *dst,
196 unsigned long doff,
197 long len, unsigned long checksum,
198 unsigned long partial_dest,
199 int *errp)
200 {
201 unsigned long carry = 0;
202 unsigned long word;
203 unsigned long second_dest;
204 int err = 0;
205
206 mskql(partial_dest, doff, partial_dest);
207 while (len >= 0) {
208 err |= __get_user(word, src);
209 len -= 8;
210 insql(word, doff, second_dest);
211 checksum += carry;
212 stq_u(partial_dest | second_dest, dst);
213 src++;
214 checksum += word;
215 insqh(word, doff, partial_dest);
216 carry = checksum < word;
217 dst++;
218 }
219 len += 8;
220 if (len) {
221 checksum += carry;
222 err |= __get_user(word, src);
223 mskql(word, len, word);
224 len -= 8;
225 checksum += word;
226 insql(word, doff, second_dest);
227 len += doff;
228 carry = checksum < word;
229 partial_dest |= second_dest;
230 if (len >= 0) {
231 stq_u(partial_dest, dst);
232 if (!len) goto out;
233 dst++;
234 insqh(word, doff, partial_dest);
235 }
236 doff = len;
237 }
238 ldq_u(second_dest, dst);
239 mskqh(second_dest, doff, second_dest);
240 stq_u(partial_dest | second_dest, dst);
241 out:
242 checksum += carry;
243 if (err) *errp = err;
244 return checksum;
245 }
246
247 /*
248 * This is so totally un-fun that it's frightening. Don't
249 * look at this too closely, you'll go blind.
250 */
251 static inline unsigned long
252 csum_partial_cfu_unaligned(const unsigned long * src, unsigned long * dst,
253 unsigned long soff, unsigned long doff,
254 long len, unsigned long checksum,
255 unsigned long partial_dest,
256 int *errp)
257 {
258 unsigned long carry = 0;
259 unsigned long first;
260 unsigned long lastsrc;
261 int err = 0;
262
263 err |= __get_user_u(first, src);
264 lastsrc = 7+len+(unsigned long)src;
265 mskql(partial_dest, doff, partial_dest);
266 while (len >= 0) {
267 unsigned long second, word;
268 unsigned long second_dest;
269
270 err |= __get_user_u(second, src+1);
271 extql(first, soff, word);
272 checksum += carry;
273 len -= 8;
274 extqh(second, soff, first);
275 src++;
276 word |= first;
277 first = second;
278 insql(word, doff, second_dest);
279 checksum += word;
280 stq_u(partial_dest | second_dest, dst);
281 carry = checksum < word;
282 insqh(word, doff, partial_dest);
283 dst++;
284 }
285 len += doff;
286 checksum += carry;
287 if (len >= 0) {
288 unsigned long second, word;
289 unsigned long second_dest;
290
291 err |= __get_user_u(second, lastsrc);
292 extql(first, soff, word);
293 extqh(second, soff, first);
294 word |= first;
295 first = second;
296 mskql(word, len-doff, word);
297 checksum += word;
298 insql(word, doff, second_dest);
299 carry = checksum < word;
300 stq_u(partial_dest | second_dest, dst);
301 if (len) {
302 ldq_u(second_dest, dst+1);
303 insqh(word, doff, partial_dest);
304 mskqh(second_dest, len, second_dest);
305 stq_u(partial_dest | second_dest, dst+1);
306 }
307 checksum += carry;
308 } else {
309 unsigned long second, word;
310 unsigned long second_dest;
311
312 err |= __get_user_u(second, lastsrc);
313 extql(first, soff, word);
314 extqh(second, soff, first);
315 word |= first;
316 ldq_u(second_dest, dst);
317 mskql(word, len-doff, word);
318 checksum += word;
319 mskqh(second_dest, len, second_dest);
320 carry = checksum < word;
321 insql(word, doff, word);
322 stq_u(partial_dest | word | second_dest, dst);
323 checksum += carry;
324 }
325 if (err) *errp = err;
326 return checksum;
327 }
328
329 static unsigned int
330 do_csum_partial_copy_from_user(const char *src, char *dst, int len,
331 unsigned int sum, int *errp)
332 {
333 unsigned long checksum = (unsigned) sum;
334 unsigned long soff = 7 & (unsigned long) src;
335 unsigned long doff = 7 & (unsigned long) dst;
336
337 if (len) {
338 if (!doff) {
339 if (!soff)
340 checksum = csum_partial_cfu_aligned(
341 (const unsigned long *) src,
342 (unsigned long *) dst,
343 len-8, checksum, errp);
344 else
345 checksum = csum_partial_cfu_dest_aligned(
346 (const unsigned long *) src,
347 (unsigned long *) dst,
348 soff, len-8, checksum, errp);
349 } else {
350 unsigned long partial_dest;
351 ldq_u(partial_dest, dst);
352 if (!soff)
353 checksum = csum_partial_cfu_src_aligned(
354 (const unsigned long *) src,
355 (unsigned long *) dst,
356 doff, len-8, checksum,
357 partial_dest, errp);
358 else
359 checksum = csum_partial_cfu_unaligned(
360 (const unsigned long *) src,
361 (unsigned long *) dst,
362 soff, doff, len-8, checksum,
363 partial_dest, errp);
364 }
365 checksum = from64to16 (checksum);
366 }
367 return checksum;
368 }
369
370 unsigned int
371 csum_partial_copy_from_user(const char *src, char *dst, int len,
372 unsigned int sum, int *errp)
373 {
374 if (!access_ok(src, len, VERIFY_READ)) {
375 *errp = -EFAULT;
376 memset(dst, 0, len);
377 return sum;
378 }
379
380 return do_csum_partial_copy_from_user(src, dst, len, sum, errp);
381 }
382
383 unsigned int
384 csum_partial_copy_nocheck(const char *src, char *dst, int len, unsigned int sum)
385 {
386 return do_csum_partial_copy_from_user(src, dst, len, sum, NULL);
387 }
388
389 unsigned int
390 csum_partial_copy (const char *src, char *dst, int len, unsigned int sum)
391 {
392 unsigned int ret;
393 int error = 0;
394
395 ret = do_csum_partial_copy_from_user(src, dst, len, sum, &error);
396 if (error)
397 printk("csum_partial_copy_old(): tell mingo to convert me!\n");
398
399 return ret;
400 }
401