Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
1da177e4 LT |
2 | /* |
3 | * User address space access functions. | |
4 | * The non-inlined parts of asm-cris/uaccess.h are here. | |
5 | * | |
6 | * Copyright (C) 2000, Axis Communications AB. | |
7 | * | |
8 | * Written by Hans-Peter Nilsson. | |
9 | * Pieces used from memcpy, originally by Kenny Ranerup long time ago. | |
10 | */ | |
11 | ||
7c0f6ba6 | 12 | #include <linux/uaccess.h> |
1da177e4 LT |
13 | |
14 | /* Asm:s have been tweaked (within the domain of correctness) to give | |
15 | satisfactory results for "gcc version 2.96 20000427 (experimental)". | |
16 | ||
17 | Check regularly... | |
18 | ||
19 | Note that the PC saved at a bus-fault is the address *after* the | |
20 | faulting instruction, which means the branch-target for instructions in | |
21 | delay-slots for taken branches. Note also that the postincrement in | |
22 | the instruction is performed regardless of bus-fault; the register is | |
23 | seen updated in fault handlers. | |
24 | ||
25 | Oh, and on the code formatting issue, to whomever feels like "fixing | |
26 | it" to Conformity: I'm too "lazy", but why don't you go ahead and "fix" | |
27 | string.c too. I just don't think too many people will hack this file | |
28 | for the code format to be an issue. */ | |
29 | ||
30 | ||
31 | /* Copy to userspace. This is based on the memcpy used for | |
32 | kernel-to-kernel copying; see "string.c". */ | |
33 | ||
dbd3c7e1 | 34 | unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn) |
1da177e4 LT |
35 | { |
36 | /* We want the parameters put in special registers. | |
37 | Make sure the compiler is able to make something useful of this. | |
38 | As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). | |
39 | ||
40 | FIXME: Comment for old gcc version. Check. | |
49b4ff33 | 41 | If gcc was alright, it really would need no temporaries, and no |
1da177e4 LT |
42 | stack space to save stuff on. */ |
43 | ||
44 | register char *dst __asm__ ("r13") = pdst; | |
45 | register const char *src __asm__ ("r11") = psrc; | |
46 | register int n __asm__ ("r12") = pn; | |
47 | register int retn __asm__ ("r10") = 0; | |
48 | ||
49 | ||
50 | /* When src is aligned but not dst, this makes a few extra needless | |
51 | cycles. I believe it would take as many to check that the | |
52 | re-alignment was unnecessary. */ | |
53 | if (((unsigned long) dst & 3) != 0 | |
54 | /* Don't align if we wouldn't copy more than a few bytes; so we | |
55 | don't have to check further for overflows. */ | |
56 | && n >= 3) | |
57 | { | |
58 | if ((unsigned long) dst & 1) | |
59 | { | |
60 | __asm_copy_to_user_1 (dst, src, retn); | |
61 | n--; | |
62 | } | |
63 | ||
64 | if ((unsigned long) dst & 2) | |
65 | { | |
66 | __asm_copy_to_user_2 (dst, src, retn); | |
67 | n -= 2; | |
68 | } | |
69 | } | |
70 | ||
71 | /* Decide which copying method to use. */ | |
72 | if (n >= 44*2) /* Break even between movem and | |
73 | move16 is at 38.7*2, but modulo 44. */ | |
74 | { | |
75 | /* For large copies we use 'movem'. */ | |
76 | ||
77 | /* It is not optimal to tell the compiler about clobbering any | |
78 | registers; that will move the saving/restoring of those registers | |
79 | to the function prologue/epilogue, and make non-movem sizes | |
80 | suboptimal. | |
81 | ||
82 | This method is not foolproof; it assumes that the "asm reg" | |
83 | declarations at the beginning of the function really are used | |
84 | here (beware: they may be moved to temporary registers). | |
85 | This way, we do not have to save/move the registers around into | |
86 | temporaries; we can safely use them straight away. | |
87 | ||
88 | If you want to check that the allocation was right; then | |
89 | check the equalities in the first comment. It should say | |
90 | "r13=r13, r11=r11, r12=r12". */ | |
91 | __asm__ volatile ("\ | |
92 | .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\ | |
93 | .err \n\ | |
94 | .endif \n\ | |
2b05d2b3 JN |
95 | \n\ |
96 | ;; Save the registers we'll use in the movem process \n\ | |
97 | ;; on the stack. \n\ | |
98 | subq 11*4,$sp \n\ | |
99 | movem $r10,[$sp] \n\ | |
100 | \n\ | |
101 | ;; Now we've got this: \n\ | |
102 | ;; r11 - src \n\ | |
103 | ;; r13 - dst \n\ | |
104 | ;; r12 - n \n\ | |
105 | \n\ | |
106 | ;; Update n for the first loop \n\ | |
107 | subq 44,$r12 \n\ | |
108 | \n\ | |
109 | ; Since the noted PC of a faulting instruction in a delay-slot of a taken \n\ | |
110 | ; branch, is that of the branch target, we actually point at the from-movem \n\ | |
111 | ; for this case. There is no ambiguity here; if there was a fault in that \n\ | |
112 | ; instruction (meaning a kernel oops), the faulted PC would be the address \n\ | |
113 | ; after *that* movem. \n\ | |
114 | \n\ | |
115 | 0: \n\ | |
116 | movem [$r11+],$r10 \n\ | |
117 | subq 44,$r12 \n\ | |
118 | bge 0b \n\ | |
119 | movem $r10,[$r13+] \n\ | |
120 | 1: \n\ | |
121 | addq 44,$r12 ;; compensate for last loop underflowing n \n\ | |
122 | \n\ | |
123 | ;; Restore registers from stack \n\ | |
124 | movem [$sp+],$r10 \n\ | |
125 | 2: \n\ | |
126 | .section .fixup,\"ax\" \n\ | |
127 | \n\ | |
128 | ; To provide a correct count in r10 of bytes that failed to be copied, \n\ | |
129 | ; we jump back into the loop if the loop-branch was taken. There is no \n\ | |
130 | ; performance penalty for sany use; the program will segfault soon enough.\n\ | |
131 | \n\ | |
132 | 3: \n\ | |
133 | move.d [$sp],$r10 \n\ | |
134 | addq 44,$r10 \n\ | |
135 | move.d $r10,[$sp] \n\ | |
136 | jump 0b \n\ | |
137 | 4: \n\ | |
138 | movem [$sp+],$r10 \n\ | |
139 | addq 44,$r10 \n\ | |
140 | addq 44,$r12 \n\ | |
141 | jump 2b \n\ | |
142 | \n\ | |
143 | .previous \n\ | |
144 | .section __ex_table,\"a\" \n\ | |
145 | .dword 0b,3b \n\ | |
146 | .dword 1b,4b \n\ | |
1da177e4 LT |
147 | .previous" |
148 | ||
149 | /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn) | |
150 | /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn)); | |
151 | ||
152 | } | |
153 | ||
154 | /* Either we directly start copying, using dword copying in a loop, or | |
155 | we copy as much as possible with 'movem' and then the last block (<44 | |
156 | bytes) is copied here. This will work since 'movem' will have | |
157 | updated SRC, DST and N. */ | |
158 | ||
159 | while (n >= 16) | |
160 | { | |
161 | __asm_copy_to_user_16 (dst, src, retn); | |
162 | n -= 16; | |
163 | } | |
164 | ||
165 | /* Having a separate by-four loops cuts down on cache footprint. | |
166 | FIXME: Test with and without; increasing switch to be 0..15. */ | |
167 | while (n >= 4) | |
168 | { | |
169 | __asm_copy_to_user_4 (dst, src, retn); | |
170 | n -= 4; | |
171 | } | |
172 | ||
173 | switch (n) | |
174 | { | |
175 | case 0: | |
176 | break; | |
177 | case 1: | |
178 | __asm_copy_to_user_1 (dst, src, retn); | |
179 | break; | |
180 | case 2: | |
181 | __asm_copy_to_user_2 (dst, src, retn); | |
182 | break; | |
183 | case 3: | |
184 | __asm_copy_to_user_3 (dst, src, retn); | |
185 | break; | |
186 | } | |
187 | ||
188 | return retn; | |
189 | } | |
dbd3c7e1 | 190 | EXPORT_SYMBOL(__copy_user); |
1da177e4 | 191 | |
b71f1bf5 | 192 | /* Copy from user to kernel. The return-value is the number of bytes that were |
1da177e4 LT |
193 | inaccessible. */ |
194 | ||
b71f1bf5 | 195 | unsigned long __copy_user_in(void *pdst, const void __user *psrc, |
dbd3c7e1 | 196 | unsigned long pn) |
1da177e4 LT |
197 | { |
198 | /* We want the parameters put in special registers. | |
199 | Make sure the compiler is able to make something useful of this. | |
200 | As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). | |
201 | ||
202 | FIXME: Comment for old gcc version. Check. | |
49b4ff33 | 203 | If gcc was alright, it really would need no temporaries, and no |
1da177e4 LT |
204 | stack space to save stuff on. */ |
205 | ||
206 | register char *dst __asm__ ("r13") = pdst; | |
207 | register const char *src __asm__ ("r11") = psrc; | |
208 | register int n __asm__ ("r12") = pn; | |
209 | register int retn __asm__ ("r10") = 0; | |
210 | ||
211 | /* The best reason to align src is that we then know that a read-fault | |
212 | was for aligned bytes; there's no 1..3 remaining good bytes to | |
213 | pickle. */ | |
214 | if (((unsigned long) src & 3) != 0) | |
215 | { | |
216 | if (((unsigned long) src & 1) && n != 0) | |
217 | { | |
218 | __asm_copy_from_user_1 (dst, src, retn); | |
219 | n--; | |
de09be34 AV |
220 | if (retn) |
221 | goto exception; | |
1da177e4 LT |
222 | } |
223 | ||
224 | if (((unsigned long) src & 2) && n >= 2) | |
225 | { | |
226 | __asm_copy_from_user_2 (dst, src, retn); | |
227 | n -= 2; | |
de09be34 AV |
228 | if (retn) |
229 | goto exception; | |
1da177e4 | 230 | } |
1da177e4 LT |
231 | } |
232 | ||
233 | /* Decide which copying method to use. */ | |
234 | if (n >= 44*2) /* Break even between movem and | |
235 | move16 is at 38.7*2, but modulo 44. | |
236 | FIXME: We use move4 now. */ | |
237 | { | |
238 | /* For large copies we use 'movem' */ | |
239 | ||
240 | /* It is not optimal to tell the compiler about clobbering any | |
241 | registers; that will move the saving/restoring of those registers | |
242 | to the function prologue/epilogue, and make non-movem sizes | |
243 | suboptimal. | |
244 | ||
245 | This method is not foolproof; it assumes that the "asm reg" | |
246 | declarations at the beginning of the function really are used | |
247 | here (beware: they may be moved to temporary registers). | |
248 | This way, we do not have to save/move the registers around into | |
249 | temporaries; we can safely use them straight away. | |
250 | ||
251 | If you want to check that the allocation was right; then | |
252 | check the equalities in the first comment. It should say | |
253 | "r13=r13, r11=r11, r12=r12" */ | |
2b05d2b3 | 254 | __asm__ volatile ("\n\ |
1da177e4 LT |
255 | .ifnc %0%1%2%3,$r13$r11$r12$r10 \n\ |
256 | .err \n\ | |
257 | .endif \n\ | |
2b05d2b3 JN |
258 | \n\ |
259 | ;; Save the registers we'll use in the movem process \n\ | |
260 | ;; on the stack. \n\ | |
261 | subq 11*4,$sp \n\ | |
262 | movem $r10,[$sp] \n\ | |
263 | \n\ | |
264 | ;; Now we've got this: \n\ | |
265 | ;; r11 - src \n\ | |
266 | ;; r13 - dst \n\ | |
267 | ;; r12 - n \n\ | |
268 | \n\ | |
269 | ;; Update n for the first loop \n\ | |
270 | subq 44,$r12 \n\ | |
271 | 0: \n\ | |
272 | movem [$r11+],$r10 \n\ | |
273 | 1: \n\ | |
274 | subq 44,$r12 \n\ | |
275 | bge 0b \n\ | |
276 | movem $r10,[$r13+] \n\ | |
277 | \n\ | |
278 | addq 44,$r12 ;; compensate for last loop underflowing n \n\ | |
279 | \n\ | |
280 | ;; Restore registers from stack \n\ | |
281 | movem [$sp+],$r10 \n\ | |
282 | 4: \n\ | |
283 | .section .fixup,\"ax\" \n\ | |
284 | \n\ | |
285 | ;; Do not jump back into the loop if we fail. For some uses, we get a \n\ | |
286 | ;; page fault somewhere on the line. Without checking for page limits, \n\ | |
287 | ;; we don't know where, but we need to copy accurately and keep an \n\ | |
288 | ;; accurate count; not just clear the whole line. To do that, we fall \n\ | |
289 | ;; down in the code below, proceeding with smaller amounts. It should \n\ | |
290 | ;; be kept in mind that we have to cater to code like what at one time \n\ | |
291 | ;; was in fs/super.c: \n\ | |
292 | ;; i = size - copy_from_user((void *)page, data, size); \n\ | |
293 | ;; which would cause repeated faults while clearing the remainder of \n\ | |
294 | ;; the SIZE bytes at PAGE after the first fault. \n\ | |
295 | ;; A caveat here is that we must not fall through from a failing page \n\ | |
296 | ;; to a valid page. \n\ | |
297 | \n\ | |
298 | 3: \n\ | |
299 | movem [$sp+],$r10 \n\ | |
300 | addq 44,$r12 ;; Get back count before faulting point. \n\ | |
301 | subq 44,$r11 ;; Get back pointer to faulting movem-line. \n\ | |
302 | jump 4b ;; Fall through, pretending the fault didn't happen.\n\ | |
303 | \n\ | |
304 | .previous \n\ | |
305 | .section __ex_table,\"a\" \n\ | |
306 | .dword 1b,3b \n\ | |
1da177e4 LT |
307 | .previous" |
308 | ||
309 | /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn) | |
310 | /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn)); | |
311 | ||
312 | } | |
313 | ||
314 | /* Either we directly start copying here, using dword copying in a loop, | |
315 | or we copy as much as possible with 'movem' and then the last block | |
316 | (<44 bytes) is copied here. This will work since 'movem' will have | |
317 | updated src, dst and n. (Except with failing src.) | |
318 | ||
319 | Since we want to keep src accurate, we can't use | |
320 | __asm_copy_from_user_N with N != (1, 2, 4); it updates dst and | |
321 | retn, but not src (by design; it's value is ignored elsewhere). */ | |
322 | ||
323 | while (n >= 4) | |
324 | { | |
325 | __asm_copy_from_user_4 (dst, src, retn); | |
326 | n -= 4; | |
327 | ||
328 | if (retn) | |
de09be34 | 329 | goto exception; |
1da177e4 LT |
330 | } |
331 | ||
332 | /* If we get here, there were no memory read faults. */ | |
333 | switch (n) | |
334 | { | |
335 | /* These copies are at least "naturally aligned" (so we don't have | |
336 | to check each byte), due to the src alignment code before the | |
337 | movem loop. The *_3 case *will* get the correct count for retn. */ | |
338 | case 0: | |
339 | /* This case deliberately left in (if you have doubts check the | |
340 | generated assembly code). */ | |
341 | break; | |
342 | case 1: | |
343 | __asm_copy_from_user_1 (dst, src, retn); | |
344 | break; | |
345 | case 2: | |
346 | __asm_copy_from_user_2 (dst, src, retn); | |
347 | break; | |
348 | case 3: | |
349 | __asm_copy_from_user_3 (dst, src, retn); | |
350 | break; | |
351 | } | |
352 | ||
353 | /* If we get here, retn correctly reflects the number of failing | |
354 | bytes. */ | |
355 | return retn; | |
356 | ||
de09be34 | 357 | exception: |
1da177e4 LT |
358 | return retn + n; |
359 | } | |
b71f1bf5 | 360 | EXPORT_SYMBOL(__copy_user_in); |
1da177e4 LT |
361 | |
362 | /* Zero userspace. */ | |
dbd3c7e1 | 363 | unsigned long __do_clear_user(void __user *pto, unsigned long pn) |
1da177e4 LT |
364 | { |
365 | /* We want the parameters put in special registers. | |
366 | Make sure the compiler is able to make something useful of this. | |
367 | As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop). | |
368 | ||
369 | FIXME: Comment for old gcc version. Check. | |
49b4ff33 | 370 | If gcc was alright, it really would need no temporaries, and no |
1da177e4 LT |
371 | stack space to save stuff on. */ |
372 | ||
373 | register char *dst __asm__ ("r13") = pto; | |
374 | register int n __asm__ ("r12") = pn; | |
375 | register int retn __asm__ ("r10") = 0; | |
376 | ||
377 | ||
378 | if (((unsigned long) dst & 3) != 0 | |
379 | /* Don't align if we wouldn't copy more than a few bytes. */ | |
380 | && n >= 3) | |
381 | { | |
382 | if ((unsigned long) dst & 1) | |
383 | { | |
384 | __asm_clear_1 (dst, retn); | |
385 | n--; | |
386 | } | |
387 | ||
388 | if ((unsigned long) dst & 2) | |
389 | { | |
390 | __asm_clear_2 (dst, retn); | |
391 | n -= 2; | |
392 | } | |
393 | } | |
394 | ||
395 | /* Decide which copying method to use. | |
396 | FIXME: This number is from the "ordinary" kernel memset. */ | |
397 | if (n >= (1*48)) | |
398 | { | |
399 | /* For large clears we use 'movem' */ | |
400 | ||
401 | /* It is not optimal to tell the compiler about clobbering any | |
402 | call-saved registers; that will move the saving/restoring of | |
403 | those registers to the function prologue/epilogue, and make | |
404 | non-movem sizes suboptimal. | |
405 | ||
406 | This method is not foolproof; it assumes that the "asm reg" | |
407 | declarations at the beginning of the function really are used | |
408 | here (beware: they may be moved to temporary registers). | |
409 | This way, we do not have to save/move the registers around into | |
410 | temporaries; we can safely use them straight away. | |
411 | ||
412 | If you want to check that the allocation was right; then | |
413 | check the equalities in the first comment. It should say | |
414 | something like "r13=r13, r11=r11, r12=r12". */ | |
2b05d2b3 | 415 | __asm__ volatile ("\n\ |
1da177e4 LT |
416 | .ifnc %0%1%2,$r13$r12$r10 \n\ |
417 | .err \n\ | |
418 | .endif \n\ | |
2b05d2b3 JN |
419 | \n\ |
420 | ;; Save the registers we'll clobber in the movem process \n\ | |
421 | ;; on the stack. Don't mention them to gcc, it will only be \n\ | |
422 | ;; upset. \n\ | |
423 | subq 11*4,$sp \n\ | |
424 | movem $r10,[$sp] \n\ | |
425 | \n\ | |
426 | clear.d $r0 \n\ | |
427 | clear.d $r1 \n\ | |
428 | clear.d $r2 \n\ | |
429 | clear.d $r3 \n\ | |
430 | clear.d $r4 \n\ | |
431 | clear.d $r5 \n\ | |
432 | clear.d $r6 \n\ | |
433 | clear.d $r7 \n\ | |
434 | clear.d $r8 \n\ | |
435 | clear.d $r9 \n\ | |
436 | clear.d $r10 \n\ | |
437 | clear.d $r11 \n\ | |
438 | \n\ | |
439 | ;; Now we've got this: \n\ | |
440 | ;; r13 - dst \n\ | |
441 | ;; r12 - n \n\ | |
442 | \n\ | |
443 | ;; Update n for the first loop \n\ | |
444 | subq 12*4,$r12 \n\ | |
445 | 0: \n\ | |
446 | subq 12*4,$r12 \n\ | |
447 | bge 0b \n\ | |
448 | movem $r11,[$r13+] \n\ | |
449 | 1: \n\ | |
450 | addq 12*4,$r12 ;; compensate for last loop underflowing n\n\ | |
451 | \n\ | |
452 | ;; Restore registers from stack \n\ | |
453 | movem [$sp+],$r10 \n\ | |
454 | 2: \n\ | |
455 | .section .fixup,\"ax\" \n\ | |
456 | 3: \n\ | |
457 | move.d [$sp],$r10 \n\ | |
458 | addq 12*4,$r10 \n\ | |
459 | move.d $r10,[$sp] \n\ | |
460 | clear.d $r10 \n\ | |
461 | jump 0b \n\ | |
462 | \n\ | |
463 | 4: \n\ | |
464 | movem [$sp+],$r10 \n\ | |
465 | addq 12*4,$r10 \n\ | |
466 | addq 12*4,$r12 \n\ | |
467 | jump 2b \n\ | |
468 | \n\ | |
469 | .previous \n\ | |
470 | .section __ex_table,\"a\" \n\ | |
471 | .dword 0b,3b \n\ | |
472 | .dword 1b,4b \n\ | |
1da177e4 LT |
473 | .previous" |
474 | ||
475 | /* Outputs */ : "=r" (dst), "=r" (n), "=r" (retn) | |
476 | /* Inputs */ : "0" (dst), "1" (n), "2" (retn) | |
477 | /* Clobber */ : "r11"); | |
478 | } | |
479 | ||
480 | while (n >= 16) | |
481 | { | |
482 | __asm_clear_16 (dst, retn); | |
483 | n -= 16; | |
484 | } | |
485 | ||
486 | /* Having a separate by-four loops cuts down on cache footprint. | |
487 | FIXME: Test with and without; increasing switch to be 0..15. */ | |
488 | while (n >= 4) | |
489 | { | |
490 | __asm_clear_4 (dst, retn); | |
491 | n -= 4; | |
492 | } | |
493 | ||
494 | switch (n) | |
495 | { | |
496 | case 0: | |
497 | break; | |
498 | case 1: | |
499 | __asm_clear_1 (dst, retn); | |
500 | break; | |
501 | case 2: | |
502 | __asm_clear_2 (dst, retn); | |
503 | break; | |
504 | case 3: | |
505 | __asm_clear_3 (dst, retn); | |
506 | break; | |
507 | } | |
508 | ||
509 | return retn; | |
510 | } | |
dbd3c7e1 | 511 | EXPORT_SYMBOL(__do_clear_user); |