Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* align.c - handle alignment exceptions for the Power PC. |
2 | * | |
3 | * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au> | |
4 | * Copyright (c) 1998-1999 TiVo, Inc. | |
5 | * PowerPC 403GCX modifications. | |
6 | * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu> | |
7 | * PowerPC 403GCX/405GP modifications. | |
8 | * Copyright (c) 2001-2002 PPC64 team, IBM Corp | |
9 | * 64-bit and Power4 support | |
10 | * | |
11 | * This program is free software; you can redistribute it and/or | |
12 | * modify it under the terms of the GNU General Public License | |
13 | * as published by the Free Software Foundation; either version | |
14 | * 2 of the License, or (at your option) any later version. | |
15 | */ | |
16 | ||
17 | #include <linux/kernel.h> | |
18 | #include <linux/mm.h> | |
19 | #include <asm/processor.h> | |
20 | #include <asm/uaccess.h> | |
21 | #include <asm/system.h> | |
22 | #include <asm/cache.h> | |
23 | #include <asm/cputable.h> | |
24 | ||
25 | struct aligninfo { | |
26 | unsigned char len; | |
27 | unsigned char flags; | |
28 | }; | |
29 | ||
30 | #define IS_XFORM(inst) (((inst) >> 26) == 31) | |
31 | #define IS_DSFORM(inst) (((inst) >> 26) >= 56) | |
32 | ||
33 | #define INVALID { 0, 0 } | |
34 | ||
35 | #define LD 1 /* load */ | |
36 | #define ST 2 /* store */ | |
37 | #define SE 4 /* sign-extend value */ | |
38 | #define F 8 /* to/from fp regs */ | |
39 | #define U 0x10 /* update index register */ | |
40 | #define M 0x20 /* multiple load/store */ | |
41 | #define SW 0x40 /* byte swap */ | |
42 | ||
43 | #define DCBZ 0x5f /* 8xx/82xx dcbz faults when cache not enabled */ | |
44 | ||
45 | /* | |
46 | * The PowerPC stores certain bits of the instruction that caused the | |
47 | * alignment exception in the DSISR register. This array maps those | |
48 | * bits to information about the operand length and what the | |
49 | * instruction would do. | |
50 | */ | |
51 | static struct aligninfo aligninfo[128] = { | |
52 | { 4, LD }, /* 00 0 0000: lwz / lwarx */ | |
53 | INVALID, /* 00 0 0001 */ | |
54 | { 4, ST }, /* 00 0 0010: stw */ | |
55 | INVALID, /* 00 0 0011 */ | |
56 | { 2, LD }, /* 00 0 0100: lhz */ | |
57 | { 2, LD+SE }, /* 00 0 0101: lha */ | |
58 | { 2, ST }, /* 00 0 0110: sth */ | |
59 | { 4, LD+M }, /* 00 0 0111: lmw */ | |
60 | { 4, LD+F }, /* 00 0 1000: lfs */ | |
61 | { 8, LD+F }, /* 00 0 1001: lfd */ | |
62 | { 4, ST+F }, /* 00 0 1010: stfs */ | |
63 | { 8, ST+F }, /* 00 0 1011: stfd */ | |
64 | INVALID, /* 00 0 1100 */ | |
65 | { 8, LD }, /* 00 0 1101: ld */ | |
66 | INVALID, /* 00 0 1110 */ | |
67 | { 8, ST }, /* 00 0 1111: std */ | |
68 | { 4, LD+U }, /* 00 1 0000: lwzu */ | |
69 | INVALID, /* 00 1 0001 */ | |
70 | { 4, ST+U }, /* 00 1 0010: stwu */ | |
71 | INVALID, /* 00 1 0011 */ | |
72 | { 2, LD+U }, /* 00 1 0100: lhzu */ | |
73 | { 2, LD+SE+U }, /* 00 1 0101: lhau */ | |
74 | { 2, ST+U }, /* 00 1 0110: sthu */ | |
75 | { 4, ST+M }, /* 00 1 0111: stmw */ | |
76 | { 4, LD+F+U }, /* 00 1 1000: lfsu */ | |
77 | { 8, LD+F+U }, /* 00 1 1001: lfdu */ | |
78 | { 4, ST+F+U }, /* 00 1 1010: stfsu */ | |
79 | { 8, ST+F+U }, /* 00 1 1011: stfdu */ | |
80 | INVALID, /* 00 1 1100 */ | |
81 | INVALID, /* 00 1 1101 */ | |
82 | INVALID, /* 00 1 1110 */ | |
83 | INVALID, /* 00 1 1111 */ | |
84 | { 8, LD }, /* 01 0 0000: ldx */ | |
85 | INVALID, /* 01 0 0001 */ | |
86 | { 8, ST }, /* 01 0 0010: stdx */ | |
87 | INVALID, /* 01 0 0011 */ | |
88 | INVALID, /* 01 0 0100 */ | |
89 | { 4, LD+SE }, /* 01 0 0101: lwax */ | |
90 | INVALID, /* 01 0 0110 */ | |
91 | INVALID, /* 01 0 0111 */ | |
92 | { 0, LD }, /* 01 0 1000: lswx */ | |
93 | { 0, LD }, /* 01 0 1001: lswi */ | |
94 | { 0, ST }, /* 01 0 1010: stswx */ | |
95 | { 0, ST }, /* 01 0 1011: stswi */ | |
96 | INVALID, /* 01 0 1100 */ | |
97 | { 8, LD+U }, /* 01 0 1101: ldu */ | |
98 | INVALID, /* 01 0 1110 */ | |
99 | { 8, ST+U }, /* 01 0 1111: stdu */ | |
100 | { 8, LD+U }, /* 01 1 0000: ldux */ | |
101 | INVALID, /* 01 1 0001 */ | |
102 | { 8, ST+U }, /* 01 1 0010: stdux */ | |
103 | INVALID, /* 01 1 0011 */ | |
104 | INVALID, /* 01 1 0100 */ | |
105 | { 4, LD+SE+U }, /* 01 1 0101: lwaux */ | |
106 | INVALID, /* 01 1 0110 */ | |
107 | INVALID, /* 01 1 0111 */ | |
108 | INVALID, /* 01 1 1000 */ | |
109 | INVALID, /* 01 1 1001 */ | |
110 | INVALID, /* 01 1 1010 */ | |
111 | INVALID, /* 01 1 1011 */ | |
112 | INVALID, /* 01 1 1100 */ | |
113 | INVALID, /* 01 1 1101 */ | |
114 | INVALID, /* 01 1 1110 */ | |
115 | INVALID, /* 01 1 1111 */ | |
116 | INVALID, /* 10 0 0000 */ | |
117 | INVALID, /* 10 0 0001 */ | |
118 | { 0, ST }, /* 10 0 0010: stwcx. */ | |
119 | INVALID, /* 10 0 0011 */ | |
120 | INVALID, /* 10 0 0100 */ | |
121 | INVALID, /* 10 0 0101 */ | |
122 | INVALID, /* 10 0 0110 */ | |
123 | INVALID, /* 10 0 0111 */ | |
124 | { 4, LD+SW }, /* 10 0 1000: lwbrx */ | |
125 | INVALID, /* 10 0 1001 */ | |
126 | { 4, ST+SW }, /* 10 0 1010: stwbrx */ | |
127 | INVALID, /* 10 0 1011 */ | |
128 | { 2, LD+SW }, /* 10 0 1100: lhbrx */ | |
129 | { 4, LD+SE }, /* 10 0 1101 lwa */ | |
130 | { 2, ST+SW }, /* 10 0 1110: sthbrx */ | |
131 | INVALID, /* 10 0 1111 */ | |
132 | INVALID, /* 10 1 0000 */ | |
133 | INVALID, /* 10 1 0001 */ | |
134 | INVALID, /* 10 1 0010 */ | |
135 | INVALID, /* 10 1 0011 */ | |
136 | INVALID, /* 10 1 0100 */ | |
137 | INVALID, /* 10 1 0101 */ | |
138 | INVALID, /* 10 1 0110 */ | |
139 | INVALID, /* 10 1 0111 */ | |
140 | INVALID, /* 10 1 1000 */ | |
141 | INVALID, /* 10 1 1001 */ | |
142 | INVALID, /* 10 1 1010 */ | |
143 | INVALID, /* 10 1 1011 */ | |
144 | INVALID, /* 10 1 1100 */ | |
145 | INVALID, /* 10 1 1101 */ | |
146 | INVALID, /* 10 1 1110 */ | |
147 | { L1_CACHE_BYTES, ST }, /* 10 1 1111: dcbz */ | |
148 | { 4, LD }, /* 11 0 0000: lwzx */ | |
149 | INVALID, /* 11 0 0001 */ | |
150 | { 4, ST }, /* 11 0 0010: stwx */ | |
151 | INVALID, /* 11 0 0011 */ | |
152 | { 2, LD }, /* 11 0 0100: lhzx */ | |
153 | { 2, LD+SE }, /* 11 0 0101: lhax */ | |
154 | { 2, ST }, /* 11 0 0110: sthx */ | |
155 | INVALID, /* 11 0 0111 */ | |
156 | { 4, LD+F }, /* 11 0 1000: lfsx */ | |
157 | { 8, LD+F }, /* 11 0 1001: lfdx */ | |
158 | { 4, ST+F }, /* 11 0 1010: stfsx */ | |
159 | { 8, ST+F }, /* 11 0 1011: stfdx */ | |
160 | INVALID, /* 11 0 1100 */ | |
161 | { 8, LD+M }, /* 11 0 1101: lmd */ | |
162 | INVALID, /* 11 0 1110 */ | |
163 | { 8, ST+M }, /* 11 0 1111: stmd */ | |
164 | { 4, LD+U }, /* 11 1 0000: lwzux */ | |
165 | INVALID, /* 11 1 0001 */ | |
166 | { 4, ST+U }, /* 11 1 0010: stwux */ | |
167 | INVALID, /* 11 1 0011 */ | |
168 | { 2, LD+U }, /* 11 1 0100: lhzux */ | |
169 | { 2, LD+SE+U }, /* 11 1 0101: lhaux */ | |
170 | { 2, ST+U }, /* 11 1 0110: sthux */ | |
171 | INVALID, /* 11 1 0111 */ | |
172 | { 4, LD+F+U }, /* 11 1 1000: lfsux */ | |
173 | { 8, LD+F+U }, /* 11 1 1001: lfdux */ | |
174 | { 4, ST+F+U }, /* 11 1 1010: stfsux */ | |
175 | { 8, ST+F+U }, /* 11 1 1011: stfdux */ | |
176 | INVALID, /* 11 1 1100 */ | |
177 | INVALID, /* 11 1 1101 */ | |
178 | INVALID, /* 11 1 1110 */ | |
179 | INVALID, /* 11 1 1111 */ | |
180 | }; | |
181 | ||
182 | #define SWAP(a, b) (t = (a), (a) = (b), (b) = t) | |
183 | ||
184 | static inline unsigned make_dsisr(unsigned instr) | |
185 | { | |
186 | unsigned dsisr; | |
187 | ||
188 | /* create a DSISR value from the instruction */ | |
189 | dsisr = (instr & 0x03ff0000) >> 16; /* bits 6:15 --> 22:31 */ | |
190 | ||
191 | if ( IS_XFORM(instr) ) { | |
192 | dsisr |= (instr & 0x00000006) << 14; /* bits 29:30 --> 15:16 */ | |
193 | dsisr |= (instr & 0x00000040) << 8; /* bit 25 --> 17 */ | |
194 | dsisr |= (instr & 0x00000780) << 3; /* bits 21:24 --> 18:21 */ | |
195 | } | |
196 | else { | |
197 | dsisr |= (instr & 0x04000000) >> 12; /* bit 5 --> 17 */ | |
198 | dsisr |= (instr & 0x78000000) >> 17; /* bits 1: 4 --> 18:21 */ | |
199 | if ( IS_DSFORM(instr) ) { | |
200 | dsisr |= (instr & 0x00000003) << 18; /* bits 30:31 --> 12:13 */ | |
201 | } | |
202 | } | |
203 | ||
204 | return dsisr; | |
205 | } | |
206 | ||
207 | int | |
208 | fix_alignment(struct pt_regs *regs) | |
209 | { | |
210 | unsigned int instr, nb, flags; | |
211 | int t; | |
212 | unsigned long reg, areg; | |
213 | unsigned long i; | |
214 | int ret; | |
215 | unsigned dsisr; | |
216 | unsigned char __user *addr; | |
217 | unsigned char __user *p; | |
218 | unsigned long __user *lp; | |
219 | union { | |
220 | long ll; | |
221 | double dd; | |
222 | unsigned char v[8]; | |
223 | struct { | |
224 | unsigned hi32; | |
225 | int low32; | |
226 | } x32; | |
227 | struct { | |
228 | unsigned char hi48[6]; | |
229 | short low16; | |
230 | } x16; | |
231 | } data; | |
232 | ||
233 | /* | |
234 | * Return 1 on success | |
235 | * Return 0 if unable to handle the interrupt | |
236 | * Return -EFAULT if data address is bad | |
237 | */ | |
238 | ||
239 | dsisr = regs->dsisr; | |
240 | ||
241 | if (cpu_has_feature(CPU_FTR_NODSISRALIGN)) { | |
242 | unsigned int real_instr; | |
243 | if (__get_user(real_instr, (unsigned int __user *)regs->nip)) | |
244 | return 0; | |
245 | dsisr = make_dsisr(real_instr); | |
246 | } | |
247 | ||
248 | /* extract the operation and registers from the dsisr */ | |
249 | reg = (dsisr >> 5) & 0x1f; /* source/dest register */ | |
250 | areg = dsisr & 0x1f; /* register to update */ | |
251 | instr = (dsisr >> 10) & 0x7f; | |
252 | instr |= (dsisr >> 13) & 0x60; | |
253 | ||
254 | /* Lookup the operation in our table */ | |
255 | nb = aligninfo[instr].len; | |
256 | flags = aligninfo[instr].flags; | |
257 | ||
258 | /* DAR has the operand effective address */ | |
259 | addr = (unsigned char __user *)regs->dar; | |
260 | ||
261 | /* A size of 0 indicates an instruction we don't support */ | |
262 | /* we also don't support the multiples (lmw, stmw, lmd, stmd) */ | |
263 | if ((nb == 0) || (flags & M)) | |
264 | return 0; /* too hard or invalid instruction */ | |
265 | ||
266 | /* | |
267 | * Special handling for dcbz | |
268 | * dcbz may give an alignment exception for accesses to caching inhibited | |
269 | * storage | |
270 | */ | |
271 | if (instr == DCBZ) | |
272 | addr = (unsigned char __user *) ((unsigned long)addr & -L1_CACHE_BYTES); | |
273 | ||
274 | /* Verify the address of the operand */ | |
275 | if (user_mode(regs)) { | |
276 | if (!access_ok((flags & ST? VERIFY_WRITE: VERIFY_READ), addr, nb)) | |
277 | return -EFAULT; /* bad address */ | |
278 | } | |
279 | ||
280 | /* Force the fprs into the save area so we can reference them */ | |
281 | if (flags & F) { | |
282 | if (!user_mode(regs)) | |
283 | return 0; | |
284 | flush_fp_to_thread(current); | |
285 | } | |
286 | ||
287 | /* If we are loading, get the data from user space */ | |
288 | if (flags & LD) { | |
289 | data.ll = 0; | |
290 | ret = 0; | |
291 | p = addr; | |
292 | switch (nb) { | |
293 | case 8: | |
294 | ret |= __get_user(data.v[0], p++); | |
295 | ret |= __get_user(data.v[1], p++); | |
296 | ret |= __get_user(data.v[2], p++); | |
297 | ret |= __get_user(data.v[3], p++); | |
298 | case 4: | |
299 | ret |= __get_user(data.v[4], p++); | |
300 | ret |= __get_user(data.v[5], p++); | |
301 | case 2: | |
302 | ret |= __get_user(data.v[6], p++); | |
303 | ret |= __get_user(data.v[7], p++); | |
304 | if (ret) | |
305 | return -EFAULT; | |
306 | } | |
307 | } | |
308 | ||
309 | /* If we are storing, get the data from the saved gpr or fpr */ | |
310 | if (flags & ST) { | |
311 | if (flags & F) { | |
312 | if (nb == 4) { | |
313 | /* Doing stfs, have to convert to single */ | |
314 | preempt_disable(); | |
315 | enable_kernel_fp(); | |
25c8a78b | 316 | cvt_df(¤t->thread.fpr[reg], (float *)&data.v[4], ¤t->thread); |
1da177e4 LT |
317 | disable_kernel_fp(); |
318 | preempt_enable(); | |
319 | } | |
320 | else | |
321 | data.dd = current->thread.fpr[reg]; | |
322 | } | |
323 | else | |
324 | data.ll = regs->gpr[reg]; | |
325 | } | |
326 | ||
327 | /* Swap bytes as needed */ | |
328 | if (flags & SW) { | |
329 | if (nb == 2) | |
330 | SWAP(data.v[6], data.v[7]); | |
331 | else { /* nb must be 4 */ | |
332 | SWAP(data.v[4], data.v[7]); | |
333 | SWAP(data.v[5], data.v[6]); | |
334 | } | |
335 | } | |
336 | ||
337 | /* Sign extend as needed */ | |
338 | if (flags & SE) { | |
339 | if ( nb == 2 ) | |
340 | data.ll = data.x16.low16; | |
341 | else /* nb must be 4 */ | |
342 | data.ll = data.x32.low32; | |
343 | } | |
344 | ||
345 | /* If we are loading, move the data to the gpr or fpr */ | |
346 | if (flags & LD) { | |
347 | if (flags & F) { | |
348 | if (nb == 4) { | |
349 | /* Doing lfs, have to convert to double */ | |
350 | preempt_disable(); | |
351 | enable_kernel_fp(); | |
25c8a78b | 352 | cvt_fd((float *)&data.v[4], ¤t->thread.fpr[reg], ¤t->thread); |
1da177e4 LT |
353 | disable_kernel_fp(); |
354 | preempt_enable(); | |
355 | } | |
356 | else | |
357 | current->thread.fpr[reg] = data.dd; | |
358 | } | |
359 | else | |
360 | regs->gpr[reg] = data.ll; | |
361 | } | |
362 | ||
363 | /* If we are storing, copy the data to the user */ | |
364 | if (flags & ST) { | |
365 | ret = 0; | |
366 | p = addr; | |
367 | switch (nb) { | |
368 | case 128: /* Special case - must be dcbz */ | |
369 | lp = (unsigned long __user *)p; | |
370 | for (i = 0; i < L1_CACHE_BYTES / sizeof(long); ++i) | |
371 | ret |= __put_user(0, lp++); | |
372 | break; | |
373 | case 8: | |
374 | ret |= __put_user(data.v[0], p++); | |
375 | ret |= __put_user(data.v[1], p++); | |
376 | ret |= __put_user(data.v[2], p++); | |
377 | ret |= __put_user(data.v[3], p++); | |
378 | case 4: | |
379 | ret |= __put_user(data.v[4], p++); | |
380 | ret |= __put_user(data.v[5], p++); | |
381 | case 2: | |
382 | ret |= __put_user(data.v[6], p++); | |
383 | ret |= __put_user(data.v[7], p++); | |
384 | } | |
385 | if (ret) | |
386 | return -EFAULT; | |
387 | } | |
388 | ||
389 | /* Update RA as needed */ | |
390 | if (flags & U) { | |
391 | regs->gpr[areg] = regs->dar; | |
392 | } | |
393 | ||
394 | return 1; | |
395 | } | |
396 |