Commit | Line | Data |
---|---|---|
a0e60b20 DG |
1 | /* |
2 | * PowerPC atomic bit operations. | |
3 | * | |
4 | * Merged version by David Gibson <david@gibson.dropbear.id.au>. | |
5 | * Based on ppc64 versions by: Dave Engebretsen, Todd Inglett, Don | |
6 | * Reed, Pat McCarthy, Peter Bergner, Anton Blanchard. They | |
7 | * originally took it from the ppc32 code. | |
8 | * | |
9 | * Within a word, bits are numbered LSB first. Lot's of places make | |
10 | * this assumption by directly testing bits with (val & (1<<nr)). | |
11 | * This can cause confusion for large (> 1 word) bitmaps on a | |
12 | * big-endian system because, unlike little endian, the number of each | |
13 | * bit depends on the word size. | |
14 | * | |
15 | * The bitop functions are defined to work on unsigned longs, so for a | |
16 | * ppc64 system the bits end up numbered: | |
17 | * |63..............0|127............64|191...........128|255...........196| | |
18 | * and on ppc32: | |
19 | * |31.....0|63....31|95....64|127...96|159..128|191..160|223..192|255..224| | |
20 | * | |
21 | * There are a few little-endian macros used mostly for filesystem | |
22 | * bitmaps, these work on similar bit arrays layouts, but | |
23 | * byte-oriented: | |
24 | * |7...0|15...8|23...16|31...24|39...32|47...40|55...48|63...56| | |
25 | * | |
26 | * The main difference is that bit 3-5 (64b) or 3-4 (32b) in the bit | |
27 | * number field needs to be reversed compared to the big-endian bit | |
28 | * fields. This can be achieved by XOR with 0x38 (64b) or 0x18 (32b). | |
29 | * | |
30 | * This program is free software; you can redistribute it and/or | |
31 | * modify it under the terms of the GNU General Public License | |
32 | * as published by the Free Software Foundation; either version | |
33 | * 2 of the License, or (at your option) any later version. | |
34 | */ | |
35 | ||
36 | #ifndef _ASM_POWERPC_BITOPS_H | |
37 | #define _ASM_POWERPC_BITOPS_H | |
38 | ||
39 | #ifdef __KERNEL__ | |
40 | ||
0624517d JS |
41 | #ifndef _LINUX_BITOPS_H |
42 | #error only <linux/bitops.h> can be included directly | |
43 | #endif | |
44 | ||
a0e60b20 | 45 | #include <linux/compiler.h> |
3ddfbcf1 | 46 | #include <asm/asm-compat.h> |
a0e60b20 DG |
47 | #include <asm/synch.h> |
48 | ||
49 | /* | |
50 | * clear_bit doesn't imply a memory barrier | |
51 | */ | |
52 | #define smp_mb__before_clear_bit() smp_mb() | |
53 | #define smp_mb__after_clear_bit() smp_mb() | |
54 | ||
a0e60b20 DG |
55 | #define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) |
56 | ||
0d2d3e38 GT |
57 | /* Macro for generating the ***_bits() functions */ |
58 | #define DEFINE_BITOP(fn, op, prefix, postfix) \ | |
59 | static __inline__ void fn(unsigned long mask, \ | |
60 | volatile unsigned long *_p) \ | |
61 | { \ | |
62 | unsigned long old; \ | |
63 | unsigned long *p = (unsigned long *)_p; \ | |
64 | __asm__ __volatile__ ( \ | |
65 | prefix \ | |
864b9e6f | 66 | "1:" PPC_LLARX(%0,0,%3,0) "\n" \ |
0d2d3e38 GT |
67 | stringify_in_c(op) "%0,%0,%2\n" \ |
68 | PPC405_ERR77(0,%3) \ | |
69 | PPC_STLCX "%0,0,%3\n" \ | |
70 | "bne- 1b\n" \ | |
71 | postfix \ | |
72 | : "=&r" (old), "+m" (*p) \ | |
73 | : "r" (mask), "r" (p) \ | |
74 | : "cc", "memory"); \ | |
75 | } | |
76 | ||
77 | DEFINE_BITOP(set_bits, or, "", "") | |
78 | DEFINE_BITOP(clear_bits, andc, "", "") | |
f10e2e5b | 79 | DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER, "") |
0d2d3e38 GT |
80 | DEFINE_BITOP(change_bits, xor, "", "") |
81 | ||
a0e60b20 DG |
82 | static __inline__ void set_bit(int nr, volatile unsigned long *addr) |
83 | { | |
2237f4f4 | 84 | set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
a0e60b20 DG |
85 | } |
86 | ||
87 | static __inline__ void clear_bit(int nr, volatile unsigned long *addr) | |
88 | { | |
2237f4f4 | 89 | clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
a0e60b20 DG |
90 | } |
91 | ||
66ffb04c NP |
92 | static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr) |
93 | { | |
2237f4f4 | 94 | clear_bits_unlock(BIT_MASK(nr), addr + BIT_WORD(nr)); |
66ffb04c NP |
95 | } |
96 | ||
a0e60b20 DG |
97 | static __inline__ void change_bit(int nr, volatile unsigned long *addr) |
98 | { | |
2237f4f4 | 99 | change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)); |
0d2d3e38 | 100 | } |
a0e60b20 | 101 | |
0d2d3e38 GT |
102 | /* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output |
103 | * operands. */ | |
864b9e6f AB |
104 | #define DEFINE_TESTOP(fn, op, prefix, postfix, eh) \ |
105 | static __inline__ unsigned long fn( \ | |
106 | unsigned long mask, \ | |
107 | volatile unsigned long *_p) \ | |
108 | { \ | |
109 | unsigned long old, t; \ | |
110 | unsigned long *p = (unsigned long *)_p; \ | |
111 | __asm__ __volatile__ ( \ | |
112 | prefix \ | |
113 | "1:" PPC_LLARX(%0,0,%3,eh) "\n" \ | |
114 | stringify_in_c(op) "%1,%0,%2\n" \ | |
115 | PPC405_ERR77(0,%3) \ | |
116 | PPC_STLCX "%1,0,%3\n" \ | |
117 | "bne- 1b\n" \ | |
118 | postfix \ | |
119 | : "=&r" (old), "=&r" (t) \ | |
120 | : "r" (mask), "r" (p) \ | |
121 | : "cc", "memory"); \ | |
122 | return (old & mask); \ | |
a0e60b20 DG |
123 | } |
124 | ||
b97021f8 BH |
125 | DEFINE_TESTOP(test_and_set_bits, or, PPC_ATOMIC_ENTRY_BARRIER, |
126 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
f10e2e5b AB |
127 | DEFINE_TESTOP(test_and_set_bits_lock, or, "", |
128 | PPC_ACQUIRE_BARRIER, 1) | |
b97021f8 BH |
129 | DEFINE_TESTOP(test_and_clear_bits, andc, PPC_ATOMIC_ENTRY_BARRIER, |
130 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
131 | DEFINE_TESTOP(test_and_change_bits, xor, PPC_ATOMIC_ENTRY_BARRIER, | |
132 | PPC_ATOMIC_EXIT_BARRIER, 0) | |
0d2d3e38 | 133 | |
a0e60b20 DG |
134 | static __inline__ int test_and_set_bit(unsigned long nr, |
135 | volatile unsigned long *addr) | |
136 | { | |
2237f4f4 | 137 | return test_and_set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
138 | } |
139 | ||
66ffb04c NP |
140 | static __inline__ int test_and_set_bit_lock(unsigned long nr, |
141 | volatile unsigned long *addr) | |
142 | { | |
2237f4f4 AM |
143 | return test_and_set_bits_lock(BIT_MASK(nr), |
144 | addr + BIT_WORD(nr)) != 0; | |
66ffb04c NP |
145 | } |
146 | ||
a0e60b20 DG |
147 | static __inline__ int test_and_clear_bit(unsigned long nr, |
148 | volatile unsigned long *addr) | |
149 | { | |
2237f4f4 | 150 | return test_and_clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
151 | } |
152 | ||
153 | static __inline__ int test_and_change_bit(unsigned long nr, | |
154 | volatile unsigned long *addr) | |
155 | { | |
2237f4f4 | 156 | return test_and_change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0; |
a0e60b20 DG |
157 | } |
158 | ||
e779b2f9 | 159 | #include <asm-generic/bitops/non-atomic.h> |
a0e60b20 | 160 | |
66ffb04c NP |
161 | static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr) |
162 | { | |
f10e2e5b | 163 | __asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory"); |
66ffb04c NP |
164 | __clear_bit(nr, addr); |
165 | } | |
166 | ||
a0e60b20 DG |
167 | /* |
168 | * Return the zero-based bit position (LE, not IBM bit numbering) of | |
169 | * the most significant 1-bit in a double word. | |
170 | */ | |
ef55d53c DH |
171 | static __inline__ __attribute__((const)) |
172 | int __ilog2(unsigned long x) | |
a0e60b20 DG |
173 | { |
174 | int lz; | |
175 | ||
3ddfbcf1 | 176 | asm (PPC_CNTLZL "%0,%1" : "=r" (lz) : "r" (x)); |
a0e60b20 DG |
177 | return BITS_PER_LONG - 1 - lz; |
178 | } | |
179 | ||
ef55d53c DH |
180 | static inline __attribute__((const)) |
181 | int __ilog2_u32(u32 n) | |
182 | { | |
183 | int bit; | |
184 | asm ("cntlzw %0,%1" : "=r" (bit) : "r" (n)); | |
185 | return 31 - bit; | |
186 | } | |
187 | ||
188 | #ifdef __powerpc64__ | |
189 | static inline __attribute__((const)) | |
02241696 | 190 | int __ilog2_u64(u64 n) |
ef55d53c DH |
191 | { |
192 | int bit; | |
193 | asm ("cntlzd %0,%1" : "=r" (bit) : "r" (n)); | |
194 | return 63 - bit; | |
195 | } | |
196 | #endif | |
197 | ||
a0e60b20 DG |
198 | /* |
199 | * Determines the bit position of the least significant 0 bit in the | |
200 | * specified double word. The returned bit position will be | |
201 | * zero-based, starting from the right side (63/31 - 0). | |
202 | */ | |
203 | static __inline__ unsigned long ffz(unsigned long x) | |
204 | { | |
205 | /* no zero exists anywhere in the 8 byte area. */ | |
206 | if ((x = ~x) == 0) | |
207 | return BITS_PER_LONG; | |
208 | ||
209 | /* | |
25985edc LDM |
210 | * Calculate the bit position of the least significant '1' bit in x |
211 | * (since x has been changed this will actually be the least significant | |
a0e60b20 DG |
212 | * '0' bit in * the original x). Note: (x & -x) gives us a mask that |
213 | * is the least significant * (RIGHT-most) 1-bit of the value in x. | |
214 | */ | |
215 | return __ilog2(x & -x); | |
216 | } | |
217 | ||
218 | static __inline__ int __ffs(unsigned long x) | |
219 | { | |
220 | return __ilog2(x & -x); | |
221 | } | |
222 | ||
223 | /* | |
224 | * ffs: find first bit set. This is defined the same way as | |
225 | * the libc and compiler builtin ffs routines, therefore | |
226 | * differs in spirit from the above ffz (man ffs). | |
227 | */ | |
228 | static __inline__ int ffs(int x) | |
229 | { | |
230 | unsigned long i = (unsigned long)x; | |
231 | return __ilog2(i & -i) + 1; | |
232 | } | |
233 | ||
234 | /* | |
235 | * fls: find last (most-significant) bit set. | |
236 | * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32. | |
237 | */ | |
238 | static __inline__ int fls(unsigned int x) | |
239 | { | |
240 | int lz; | |
241 | ||
242 | asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x)); | |
243 | return 32 - lz; | |
244 | } | |
9f264be6 | 245 | |
56a6b1eb AH |
246 | static __inline__ unsigned long __fls(unsigned long x) |
247 | { | |
248 | return __ilog2(x); | |
249 | } | |
250 | ||
9f264be6 PM |
251 | /* |
252 | * 64-bit can do this using one cntlzd (count leading zeroes doubleword) | |
253 | * instruction; for 32-bit we use the generic version, which does two | |
254 | * 32-bit fls calls. | |
255 | */ | |
256 | #ifdef __powerpc64__ | |
257 | static __inline__ int fls64(__u64 x) | |
258 | { | |
259 | int lz; | |
260 | ||
261 | asm ("cntlzd %0,%1" : "=r" (lz) : "r" (x)); | |
262 | return 64 - lz; | |
263 | } | |
264 | #else | |
e779b2f9 | 265 | #include <asm-generic/bitops/fls64.h> |
9f264be6 PM |
266 | #endif /* __powerpc64__ */ |
267 | ||
64ff3128 AB |
268 | #ifdef CONFIG_PPC64 |
269 | unsigned int __arch_hweight8(unsigned int w); | |
270 | unsigned int __arch_hweight16(unsigned int w); | |
271 | unsigned int __arch_hweight32(unsigned int w); | |
272 | unsigned long __arch_hweight64(__u64 w); | |
273 | #include <asm-generic/bitops/const_hweight.h> | |
274 | #else | |
e779b2f9 | 275 | #include <asm-generic/bitops/hweight.h> |
64ff3128 AB |
276 | #endif |
277 | ||
47b9d9bd | 278 | #include <asm-generic/bitops/find.h> |
a0e60b20 DG |
279 | |
280 | /* Little-endian versions */ | |
79597be9 | 281 | #include <asm-generic/bitops/le.h> |
a0e60b20 | 282 | |
a0e60b20 DG |
283 | /* Bitmap functions for the ext2 filesystem */ |
284 | ||
148817ba | 285 | #include <asm-generic/bitops/ext2-atomic-setbit.h> |
a0e60b20 | 286 | |
e779b2f9 | 287 | #include <asm-generic/bitops/sched.h> |
a0e60b20 DG |
288 | |
289 | #endif /* __KERNEL__ */ | |
290 | ||
291 | #endif /* _ASM_POWERPC_BITOPS_H */ |