Commit | Line | Data |
---|---|---|
9031fefd WD |
1 | /* |
2 | * Userspace implementations of gettimeofday() and friends. | |
3 | * | |
4 | * Copyright (C) 2012 ARM Limited | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
17 | * | |
18 | * Author: Will Deacon <will.deacon@arm.com> | |
19 | */ | |
20 | ||
21 | #include <linux/linkage.h> | |
22 | #include <asm/asm-offsets.h> | |
23 | #include <asm/unistd.h> | |
24 | ||
25 | #define NSEC_PER_SEC_LO16 0xca00 | |
26 | #define NSEC_PER_SEC_HI16 0x3b9a | |
27 | ||
28 | vdso_data .req x6 | |
9d2622be KB |
29 | seqcnt .req w7 |
30 | w_tmp .req w8 | |
31 | x_tmp .req x8 | |
32 | ||
33 | /* | |
34 | * Conventions for macro arguments: | |
35 | * - An argument is write-only if its name starts with "res". | |
36 | * - All other arguments are read-only, unless otherwise specified. | |
37 | */ | |
9031fefd WD |
38 | |
39 | .macro seqcnt_acquire | |
40 | 9999: ldr seqcnt, [vdso_data, #VDSO_TB_SEQ_COUNT] | |
41 | tbnz seqcnt, #0, 9999b | |
42 | dmb ishld | |
9031fefd WD |
43 | .endm |
44 | ||
9d2622be | 45 | .macro seqcnt_check fail |
9031fefd | 46 | dmb ishld |
9d2622be KB |
47 | ldr w_tmp, [vdso_data, #VDSO_TB_SEQ_COUNT] |
48 | cmp w_tmp, seqcnt | |
49 | b.ne \fail | |
9031fefd WD |
50 | .endm |
51 | ||
9d2622be KB |
52 | .macro syscall_check fail |
53 | ldr w_tmp, [vdso_data, #VDSO_USE_SYSCALL] | |
54 | cbnz w_tmp, \fail | |
55 | .endm | |
56 | ||
57 | .macro get_nsec_per_sec res | |
58 | mov \res, #NSEC_PER_SEC_LO16 | |
59 | movk \res, #NSEC_PER_SEC_HI16, lsl #16 | |
60 | .endm | |
61 | ||
62 | /* | |
63 | * Returns the clock delta, in nanoseconds left-shifted by the clock | |
64 | * shift. | |
65 | */ | |
66 | .macro get_clock_shifted_nsec res, cycle_last, mult | |
67 | /* Read the virtual counter. */ | |
68 | isb | |
69 | mrs x_tmp, cntvct_el0 | |
70 | /* Calculate cycle delta and convert to ns. */ | |
71 | sub \res, x_tmp, \cycle_last | |
72 | /* We can only guarantee 56 bits of precision. */ | |
73 | movn x_tmp, #0xff00, lsl #48 | |
74 | and \res, x_tmp, \res | |
75 | mul \res, \res, \mult | |
76 | .endm | |
77 | ||
78 | /* | |
79 | * Returns in res_{sec,nsec} the REALTIME timespec, based on the | |
80 | * "wall time" (xtime) and the clock_mono delta. | |
81 | */ | |
82 | .macro get_ts_realtime res_sec, res_nsec, \ | |
83 | clock_nsec, xtime_sec, xtime_nsec, nsec_to_sec | |
84 | add \res_nsec, \clock_nsec, \xtime_nsec | |
85 | udiv x_tmp, \res_nsec, \nsec_to_sec | |
86 | add \res_sec, \xtime_sec, x_tmp | |
87 | msub \res_nsec, x_tmp, \nsec_to_sec, \res_nsec | |
88 | .endm | |
89 | ||
c2e0fb35 KB |
90 | /* |
91 | * Returns in res_{sec,nsec} the timespec based on the clock_raw delta, | |
92 | * used for CLOCK_MONOTONIC_RAW. | |
93 | */ | |
94 | .macro get_ts_clock_raw res_sec, res_nsec, clock_nsec, nsec_to_sec | |
95 | udiv \res_sec, \clock_nsec, \nsec_to_sec | |
96 | msub \res_nsec, \res_sec, \nsec_to_sec, \clock_nsec | |
97 | .endm | |
98 | ||
9d2622be KB |
99 | /* sec and nsec are modified in place. */ |
100 | .macro add_ts sec, nsec, ts_sec, ts_nsec, nsec_to_sec | |
101 | /* Add timespec. */ | |
102 | add \sec, \sec, \ts_sec | |
103 | add \nsec, \nsec, \ts_nsec | |
104 | ||
105 | /* Normalise the new timespec. */ | |
106 | cmp \nsec, \nsec_to_sec | |
107 | b.lt 9999f | |
108 | sub \nsec, \nsec, \nsec_to_sec | |
109 | add \sec, \sec, #1 | |
110 | 9999: | |
111 | cmp \nsec, #0 | |
112 | b.ge 9998f | |
113 | add \nsec, \nsec, \nsec_to_sec | |
114 | sub \sec, \sec, #1 | |
115 | 9998: | |
116 | .endm | |
117 | ||
118 | .macro clock_gettime_return, shift=0 | |
119 | .if \shift == 1 | |
120 | lsr x11, x11, x12 | |
121 | .endif | |
122 | stp x10, x11, [x1, #TSPEC_TV_SEC] | |
123 | mov x0, xzr | |
124 | ret | |
125 | .endm | |
126 | ||
127 | .macro jump_slot jumptable, index, label | |
128 | .if (. - \jumptable) != 4 * (\index) | |
129 | .error "Jump slot index mismatch" | |
130 | .endif | |
131 | b \label | |
9031fefd WD |
132 | .endm |
133 | ||
134 | .text | |
135 | ||
136 | /* int __kernel_gettimeofday(struct timeval *tv, struct timezone *tz); */ | |
137 | ENTRY(__kernel_gettimeofday) | |
138 | .cfi_startproc | |
9031fefd | 139 | adr vdso_data, _vdso_data |
9031fefd WD |
140 | /* If tv is NULL, skip to the timezone code. */ |
141 | cbz x0, 2f | |
9d2622be KB |
142 | |
143 | /* Compute the time of day. */ | |
144 | 1: seqcnt_acquire | |
145 | syscall_check fail=4f | |
146 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] | |
c2e0fb35 KB |
147 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
148 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] | |
9d2622be KB |
149 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
150 | seqcnt_check fail=1b | |
151 | ||
152 | get_nsec_per_sec res=x9 | |
153 | lsl x9, x9, x12 | |
154 | ||
155 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 | |
156 | get_ts_realtime res_sec=x10, res_nsec=x11, \ | |
157 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 | |
9031fefd WD |
158 | |
159 | /* Convert ns to us. */ | |
d91fb5c2 | 160 | mov x13, #1000 |
45a7905f | 161 | lsl x13, x13, x12 |
d91fb5c2 WD |
162 | udiv x11, x11, x13 |
163 | stp x10, x11, [x0, #TVAL_TV_SEC] | |
9031fefd WD |
164 | 2: |
165 | /* If tz is NULL, return 0. */ | |
166 | cbz x1, 3f | |
167 | ldp w4, w5, [vdso_data, #VDSO_TZ_MINWEST] | |
9031fefd WD |
168 | stp w4, w5, [x1, #TZ_MINWEST] |
169 | 3: | |
170 | mov x0, xzr | |
9d2622be | 171 | ret |
9031fefd WD |
172 | 4: |
173 | /* Syscall fallback. */ | |
174 | mov x8, #__NR_gettimeofday | |
175 | svc #0 | |
9d2622be | 176 | ret |
9031fefd WD |
177 | .cfi_endproc |
178 | ENDPROC(__kernel_gettimeofday) | |
179 | ||
9d2622be KB |
180 | #define JUMPSLOT_MAX CLOCK_MONOTONIC_COARSE |
181 | ||
9031fefd WD |
182 | /* int __kernel_clock_gettime(clockid_t clock_id, struct timespec *tp); */ |
183 | ENTRY(__kernel_clock_gettime) | |
184 | .cfi_startproc | |
c2e0fb35 KB |
185 | cmp w0, #JUMPSLOT_MAX |
186 | b.hi syscall | |
9031fefd | 187 | adr vdso_data, _vdso_data |
c2e0fb35 KB |
188 | adr x_tmp, jumptable |
189 | add x_tmp, x_tmp, w0, uxtw #2 | |
190 | br x_tmp | |
9d2622be KB |
191 | |
192 | ALIGN | |
193 | jumptable: | |
194 | jump_slot jumptable, CLOCK_REALTIME, realtime | |
195 | jump_slot jumptable, CLOCK_MONOTONIC, monotonic | |
c2e0fb35 KB |
196 | b syscall |
197 | b syscall | |
198 | jump_slot jumptable, CLOCK_MONOTONIC_RAW, monotonic_raw | |
9d2622be KB |
199 | jump_slot jumptable, CLOCK_REALTIME_COARSE, realtime_coarse |
200 | jump_slot jumptable, CLOCK_MONOTONIC_COARSE, monotonic_coarse | |
201 | ||
202 | .if (. - jumptable) != 4 * (JUMPSLOT_MAX + 1) | |
203 | .error "Wrong jumptable size" | |
204 | .endif | |
205 | ||
206 | ALIGN | |
207 | realtime: | |
208 | seqcnt_acquire | |
209 | syscall_check fail=syscall | |
210 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] | |
c2e0fb35 KB |
211 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
212 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] | |
9d2622be KB |
213 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
214 | seqcnt_check fail=realtime | |
9031fefd | 215 | |
9d2622be KB |
216 | /* All computations are done with left-shifted nsecs. */ |
217 | get_nsec_per_sec res=x9 | |
218 | lsl x9, x9, x12 | |
9031fefd | 219 | |
9d2622be KB |
220 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
221 | get_ts_realtime res_sec=x10, res_nsec=x11, \ | |
222 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 | |
223 | clock_gettime_return, shift=1 | |
069b9186 | 224 | |
9d2622be KB |
225 | ALIGN |
226 | monotonic: | |
227 | seqcnt_acquire | |
228 | syscall_check fail=syscall | |
229 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] | |
c2e0fb35 KB |
230 | /* w11 = cs_mono_mult, w12 = cs_shift */ |
231 | ldp w11, w12, [vdso_data, #VDSO_CS_MONO_MULT] | |
9d2622be KB |
232 | ldp x13, x14, [vdso_data, #VDSO_XTIME_CLK_SEC] |
233 | ldp x3, x4, [vdso_data, #VDSO_WTM_CLK_SEC] | |
234 | seqcnt_check fail=monotonic | |
9031fefd | 235 | |
9d2622be KB |
236 | /* All computations are done with left-shifted nsecs. */ |
237 | lsl x4, x4, x12 | |
238 | get_nsec_per_sec res=x9 | |
239 | lsl x9, x9, x12 | |
9031fefd | 240 | |
9d2622be KB |
241 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 |
242 | get_ts_realtime res_sec=x10, res_nsec=x11, \ | |
243 | clock_nsec=x15, xtime_sec=x13, xtime_nsec=x14, nsec_to_sec=x9 | |
9031fefd | 244 | |
9d2622be KB |
245 | add_ts sec=x10, nsec=x11, ts_sec=x3, ts_nsec=x4, nsec_to_sec=x9 |
246 | clock_gettime_return, shift=1 | |
069b9186 | 247 | |
c2e0fb35 KB |
248 | ALIGN |
249 | monotonic_raw: | |
250 | seqcnt_acquire | |
251 | syscall_check fail=syscall | |
252 | ldr x10, [vdso_data, #VDSO_CS_CYCLE_LAST] | |
253 | /* w11 = cs_raw_mult, w12 = cs_shift */ | |
254 | ldp w12, w11, [vdso_data, #VDSO_CS_SHIFT] | |
255 | ldp x13, x14, [vdso_data, #VDSO_RAW_TIME_SEC] | |
256 | seqcnt_check fail=monotonic_raw | |
257 | ||
258 | /* All computations are done with left-shifted nsecs. */ | |
c2e0fb35 KB |
259 | get_nsec_per_sec res=x9 |
260 | lsl x9, x9, x12 | |
261 | ||
262 | get_clock_shifted_nsec res=x15, cycle_last=x10, mult=x11 | |
263 | get_ts_clock_raw res_sec=x10, res_nsec=x11, \ | |
264 | clock_nsec=x15, nsec_to_sec=x9 | |
265 | ||
266 | add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 | |
267 | clock_gettime_return, shift=1 | |
268 | ||
9d2622be KB |
269 | ALIGN |
270 | realtime_coarse: | |
271 | seqcnt_acquire | |
d91fb5c2 | 272 | ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] |
9d2622be KB |
273 | seqcnt_check fail=realtime_coarse |
274 | clock_gettime_return | |
9031fefd | 275 | |
9d2622be KB |
276 | ALIGN |
277 | monotonic_coarse: | |
278 | seqcnt_acquire | |
279 | ldp x10, x11, [vdso_data, #VDSO_XTIME_CRS_SEC] | |
d91fb5c2 | 280 | ldp x13, x14, [vdso_data, #VDSO_WTM_CLK_SEC] |
9d2622be | 281 | seqcnt_check fail=monotonic_coarse |
9031fefd | 282 | |
9d2622be KB |
283 | /* Computations are done in (non-shifted) nsecs. */ |
284 | get_nsec_per_sec res=x9 | |
285 | add_ts sec=x10, nsec=x11, ts_sec=x13, ts_nsec=x14, nsec_to_sec=x9 | |
286 | clock_gettime_return | |
9031fefd | 287 | |
9d2622be KB |
288 | ALIGN |
289 | syscall: /* Syscall fallback. */ | |
9031fefd WD |
290 | mov x8, #__NR_clock_gettime |
291 | svc #0 | |
292 | ret | |
293 | .cfi_endproc | |
294 | ENDPROC(__kernel_clock_gettime) | |
295 | ||
296 | /* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */ | |
297 | ENTRY(__kernel_clock_getres) | |
298 | .cfi_startproc | |
9031fefd WD |
299 | cmp w0, #CLOCK_REALTIME |
300 | ccmp w0, #CLOCK_MONOTONIC, #0x4, ne | |
c2e0fb35 | 301 | ccmp w0, #CLOCK_MONOTONIC_RAW, #0x4, ne |
9031fefd WD |
302 | b.ne 1f |
303 | ||
304 | ldr x2, 5f | |
305 | b 2f | |
306 | 1: | |
307 | cmp w0, #CLOCK_REALTIME_COARSE | |
308 | ccmp w0, #CLOCK_MONOTONIC_COARSE, #0x4, ne | |
309 | b.ne 4f | |
310 | ldr x2, 6f | |
311 | 2: | |
67a8ab4a | 312 | cbz x1, 3f |
9031fefd WD |
313 | stp xzr, x2, [x1] |
314 | ||
315 | 3: /* res == NULL. */ | |
316 | mov w0, wzr | |
317 | ret | |
318 | ||
319 | 4: /* Syscall fallback. */ | |
320 | mov x8, #__NR_clock_getres | |
321 | svc #0 | |
322 | ret | |
323 | 5: | |
324 | .quad CLOCK_REALTIME_RES | |
325 | 6: | |
326 | .quad CLOCK_COARSE_RES | |
327 | .cfi_endproc | |
328 | ENDPROC(__kernel_clock_getres) |