Commit | Line | Data |
---|---|---|
8512287a | 1 | /* |
ba88f57f | 2 | * Userspace implementations of gettimeofday() and friends. |
8512287a | 3 | * |
ba88f57f MS |
4 | * Copyright (C) 2017 Cavium, Inc. |
5 | * Copyright (C) 2015 Mentor Graphics Corporation | |
6 | * Copyright (C) 2012 ARM Limited | |
8512287a | 7 | * |
ba88f57f MS |
8 | * This program is free software; you can redistribute it and/or modify |
9 | * it under the terms of the GNU General Public License version 2 as | |
10 | * published by the Free Software Foundation. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
8512287a NL |
16 | * |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
ba88f57f MS |
19 | * |
20 | * Author: Will Deacon <will.deacon@arm.com> | |
21 | * Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com> | |
22 | * Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com> | |
8512287a NL |
23 | */ |
24 | ||
8512287a | 25 | #include <asm/barrier.h> |
a866f568 | 26 | #include <linux/compiler.h> /* for notrace */ |
f21c33d6 MS |
27 | #include <linux/math64.h> /* for __iter_div_u64_rem() */ |
28 | #include <uapi/linux/time.h> /* for struct timespec */ | |
8512287a | 29 | |
a866f568 | 30 | #include "compiler.h" |
5441a8aa | 31 | #include "datapage.h" |
8512287a | 32 | |
a866f568 MS |
33 | DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz) |
34 | DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts) | |
e97c5515 | 35 | DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts) |
a866f568 | 36 | |
5441a8aa | 37 | static notrace u32 vdso_read_begin(const struct vdso_data *vd) |
8512287a NL |
38 | { |
39 | u32 seq; | |
40 | ||
ba88f57f | 41 | do { |
5441a8aa | 42 | seq = READ_ONCE(vd->tb_seq_count); |
ba88f57f MS |
43 | |
44 | if ((seq & 1) == 0) | |
45 | break; | |
8512287a | 46 | |
ba88f57f MS |
47 | cpu_relax(); |
48 | } while (true); | |
49 | ||
50 | smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */ | |
8512287a NL |
51 | return seq; |
52 | } | |
53 | ||
5441a8aa | 54 | static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start) |
8512287a | 55 | { |
ba88f57f MS |
56 | u32 seq; |
57 | ||
58 | smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */ | |
5441a8aa | 59 | seq = READ_ONCE(vd->tb_seq_count); |
ba88f57f | 60 | return seq != start; |
8512287a NL |
61 | } |
62 | ||
5441a8aa MS |
63 | static notrace int do_realtime_coarse(const struct vdso_data *vd, |
64 | struct timespec *ts) | |
8512287a NL |
65 | { |
66 | u32 seq; | |
67 | ||
68 | do { | |
5441a8aa | 69 | seq = vdso_read_begin(vd); |
8512287a | 70 | |
5441a8aa MS |
71 | ts->tv_sec = vd->xtime_coarse_sec; |
72 | ts->tv_nsec = vd->xtime_coarse_nsec; | |
8512287a | 73 | |
5441a8aa | 74 | } while (vdso_read_retry(vd, seq)); |
8512287a NL |
75 | |
76 | return 0; | |
77 | } | |
78 | ||
5441a8aa MS |
79 | static notrace int do_monotonic_coarse(const struct vdso_data *vd, |
80 | struct timespec *ts) | |
8512287a NL |
81 | { |
82 | struct timespec tomono; | |
83 | u32 seq; | |
f21c33d6 | 84 | u64 nsec; |
8512287a NL |
85 | |
86 | do { | |
5441a8aa | 87 | seq = vdso_read_begin(vd); |
8512287a | 88 | |
5441a8aa MS |
89 | ts->tv_sec = vd->xtime_coarse_sec; |
90 | ts->tv_nsec = vd->xtime_coarse_nsec; | |
8512287a | 91 | |
5441a8aa MS |
92 | tomono.tv_sec = vd->wtm_clock_sec; |
93 | tomono.tv_nsec = vd->wtm_clock_nsec; | |
8512287a | 94 | |
5441a8aa | 95 | } while (vdso_read_retry(vd, seq)); |
8512287a NL |
96 | |
97 | ts->tv_sec += tomono.tv_sec; | |
f21c33d6 MS |
98 | /* open coding timespec_add_ns */ |
99 | ts->tv_sec += __iter_div_u64_rem(ts->tv_nsec + tomono.tv_nsec, | |
100 | NSEC_PER_SEC, &nsec); | |
101 | ts->tv_nsec = nsec; | |
8512287a NL |
102 | |
103 | return 0; | |
104 | } | |
105 | ||
106 | #ifdef CONFIG_ARM_ARCH_TIMER | |
107 | ||
f21c33d6 MS |
108 | /* |
109 | * Returns the clock delta, in nanoseconds left-shifted by the clock | |
110 | * shift. | |
111 | */ | |
112 | static notrace u64 get_clock_shifted_nsec(const u64 cycle_last, | |
113 | const u32 mult, | |
114 | const u64 mask) | |
8512287a | 115 | { |
f21c33d6 | 116 | u64 res; |
8512287a | 117 | |
f21c33d6 MS |
118 | /* Read the virtual counter. */ |
119 | res = arch_vdso_read_counter(); | |
8512287a | 120 | |
f21c33d6 | 121 | res = res - cycle_last; |
8512287a | 122 | |
f21c33d6 MS |
123 | res &= mask; |
124 | return res * mult; | |
8512287a NL |
125 | } |
126 | ||
5441a8aa | 127 | static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts) |
8512287a | 128 | { |
f21c33d6 MS |
129 | u32 seq, mult, shift; |
130 | u64 nsec, cycle_last; | |
330b5740 MS |
131 | #ifdef ARCH_CLOCK_FIXED_MASK |
132 | static const u64 mask = ARCH_CLOCK_FIXED_MASK; | |
133 | #else | |
f21c33d6 | 134 | u64 mask; |
330b5740 | 135 | #endif |
f21c33d6 | 136 | vdso_xtime_clock_sec_t sec; |
8512287a NL |
137 | |
138 | do { | |
5441a8aa | 139 | seq = vdso_read_begin(vd); |
8512287a | 140 | |
5441a8aa | 141 | if (vd->use_syscall) |
8512287a NL |
142 | return -1; |
143 | ||
f21c33d6 | 144 | cycle_last = vd->cs_cycle_last; |
8512287a | 145 | |
f21c33d6 MS |
146 | mult = vd->cs_mono_mult; |
147 | shift = vd->cs_shift; | |
330b5740 | 148 | #ifndef ARCH_CLOCK_FIXED_MASK |
f21c33d6 | 149 | mask = vd->cs_mask; |
330b5740 | 150 | #endif |
f21c33d6 MS |
151 | |
152 | sec = vd->xtime_clock_sec; | |
153 | nsec = vd->xtime_clock_snsec; | |
154 | ||
155 | } while (unlikely(vdso_read_retry(vd, seq))); | |
8512287a | 156 | |
f21c33d6 MS |
157 | nsec += get_clock_shifted_nsec(cycle_last, mult, mask); |
158 | nsec >>= shift; | |
159 | /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */ | |
160 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
161 | ts->tv_nsec = nsec; | |
8512287a NL |
162 | |
163 | return 0; | |
164 | } | |
165 | ||
5441a8aa | 166 | static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts) |
8512287a | 167 | { |
f21c33d6 MS |
168 | u32 seq, mult, shift; |
169 | u64 nsec, cycle_last; | |
330b5740 MS |
170 | #ifdef ARCH_CLOCK_FIXED_MASK |
171 | static const u64 mask = ARCH_CLOCK_FIXED_MASK; | |
172 | #else | |
f21c33d6 | 173 | u64 mask; |
330b5740 | 174 | #endif |
f21c33d6 MS |
175 | vdso_wtm_clock_nsec_t wtm_nsec; |
176 | __kernel_time_t sec; | |
8512287a NL |
177 | |
178 | do { | |
5441a8aa | 179 | seq = vdso_read_begin(vd); |
8512287a | 180 | |
5441a8aa | 181 | if (vd->use_syscall) |
8512287a NL |
182 | return -1; |
183 | ||
f21c33d6 | 184 | cycle_last = vd->cs_cycle_last; |
8512287a | 185 | |
f21c33d6 MS |
186 | mult = vd->cs_mono_mult; |
187 | shift = vd->cs_shift; | |
330b5740 | 188 | #ifndef ARCH_CLOCK_FIXED_MASK |
f21c33d6 | 189 | mask = vd->cs_mask; |
330b5740 | 190 | #endif |
8512287a | 191 | |
f21c33d6 MS |
192 | sec = vd->xtime_clock_sec; |
193 | nsec = vd->xtime_clock_snsec; | |
8512287a | 194 | |
f21c33d6 MS |
195 | sec += vd->wtm_clock_sec; |
196 | wtm_nsec = vd->wtm_clock_nsec; | |
197 | ||
198 | } while (unlikely(vdso_read_retry(vd, seq))); | |
199 | ||
200 | nsec += get_clock_shifted_nsec(cycle_last, mult, mask); | |
201 | nsec >>= shift; | |
202 | nsec += wtm_nsec; | |
203 | /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */ | |
204 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
205 | ts->tv_nsec = nsec; | |
8512287a NL |
206 | |
207 | return 0; | |
208 | } | |
209 | ||
b484680c MS |
210 | static notrace int do_monotonic_raw(const struct vdso_data *vd, |
211 | struct timespec *ts) | |
212 | { | |
213 | u32 seq, mult, shift; | |
214 | u64 nsec, cycle_last; | |
330b5740 MS |
215 | #ifdef ARCH_CLOCK_FIXED_MASK |
216 | static const u64 mask = ARCH_CLOCK_FIXED_MASK; | |
217 | #else | |
b484680c | 218 | u64 mask; |
330b5740 | 219 | #endif |
b484680c MS |
220 | vdso_raw_time_sec_t sec; |
221 | ||
222 | do { | |
223 | seq = vdso_read_begin(vd); | |
224 | ||
225 | if (vd->use_syscall) | |
226 | return -1; | |
227 | ||
228 | cycle_last = vd->cs_cycle_last; | |
229 | ||
230 | mult = vd->cs_raw_mult; | |
231 | shift = vd->cs_shift; | |
330b5740 | 232 | #ifndef ARCH_CLOCK_FIXED_MASK |
b484680c | 233 | mask = vd->cs_mask; |
330b5740 | 234 | #endif |
b484680c MS |
235 | |
236 | sec = vd->raw_time_sec; | |
237 | nsec = vd->raw_time_nsec; | |
238 | ||
239 | } while (unlikely(vdso_read_retry(vd, seq))); | |
240 | ||
241 | nsec += get_clock_shifted_nsec(cycle_last, mult, mask); | |
242 | nsec >>= shift; | |
243 | /* open coding timespec_add_ns to save a ts->tv_nsec = 0 */ | |
244 | ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec); | |
245 | ts->tv_nsec = nsec; | |
246 | ||
247 | return 0; | |
248 | } | |
249 | ||
8512287a NL |
250 | #else /* CONFIG_ARM_ARCH_TIMER */ |
251 | ||
5441a8aa | 252 | static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts) |
8512287a NL |
253 | { |
254 | return -1; | |
255 | } | |
256 | ||
5441a8aa | 257 | static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts) |
8512287a NL |
258 | { |
259 | return -1; | |
260 | } | |
261 | ||
b484680c MS |
262 | static notrace int do_monotonic_raw(const struct vdso_data *vd, |
263 | struct timespec *ts) | |
264 | { | |
265 | return -1; | |
266 | } | |
267 | ||
8512287a NL |
268 | #endif /* CONFIG_ARM_ARCH_TIMER */ |
269 | ||
a866f568 | 270 | notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts) |
8512287a | 271 | { |
5441a8aa | 272 | const struct vdso_data *vd = __get_datapage(); |
8512287a | 273 | |
a866f568 | 274 | switch (clock) { |
8512287a | 275 | case CLOCK_REALTIME_COARSE: |
a866f568 | 276 | do_realtime_coarse(vd, ts); |
8512287a NL |
277 | break; |
278 | case CLOCK_MONOTONIC_COARSE: | |
a866f568 | 279 | do_monotonic_coarse(vd, ts); |
8512287a NL |
280 | break; |
281 | case CLOCK_REALTIME: | |
a866f568 MS |
282 | if (do_realtime(vd, ts)) |
283 | goto fallback; | |
8512287a NL |
284 | break; |
285 | case CLOCK_MONOTONIC: | |
a866f568 MS |
286 | if (do_monotonic(vd, ts)) |
287 | goto fallback; | |
8512287a | 288 | break; |
b484680c MS |
289 | case CLOCK_MONOTONIC_RAW: |
290 | if (do_monotonic_raw(vd, ts)) | |
291 | goto fallback; | |
292 | break; | |
8512287a | 293 | default: |
a866f568 | 294 | goto fallback; |
8512287a NL |
295 | } |
296 | ||
a866f568 MS |
297 | return 0; |
298 | fallback: | |
299 | return clock_gettime_fallback(clock, ts); | |
8512287a NL |
300 | } |
301 | ||
302 | notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) | |
303 | { | |
5441a8aa | 304 | const struct vdso_data *vd = __get_datapage(); |
8512287a | 305 | |
a866f568 MS |
306 | if (likely(tv != NULL)) { |
307 | struct timespec ts; | |
308 | ||
309 | if (do_realtime(vd, &ts)) | |
310 | return gettimeofday_fallback(tv, tz); | |
8512287a | 311 | |
8512287a NL |
312 | tv->tv_sec = ts.tv_sec; |
313 | tv->tv_usec = ts.tv_nsec / 1000; | |
314 | } | |
a866f568 MS |
315 | |
316 | if (unlikely(tz != NULL)) { | |
5441a8aa MS |
317 | tz->tz_minuteswest = vd->tz_minuteswest; |
318 | tz->tz_dsttime = vd->tz_dsttime; | |
8512287a NL |
319 | } |
320 | ||
a866f568 | 321 | return 0; |
8512287a | 322 | } |
e97c5515 MS |
323 | |
324 | int __vdso_clock_getres(clockid_t clock, struct timespec *res) | |
325 | { | |
326 | long nsec; | |
327 | ||
328 | if (clock == CLOCK_REALTIME || | |
329 | clock == CLOCK_MONOTONIC || | |
330 | clock == CLOCK_MONOTONIC_RAW) | |
331 | nsec = MONOTONIC_RES_NSEC; | |
332 | else if (clock == CLOCK_REALTIME_COARSE || | |
333 | clock == CLOCK_MONOTONIC_COARSE) | |
334 | nsec = LOW_RES_NSEC; | |
335 | else | |
336 | return clock_getres_fallback(clock, res); | |
337 | ||
338 | if (likely(res != NULL)) { | |
339 | res->tv_sec = 0; | |
340 | res->tv_nsec = nsec; | |
341 | } | |
342 | ||
343 | return 0; | |
344 | } |