Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/kernel/timer.c | |
3 | * | |
4 | * Kernel internal timers, kernel timekeeping, basic process system calls | |
5 | * | |
6 | * Copyright (C) 1991, 1992 Linus Torvalds | |
7 | * | |
8 | * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better. | |
9 | * | |
10 | * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 | |
11 | * "A Kernel Model for Precision Timekeeping" by Dave Mills | |
12 | * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to | |
13 | * serialize accesses to xtime/lost_ticks). | |
14 | * Copyright (C) 1998 Andrea Arcangeli | |
15 | * 1999-03-10 Improved NTP compatibility by Ulrich Windl | |
16 | * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love | |
17 | * 2000-10-05 Implemented scalable SMP per-CPU timer handling. | |
18 | * Copyright (C) 2000, 2001, 2002 Ingo Molnar | |
19 | * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar | |
20 | */ | |
21 | ||
22 | #include <linux/kernel_stat.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/interrupt.h> | |
25 | #include <linux/percpu.h> | |
26 | #include <linux/init.h> | |
27 | #include <linux/mm.h> | |
28 | #include <linux/swap.h> | |
29 | #include <linux/notifier.h> | |
30 | #include <linux/thread_info.h> | |
31 | #include <linux/time.h> | |
32 | #include <linux/jiffies.h> | |
33 | #include <linux/posix-timers.h> | |
34 | #include <linux/cpu.h> | |
35 | #include <linux/syscalls.h> | |
97a41e26 | 36 | #include <linux/delay.h> |
1da177e4 LT |
37 | |
38 | #include <asm/uaccess.h> | |
39 | #include <asm/unistd.h> | |
40 | #include <asm/div64.h> | |
41 | #include <asm/timex.h> | |
42 | #include <asm/io.h> | |
43 | ||
44 | #ifdef CONFIG_TIME_INTERPOLATION | |
45 | static void time_interpolator_update(long delta_nsec); | |
46 | #else | |
47 | #define time_interpolator_update(x) | |
48 | #endif | |
49 | ||
ecea8d19 TG |
50 | u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; |
51 | ||
52 | EXPORT_SYMBOL(jiffies_64); | |
53 | ||
1da177e4 LT |
54 | /* |
55 | * per-CPU timer vector definitions: | |
56 | */ | |
1da177e4 LT |
57 | #define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6) |
58 | #define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8) | |
59 | #define TVN_SIZE (1 << TVN_BITS) | |
60 | #define TVR_SIZE (1 << TVR_BITS) | |
61 | #define TVN_MASK (TVN_SIZE - 1) | |
62 | #define TVR_MASK (TVR_SIZE - 1) | |
63 | ||
64 | typedef struct tvec_s { | |
65 | struct list_head vec[TVN_SIZE]; | |
66 | } tvec_t; | |
67 | ||
68 | typedef struct tvec_root_s { | |
69 | struct list_head vec[TVR_SIZE]; | |
70 | } tvec_root_t; | |
71 | ||
72 | struct tvec_t_base_s { | |
3691c519 ON |
73 | spinlock_t lock; |
74 | struct timer_list *running_timer; | |
1da177e4 | 75 | unsigned long timer_jiffies; |
1da177e4 LT |
76 | tvec_root_t tv1; |
77 | tvec_t tv2; | |
78 | tvec_t tv3; | |
79 | tvec_t tv4; | |
80 | tvec_t tv5; | |
81 | } ____cacheline_aligned_in_smp; | |
82 | ||
83 | typedef struct tvec_t_base_s tvec_base_t; | |
ba6edfcd | 84 | |
3691c519 ON |
85 | tvec_base_t boot_tvec_bases; |
86 | EXPORT_SYMBOL(boot_tvec_bases); | |
ba6edfcd | 87 | static DEFINE_PER_CPU(tvec_base_t *, tvec_bases) = { &boot_tvec_bases }; |
1da177e4 LT |
88 | |
89 | static inline void set_running_timer(tvec_base_t *base, | |
90 | struct timer_list *timer) | |
91 | { | |
92 | #ifdef CONFIG_SMP | |
3691c519 | 93 | base->running_timer = timer; |
1da177e4 LT |
94 | #endif |
95 | } | |
96 | ||
1da177e4 LT |
97 | static void internal_add_timer(tvec_base_t *base, struct timer_list *timer) |
98 | { | |
99 | unsigned long expires = timer->expires; | |
100 | unsigned long idx = expires - base->timer_jiffies; | |
101 | struct list_head *vec; | |
102 | ||
103 | if (idx < TVR_SIZE) { | |
104 | int i = expires & TVR_MASK; | |
105 | vec = base->tv1.vec + i; | |
106 | } else if (idx < 1 << (TVR_BITS + TVN_BITS)) { | |
107 | int i = (expires >> TVR_BITS) & TVN_MASK; | |
108 | vec = base->tv2.vec + i; | |
109 | } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) { | |
110 | int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK; | |
111 | vec = base->tv3.vec + i; | |
112 | } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) { | |
113 | int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK; | |
114 | vec = base->tv4.vec + i; | |
115 | } else if ((signed long) idx < 0) { | |
116 | /* | |
117 | * Can happen if you add a timer with expires == jiffies, | |
118 | * or you set a timer to go off in the past | |
119 | */ | |
120 | vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK); | |
121 | } else { | |
122 | int i; | |
123 | /* If the timeout is larger than 0xffffffff on 64-bit | |
124 | * architectures then we use the maximum timeout: | |
125 | */ | |
126 | if (idx > 0xffffffffUL) { | |
127 | idx = 0xffffffffUL; | |
128 | expires = idx + base->timer_jiffies; | |
129 | } | |
130 | i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK; | |
131 | vec = base->tv5.vec + i; | |
132 | } | |
133 | /* | |
134 | * Timers are FIFO: | |
135 | */ | |
136 | list_add_tail(&timer->entry, vec); | |
137 | } | |
138 | ||
55c888d6 ON |
139 | /*** |
140 | * init_timer - initialize a timer. | |
141 | * @timer: the timer to be initialized | |
142 | * | |
143 | * init_timer() must be done to a timer prior calling *any* of the | |
144 | * other timer functions. | |
145 | */ | |
146 | void fastcall init_timer(struct timer_list *timer) | |
147 | { | |
148 | timer->entry.next = NULL; | |
bfe5d834 | 149 | timer->base = __raw_get_cpu_var(tvec_bases); |
55c888d6 ON |
150 | } |
151 | EXPORT_SYMBOL(init_timer); | |
152 | ||
153 | static inline void detach_timer(struct timer_list *timer, | |
154 | int clear_pending) | |
155 | { | |
156 | struct list_head *entry = &timer->entry; | |
157 | ||
158 | __list_del(entry->prev, entry->next); | |
159 | if (clear_pending) | |
160 | entry->next = NULL; | |
161 | entry->prev = LIST_POISON2; | |
162 | } | |
163 | ||
164 | /* | |
3691c519 | 165 | * We are using hashed locking: holding per_cpu(tvec_bases).lock |
55c888d6 ON |
166 | * means that all timers which are tied to this base via timer->base are |
167 | * locked, and the base itself is locked too. | |
168 | * | |
169 | * So __run_timers/migrate_timers can safely modify all timers which could | |
170 | * be found on ->tvX lists. | |
171 | * | |
172 | * When the timer's base is locked, and the timer removed from list, it is | |
173 | * possible to set timer->base = NULL and drop the lock: the timer remains | |
174 | * locked. | |
175 | */ | |
3691c519 | 176 | static tvec_base_t *lock_timer_base(struct timer_list *timer, |
55c888d6 ON |
177 | unsigned long *flags) |
178 | { | |
3691c519 | 179 | tvec_base_t *base; |
55c888d6 ON |
180 | |
181 | for (;;) { | |
182 | base = timer->base; | |
183 | if (likely(base != NULL)) { | |
184 | spin_lock_irqsave(&base->lock, *flags); | |
185 | if (likely(base == timer->base)) | |
186 | return base; | |
187 | /* The timer has migrated to another CPU */ | |
188 | spin_unlock_irqrestore(&base->lock, *flags); | |
189 | } | |
190 | cpu_relax(); | |
191 | } | |
192 | } | |
193 | ||
1da177e4 LT |
194 | int __mod_timer(struct timer_list *timer, unsigned long expires) |
195 | { | |
3691c519 | 196 | tvec_base_t *base, *new_base; |
1da177e4 LT |
197 | unsigned long flags; |
198 | int ret = 0; | |
199 | ||
200 | BUG_ON(!timer->function); | |
1da177e4 | 201 | |
55c888d6 ON |
202 | base = lock_timer_base(timer, &flags); |
203 | ||
204 | if (timer_pending(timer)) { | |
205 | detach_timer(timer, 0); | |
206 | ret = 1; | |
207 | } | |
208 | ||
a4a6198b | 209 | new_base = __get_cpu_var(tvec_bases); |
1da177e4 | 210 | |
3691c519 | 211 | if (base != new_base) { |
1da177e4 | 212 | /* |
55c888d6 ON |
213 | * We are trying to schedule the timer on the local CPU. |
214 | * However we can't change timer's base while it is running, | |
215 | * otherwise del_timer_sync() can't detect that the timer's | |
216 | * handler yet has not finished. This also guarantees that | |
217 | * the timer is serialized wrt itself. | |
1da177e4 | 218 | */ |
a2c348fe | 219 | if (likely(base->running_timer != timer)) { |
55c888d6 ON |
220 | /* See the comment in lock_timer_base() */ |
221 | timer->base = NULL; | |
222 | spin_unlock(&base->lock); | |
a2c348fe ON |
223 | base = new_base; |
224 | spin_lock(&base->lock); | |
225 | timer->base = base; | |
1da177e4 LT |
226 | } |
227 | } | |
228 | ||
1da177e4 | 229 | timer->expires = expires; |
a2c348fe ON |
230 | internal_add_timer(base, timer); |
231 | spin_unlock_irqrestore(&base->lock, flags); | |
1da177e4 LT |
232 | |
233 | return ret; | |
234 | } | |
235 | ||
236 | EXPORT_SYMBOL(__mod_timer); | |
237 | ||
238 | /*** | |
239 | * add_timer_on - start a timer on a particular CPU | |
240 | * @timer: the timer to be added | |
241 | * @cpu: the CPU to start it on | |
242 | * | |
243 | * This is not very scalable on SMP. Double adds are not possible. | |
244 | */ | |
245 | void add_timer_on(struct timer_list *timer, int cpu) | |
246 | { | |
a4a6198b | 247 | tvec_base_t *base = per_cpu(tvec_bases, cpu); |
1da177e4 | 248 | unsigned long flags; |
55c888d6 | 249 | |
1da177e4 | 250 | BUG_ON(timer_pending(timer) || !timer->function); |
3691c519 ON |
251 | spin_lock_irqsave(&base->lock, flags); |
252 | timer->base = base; | |
1da177e4 | 253 | internal_add_timer(base, timer); |
3691c519 | 254 | spin_unlock_irqrestore(&base->lock, flags); |
1da177e4 LT |
255 | } |
256 | ||
257 | ||
258 | /*** | |
259 | * mod_timer - modify a timer's timeout | |
260 | * @timer: the timer to be modified | |
261 | * | |
262 | * mod_timer is a more efficient way to update the expire field of an | |
263 | * active timer (if the timer is inactive it will be activated) | |
264 | * | |
265 | * mod_timer(timer, expires) is equivalent to: | |
266 | * | |
267 | * del_timer(timer); timer->expires = expires; add_timer(timer); | |
268 | * | |
269 | * Note that if there are multiple unserialized concurrent users of the | |
270 | * same timer, then mod_timer() is the only safe way to modify the timeout, | |
271 | * since add_timer() cannot modify an already running timer. | |
272 | * | |
273 | * The function returns whether it has modified a pending timer or not. | |
274 | * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an | |
275 | * active timer returns 1.) | |
276 | */ | |
277 | int mod_timer(struct timer_list *timer, unsigned long expires) | |
278 | { | |
279 | BUG_ON(!timer->function); | |
280 | ||
1da177e4 LT |
281 | /* |
282 | * This is a common optimization triggered by the | |
283 | * networking code - if the timer is re-modified | |
284 | * to be the same thing then just return: | |
285 | */ | |
286 | if (timer->expires == expires && timer_pending(timer)) | |
287 | return 1; | |
288 | ||
289 | return __mod_timer(timer, expires); | |
290 | } | |
291 | ||
292 | EXPORT_SYMBOL(mod_timer); | |
293 | ||
294 | /*** | |
295 | * del_timer - deactive a timer. | |
296 | * @timer: the timer to be deactivated | |
297 | * | |
298 | * del_timer() deactivates a timer - this works on both active and inactive | |
299 | * timers. | |
300 | * | |
301 | * The function returns whether it has deactivated a pending timer or not. | |
302 | * (ie. del_timer() of an inactive timer returns 0, del_timer() of an | |
303 | * active timer returns 1.) | |
304 | */ | |
305 | int del_timer(struct timer_list *timer) | |
306 | { | |
3691c519 | 307 | tvec_base_t *base; |
1da177e4 | 308 | unsigned long flags; |
55c888d6 | 309 | int ret = 0; |
1da177e4 | 310 | |
55c888d6 ON |
311 | if (timer_pending(timer)) { |
312 | base = lock_timer_base(timer, &flags); | |
313 | if (timer_pending(timer)) { | |
314 | detach_timer(timer, 1); | |
315 | ret = 1; | |
316 | } | |
1da177e4 | 317 | spin_unlock_irqrestore(&base->lock, flags); |
1da177e4 | 318 | } |
1da177e4 | 319 | |
55c888d6 | 320 | return ret; |
1da177e4 LT |
321 | } |
322 | ||
323 | EXPORT_SYMBOL(del_timer); | |
324 | ||
325 | #ifdef CONFIG_SMP | |
fd450b73 ON |
326 | /* |
327 | * This function tries to deactivate a timer. Upon successful (ret >= 0) | |
328 | * exit the timer is not queued and the handler is not running on any CPU. | |
329 | * | |
330 | * It must not be called from interrupt contexts. | |
331 | */ | |
332 | int try_to_del_timer_sync(struct timer_list *timer) | |
333 | { | |
3691c519 | 334 | tvec_base_t *base; |
fd450b73 ON |
335 | unsigned long flags; |
336 | int ret = -1; | |
337 | ||
338 | base = lock_timer_base(timer, &flags); | |
339 | ||
340 | if (base->running_timer == timer) | |
341 | goto out; | |
342 | ||
343 | ret = 0; | |
344 | if (timer_pending(timer)) { | |
345 | detach_timer(timer, 1); | |
346 | ret = 1; | |
347 | } | |
348 | out: | |
349 | spin_unlock_irqrestore(&base->lock, flags); | |
350 | ||
351 | return ret; | |
352 | } | |
353 | ||
1da177e4 LT |
354 | /*** |
355 | * del_timer_sync - deactivate a timer and wait for the handler to finish. | |
356 | * @timer: the timer to be deactivated | |
357 | * | |
358 | * This function only differs from del_timer() on SMP: besides deactivating | |
359 | * the timer it also makes sure the handler has finished executing on other | |
360 | * CPUs. | |
361 | * | |
362 | * Synchronization rules: callers must prevent restarting of the timer, | |
363 | * otherwise this function is meaningless. It must not be called from | |
364 | * interrupt contexts. The caller must not hold locks which would prevent | |
55c888d6 ON |
365 | * completion of the timer's handler. The timer's handler must not call |
366 | * add_timer_on(). Upon exit the timer is not queued and the handler is | |
367 | * not running on any CPU. | |
1da177e4 LT |
368 | * |
369 | * The function returns whether it has deactivated a pending timer or not. | |
1da177e4 LT |
370 | */ |
371 | int del_timer_sync(struct timer_list *timer) | |
372 | { | |
fd450b73 ON |
373 | for (;;) { |
374 | int ret = try_to_del_timer_sync(timer); | |
375 | if (ret >= 0) | |
376 | return ret; | |
a0009652 | 377 | cpu_relax(); |
fd450b73 | 378 | } |
1da177e4 | 379 | } |
1da177e4 | 380 | |
55c888d6 | 381 | EXPORT_SYMBOL(del_timer_sync); |
1da177e4 LT |
382 | #endif |
383 | ||
384 | static int cascade(tvec_base_t *base, tvec_t *tv, int index) | |
385 | { | |
386 | /* cascade all the timers from tv up one level */ | |
3439dd86 P |
387 | struct timer_list *timer, *tmp; |
388 | struct list_head tv_list; | |
389 | ||
390 | list_replace_init(tv->vec + index, &tv_list); | |
1da177e4 | 391 | |
1da177e4 | 392 | /* |
3439dd86 P |
393 | * We are removing _all_ timers from the list, so we |
394 | * don't have to detach them individually. | |
1da177e4 | 395 | */ |
3439dd86 P |
396 | list_for_each_entry_safe(timer, tmp, &tv_list, entry) { |
397 | BUG_ON(timer->base != base); | |
398 | internal_add_timer(base, timer); | |
1da177e4 | 399 | } |
1da177e4 LT |
400 | |
401 | return index; | |
402 | } | |
403 | ||
404 | /*** | |
405 | * __run_timers - run all expired timers (if any) on this CPU. | |
406 | * @base: the timer vector to be processed. | |
407 | * | |
408 | * This function cascades all vectors and executes all expired timer | |
409 | * vectors. | |
410 | */ | |
411 | #define INDEX(N) (base->timer_jiffies >> (TVR_BITS + N * TVN_BITS)) & TVN_MASK | |
412 | ||
413 | static inline void __run_timers(tvec_base_t *base) | |
414 | { | |
415 | struct timer_list *timer; | |
416 | ||
3691c519 | 417 | spin_lock_irq(&base->lock); |
1da177e4 | 418 | while (time_after_eq(jiffies, base->timer_jiffies)) { |
626ab0e6 | 419 | struct list_head work_list; |
1da177e4 LT |
420 | struct list_head *head = &work_list; |
421 | int index = base->timer_jiffies & TVR_MASK; | |
626ab0e6 | 422 | |
1da177e4 LT |
423 | /* |
424 | * Cascade timers: | |
425 | */ | |
426 | if (!index && | |
427 | (!cascade(base, &base->tv2, INDEX(0))) && | |
428 | (!cascade(base, &base->tv3, INDEX(1))) && | |
429 | !cascade(base, &base->tv4, INDEX(2))) | |
430 | cascade(base, &base->tv5, INDEX(3)); | |
626ab0e6 ON |
431 | ++base->timer_jiffies; |
432 | list_replace_init(base->tv1.vec + index, &work_list); | |
55c888d6 | 433 | while (!list_empty(head)) { |
1da177e4 LT |
434 | void (*fn)(unsigned long); |
435 | unsigned long data; | |
436 | ||
437 | timer = list_entry(head->next,struct timer_list,entry); | |
438 | fn = timer->function; | |
439 | data = timer->data; | |
440 | ||
1da177e4 | 441 | set_running_timer(base, timer); |
55c888d6 | 442 | detach_timer(timer, 1); |
3691c519 | 443 | spin_unlock_irq(&base->lock); |
1da177e4 | 444 | { |
be5b4fbd | 445 | int preempt_count = preempt_count(); |
1da177e4 LT |
446 | fn(data); |
447 | if (preempt_count != preempt_count()) { | |
be5b4fbd JJ |
448 | printk(KERN_WARNING "huh, entered %p " |
449 | "with preempt_count %08x, exited" | |
450 | " with %08x?\n", | |
451 | fn, preempt_count, | |
452 | preempt_count()); | |
1da177e4 LT |
453 | BUG(); |
454 | } | |
455 | } | |
3691c519 | 456 | spin_lock_irq(&base->lock); |
1da177e4 LT |
457 | } |
458 | } | |
459 | set_running_timer(base, NULL); | |
3691c519 | 460 | spin_unlock_irq(&base->lock); |
1da177e4 LT |
461 | } |
462 | ||
463 | #ifdef CONFIG_NO_IDLE_HZ | |
464 | /* | |
465 | * Find out when the next timer event is due to happen. This | |
466 | * is used on S/390 to stop all activity when a cpus is idle. | |
467 | * This functions needs to be called disabled. | |
468 | */ | |
469 | unsigned long next_timer_interrupt(void) | |
470 | { | |
471 | tvec_base_t *base; | |
472 | struct list_head *list; | |
473 | struct timer_list *nte; | |
474 | unsigned long expires; | |
69239749 TL |
475 | unsigned long hr_expires = MAX_JIFFY_OFFSET; |
476 | ktime_t hr_delta; | |
1da177e4 LT |
477 | tvec_t *varray[4]; |
478 | int i, j; | |
479 | ||
69239749 TL |
480 | hr_delta = hrtimer_get_next_event(); |
481 | if (hr_delta.tv64 != KTIME_MAX) { | |
482 | struct timespec tsdelta; | |
483 | tsdelta = ktime_to_timespec(hr_delta); | |
484 | hr_expires = timespec_to_jiffies(&tsdelta); | |
485 | if (hr_expires < 3) | |
486 | return hr_expires + jiffies; | |
487 | } | |
488 | hr_expires += jiffies; | |
489 | ||
a4a6198b | 490 | base = __get_cpu_var(tvec_bases); |
3691c519 | 491 | spin_lock(&base->lock); |
1da177e4 | 492 | expires = base->timer_jiffies + (LONG_MAX >> 1); |
53f087fe | 493 | list = NULL; |
1da177e4 LT |
494 | |
495 | /* Look for timer events in tv1. */ | |
496 | j = base->timer_jiffies & TVR_MASK; | |
497 | do { | |
498 | list_for_each_entry(nte, base->tv1.vec + j, entry) { | |
499 | expires = nte->expires; | |
500 | if (j < (base->timer_jiffies & TVR_MASK)) | |
501 | list = base->tv2.vec + (INDEX(0)); | |
502 | goto found; | |
503 | } | |
504 | j = (j + 1) & TVR_MASK; | |
505 | } while (j != (base->timer_jiffies & TVR_MASK)); | |
506 | ||
507 | /* Check tv2-tv5. */ | |
508 | varray[0] = &base->tv2; | |
509 | varray[1] = &base->tv3; | |
510 | varray[2] = &base->tv4; | |
511 | varray[3] = &base->tv5; | |
512 | for (i = 0; i < 4; i++) { | |
513 | j = INDEX(i); | |
514 | do { | |
515 | if (list_empty(varray[i]->vec + j)) { | |
516 | j = (j + 1) & TVN_MASK; | |
517 | continue; | |
518 | } | |
519 | list_for_each_entry(nte, varray[i]->vec + j, entry) | |
520 | if (time_before(nte->expires, expires)) | |
521 | expires = nte->expires; | |
522 | if (j < (INDEX(i)) && i < 3) | |
523 | list = varray[i + 1]->vec + (INDEX(i + 1)); | |
524 | goto found; | |
525 | } while (j != (INDEX(i))); | |
526 | } | |
527 | found: | |
528 | if (list) { | |
529 | /* | |
530 | * The search wrapped. We need to look at the next list | |
531 | * from next tv element that would cascade into tv element | |
532 | * where we found the timer element. | |
533 | */ | |
534 | list_for_each_entry(nte, list, entry) { | |
535 | if (time_before(nte->expires, expires)) | |
536 | expires = nte->expires; | |
537 | } | |
538 | } | |
3691c519 | 539 | spin_unlock(&base->lock); |
69239749 | 540 | |
0662b713 ZA |
541 | /* |
542 | * It can happen that other CPUs service timer IRQs and increment | |
543 | * jiffies, but we have not yet got a local timer tick to process | |
544 | * the timer wheels. In that case, the expiry time can be before | |
545 | * jiffies, but since the high-resolution timer here is relative to | |
546 | * jiffies, the default expression when high-resolution timers are | |
547 | * not active, | |
548 | * | |
549 | * time_before(MAX_JIFFY_OFFSET + jiffies, expires) | |
550 | * | |
551 | * would falsely evaluate to true. If that is the case, just | |
552 | * return jiffies so that we can immediately fire the local timer | |
553 | */ | |
554 | if (time_before(expires, jiffies)) | |
555 | return jiffies; | |
556 | ||
69239749 TL |
557 | if (time_before(hr_expires, expires)) |
558 | return hr_expires; | |
559 | ||
1da177e4 LT |
560 | return expires; |
561 | } | |
562 | #endif | |
563 | ||
564 | /******************************************************************/ | |
565 | ||
566 | /* | |
567 | * Timekeeping variables | |
568 | */ | |
569 | unsigned long tick_usec = TICK_USEC; /* USER_HZ period (usec) */ | |
570 | unsigned long tick_nsec = TICK_NSEC; /* ACTHZ period (nsec) */ | |
571 | ||
572 | /* | |
573 | * The current time | |
574 | * wall_to_monotonic is what we need to add to xtime (or xtime corrected | |
575 | * for sub jiffie times) to get to monotonic time. Monotonic is pegged | |
576 | * at zero at system boot time, so wall_to_monotonic will be negative, | |
577 | * however, we will ALWAYS keep the tv_nsec part positive so we can use | |
578 | * the usual normalization. | |
579 | */ | |
580 | struct timespec xtime __attribute__ ((aligned (16))); | |
581 | struct timespec wall_to_monotonic __attribute__ ((aligned (16))); | |
582 | ||
583 | EXPORT_SYMBOL(xtime); | |
584 | ||
585 | /* Don't completely fail for HZ > 500. */ | |
586 | int tickadj = 500/HZ ? : 1; /* microsecs */ | |
587 | ||
588 | ||
589 | /* | |
590 | * phase-lock loop variables | |
591 | */ | |
592 | /* TIME_ERROR prevents overwriting the CMOS clock */ | |
593 | int time_state = TIME_OK; /* clock synchronization status */ | |
594 | int time_status = STA_UNSYNC; /* clock status bits */ | |
595 | long time_offset; /* time adjustment (us) */ | |
596 | long time_constant = 2; /* pll time constant */ | |
597 | long time_tolerance = MAXFREQ; /* frequency tolerance (ppm) */ | |
598 | long time_precision = 1; /* clock precision (us) */ | |
599 | long time_maxerror = NTP_PHASE_LIMIT; /* maximum error (us) */ | |
600 | long time_esterror = NTP_PHASE_LIMIT; /* estimated error (us) */ | |
1da177e4 LT |
601 | long time_freq = (((NSEC_PER_SEC + HZ/2) % HZ - HZ/2) << SHIFT_USEC) / NSEC_PER_USEC; |
602 | /* frequency offset (scaled ppm)*/ | |
603 | static long time_adj; /* tick adjust (scaled 1 / HZ) */ | |
604 | long time_reftime; /* time at last adjustment (s) */ | |
605 | long time_adjust; | |
606 | long time_next_adjust; | |
607 | ||
608 | /* | |
609 | * this routine handles the overflow of the microsecond field | |
610 | * | |
611 | * The tricky bits of code to handle the accurate clock support | |
612 | * were provided by Dave Mills (Mills@UDEL.EDU) of NTP fame. | |
613 | * They were originally developed for SUN and DEC kernels. | |
614 | * All the kudos should go to Dave for this stuff. | |
615 | * | |
616 | */ | |
617 | static void second_overflow(void) | |
618 | { | |
a5a0d52c AM |
619 | long ltemp; |
620 | ||
621 | /* Bump the maxerror field */ | |
622 | time_maxerror += time_tolerance >> SHIFT_USEC; | |
623 | if (time_maxerror > NTP_PHASE_LIMIT) { | |
624 | time_maxerror = NTP_PHASE_LIMIT; | |
625 | time_status |= STA_UNSYNC; | |
1da177e4 | 626 | } |
a5a0d52c AM |
627 | |
628 | /* | |
629 | * Leap second processing. If in leap-insert state at the end of the | |
630 | * day, the system clock is set back one second; if in leap-delete | |
631 | * state, the system clock is set ahead one second. The microtime() | |
632 | * routine or external clock driver will insure that reported time is | |
633 | * always monotonic. The ugly divides should be replaced. | |
634 | */ | |
635 | switch (time_state) { | |
636 | case TIME_OK: | |
637 | if (time_status & STA_INS) | |
638 | time_state = TIME_INS; | |
639 | else if (time_status & STA_DEL) | |
640 | time_state = TIME_DEL; | |
641 | break; | |
642 | case TIME_INS: | |
643 | if (xtime.tv_sec % 86400 == 0) { | |
644 | xtime.tv_sec--; | |
645 | wall_to_monotonic.tv_sec++; | |
646 | /* | |
647 | * The timer interpolator will make time change | |
648 | * gradually instead of an immediate jump by one second | |
649 | */ | |
650 | time_interpolator_update(-NSEC_PER_SEC); | |
651 | time_state = TIME_OOP; | |
652 | clock_was_set(); | |
653 | printk(KERN_NOTICE "Clock: inserting leap second " | |
654 | "23:59:60 UTC\n"); | |
655 | } | |
656 | break; | |
657 | case TIME_DEL: | |
658 | if ((xtime.tv_sec + 1) % 86400 == 0) { | |
659 | xtime.tv_sec++; | |
660 | wall_to_monotonic.tv_sec--; | |
661 | /* | |
662 | * Use of time interpolator for a gradual change of | |
663 | * time | |
664 | */ | |
665 | time_interpolator_update(NSEC_PER_SEC); | |
666 | time_state = TIME_WAIT; | |
667 | clock_was_set(); | |
668 | printk(KERN_NOTICE "Clock: deleting leap second " | |
669 | "23:59:59 UTC\n"); | |
670 | } | |
671 | break; | |
672 | case TIME_OOP: | |
673 | time_state = TIME_WAIT; | |
674 | break; | |
675 | case TIME_WAIT: | |
676 | if (!(time_status & (STA_INS | STA_DEL))) | |
677 | time_state = TIME_OK; | |
1da177e4 | 678 | } |
a5a0d52c AM |
679 | |
680 | /* | |
681 | * Compute the phase adjustment for the next second. In PLL mode, the | |
682 | * offset is reduced by a fixed factor times the time constant. In FLL | |
683 | * mode the offset is used directly. In either mode, the maximum phase | |
684 | * adjustment for each second is clamped so as to spread the adjustment | |
685 | * over not more than the number of seconds between updates. | |
686 | */ | |
1da177e4 LT |
687 | ltemp = time_offset; |
688 | if (!(time_status & STA_FLL)) | |
1bb34a41 | 689 | ltemp = shift_right(ltemp, SHIFT_KG + time_constant); |
690 | ltemp = min(ltemp, (MAXPHASE / MINSEC) << SHIFT_UPDATE); | |
691 | ltemp = max(ltemp, -(MAXPHASE / MINSEC) << SHIFT_UPDATE); | |
1da177e4 LT |
692 | time_offset -= ltemp; |
693 | time_adj = ltemp << (SHIFT_SCALE - SHIFT_HZ - SHIFT_UPDATE); | |
1da177e4 | 694 | |
a5a0d52c AM |
695 | /* |
696 | * Compute the frequency estimate and additional phase adjustment due | |
5ddcfa87 | 697 | * to frequency error for the next second. |
a5a0d52c | 698 | */ |
5ddcfa87 | 699 | ltemp = time_freq; |
a5a0d52c | 700 | time_adj += shift_right(ltemp,(SHIFT_USEC + SHIFT_HZ - SHIFT_SCALE)); |
1da177e4 LT |
701 | |
702 | #if HZ == 100 | |
a5a0d52c AM |
703 | /* |
704 | * Compensate for (HZ==100) != (1 << SHIFT_HZ). Add 25% and 3.125% to | |
705 | * get 128.125; => only 0.125% error (p. 14) | |
706 | */ | |
707 | time_adj += shift_right(time_adj, 2) + shift_right(time_adj, 5); | |
1da177e4 | 708 | #endif |
4b8f573b | 709 | #if HZ == 250 |
a5a0d52c AM |
710 | /* |
711 | * Compensate for (HZ==250) != (1 << SHIFT_HZ). Add 1.5625% and | |
712 | * 0.78125% to get 255.85938; => only 0.05% error (p. 14) | |
713 | */ | |
714 | time_adj += shift_right(time_adj, 6) + shift_right(time_adj, 7); | |
4b8f573b | 715 | #endif |
1da177e4 | 716 | #if HZ == 1000 |
a5a0d52c AM |
717 | /* |
718 | * Compensate for (HZ==1000) != (1 << SHIFT_HZ). Add 1.5625% and | |
719 | * 0.78125% to get 1023.4375; => only 0.05% error (p. 14) | |
720 | */ | |
721 | time_adj += shift_right(time_adj, 6) + shift_right(time_adj, 7); | |
1da177e4 LT |
722 | #endif |
723 | } | |
724 | ||
726c14bf PM |
725 | /* |
726 | * Returns how many microseconds we need to add to xtime this tick | |
727 | * in doing an adjustment requested with adjtime. | |
728 | */ | |
729 | static long adjtime_adjustment(void) | |
1da177e4 | 730 | { |
726c14bf | 731 | long time_adjust_step; |
1da177e4 | 732 | |
726c14bf PM |
733 | time_adjust_step = time_adjust; |
734 | if (time_adjust_step) { | |
a5a0d52c AM |
735 | /* |
736 | * We are doing an adjtime thing. Prepare time_adjust_step to | |
737 | * be within bounds. Note that a positive time_adjust means we | |
738 | * want the clock to run faster. | |
739 | * | |
740 | * Limit the amount of the step to be in the range | |
741 | * -tickadj .. +tickadj | |
742 | */ | |
743 | time_adjust_step = min(time_adjust_step, (long)tickadj); | |
744 | time_adjust_step = max(time_adjust_step, (long)-tickadj); | |
726c14bf PM |
745 | } |
746 | return time_adjust_step; | |
747 | } | |
a5a0d52c | 748 | |
726c14bf | 749 | /* in the NTP reference this is called "hardclock()" */ |
5eb6d205 | 750 | static void update_ntp_one_tick(void) |
726c14bf | 751 | { |
5eb6d205 | 752 | long time_adjust_step; |
726c14bf PM |
753 | |
754 | time_adjust_step = adjtime_adjustment(); | |
755 | if (time_adjust_step) | |
a5a0d52c AM |
756 | /* Reduce by this step the amount of time left */ |
757 | time_adjust -= time_adjust_step; | |
1da177e4 LT |
758 | |
759 | /* Changes by adjtime() do not take effect till next tick. */ | |
760 | if (time_next_adjust != 0) { | |
761 | time_adjust = time_next_adjust; | |
762 | time_next_adjust = 0; | |
763 | } | |
764 | } | |
765 | ||
726c14bf PM |
766 | /* |
767 | * Return how long ticks are at the moment, that is, how much time | |
768 | * update_wall_time_one_tick will add to xtime next time we call it | |
769 | * (assuming no calls to do_adjtimex in the meantime). | |
260a4230 | 770 | * The return value is in fixed-point nanoseconds shifted by the |
771 | * specified number of bits to the right of the binary point. | |
726c14bf PM |
772 | * This function has no side-effects. |
773 | */ | |
19923c19 | 774 | u64 current_tick_length(void) |
726c14bf PM |
775 | { |
776 | long delta_nsec; | |
260a4230 | 777 | u64 ret; |
726c14bf | 778 | |
260a4230 | 779 | /* calculate the finest interval NTP will allow. |
780 | * ie: nanosecond value shifted by (SHIFT_SCALE - 10) | |
781 | */ | |
726c14bf | 782 | delta_nsec = tick_nsec + adjtime_adjustment() * 1000; |
19923c19 RZ |
783 | ret = (u64)delta_nsec << TICK_LENGTH_SHIFT; |
784 | ret += (s64)time_adj << (TICK_LENGTH_SHIFT - (SHIFT_SCALE - 10)); | |
260a4230 | 785 | |
786 | return ret; | |
726c14bf PM |
787 | } |
788 | ||
ad596171 | 789 | /* XXX - all of this timekeeping code should be later moved to time.c */ |
790 | #include <linux/clocksource.h> | |
791 | static struct clocksource *clock; /* pointer to current clocksource */ | |
cf3c769b | 792 | |
793 | #ifdef CONFIG_GENERIC_TIME | |
794 | /** | |
795 | * __get_nsec_offset - Returns nanoseconds since last call to periodic_hook | |
796 | * | |
797 | * private function, must hold xtime_lock lock when being | |
798 | * called. Returns the number of nanoseconds since the | |
799 | * last call to update_wall_time() (adjusted by NTP scaling) | |
800 | */ | |
801 | static inline s64 __get_nsec_offset(void) | |
802 | { | |
803 | cycle_t cycle_now, cycle_delta; | |
804 | s64 ns_offset; | |
805 | ||
806 | /* read clocksource: */ | |
a2752549 | 807 | cycle_now = clocksource_read(clock); |
cf3c769b | 808 | |
809 | /* calculate the delta since the last update_wall_time: */ | |
19923c19 | 810 | cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; |
cf3c769b | 811 | |
812 | /* convert to nanoseconds: */ | |
813 | ns_offset = cyc2ns(clock, cycle_delta); | |
814 | ||
815 | return ns_offset; | |
816 | } | |
817 | ||
818 | /** | |
819 | * __get_realtime_clock_ts - Returns the time of day in a timespec | |
820 | * @ts: pointer to the timespec to be set | |
821 | * | |
822 | * Returns the time of day in a timespec. Used by | |
823 | * do_gettimeofday() and get_realtime_clock_ts(). | |
824 | */ | |
825 | static inline void __get_realtime_clock_ts(struct timespec *ts) | |
826 | { | |
827 | unsigned long seq; | |
828 | s64 nsecs; | |
829 | ||
830 | do { | |
831 | seq = read_seqbegin(&xtime_lock); | |
832 | ||
833 | *ts = xtime; | |
834 | nsecs = __get_nsec_offset(); | |
835 | ||
836 | } while (read_seqretry(&xtime_lock, seq)); | |
837 | ||
838 | timespec_add_ns(ts, nsecs); | |
839 | } | |
840 | ||
841 | /** | |
a2752549 | 842 | * getnstimeofday - Returns the time of day in a timespec |
cf3c769b | 843 | * @ts: pointer to the timespec to be set |
844 | * | |
845 | * Returns the time of day in a timespec. | |
846 | */ | |
847 | void getnstimeofday(struct timespec *ts) | |
848 | { | |
849 | __get_realtime_clock_ts(ts); | |
850 | } | |
851 | ||
852 | EXPORT_SYMBOL(getnstimeofday); | |
853 | ||
854 | /** | |
855 | * do_gettimeofday - Returns the time of day in a timeval | |
856 | * @tv: pointer to the timeval to be set | |
857 | * | |
858 | * NOTE: Users should be converted to using get_realtime_clock_ts() | |
859 | */ | |
860 | void do_gettimeofday(struct timeval *tv) | |
861 | { | |
862 | struct timespec now; | |
863 | ||
864 | __get_realtime_clock_ts(&now); | |
865 | tv->tv_sec = now.tv_sec; | |
866 | tv->tv_usec = now.tv_nsec/1000; | |
867 | } | |
868 | ||
869 | EXPORT_SYMBOL(do_gettimeofday); | |
870 | /** | |
871 | * do_settimeofday - Sets the time of day | |
872 | * @tv: pointer to the timespec variable containing the new time | |
873 | * | |
874 | * Sets the time of day to the new time and update NTP and notify hrtimers | |
875 | */ | |
876 | int do_settimeofday(struct timespec *tv) | |
877 | { | |
878 | unsigned long flags; | |
879 | time_t wtm_sec, sec = tv->tv_sec; | |
880 | long wtm_nsec, nsec = tv->tv_nsec; | |
881 | ||
882 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | |
883 | return -EINVAL; | |
884 | ||
885 | write_seqlock_irqsave(&xtime_lock, flags); | |
886 | ||
887 | nsec -= __get_nsec_offset(); | |
888 | ||
889 | wtm_sec = wall_to_monotonic.tv_sec + (xtime.tv_sec - sec); | |
890 | wtm_nsec = wall_to_monotonic.tv_nsec + (xtime.tv_nsec - nsec); | |
891 | ||
892 | set_normalized_timespec(&xtime, sec, nsec); | |
893 | set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec); | |
894 | ||
e154ff3d | 895 | clock->error = 0; |
cf3c769b | 896 | ntp_clear(); |
897 | ||
898 | write_sequnlock_irqrestore(&xtime_lock, flags); | |
899 | ||
900 | /* signal hrtimers about time change */ | |
901 | clock_was_set(); | |
902 | ||
903 | return 0; | |
904 | } | |
905 | ||
906 | EXPORT_SYMBOL(do_settimeofday); | |
907 | ||
908 | /** | |
909 | * change_clocksource - Swaps clocksources if a new one is available | |
910 | * | |
911 | * Accumulates current time interval and initializes new clocksource | |
912 | */ | |
913 | static int change_clocksource(void) | |
914 | { | |
915 | struct clocksource *new; | |
916 | cycle_t now; | |
917 | u64 nsec; | |
a2752549 | 918 | new = clocksource_get_next(); |
cf3c769b | 919 | if (clock != new) { |
a2752549 | 920 | now = clocksource_read(new); |
cf3c769b | 921 | nsec = __get_nsec_offset(); |
922 | timespec_add_ns(&xtime, nsec); | |
923 | ||
924 | clock = new; | |
19923c19 | 925 | clock->cycle_last = now; |
cf3c769b | 926 | printk(KERN_INFO "Time: %s clocksource has been installed.\n", |
927 | clock->name); | |
928 | return 1; | |
929 | } else if (clock->update_callback) { | |
930 | return clock->update_callback(); | |
931 | } | |
932 | return 0; | |
933 | } | |
934 | #else | |
935 | #define change_clocksource() (0) | |
936 | #endif | |
937 | ||
938 | /** | |
939 | * timeofday_is_continuous - check to see if timekeeping is free running | |
940 | */ | |
941 | int timekeeping_is_continuous(void) | |
942 | { | |
943 | unsigned long seq; | |
944 | int ret; | |
945 | ||
946 | do { | |
947 | seq = read_seqbegin(&xtime_lock); | |
948 | ||
949 | ret = clock->is_continuous; | |
950 | ||
951 | } while (read_seqretry(&xtime_lock, seq)); | |
952 | ||
953 | return ret; | |
954 | } | |
955 | ||
1da177e4 | 956 | /* |
ad596171 | 957 | * timekeeping_init - Initializes the clocksource and common timekeeping values |
1da177e4 | 958 | */ |
ad596171 | 959 | void __init timekeeping_init(void) |
1da177e4 | 960 | { |
ad596171 | 961 | unsigned long flags; |
962 | ||
963 | write_seqlock_irqsave(&xtime_lock, flags); | |
a2752549 | 964 | clock = clocksource_get_next(); |
965 | clocksource_calculate_interval(clock, tick_nsec); | |
19923c19 | 966 | clock->cycle_last = clocksource_read(clock); |
ad596171 | 967 | ntp_clear(); |
968 | write_sequnlock_irqrestore(&xtime_lock, flags); | |
969 | } | |
970 | ||
971 | ||
3e143475 | 972 | static int timekeeping_suspended; |
ad596171 | 973 | /* |
974 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | |
975 | * @dev: unused | |
976 | * | |
977 | * This is for the generic clocksource timekeeping. | |
978 | * xtime/wall_to_monotonic/jiffies/wall_jiffies/etc are | |
979 | * still managed by arch specific suspend/resume code. | |
980 | */ | |
981 | static int timekeeping_resume(struct sys_device *dev) | |
982 | { | |
983 | unsigned long flags; | |
984 | ||
985 | write_seqlock_irqsave(&xtime_lock, flags); | |
986 | /* restart the last cycle value */ | |
19923c19 | 987 | clock->cycle_last = clocksource_read(clock); |
3e143475 | 988 | clock->error = 0; |
989 | timekeeping_suspended = 0; | |
990 | write_sequnlock_irqrestore(&xtime_lock, flags); | |
991 | return 0; | |
992 | } | |
993 | ||
994 | static int timekeeping_suspend(struct sys_device *dev, pm_message_t state) | |
995 | { | |
996 | unsigned long flags; | |
997 | ||
998 | write_seqlock_irqsave(&xtime_lock, flags); | |
999 | timekeeping_suspended = 1; | |
ad596171 | 1000 | write_sequnlock_irqrestore(&xtime_lock, flags); |
1001 | return 0; | |
1002 | } | |
1003 | ||
1004 | /* sysfs resume/suspend bits for timekeeping */ | |
1005 | static struct sysdev_class timekeeping_sysclass = { | |
1006 | .resume = timekeeping_resume, | |
3e143475 | 1007 | .suspend = timekeeping_suspend, |
ad596171 | 1008 | set_kset_name("timekeeping"), |
1009 | }; | |
1010 | ||
1011 | static struct sys_device device_timer = { | |
1012 | .id = 0, | |
1013 | .cls = &timekeeping_sysclass, | |
1014 | }; | |
1015 | ||
1016 | static int __init timekeeping_init_device(void) | |
1017 | { | |
1018 | int error = sysdev_class_register(&timekeeping_sysclass); | |
1019 | if (!error) | |
1020 | error = sysdev_register(&device_timer); | |
1021 | return error; | |
1022 | } | |
1023 | ||
1024 | device_initcall(timekeeping_init_device); | |
1025 | ||
19923c19 | 1026 | /* |
e154ff3d | 1027 | * If the error is already larger, we look ahead even further |
19923c19 RZ |
1028 | * to compensate for late or lost adjustments. |
1029 | */ | |
e154ff3d | 1030 | static __always_inline int clocksource_bigadjust(s64 error, s64 *interval, s64 *offset) |
19923c19 | 1031 | { |
e154ff3d RZ |
1032 | s64 tick_error, i; |
1033 | u32 look_ahead, adj; | |
1034 | s32 error2, mult; | |
19923c19 RZ |
1035 | |
1036 | /* | |
e154ff3d RZ |
1037 | * Use the current error value to determine how much to look ahead. |
1038 | * The larger the error the slower we adjust for it to avoid problems | |
1039 | * with losing too many ticks, otherwise we would overadjust and | |
1040 | * produce an even larger error. The smaller the adjustment the | |
1041 | * faster we try to adjust for it, as lost ticks can do less harm | |
1042 | * here. This is tuned so that an error of about 1 msec is adusted | |
1043 | * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks). | |
19923c19 | 1044 | */ |
e154ff3d RZ |
1045 | error2 = clock->error >> (TICK_LENGTH_SHIFT + 22 - 2 * SHIFT_HZ); |
1046 | error2 = abs(error2); | |
1047 | for (look_ahead = 0; error2 > 0; look_ahead++) | |
1048 | error2 >>= 2; | |
19923c19 RZ |
1049 | |
1050 | /* | |
e154ff3d RZ |
1051 | * Now calculate the error in (1 << look_ahead) ticks, but first |
1052 | * remove the single look ahead already included in the error. | |
19923c19 | 1053 | */ |
e154ff3d RZ |
1054 | tick_error = current_tick_length() >> (TICK_LENGTH_SHIFT - clock->shift + 1); |
1055 | tick_error -= clock->xtime_interval >> 1; | |
1056 | error = ((error - tick_error) >> look_ahead) + tick_error; | |
1057 | ||
1058 | /* Finally calculate the adjustment shift value. */ | |
1059 | i = *interval; | |
1060 | mult = 1; | |
1061 | if (error < 0) { | |
1062 | error = -error; | |
1063 | *interval = -*interval; | |
1064 | *offset = -*offset; | |
1065 | mult = -1; | |
19923c19 | 1066 | } |
e154ff3d RZ |
1067 | for (adj = 0; error > i; adj++) |
1068 | error >>= 1; | |
19923c19 RZ |
1069 | |
1070 | *interval <<= adj; | |
1071 | *offset <<= adj; | |
e154ff3d | 1072 | return mult << adj; |
19923c19 RZ |
1073 | } |
1074 | ||
1075 | /* | |
1076 | * Adjust the multiplier to reduce the error value, | |
1077 | * this is optimized for the most common adjustments of -1,0,1, | |
1078 | * for other values we can do a bit more work. | |
1079 | */ | |
1080 | static void clocksource_adjust(struct clocksource *clock, s64 offset) | |
1081 | { | |
1082 | s64 error, interval = clock->cycle_interval; | |
1083 | int adj; | |
1084 | ||
1085 | error = clock->error >> (TICK_LENGTH_SHIFT - clock->shift - 1); | |
1086 | if (error > interval) { | |
e154ff3d RZ |
1087 | error >>= 2; |
1088 | if (likely(error <= interval)) | |
1089 | adj = 1; | |
1090 | else | |
1091 | adj = clocksource_bigadjust(error, &interval, &offset); | |
19923c19 | 1092 | } else if (error < -interval) { |
e154ff3d RZ |
1093 | error >>= 2; |
1094 | if (likely(error >= -interval)) { | |
1095 | adj = -1; | |
1096 | interval = -interval; | |
1097 | offset = -offset; | |
1098 | } else | |
1099 | adj = clocksource_bigadjust(error, &interval, &offset); | |
19923c19 RZ |
1100 | } else |
1101 | return; | |
1102 | ||
1103 | clock->mult += adj; | |
1104 | clock->xtime_interval += interval; | |
1105 | clock->xtime_nsec -= offset; | |
1106 | clock->error -= (interval - offset) << (TICK_LENGTH_SHIFT - clock->shift); | |
1107 | } | |
1108 | ||
ad596171 | 1109 | /* |
1110 | * update_wall_time - Uses the current clocksource to increment the wall time | |
1111 | * | |
1112 | * Called from the timer interrupt, must hold a write on xtime_lock. | |
1113 | */ | |
1114 | static void update_wall_time(void) | |
1115 | { | |
19923c19 | 1116 | cycle_t offset; |
ad596171 | 1117 | |
3e143475 | 1118 | /* Make sure we're fully resumed: */ |
1119 | if (unlikely(timekeeping_suspended)) | |
1120 | return; | |
5eb6d205 | 1121 | |
19923c19 RZ |
1122 | #ifdef CONFIG_GENERIC_TIME |
1123 | offset = (clocksource_read(clock) - clock->cycle_last) & clock->mask; | |
1124 | #else | |
1125 | offset = clock->cycle_interval; | |
1126 | #endif | |
3e143475 | 1127 | clock->xtime_nsec += (s64)xtime.tv_nsec << clock->shift; |
ad596171 | 1128 | |
1129 | /* normally this loop will run just once, however in the | |
1130 | * case of lost or late ticks, it will accumulate correctly. | |
1131 | */ | |
19923c19 | 1132 | while (offset >= clock->cycle_interval) { |
ad596171 | 1133 | /* accumulate one interval */ |
19923c19 RZ |
1134 | clock->xtime_nsec += clock->xtime_interval; |
1135 | clock->cycle_last += clock->cycle_interval; | |
1136 | offset -= clock->cycle_interval; | |
1137 | ||
1138 | if (clock->xtime_nsec >= (u64)NSEC_PER_SEC << clock->shift) { | |
1139 | clock->xtime_nsec -= (u64)NSEC_PER_SEC << clock->shift; | |
1140 | xtime.tv_sec++; | |
1141 | second_overflow(); | |
1142 | } | |
ad596171 | 1143 | |
5eb6d205 | 1144 | /* interpolator bits */ |
19923c19 | 1145 | time_interpolator_update(clock->xtime_interval |
5eb6d205 | 1146 | >> clock->shift); |
1147 | /* increment the NTP state machine */ | |
1148 | update_ntp_one_tick(); | |
1149 | ||
1150 | /* accumulate error between NTP and clock interval */ | |
19923c19 RZ |
1151 | clock->error += current_tick_length(); |
1152 | clock->error -= clock->xtime_interval << (TICK_LENGTH_SHIFT - clock->shift); | |
1153 | } | |
5eb6d205 | 1154 | |
19923c19 RZ |
1155 | /* correct the clock when NTP error is too big */ |
1156 | clocksource_adjust(clock, offset); | |
5eb6d205 | 1157 | |
5eb6d205 | 1158 | /* store full nanoseconds into xtime */ |
e154ff3d | 1159 | xtime.tv_nsec = (s64)clock->xtime_nsec >> clock->shift; |
19923c19 | 1160 | clock->xtime_nsec -= (s64)xtime.tv_nsec << clock->shift; |
cf3c769b | 1161 | |
1162 | /* check to see if there is a new clocksource to use */ | |
1163 | if (change_clocksource()) { | |
19923c19 RZ |
1164 | clock->error = 0; |
1165 | clock->xtime_nsec = 0; | |
a2752549 | 1166 | clocksource_calculate_interval(clock, tick_nsec); |
cf3c769b | 1167 | } |
1da177e4 LT |
1168 | } |
1169 | ||
1170 | /* | |
1171 | * Called from the timer interrupt handler to charge one tick to the current | |
1172 | * process. user_tick is 1 if the tick is user time, 0 for system. | |
1173 | */ | |
1174 | void update_process_times(int user_tick) | |
1175 | { | |
1176 | struct task_struct *p = current; | |
1177 | int cpu = smp_processor_id(); | |
1178 | ||
1179 | /* Note: this timer irq context must be accounted for as well. */ | |
1180 | if (user_tick) | |
1181 | account_user_time(p, jiffies_to_cputime(1)); | |
1182 | else | |
1183 | account_system_time(p, HARDIRQ_OFFSET, jiffies_to_cputime(1)); | |
1184 | run_local_timers(); | |
1185 | if (rcu_pending(cpu)) | |
1186 | rcu_check_callbacks(cpu, user_tick); | |
1187 | scheduler_tick(); | |
1188 | run_posix_cpu_timers(p); | |
1189 | } | |
1190 | ||
1191 | /* | |
1192 | * Nr of active tasks - counted in fixed-point numbers | |
1193 | */ | |
1194 | static unsigned long count_active_tasks(void) | |
1195 | { | |
db1b1fef | 1196 | return nr_active() * FIXED_1; |
1da177e4 LT |
1197 | } |
1198 | ||
1199 | /* | |
1200 | * Hmm.. Changed this, as the GNU make sources (load.c) seems to | |
1201 | * imply that avenrun[] is the standard name for this kind of thing. | |
1202 | * Nothing else seems to be standardized: the fractional size etc | |
1203 | * all seem to differ on different machines. | |
1204 | * | |
1205 | * Requires xtime_lock to access. | |
1206 | */ | |
1207 | unsigned long avenrun[3]; | |
1208 | ||
1209 | EXPORT_SYMBOL(avenrun); | |
1210 | ||
1211 | /* | |
1212 | * calc_load - given tick count, update the avenrun load estimates. | |
1213 | * This is called while holding a write_lock on xtime_lock. | |
1214 | */ | |
1215 | static inline void calc_load(unsigned long ticks) | |
1216 | { | |
1217 | unsigned long active_tasks; /* fixed-point */ | |
1218 | static int count = LOAD_FREQ; | |
1219 | ||
1220 | count -= ticks; | |
1221 | if (count < 0) { | |
1222 | count += LOAD_FREQ; | |
1223 | active_tasks = count_active_tasks(); | |
1224 | CALC_LOAD(avenrun[0], EXP_1, active_tasks); | |
1225 | CALC_LOAD(avenrun[1], EXP_5, active_tasks); | |
1226 | CALC_LOAD(avenrun[2], EXP_15, active_tasks); | |
1227 | } | |
1228 | } | |
1229 | ||
1230 | /* jiffies at the most recent update of wall time */ | |
1231 | unsigned long wall_jiffies = INITIAL_JIFFIES; | |
1232 | ||
1233 | /* | |
1234 | * This read-write spinlock protects us from races in SMP while | |
1235 | * playing with xtime and avenrun. | |
1236 | */ | |
1237 | #ifndef ARCH_HAVE_XTIME_LOCK | |
e4d91918 | 1238 | __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock); |
1da177e4 LT |
1239 | |
1240 | EXPORT_SYMBOL(xtime_lock); | |
1241 | #endif | |
1242 | ||
1243 | /* | |
1244 | * This function runs timers and the timer-tq in bottom half context. | |
1245 | */ | |
1246 | static void run_timer_softirq(struct softirq_action *h) | |
1247 | { | |
a4a6198b | 1248 | tvec_base_t *base = __get_cpu_var(tvec_bases); |
1da177e4 | 1249 | |
c0a31329 | 1250 | hrtimer_run_queues(); |
1da177e4 LT |
1251 | if (time_after_eq(jiffies, base->timer_jiffies)) |
1252 | __run_timers(base); | |
1253 | } | |
1254 | ||
1255 | /* | |
1256 | * Called by the local, per-CPU timer interrupt on SMP. | |
1257 | */ | |
1258 | void run_local_timers(void) | |
1259 | { | |
1260 | raise_softirq(TIMER_SOFTIRQ); | |
6687a97d | 1261 | softlockup_tick(); |
1da177e4 LT |
1262 | } |
1263 | ||
1264 | /* | |
1265 | * Called by the timer interrupt. xtime_lock must already be taken | |
1266 | * by the timer IRQ! | |
1267 | */ | |
1268 | static inline void update_times(void) | |
1269 | { | |
1270 | unsigned long ticks; | |
1271 | ||
1272 | ticks = jiffies - wall_jiffies; | |
ad596171 | 1273 | wall_jiffies += ticks; |
1274 | update_wall_time(); | |
1da177e4 LT |
1275 | calc_load(ticks); |
1276 | } | |
1277 | ||
1278 | /* | |
1279 | * The 64-bit jiffies value is not atomic - you MUST NOT read it | |
1280 | * without sampling the sequence number in xtime_lock. | |
1281 | * jiffies is defined in the linker script... | |
1282 | */ | |
1283 | ||
1284 | void do_timer(struct pt_regs *regs) | |
1285 | { | |
1286 | jiffies_64++; | |
5aee405c AN |
1287 | /* prevent loading jiffies before storing new jiffies_64 value. */ |
1288 | barrier(); | |
1da177e4 LT |
1289 | update_times(); |
1290 | } | |
1291 | ||
1292 | #ifdef __ARCH_WANT_SYS_ALARM | |
1293 | ||
1294 | /* | |
1295 | * For backwards compatibility? This can be done in libc so Alpha | |
1296 | * and all newer ports shouldn't need it. | |
1297 | */ | |
1298 | asmlinkage unsigned long sys_alarm(unsigned int seconds) | |
1299 | { | |
c08b8a49 | 1300 | return alarm_setitimer(seconds); |
1da177e4 LT |
1301 | } |
1302 | ||
1303 | #endif | |
1304 | ||
1305 | #ifndef __alpha__ | |
1306 | ||
1307 | /* | |
1308 | * The Alpha uses getxpid, getxuid, and getxgid instead. Maybe this | |
1309 | * should be moved into arch/i386 instead? | |
1310 | */ | |
1311 | ||
1312 | /** | |
1313 | * sys_getpid - return the thread group id of the current process | |
1314 | * | |
1315 | * Note, despite the name, this returns the tgid not the pid. The tgid and | |
1316 | * the pid are identical unless CLONE_THREAD was specified on clone() in | |
1317 | * which case the tgid is the same in all threads of the same group. | |
1318 | * | |
1319 | * This is SMP safe as current->tgid does not change. | |
1320 | */ | |
1321 | asmlinkage long sys_getpid(void) | |
1322 | { | |
1323 | return current->tgid; | |
1324 | } | |
1325 | ||
1326 | /* | |
1327 | * Accessing ->group_leader->real_parent is not SMP-safe, it could | |
1328 | * change from under us. However, rather than getting any lock | |
1329 | * we can use an optimistic algorithm: get the parent | |
1330 | * pid, and go back and check that the parent is still | |
1331 | * the same. If it has changed (which is extremely unlikely | |
1332 | * indeed), we just try again.. | |
1333 | * | |
1334 | * NOTE! This depends on the fact that even if we _do_ | |
1335 | * get an old value of "parent", we can happily dereference | |
1336 | * the pointer (it was and remains a dereferencable kernel pointer | |
1337 | * no matter what): we just can't necessarily trust the result | |
1338 | * until we know that the parent pointer is valid. | |
1339 | * | |
1340 | * NOTE2: ->group_leader never changes from under us. | |
1341 | */ | |
1342 | asmlinkage long sys_getppid(void) | |
1343 | { | |
1344 | int pid; | |
1345 | struct task_struct *me = current; | |
1346 | struct task_struct *parent; | |
1347 | ||
1348 | parent = me->group_leader->real_parent; | |
1349 | for (;;) { | |
1350 | pid = parent->tgid; | |
4c5640cb | 1351 | #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT) |
1da177e4 LT |
1352 | { |
1353 | struct task_struct *old = parent; | |
1354 | ||
1355 | /* | |
1356 | * Make sure we read the pid before re-reading the | |
1357 | * parent pointer: | |
1358 | */ | |
d59dd462 | 1359 | smp_rmb(); |
1da177e4 LT |
1360 | parent = me->group_leader->real_parent; |
1361 | if (old != parent) | |
1362 | continue; | |
1363 | } | |
1364 | #endif | |
1365 | break; | |
1366 | } | |
1367 | return pid; | |
1368 | } | |
1369 | ||
1370 | asmlinkage long sys_getuid(void) | |
1371 | { | |
1372 | /* Only we change this so SMP safe */ | |
1373 | return current->uid; | |
1374 | } | |
1375 | ||
1376 | asmlinkage long sys_geteuid(void) | |
1377 | { | |
1378 | /* Only we change this so SMP safe */ | |
1379 | return current->euid; | |
1380 | } | |
1381 | ||
1382 | asmlinkage long sys_getgid(void) | |
1383 | { | |
1384 | /* Only we change this so SMP safe */ | |
1385 | return current->gid; | |
1386 | } | |
1387 | ||
1388 | asmlinkage long sys_getegid(void) | |
1389 | { | |
1390 | /* Only we change this so SMP safe */ | |
1391 | return current->egid; | |
1392 | } | |
1393 | ||
1394 | #endif | |
1395 | ||
1396 | static void process_timeout(unsigned long __data) | |
1397 | { | |
36c8b586 | 1398 | wake_up_process((struct task_struct *)__data); |
1da177e4 LT |
1399 | } |
1400 | ||
1401 | /** | |
1402 | * schedule_timeout - sleep until timeout | |
1403 | * @timeout: timeout value in jiffies | |
1404 | * | |
1405 | * Make the current task sleep until @timeout jiffies have | |
1406 | * elapsed. The routine will return immediately unless | |
1407 | * the current task state has been set (see set_current_state()). | |
1408 | * | |
1409 | * You can set the task state as follows - | |
1410 | * | |
1411 | * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to | |
1412 | * pass before the routine returns. The routine will return 0 | |
1413 | * | |
1414 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | |
1415 | * delivered to the current task. In this case the remaining time | |
1416 | * in jiffies will be returned, or 0 if the timer expired in time | |
1417 | * | |
1418 | * The current task state is guaranteed to be TASK_RUNNING when this | |
1419 | * routine returns. | |
1420 | * | |
1421 | * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule | |
1422 | * the CPU away without a bound on the timeout. In this case the return | |
1423 | * value will be %MAX_SCHEDULE_TIMEOUT. | |
1424 | * | |
1425 | * In all cases the return value is guaranteed to be non-negative. | |
1426 | */ | |
1427 | fastcall signed long __sched schedule_timeout(signed long timeout) | |
1428 | { | |
1429 | struct timer_list timer; | |
1430 | unsigned long expire; | |
1431 | ||
1432 | switch (timeout) | |
1433 | { | |
1434 | case MAX_SCHEDULE_TIMEOUT: | |
1435 | /* | |
1436 | * These two special cases are useful to be comfortable | |
1437 | * in the caller. Nothing more. We could take | |
1438 | * MAX_SCHEDULE_TIMEOUT from one of the negative value | |
1439 | * but I' d like to return a valid offset (>=0) to allow | |
1440 | * the caller to do everything it want with the retval. | |
1441 | */ | |
1442 | schedule(); | |
1443 | goto out; | |
1444 | default: | |
1445 | /* | |
1446 | * Another bit of PARANOID. Note that the retval will be | |
1447 | * 0 since no piece of kernel is supposed to do a check | |
1448 | * for a negative retval of schedule_timeout() (since it | |
1449 | * should never happens anyway). You just have the printk() | |
1450 | * that will tell you if something is gone wrong and where. | |
1451 | */ | |
1452 | if (timeout < 0) | |
1453 | { | |
1454 | printk(KERN_ERR "schedule_timeout: wrong timeout " | |
a5a0d52c AM |
1455 | "value %lx from %p\n", timeout, |
1456 | __builtin_return_address(0)); | |
1da177e4 LT |
1457 | current->state = TASK_RUNNING; |
1458 | goto out; | |
1459 | } | |
1460 | } | |
1461 | ||
1462 | expire = timeout + jiffies; | |
1463 | ||
a8db2db1 ON |
1464 | setup_timer(&timer, process_timeout, (unsigned long)current); |
1465 | __mod_timer(&timer, expire); | |
1da177e4 LT |
1466 | schedule(); |
1467 | del_singleshot_timer_sync(&timer); | |
1468 | ||
1469 | timeout = expire - jiffies; | |
1470 | ||
1471 | out: | |
1472 | return timeout < 0 ? 0 : timeout; | |
1473 | } | |
1da177e4 LT |
1474 | EXPORT_SYMBOL(schedule_timeout); |
1475 | ||
8a1c1757 AM |
1476 | /* |
1477 | * We can use __set_current_state() here because schedule_timeout() calls | |
1478 | * schedule() unconditionally. | |
1479 | */ | |
64ed93a2 NA |
1480 | signed long __sched schedule_timeout_interruptible(signed long timeout) |
1481 | { | |
a5a0d52c AM |
1482 | __set_current_state(TASK_INTERRUPTIBLE); |
1483 | return schedule_timeout(timeout); | |
64ed93a2 NA |
1484 | } |
1485 | EXPORT_SYMBOL(schedule_timeout_interruptible); | |
1486 | ||
1487 | signed long __sched schedule_timeout_uninterruptible(signed long timeout) | |
1488 | { | |
a5a0d52c AM |
1489 | __set_current_state(TASK_UNINTERRUPTIBLE); |
1490 | return schedule_timeout(timeout); | |
64ed93a2 NA |
1491 | } |
1492 | EXPORT_SYMBOL(schedule_timeout_uninterruptible); | |
1493 | ||
1da177e4 LT |
1494 | /* Thread ID - the internal kernel "pid" */ |
1495 | asmlinkage long sys_gettid(void) | |
1496 | { | |
1497 | return current->pid; | |
1498 | } | |
1499 | ||
1da177e4 LT |
1500 | /* |
1501 | * sys_sysinfo - fill in sysinfo struct | |
1502 | */ | |
1503 | asmlinkage long sys_sysinfo(struct sysinfo __user *info) | |
1504 | { | |
1505 | struct sysinfo val; | |
1506 | unsigned long mem_total, sav_total; | |
1507 | unsigned int mem_unit, bitcount; | |
1508 | unsigned long seq; | |
1509 | ||
1510 | memset((char *)&val, 0, sizeof(struct sysinfo)); | |
1511 | ||
1512 | do { | |
1513 | struct timespec tp; | |
1514 | seq = read_seqbegin(&xtime_lock); | |
1515 | ||
1516 | /* | |
1517 | * This is annoying. The below is the same thing | |
1518 | * posix_get_clock_monotonic() does, but it wants to | |
1519 | * take the lock which we want to cover the loads stuff | |
1520 | * too. | |
1521 | */ | |
1522 | ||
1523 | getnstimeofday(&tp); | |
1524 | tp.tv_sec += wall_to_monotonic.tv_sec; | |
1525 | tp.tv_nsec += wall_to_monotonic.tv_nsec; | |
1526 | if (tp.tv_nsec - NSEC_PER_SEC >= 0) { | |
1527 | tp.tv_nsec = tp.tv_nsec - NSEC_PER_SEC; | |
1528 | tp.tv_sec++; | |
1529 | } | |
1530 | val.uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0); | |
1531 | ||
1532 | val.loads[0] = avenrun[0] << (SI_LOAD_SHIFT - FSHIFT); | |
1533 | val.loads[1] = avenrun[1] << (SI_LOAD_SHIFT - FSHIFT); | |
1534 | val.loads[2] = avenrun[2] << (SI_LOAD_SHIFT - FSHIFT); | |
1535 | ||
1536 | val.procs = nr_threads; | |
1537 | } while (read_seqretry(&xtime_lock, seq)); | |
1538 | ||
1539 | si_meminfo(&val); | |
1540 | si_swapinfo(&val); | |
1541 | ||
1542 | /* | |
1543 | * If the sum of all the available memory (i.e. ram + swap) | |
1544 | * is less than can be stored in a 32 bit unsigned long then | |
1545 | * we can be binary compatible with 2.2.x kernels. If not, | |
1546 | * well, in that case 2.2.x was broken anyways... | |
1547 | * | |
1548 | * -Erik Andersen <andersee@debian.org> | |
1549 | */ | |
1550 | ||
1551 | mem_total = val.totalram + val.totalswap; | |
1552 | if (mem_total < val.totalram || mem_total < val.totalswap) | |
1553 | goto out; | |
1554 | bitcount = 0; | |
1555 | mem_unit = val.mem_unit; | |
1556 | while (mem_unit > 1) { | |
1557 | bitcount++; | |
1558 | mem_unit >>= 1; | |
1559 | sav_total = mem_total; | |
1560 | mem_total <<= 1; | |
1561 | if (mem_total < sav_total) | |
1562 | goto out; | |
1563 | } | |
1564 | ||
1565 | /* | |
1566 | * If mem_total did not overflow, multiply all memory values by | |
1567 | * val.mem_unit and set it to 1. This leaves things compatible | |
1568 | * with 2.2.x, and also retains compatibility with earlier 2.4.x | |
1569 | * kernels... | |
1570 | */ | |
1571 | ||
1572 | val.mem_unit = 1; | |
1573 | val.totalram <<= bitcount; | |
1574 | val.freeram <<= bitcount; | |
1575 | val.sharedram <<= bitcount; | |
1576 | val.bufferram <<= bitcount; | |
1577 | val.totalswap <<= bitcount; | |
1578 | val.freeswap <<= bitcount; | |
1579 | val.totalhigh <<= bitcount; | |
1580 | val.freehigh <<= bitcount; | |
1581 | ||
1582 | out: | |
1583 | if (copy_to_user(info, &val, sizeof(struct sysinfo))) | |
1584 | return -EFAULT; | |
1585 | ||
1586 | return 0; | |
1587 | } | |
1588 | ||
d730e882 IM |
1589 | /* |
1590 | * lockdep: we want to track each per-CPU base as a separate lock-class, | |
1591 | * but timer-bases are kmalloc()-ed, so we need to attach separate | |
1592 | * keys to them: | |
1593 | */ | |
1594 | static struct lock_class_key base_lock_keys[NR_CPUS]; | |
1595 | ||
a4a6198b | 1596 | static int __devinit init_timers_cpu(int cpu) |
1da177e4 LT |
1597 | { |
1598 | int j; | |
1599 | tvec_base_t *base; | |
ba6edfcd | 1600 | static char __devinitdata tvec_base_done[NR_CPUS]; |
55c888d6 | 1601 | |
ba6edfcd | 1602 | if (!tvec_base_done[cpu]) { |
a4a6198b JB |
1603 | static char boot_done; |
1604 | ||
a4a6198b | 1605 | if (boot_done) { |
ba6edfcd AM |
1606 | /* |
1607 | * The APs use this path later in boot | |
1608 | */ | |
a4a6198b JB |
1609 | base = kmalloc_node(sizeof(*base), GFP_KERNEL, |
1610 | cpu_to_node(cpu)); | |
1611 | if (!base) | |
1612 | return -ENOMEM; | |
1613 | memset(base, 0, sizeof(*base)); | |
ba6edfcd | 1614 | per_cpu(tvec_bases, cpu) = base; |
a4a6198b | 1615 | } else { |
ba6edfcd AM |
1616 | /* |
1617 | * This is for the boot CPU - we use compile-time | |
1618 | * static initialisation because per-cpu memory isn't | |
1619 | * ready yet and because the memory allocators are not | |
1620 | * initialised either. | |
1621 | */ | |
a4a6198b | 1622 | boot_done = 1; |
ba6edfcd | 1623 | base = &boot_tvec_bases; |
a4a6198b | 1624 | } |
ba6edfcd AM |
1625 | tvec_base_done[cpu] = 1; |
1626 | } else { | |
1627 | base = per_cpu(tvec_bases, cpu); | |
a4a6198b | 1628 | } |
ba6edfcd | 1629 | |
3691c519 | 1630 | spin_lock_init(&base->lock); |
d730e882 IM |
1631 | lockdep_set_class(&base->lock, base_lock_keys + cpu); |
1632 | ||
1da177e4 LT |
1633 | for (j = 0; j < TVN_SIZE; j++) { |
1634 | INIT_LIST_HEAD(base->tv5.vec + j); | |
1635 | INIT_LIST_HEAD(base->tv4.vec + j); | |
1636 | INIT_LIST_HEAD(base->tv3.vec + j); | |
1637 | INIT_LIST_HEAD(base->tv2.vec + j); | |
1638 | } | |
1639 | for (j = 0; j < TVR_SIZE; j++) | |
1640 | INIT_LIST_HEAD(base->tv1.vec + j); | |
1641 | ||
1642 | base->timer_jiffies = jiffies; | |
a4a6198b | 1643 | return 0; |
1da177e4 LT |
1644 | } |
1645 | ||
1646 | #ifdef CONFIG_HOTPLUG_CPU | |
55c888d6 | 1647 | static void migrate_timer_list(tvec_base_t *new_base, struct list_head *head) |
1da177e4 LT |
1648 | { |
1649 | struct timer_list *timer; | |
1650 | ||
1651 | while (!list_empty(head)) { | |
1652 | timer = list_entry(head->next, struct timer_list, entry); | |
55c888d6 | 1653 | detach_timer(timer, 0); |
3691c519 | 1654 | timer->base = new_base; |
1da177e4 | 1655 | internal_add_timer(new_base, timer); |
1da177e4 | 1656 | } |
1da177e4 LT |
1657 | } |
1658 | ||
1659 | static void __devinit migrate_timers(int cpu) | |
1660 | { | |
1661 | tvec_base_t *old_base; | |
1662 | tvec_base_t *new_base; | |
1663 | int i; | |
1664 | ||
1665 | BUG_ON(cpu_online(cpu)); | |
a4a6198b JB |
1666 | old_base = per_cpu(tvec_bases, cpu); |
1667 | new_base = get_cpu_var(tvec_bases); | |
1da177e4 LT |
1668 | |
1669 | local_irq_disable(); | |
3691c519 ON |
1670 | spin_lock(&new_base->lock); |
1671 | spin_lock(&old_base->lock); | |
1672 | ||
1673 | BUG_ON(old_base->running_timer); | |
1da177e4 | 1674 | |
1da177e4 | 1675 | for (i = 0; i < TVR_SIZE; i++) |
55c888d6 ON |
1676 | migrate_timer_list(new_base, old_base->tv1.vec + i); |
1677 | for (i = 0; i < TVN_SIZE; i++) { | |
1678 | migrate_timer_list(new_base, old_base->tv2.vec + i); | |
1679 | migrate_timer_list(new_base, old_base->tv3.vec + i); | |
1680 | migrate_timer_list(new_base, old_base->tv4.vec + i); | |
1681 | migrate_timer_list(new_base, old_base->tv5.vec + i); | |
1682 | } | |
1683 | ||
3691c519 ON |
1684 | spin_unlock(&old_base->lock); |
1685 | spin_unlock(&new_base->lock); | |
1da177e4 LT |
1686 | local_irq_enable(); |
1687 | put_cpu_var(tvec_bases); | |
1da177e4 LT |
1688 | } |
1689 | #endif /* CONFIG_HOTPLUG_CPU */ | |
1690 | ||
9c7b216d | 1691 | static int __devinit timer_cpu_notify(struct notifier_block *self, |
1da177e4 LT |
1692 | unsigned long action, void *hcpu) |
1693 | { | |
1694 | long cpu = (long)hcpu; | |
1695 | switch(action) { | |
1696 | case CPU_UP_PREPARE: | |
a4a6198b JB |
1697 | if (init_timers_cpu(cpu) < 0) |
1698 | return NOTIFY_BAD; | |
1da177e4 LT |
1699 | break; |
1700 | #ifdef CONFIG_HOTPLUG_CPU | |
1701 | case CPU_DEAD: | |
1702 | migrate_timers(cpu); | |
1703 | break; | |
1704 | #endif | |
1705 | default: | |
1706 | break; | |
1707 | } | |
1708 | return NOTIFY_OK; | |
1709 | } | |
1710 | ||
054cc8a2 | 1711 | static struct notifier_block __devinitdata timers_nb = { |
1da177e4 LT |
1712 | .notifier_call = timer_cpu_notify, |
1713 | }; | |
1714 | ||
1715 | ||
1716 | void __init init_timers(void) | |
1717 | { | |
1718 | timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE, | |
1719 | (void *)(long)smp_processor_id()); | |
1720 | register_cpu_notifier(&timers_nb); | |
1721 | open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL); | |
1722 | } | |
1723 | ||
1724 | #ifdef CONFIG_TIME_INTERPOLATION | |
1725 | ||
67890d70 CL |
1726 | struct time_interpolator *time_interpolator __read_mostly; |
1727 | static struct time_interpolator *time_interpolator_list __read_mostly; | |
1da177e4 LT |
1728 | static DEFINE_SPINLOCK(time_interpolator_lock); |
1729 | ||
1730 | static inline u64 time_interpolator_get_cycles(unsigned int src) | |
1731 | { | |
1732 | unsigned long (*x)(void); | |
1733 | ||
1734 | switch (src) | |
1735 | { | |
1736 | case TIME_SOURCE_FUNCTION: | |
1737 | x = time_interpolator->addr; | |
1738 | return x(); | |
1739 | ||
1740 | case TIME_SOURCE_MMIO64 : | |
685db65e | 1741 | return readq_relaxed((void __iomem *)time_interpolator->addr); |
1da177e4 LT |
1742 | |
1743 | case TIME_SOURCE_MMIO32 : | |
685db65e | 1744 | return readl_relaxed((void __iomem *)time_interpolator->addr); |
1da177e4 LT |
1745 | |
1746 | default: return get_cycles(); | |
1747 | } | |
1748 | } | |
1749 | ||
486d46ae | 1750 | static inline u64 time_interpolator_get_counter(int writelock) |
1da177e4 LT |
1751 | { |
1752 | unsigned int src = time_interpolator->source; | |
1753 | ||
1754 | if (time_interpolator->jitter) | |
1755 | { | |
1756 | u64 lcycle; | |
1757 | u64 now; | |
1758 | ||
1759 | do { | |
1760 | lcycle = time_interpolator->last_cycle; | |
1761 | now = time_interpolator_get_cycles(src); | |
1762 | if (lcycle && time_after(lcycle, now)) | |
1763 | return lcycle; | |
486d46ae AW |
1764 | |
1765 | /* When holding the xtime write lock, there's no need | |
1766 | * to add the overhead of the cmpxchg. Readers are | |
1767 | * force to retry until the write lock is released. | |
1768 | */ | |
1769 | if (writelock) { | |
1770 | time_interpolator->last_cycle = now; | |
1771 | return now; | |
1772 | } | |
1da177e4 LT |
1773 | /* Keep track of the last timer value returned. The use of cmpxchg here |
1774 | * will cause contention in an SMP environment. | |
1775 | */ | |
1776 | } while (unlikely(cmpxchg(&time_interpolator->last_cycle, lcycle, now) != lcycle)); | |
1777 | return now; | |
1778 | } | |
1779 | else | |
1780 | return time_interpolator_get_cycles(src); | |
1781 | } | |
1782 | ||
1783 | void time_interpolator_reset(void) | |
1784 | { | |
1785 | time_interpolator->offset = 0; | |
486d46ae | 1786 | time_interpolator->last_counter = time_interpolator_get_counter(1); |
1da177e4 LT |
1787 | } |
1788 | ||
1789 | #define GET_TI_NSECS(count,i) (((((count) - i->last_counter) & (i)->mask) * (i)->nsec_per_cyc) >> (i)->shift) | |
1790 | ||
1791 | unsigned long time_interpolator_get_offset(void) | |
1792 | { | |
1793 | /* If we do not have a time interpolator set up then just return zero */ | |
1794 | if (!time_interpolator) | |
1795 | return 0; | |
1796 | ||
1797 | return time_interpolator->offset + | |
486d46ae | 1798 | GET_TI_NSECS(time_interpolator_get_counter(0), time_interpolator); |
1da177e4 LT |
1799 | } |
1800 | ||
1801 | #define INTERPOLATOR_ADJUST 65536 | |
1802 | #define INTERPOLATOR_MAX_SKIP 10*INTERPOLATOR_ADJUST | |
1803 | ||
1804 | static void time_interpolator_update(long delta_nsec) | |
1805 | { | |
1806 | u64 counter; | |
1807 | unsigned long offset; | |
1808 | ||
1809 | /* If there is no time interpolator set up then do nothing */ | |
1810 | if (!time_interpolator) | |
1811 | return; | |
1812 | ||
a5a0d52c AM |
1813 | /* |
1814 | * The interpolator compensates for late ticks by accumulating the late | |
1815 | * time in time_interpolator->offset. A tick earlier than expected will | |
1816 | * lead to a reset of the offset and a corresponding jump of the clock | |
1817 | * forward. Again this only works if the interpolator clock is running | |
1818 | * slightly slower than the regular clock and the tuning logic insures | |
1819 | * that. | |
1820 | */ | |
1da177e4 | 1821 | |
486d46ae | 1822 | counter = time_interpolator_get_counter(1); |
a5a0d52c AM |
1823 | offset = time_interpolator->offset + |
1824 | GET_TI_NSECS(counter, time_interpolator); | |
1da177e4 LT |
1825 | |
1826 | if (delta_nsec < 0 || (unsigned long) delta_nsec < offset) | |
1827 | time_interpolator->offset = offset - delta_nsec; | |
1828 | else { | |
1829 | time_interpolator->skips++; | |
1830 | time_interpolator->ns_skipped += delta_nsec - offset; | |
1831 | time_interpolator->offset = 0; | |
1832 | } | |
1833 | time_interpolator->last_counter = counter; | |
1834 | ||
1835 | /* Tuning logic for time interpolator invoked every minute or so. | |
1836 | * Decrease interpolator clock speed if no skips occurred and an offset is carried. | |
1837 | * Increase interpolator clock speed if we skip too much time. | |
1838 | */ | |
1839 | if (jiffies % INTERPOLATOR_ADJUST == 0) | |
1840 | { | |
b20367a6 | 1841 | if (time_interpolator->skips == 0 && time_interpolator->offset > tick_nsec) |
1da177e4 LT |
1842 | time_interpolator->nsec_per_cyc--; |
1843 | if (time_interpolator->ns_skipped > INTERPOLATOR_MAX_SKIP && time_interpolator->offset == 0) | |
1844 | time_interpolator->nsec_per_cyc++; | |
1845 | time_interpolator->skips = 0; | |
1846 | time_interpolator->ns_skipped = 0; | |
1847 | } | |
1848 | } | |
1849 | ||
1850 | static inline int | |
1851 | is_better_time_interpolator(struct time_interpolator *new) | |
1852 | { | |
1853 | if (!time_interpolator) | |
1854 | return 1; | |
1855 | return new->frequency > 2*time_interpolator->frequency || | |
1856 | (unsigned long)new->drift < (unsigned long)time_interpolator->drift; | |
1857 | } | |
1858 | ||
1859 | void | |
1860 | register_time_interpolator(struct time_interpolator *ti) | |
1861 | { | |
1862 | unsigned long flags; | |
1863 | ||
1864 | /* Sanity check */ | |
9f31252c | 1865 | BUG_ON(ti->frequency == 0 || ti->mask == 0); |
1da177e4 LT |
1866 | |
1867 | ti->nsec_per_cyc = ((u64)NSEC_PER_SEC << ti->shift) / ti->frequency; | |
1868 | spin_lock(&time_interpolator_lock); | |
1869 | write_seqlock_irqsave(&xtime_lock, flags); | |
1870 | if (is_better_time_interpolator(ti)) { | |
1871 | time_interpolator = ti; | |
1872 | time_interpolator_reset(); | |
1873 | } | |
1874 | write_sequnlock_irqrestore(&xtime_lock, flags); | |
1875 | ||
1876 | ti->next = time_interpolator_list; | |
1877 | time_interpolator_list = ti; | |
1878 | spin_unlock(&time_interpolator_lock); | |
1879 | } | |
1880 | ||
1881 | void | |
1882 | unregister_time_interpolator(struct time_interpolator *ti) | |
1883 | { | |
1884 | struct time_interpolator *curr, **prev; | |
1885 | unsigned long flags; | |
1886 | ||
1887 | spin_lock(&time_interpolator_lock); | |
1888 | prev = &time_interpolator_list; | |
1889 | for (curr = *prev; curr; curr = curr->next) { | |
1890 | if (curr == ti) { | |
1891 | *prev = curr->next; | |
1892 | break; | |
1893 | } | |
1894 | prev = &curr->next; | |
1895 | } | |
1896 | ||
1897 | write_seqlock_irqsave(&xtime_lock, flags); | |
1898 | if (ti == time_interpolator) { | |
1899 | /* we lost the best time-interpolator: */ | |
1900 | time_interpolator = NULL; | |
1901 | /* find the next-best interpolator */ | |
1902 | for (curr = time_interpolator_list; curr; curr = curr->next) | |
1903 | if (is_better_time_interpolator(curr)) | |
1904 | time_interpolator = curr; | |
1905 | time_interpolator_reset(); | |
1906 | } | |
1907 | write_sequnlock_irqrestore(&xtime_lock, flags); | |
1908 | spin_unlock(&time_interpolator_lock); | |
1909 | } | |
1910 | #endif /* CONFIG_TIME_INTERPOLATION */ | |
1911 | ||
1912 | /** | |
1913 | * msleep - sleep safely even with waitqueue interruptions | |
1914 | * @msecs: Time in milliseconds to sleep for | |
1915 | */ | |
1916 | void msleep(unsigned int msecs) | |
1917 | { | |
1918 | unsigned long timeout = msecs_to_jiffies(msecs) + 1; | |
1919 | ||
75bcc8c5 NA |
1920 | while (timeout) |
1921 | timeout = schedule_timeout_uninterruptible(timeout); | |
1da177e4 LT |
1922 | } |
1923 | ||
1924 | EXPORT_SYMBOL(msleep); | |
1925 | ||
1926 | /** | |
96ec3efd | 1927 | * msleep_interruptible - sleep waiting for signals |
1da177e4 LT |
1928 | * @msecs: Time in milliseconds to sleep for |
1929 | */ | |
1930 | unsigned long msleep_interruptible(unsigned int msecs) | |
1931 | { | |
1932 | unsigned long timeout = msecs_to_jiffies(msecs) + 1; | |
1933 | ||
75bcc8c5 NA |
1934 | while (timeout && !signal_pending(current)) |
1935 | timeout = schedule_timeout_interruptible(timeout); | |
1da177e4 LT |
1936 | return jiffies_to_msecs(timeout); |
1937 | } | |
1938 | ||
1939 | EXPORT_SYMBOL(msleep_interruptible); |