Commit | Line | Data |
---|---|---|
15c84731 JF |
1 | /* |
2 | * Xen time implementation. | |
3 | * | |
4 | * This is implemented in terms of a clocksource driver which uses | |
5 | * the hypervisor clock as a nanosecond timebase, and a clockevent | |
6 | * driver which uses the hypervisor's timer mechanism. | |
7 | * | |
8 | * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 | |
9 | */ | |
10 | #include <linux/kernel.h> | |
11 | #include <linux/interrupt.h> | |
12 | #include <linux/clocksource.h> | |
13 | #include <linux/clockchips.h> | |
f91a8b44 | 14 | #include <linux/kernel_stat.h> |
f595ec96 | 15 | #include <linux/math64.h> |
5a0e3ad6 | 16 | #include <linux/gfp.h> |
15c84731 | 17 | |
1c7b67f7 | 18 | #include <asm/pvclock.h> |
15c84731 JF |
19 | #include <asm/xen/hypervisor.h> |
20 | #include <asm/xen/hypercall.h> | |
21 | ||
22 | #include <xen/events.h> | |
409771d2 | 23 | #include <xen/features.h> |
15c84731 JF |
24 | #include <xen/interface/xen.h> |
25 | #include <xen/interface/vcpu.h> | |
26 | ||
27 | #include "xen-ops.h" | |
28 | ||
15c84731 JF |
29 | /* Xen may fire a timer up to this many ns early */ |
30 | #define TIMER_SLOP 100000 | |
f91a8b44 | 31 | #define NS_PER_TICK (1000000000LL / HZ) |
15c84731 | 32 | |
f91a8b44 | 33 | /* runstate info updated by Xen */ |
c6e22f9e | 34 | static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); |
f91a8b44 JF |
35 | |
36 | /* snapshots of runstate info */ | |
c6e22f9e | 37 | static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot); |
f91a8b44 | 38 | |
ac9349c4 | 39 | /* unused ns of stolen time */ |
c6e22f9e | 40 | static DEFINE_PER_CPU(u64, xen_residual_stolen); |
f91a8b44 JF |
41 | |
42 | /* return an consistent snapshot of 64-bit time/counter value */ | |
43 | static u64 get64(const u64 *p) | |
44 | { | |
45 | u64 ret; | |
46 | ||
47 | if (BITS_PER_LONG < 64) { | |
48 | u32 *p32 = (u32 *)p; | |
49 | u32 h, l; | |
50 | ||
51 | /* | |
52 | * Read high then low, and then make sure high is | |
53 | * still the same; this will only loop if low wraps | |
54 | * and carries into high. | |
55 | * XXX some clean way to make this endian-proof? | |
56 | */ | |
57 | do { | |
58 | h = p32[1]; | |
59 | barrier(); | |
60 | l = p32[0]; | |
61 | barrier(); | |
62 | } while (p32[1] != h); | |
63 | ||
64 | ret = (((u64)h) << 32) | l; | |
65 | } else | |
66 | ret = *p; | |
67 | ||
68 | return ret; | |
69 | } | |
70 | ||
71 | /* | |
72 | * Runstate accounting | |
73 | */ | |
74 | static void get_runstate_snapshot(struct vcpu_runstate_info *res) | |
75 | { | |
76 | u64 state_time; | |
77 | struct vcpu_runstate_info *state; | |
78 | ||
f120f13e | 79 | BUG_ON(preemptible()); |
f91a8b44 | 80 | |
c6e22f9e | 81 | state = &__get_cpu_var(xen_runstate); |
f91a8b44 JF |
82 | |
83 | /* | |
84 | * The runstate info is always updated by the hypervisor on | |
85 | * the current CPU, so there's no need to use anything | |
86 | * stronger than a compiler barrier when fetching it. | |
87 | */ | |
88 | do { | |
89 | state_time = get64(&state->state_entry_time); | |
90 | barrier(); | |
91 | *res = *state; | |
92 | barrier(); | |
93 | } while (get64(&state->state_entry_time) != state_time); | |
f91a8b44 JF |
94 | } |
95 | ||
f0d73394 JF |
96 | /* return true when a vcpu could run but has no real cpu to run on */ |
97 | bool xen_vcpu_stolen(int vcpu) | |
98 | { | |
c6e22f9e | 99 | return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; |
f0d73394 JF |
100 | } |
101 | ||
be012920 | 102 | void xen_setup_runstate_info(int cpu) |
f91a8b44 JF |
103 | { |
104 | struct vcpu_register_runstate_memory_area area; | |
105 | ||
c6e22f9e | 106 | area.addr.v = &per_cpu(xen_runstate, cpu); |
f91a8b44 JF |
107 | |
108 | if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, | |
109 | cpu, &area)) | |
110 | BUG(); | |
111 | } | |
112 | ||
113 | static void do_stolen_accounting(void) | |
114 | { | |
115 | struct vcpu_runstate_info state; | |
116 | struct vcpu_runstate_info *snap; | |
ac9349c4 | 117 | s64 runnable, offline, stolen; |
f91a8b44 JF |
118 | cputime_t ticks; |
119 | ||
120 | get_runstate_snapshot(&state); | |
121 | ||
122 | WARN_ON(state.state != RUNSTATE_running); | |
123 | ||
c6e22f9e | 124 | snap = &__get_cpu_var(xen_runstate_snapshot); |
f91a8b44 JF |
125 | |
126 | /* work out how much time the VCPU has not been runn*ing* */ | |
f91a8b44 JF |
127 | runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable]; |
128 | offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline]; | |
129 | ||
130 | *snap = state; | |
131 | ||
132 | /* Add the appropriate number of ticks of stolen time, | |
79741dd3 | 133 | including any left-overs from last time. */ |
780f36d8 | 134 | stolen = runnable + offline + __this_cpu_read(xen_residual_stolen); |
f91a8b44 JF |
135 | |
136 | if (stolen < 0) | |
137 | stolen = 0; | |
138 | ||
f595ec96 | 139 | ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen); |
780f36d8 | 140 | __this_cpu_write(xen_residual_stolen, stolen); |
79741dd3 | 141 | account_steal_ticks(ticks); |
f91a8b44 JF |
142 | } |
143 | ||
e93ef949 | 144 | /* Get the TSC speed from Xen */ |
409771d2 | 145 | static unsigned long xen_tsc_khz(void) |
15c84731 | 146 | { |
3807f345 | 147 | struct pvclock_vcpu_time_info *info = |
15c84731 JF |
148 | &HYPERVISOR_shared_info->vcpu_info[0].time; |
149 | ||
3807f345 | 150 | return pvclock_tsc_khz(info); |
15c84731 JF |
151 | } |
152 | ||
ee7686bc | 153 | cycle_t xen_clocksource_read(void) |
15c84731 | 154 | { |
1c7b67f7 | 155 | struct pvclock_vcpu_time_info *src; |
15c84731 | 156 | cycle_t ret; |
15c84731 | 157 | |
f1c39625 JF |
158 | preempt_disable_notrace(); |
159 | src = &__get_cpu_var(xen_vcpu)->time; | |
1c7b67f7 | 160 | ret = pvclock_clocksource_read(src); |
f1c39625 | 161 | preempt_enable_notrace(); |
15c84731 JF |
162 | return ret; |
163 | } | |
164 | ||
8e19608e MD |
165 | static cycle_t xen_clocksource_get_cycles(struct clocksource *cs) |
166 | { | |
167 | return xen_clocksource_read(); | |
168 | } | |
169 | ||
15c84731 JF |
170 | static void xen_read_wallclock(struct timespec *ts) |
171 | { | |
1c7b67f7 GH |
172 | struct shared_info *s = HYPERVISOR_shared_info; |
173 | struct pvclock_wall_clock *wall_clock = &(s->wc); | |
174 | struct pvclock_vcpu_time_info *vcpu_time; | |
15c84731 | 175 | |
1c7b67f7 GH |
176 | vcpu_time = &get_cpu_var(xen_vcpu)->time; |
177 | pvclock_read_wallclock(wall_clock, vcpu_time, ts); | |
178 | put_cpu_var(xen_vcpu); | |
15c84731 JF |
179 | } |
180 | ||
409771d2 | 181 | static unsigned long xen_get_wallclock(void) |
15c84731 JF |
182 | { |
183 | struct timespec ts; | |
184 | ||
185 | xen_read_wallclock(&ts); | |
15c84731 JF |
186 | return ts.tv_sec; |
187 | } | |
188 | ||
409771d2 | 189 | static int xen_set_wallclock(unsigned long now) |
15c84731 | 190 | { |
fdb9eb9f JF |
191 | struct xen_platform_op op; |
192 | int rc; | |
193 | ||
15c84731 | 194 | /* do nothing for domU */ |
fdb9eb9f JF |
195 | if (!xen_initial_domain()) |
196 | return -1; | |
197 | ||
198 | op.cmd = XENPF_settime; | |
199 | op.u.settime.secs = now; | |
200 | op.u.settime.nsecs = 0; | |
201 | op.u.settime.system_time = xen_clocksource_read(); | |
202 | ||
203 | rc = HYPERVISOR_dom0_op(&op); | |
204 | WARN(rc != 0, "XENPF_settime failed: now=%ld\n", now); | |
205 | ||
206 | return rc; | |
15c84731 JF |
207 | } |
208 | ||
209 | static struct clocksource xen_clocksource __read_mostly = { | |
210 | .name = "xen", | |
211 | .rating = 400, | |
8e19608e | 212 | .read = xen_clocksource_get_cycles, |
15c84731 | 213 | .mask = ~0, |
15c84731 JF |
214 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
215 | }; | |
216 | ||
217 | /* | |
218 | Xen clockevent implementation | |
219 | ||
220 | Xen has two clockevent implementations: | |
221 | ||
222 | The old timer_op one works with all released versions of Xen prior | |
223 | to version 3.0.4. This version of the hypervisor provides a | |
224 | single-shot timer with nanosecond resolution. However, sharing the | |
225 | same event channel is a 100Hz tick which is delivered while the | |
226 | vcpu is running. We don't care about or use this tick, but it will | |
227 | cause the core time code to think the timer fired too soon, and | |
228 | will end up resetting it each time. It could be filtered, but | |
229 | doing so has complications when the ktime clocksource is not yet | |
230 | the xen clocksource (ie, at boot time). | |
231 | ||
232 | The new vcpu_op-based timer interface allows the tick timer period | |
233 | to be changed or turned off. The tick timer is not useful as a | |
234 | periodic timer because events are only delivered to running vcpus. | |
235 | The one-shot timer can report when a timeout is in the past, so | |
236 | set_next_event is capable of returning -ETIME when appropriate. | |
237 | This interface is used when available. | |
238 | */ | |
239 | ||
240 | ||
241 | /* | |
242 | Get a hypervisor absolute time. In theory we could maintain an | |
243 | offset between the kernel's time and the hypervisor's time, and | |
244 | apply that to a kernel's absolute timeout. Unfortunately the | |
245 | hypervisor and kernel times can drift even if the kernel is using | |
246 | the Xen clocksource, because ntp can warp the kernel's clocksource. | |
247 | */ | |
248 | static s64 get_abs_timeout(unsigned long delta) | |
249 | { | |
250 | return xen_clocksource_read() + delta; | |
251 | } | |
252 | ||
253 | static void xen_timerop_set_mode(enum clock_event_mode mode, | |
254 | struct clock_event_device *evt) | |
255 | { | |
256 | switch (mode) { | |
257 | case CLOCK_EVT_MODE_PERIODIC: | |
258 | /* unsupported */ | |
259 | WARN_ON(1); | |
260 | break; | |
261 | ||
262 | case CLOCK_EVT_MODE_ONESHOT: | |
18de5bc4 | 263 | case CLOCK_EVT_MODE_RESUME: |
15c84731 JF |
264 | break; |
265 | ||
266 | case CLOCK_EVT_MODE_UNUSED: | |
267 | case CLOCK_EVT_MODE_SHUTDOWN: | |
268 | HYPERVISOR_set_timer_op(0); /* cancel timeout */ | |
269 | break; | |
270 | } | |
271 | } | |
272 | ||
273 | static int xen_timerop_set_next_event(unsigned long delta, | |
274 | struct clock_event_device *evt) | |
275 | { | |
276 | WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); | |
277 | ||
278 | if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0) | |
279 | BUG(); | |
280 | ||
281 | /* We may have missed the deadline, but there's no real way of | |
282 | knowing for sure. If the event was in the past, then we'll | |
283 | get an immediate interrupt. */ | |
284 | ||
285 | return 0; | |
286 | } | |
287 | ||
288 | static const struct clock_event_device xen_timerop_clockevent = { | |
289 | .name = "xen", | |
290 | .features = CLOCK_EVT_FEAT_ONESHOT, | |
291 | ||
292 | .max_delta_ns = 0xffffffff, | |
293 | .min_delta_ns = TIMER_SLOP, | |
294 | ||
295 | .mult = 1, | |
296 | .shift = 0, | |
297 | .rating = 500, | |
298 | ||
299 | .set_mode = xen_timerop_set_mode, | |
300 | .set_next_event = xen_timerop_set_next_event, | |
301 | }; | |
302 | ||
303 | ||
304 | ||
305 | static void xen_vcpuop_set_mode(enum clock_event_mode mode, | |
306 | struct clock_event_device *evt) | |
307 | { | |
308 | int cpu = smp_processor_id(); | |
309 | ||
310 | switch (mode) { | |
311 | case CLOCK_EVT_MODE_PERIODIC: | |
312 | WARN_ON(1); /* unsupported */ | |
313 | break; | |
314 | ||
315 | case CLOCK_EVT_MODE_ONESHOT: | |
316 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) | |
317 | BUG(); | |
318 | break; | |
319 | ||
320 | case CLOCK_EVT_MODE_UNUSED: | |
321 | case CLOCK_EVT_MODE_SHUTDOWN: | |
322 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) || | |
323 | HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) | |
324 | BUG(); | |
325 | break; | |
18de5bc4 TG |
326 | case CLOCK_EVT_MODE_RESUME: |
327 | break; | |
15c84731 JF |
328 | } |
329 | } | |
330 | ||
331 | static int xen_vcpuop_set_next_event(unsigned long delta, | |
332 | struct clock_event_device *evt) | |
333 | { | |
334 | int cpu = smp_processor_id(); | |
335 | struct vcpu_set_singleshot_timer single; | |
336 | int ret; | |
337 | ||
338 | WARN_ON(evt->mode != CLOCK_EVT_MODE_ONESHOT); | |
339 | ||
340 | single.timeout_abs_ns = get_abs_timeout(delta); | |
1e46601d SS |
341 | /* Get an event anyway, even if the timeout is already expired */ |
342 | single.flags = 0; | |
15c84731 JF |
343 | |
344 | ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single); | |
1e46601d | 345 | BUG_ON(ret != 0); |
15c84731 JF |
346 | |
347 | return ret; | |
348 | } | |
349 | ||
350 | static const struct clock_event_device xen_vcpuop_clockevent = { | |
351 | .name = "xen", | |
352 | .features = CLOCK_EVT_FEAT_ONESHOT, | |
353 | ||
354 | .max_delta_ns = 0xffffffff, | |
355 | .min_delta_ns = TIMER_SLOP, | |
356 | ||
357 | .mult = 1, | |
358 | .shift = 0, | |
359 | .rating = 500, | |
360 | ||
361 | .set_mode = xen_vcpuop_set_mode, | |
362 | .set_next_event = xen_vcpuop_set_next_event, | |
363 | }; | |
364 | ||
365 | static const struct clock_event_device *xen_clockevent = | |
366 | &xen_timerop_clockevent; | |
ef35a4e6 | 367 | static DEFINE_PER_CPU(struct clock_event_device, xen_clock_events) = { .irq = -1 }; |
15c84731 JF |
368 | |
369 | static irqreturn_t xen_timer_interrupt(int irq, void *dev_id) | |
370 | { | |
371 | struct clock_event_device *evt = &__get_cpu_var(xen_clock_events); | |
372 | irqreturn_t ret; | |
373 | ||
374 | ret = IRQ_NONE; | |
375 | if (evt->event_handler) { | |
376 | evt->event_handler(evt); | |
377 | ret = IRQ_HANDLED; | |
378 | } | |
379 | ||
f91a8b44 JF |
380 | do_stolen_accounting(); |
381 | ||
15c84731 JF |
382 | return ret; |
383 | } | |
384 | ||
f87e4cac | 385 | void xen_setup_timer(int cpu) |
15c84731 JF |
386 | { |
387 | const char *name; | |
388 | struct clock_event_device *evt; | |
389 | int irq; | |
390 | ||
ef35a4e6 KRW |
391 | evt = &per_cpu(xen_clock_events, cpu); |
392 | WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu); | |
393 | ||
15c84731 JF |
394 | printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu); |
395 | ||
396 | name = kasprintf(GFP_KERNEL, "timer%d", cpu); | |
397 | if (!name) | |
398 | name = "<timer kasprintf failed>"; | |
399 | ||
400 | irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt, | |
f611f2da IC |
401 | IRQF_DISABLED|IRQF_PERCPU| |
402 | IRQF_NOBALANCING|IRQF_TIMER| | |
403 | IRQF_FORCE_RESUME, | |
15c84731 JF |
404 | name, NULL); |
405 | ||
15c84731 JF |
406 | memcpy(evt, xen_clockevent, sizeof(*evt)); |
407 | ||
320ab2b0 | 408 | evt->cpumask = cpumask_of(cpu); |
15c84731 | 409 | evt->irq = irq; |
f87e4cac JF |
410 | } |
411 | ||
d68d82af AN |
412 | void xen_teardown_timer(int cpu) |
413 | { | |
414 | struct clock_event_device *evt; | |
415 | BUG_ON(cpu == 0); | |
416 | evt = &per_cpu(xen_clock_events, cpu); | |
417 | unbind_from_irqhandler(evt->irq, NULL); | |
ef35a4e6 | 418 | evt->irq = -1; |
d68d82af AN |
419 | } |
420 | ||
f87e4cac JF |
421 | void xen_setup_cpu_clockevents(void) |
422 | { | |
423 | BUG_ON(preemptible()); | |
f91a8b44 | 424 | |
f87e4cac | 425 | clockevents_register_device(&__get_cpu_var(xen_clock_events)); |
15c84731 JF |
426 | } |
427 | ||
d07af1f0 JF |
428 | void xen_timer_resume(void) |
429 | { | |
430 | int cpu; | |
431 | ||
e7a3481c JF |
432 | pvclock_resume(); |
433 | ||
d07af1f0 JF |
434 | if (xen_clockevent != &xen_vcpuop_clockevent) |
435 | return; | |
436 | ||
437 | for_each_online_cpu(cpu) { | |
438 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL)) | |
439 | BUG(); | |
440 | } | |
441 | } | |
442 | ||
fb6ce5de | 443 | static const struct pv_time_ops xen_time_ops __initconst = { |
ca50a5f3 | 444 | .sched_clock = xen_clocksource_read, |
409771d2 SS |
445 | }; |
446 | ||
fb6ce5de | 447 | static void __init xen_time_init(void) |
15c84731 JF |
448 | { |
449 | int cpu = smp_processor_id(); | |
c4507257 | 450 | struct timespec tp; |
15c84731 | 451 | |
b01cc1b0 | 452 | clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); |
15c84731 JF |
453 | |
454 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { | |
f91a8b44 | 455 | /* Successfully turned off 100Hz tick, so we have the |
15c84731 JF |
456 | vcpuop-based timer interface */ |
457 | printk(KERN_DEBUG "Xen: using vcpuop timer interface\n"); | |
458 | xen_clockevent = &xen_vcpuop_clockevent; | |
459 | } | |
460 | ||
461 | /* Set initial system time with full resolution */ | |
c4507257 JS |
462 | xen_read_wallclock(&tp); |
463 | do_settimeofday(&tp); | |
15c84731 | 464 | |
404ee5b1 | 465 | setup_force_cpu_cap(X86_FEATURE_TSC); |
15c84731 | 466 | |
be012920 | 467 | xen_setup_runstate_info(cpu); |
15c84731 | 468 | xen_setup_timer(cpu); |
f87e4cac | 469 | xen_setup_cpu_clockevents(); |
15c84731 | 470 | } |
409771d2 | 471 | |
fb6ce5de | 472 | void __init xen_init_time_ops(void) |
409771d2 SS |
473 | { |
474 | pv_time_ops = xen_time_ops; | |
475 | ||
476 | x86_init.timers.timer_init = xen_time_init; | |
477 | x86_init.timers.setup_percpu_clockev = x86_init_noop; | |
478 | x86_cpuinit.setup_percpu_clockev = x86_init_noop; | |
479 | ||
480 | x86_platform.calibrate_tsc = xen_tsc_khz; | |
481 | x86_platform.get_wallclock = xen_get_wallclock; | |
482 | x86_platform.set_wallclock = xen_set_wallclock; | |
483 | } | |
484 | ||
ca65f9fc | 485 | #ifdef CONFIG_XEN_PVHVM |
409771d2 SS |
486 | static void xen_hvm_setup_cpu_clockevents(void) |
487 | { | |
488 | int cpu = smp_processor_id(); | |
489 | xen_setup_runstate_info(cpu); | |
7918c92a KRW |
490 | /* |
491 | * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence | |
492 | * doing it xen_hvm_cpu_notify (which gets called by smp_init during | |
493 | * early bootup and also during CPU hotplug events). | |
494 | */ | |
409771d2 SS |
495 | xen_setup_cpu_clockevents(); |
496 | } | |
497 | ||
fb6ce5de | 498 | void __init xen_hvm_init_time_ops(void) |
409771d2 SS |
499 | { |
500 | /* vector callback is needed otherwise we cannot receive interrupts | |
31e7e931 SS |
501 | * on cpu > 0 and at this point we don't know how many cpus are |
502 | * available */ | |
503 | if (!xen_have_vector_callback) | |
409771d2 SS |
504 | return; |
505 | if (!xen_feature(XENFEAT_hvm_safe_pvclock)) { | |
506 | printk(KERN_INFO "Xen doesn't support pvclock on HVM," | |
507 | "disable pv timer\n"); | |
508 | return; | |
509 | } | |
510 | ||
511 | pv_time_ops = xen_time_ops; | |
512 | x86_init.timers.setup_percpu_clockev = xen_time_init; | |
513 | x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents; | |
514 | ||
515 | x86_platform.calibrate_tsc = xen_tsc_khz; | |
516 | x86_platform.get_wallclock = xen_get_wallclock; | |
517 | x86_platform.set_wallclock = xen_set_wallclock; | |
518 | } | |
ca65f9fc | 519 | #endif |