x86/tsc, sched/clock, clocksource: Use clocksource watchdog to provide stable sync...

Currently we keep sched_clock_tick() active for stable TSC in order to
keep the per-CPU state semi up-to-date. The (obvious) problem is that
by the time we detect TSC is borked, our per-CPU state is also borked.

So hook into the clocksource watchdog and call a method after we've
found it to still be stable.

There's the obvious race where the TSC goes wonky between finding it
stable and us running the callback, but closing that is too much work
and not really worth it, since we're already detecting TSC wobbles
after the fact, so we cannot, per definition, fully avoid funny clock
values.

And since the watchdog runs less often than the tick, this is also an
optimization.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 66015195bd18c65e147084840ba8ea7d898c9be9..c1b16b328abe370e27dc6174c4ad857854ca2825 100644 (file)
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -1033,6 +1033,15 @@ static void tsc_cs_mark_unstable(struct clocksource *cs)
        pr_info("Marking TSC unstable due to clocksource watchdog\n");
 }
 
+static void tsc_cs_tick_stable(struct clocksource *cs)
+{
+       if (tsc_unstable)
+               return;
+
+       if (using_native_sched_clock())
+               sched_clock_tick_stable();
+}
+
 /*
  * .mask MUST be CLOCKSOURCE_MASK(64). See comment above read_tsc()
  */
@@ -1046,6 +1055,7 @@ static struct clocksource clocksource_tsc = {
        .archdata               = { .vclock_mode = VCLOCK_TSC },
        .resume                 = tsc_resume,
        .mark_unstable          = tsc_cs_mark_unstable,
+       .tick_stable            = tsc_cs_tick_stable,
 };
 
 void mark_tsc_unstable(char *reason)

diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index f2b10d9ebd04e7d49d032b5cb097e7f2ae630e56..81490456c242936125205a99e68ecc3ec86c70fa 100644 (file)
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -96,6 +96,7 @@ struct clocksource {
        void (*suspend)(struct clocksource *cs);
        void (*resume)(struct clocksource *cs);
        void (*mark_unstable)(struct clocksource *cs);
+       void (*tick_stable)(struct clocksource *cs);
 
        /* private: */
 #ifdef CONFIG_CLOCKSOURCE_WATCHDOG

diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h
index 34fe92ce1ebd7c6e9dfa0dac94ff4512deb6f95f..978cbb0af5f38c2502fb1611d024c567a416d353 100644 (file)
--- a/include/linux/sched/clock.h
+++ b/include/linux/sched/clock.h
@@ -63,8 +63,8 @@ extern void clear_sched_clock_stable(void);
  */
 extern u64 __sched_clock_offset;
 
-
 extern void sched_clock_tick(void);
+extern void sched_clock_tick_stable(void);
 extern void sched_clock_idle_sleep_event(void);
 extern void sched_clock_idle_wakeup_event(u64 delta_ns);
 

diff --git a/kernel/sched/clock.c b/kernel/sched/clock.c
index dc650851935f80cc33d64d317b6cf579fd7054af..f861637f7fdc6b4c43c4b1551e66285c6a0dc104 100644 (file)
--- a/kernel/sched/clock.c
+++ b/kernel/sched/clock.c
@@ -366,20 +366,38 @@ void sched_clock_tick(void)
 {
        struct sched_clock_data *scd;
 
+       if (sched_clock_stable())
+               return;
+
+       if (unlikely(!sched_clock_running))
+               return;
+
        WARN_ON_ONCE(!irqs_disabled());
 
-       /*
-        * Update these values even if sched_clock_stable(), because it can
-        * become unstable at any point in time at which point we need some
-        * values to fall back on.
-        *
-        * XXX arguably we can skip this if we expose tsc_clocksource_reliable
-        */
        scd = this_scd();
        __scd_stamp(scd);
+       sched_clock_local(scd);
+}
+
+void sched_clock_tick_stable(void)
+{
+       u64 gtod, clock;
 
-       if (!sched_clock_stable() && likely(sched_clock_running))
-               sched_clock_local(scd);
+       if (!sched_clock_stable())
+               return;
+
+       /*
+        * Called under watchdog_lock.
+        *
+        * The watchdog just found this TSC to (still) be stable, so now is a
+        * good moment to update our __gtod_offset. Because once we find the
+        * TSC to be unstable, any computation will be computing crap.
+        */
+       local_irq_disable();
+       gtod = ktime_get_ns();
+       clock = sched_clock();
+       __gtod_offset = (clock + __sched_clock_offset) - gtod;
+       local_irq_enable();
 }
 
 /*

diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 93621ae718d391ac6a95cd561dfc22bd6dc15a18..03918a19cf2da854bcefa9f8188daa53a7db82f4 100644 (file)
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -233,6 +233,9 @@ static void clocksource_watchdog(unsigned long data)
                        continue;
                }
 
+               if (cs == curr_clocksource && cs->tick_stable)
+                       cs->tick_stable(cs);
+
                if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&
                    (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
                    (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {