timers: Make the cpu base lock raw

The timers cpu base lock could not be converted to a raw spinlock becaue
the lock held time was non-deterministic due to cascading and long lasting
timer wheel traversals.

The rework of the timer wheel to the new non-cascading model removed also
the wheel traversals and the lock held times are deterministic now. This
allows to make the lock raw and thereby unbreaks NOHz* on preempt-RT.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Link: http://lkml.kernel.org/r/20170627161538.30257-1-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 709a404bd133523fad3b72ecf431c92a05440ef1..71ce3f4eead34afc12fa2743c69e8e9e77c544ad 100644 (file)
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -195,7 +195,7 @@ EXPORT_SYMBOL(jiffies_64);
 #endif
 
 struct timer_base {
-       spinlock_t              lock;
+       raw_spinlock_t          lock;
        struct timer_list       *running_timer;
        unsigned long           clk;
        unsigned long           next_expiry;
@@ -913,10 +913,10 @@ static struct timer_base *lock_timer_base(struct timer_list *timer,
 
                if (!(tf & TIMER_MIGRATING)) {
                        base = get_timer_base(tf);
-                       spin_lock_irqsave(&base->lock, *flags);
+                       raw_spin_lock_irqsave(&base->lock, *flags);
                        if (timer->flags == tf)
                                return base;
-                       spin_unlock_irqrestore(&base->lock, *flags);
+                       raw_spin_unlock_irqrestore(&base->lock, *flags);
                }
                cpu_relax();
        }
@@ -986,9 +986,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
                        /* See the comment in lock_timer_base() */
                        timer->flags |= TIMER_MIGRATING;
 
-                       spin_unlock(&base->lock);
+                       raw_spin_unlock(&base->lock);
                        base = new_base;
-                       spin_lock(&base->lock);
+                       raw_spin_lock(&base->lock);
                        WRITE_ONCE(timer->flags,
                                   (timer->flags & ~TIMER_BASEMASK) | base->cpu);
                }
@@ -1013,7 +1013,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only)
        }
 
 out_unlock:
-       spin_unlock_irqrestore(&base->lock, flags);
+       raw_spin_unlock_irqrestore(&base->lock, flags);
 
        return ret;
 }
@@ -1106,16 +1106,16 @@ void add_timer_on(struct timer_list *timer, int cpu)
        if (base != new_base) {
                timer->flags |= TIMER_MIGRATING;
 
-               spin_unlock(&base->lock);
+               raw_spin_unlock(&base->lock);
                base = new_base;
-               spin_lock(&base->lock);
+               raw_spin_lock(&base->lock);
                WRITE_ONCE(timer->flags,
                           (timer->flags & ~TIMER_BASEMASK) | cpu);
        }
 
        debug_activate(timer, timer->expires);
        internal_add_timer(base, timer);
-       spin_unlock_irqrestore(&base->lock, flags);
+       raw_spin_unlock_irqrestore(&base->lock, flags);
 }
 EXPORT_SYMBOL_GPL(add_timer_on);
 
@@ -1141,7 +1141,7 @@ int del_timer(struct timer_list *timer)
        if (timer_pending(timer)) {
                base = lock_timer_base(timer, &flags);
                ret = detach_if_pending(timer, base, true);
-               spin_unlock_irqrestore(&base->lock, flags);
+               raw_spin_unlock_irqrestore(&base->lock, flags);
        }
 
        return ret;
@@ -1168,7 +1168,7 @@ int try_to_del_timer_sync(struct timer_list *timer)
        if (base->running_timer != timer)
                ret = detach_if_pending(timer, base, true);
 
-       spin_unlock_irqrestore(&base->lock, flags);
+       raw_spin_unlock_irqrestore(&base->lock, flags);
 
        return ret;
 }
@@ -1299,13 +1299,13 @@ static void expire_timers(struct timer_base *base, struct hlist_head *head)
                data = timer->data;
 
                if (timer->flags & TIMER_IRQSAFE) {
-                       spin_unlock(&base->lock);
+                       raw_spin_unlock(&base->lock);
                        call_timer_fn(timer, fn, data);
-                       spin_lock(&base->lock);
+                       raw_spin_lock(&base->lock);
                } else {
-                       spin_unlock_irq(&base->lock);
+                       raw_spin_unlock_irq(&base->lock);
                        call_timer_fn(timer, fn, data);
-                       spin_lock_irq(&base->lock);
+                       raw_spin_lock_irq(&base->lock);
                }
        }
 }
@@ -1474,7 +1474,7 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
        if (cpu_is_offline(smp_processor_id()))
                return expires;
 
-       spin_lock(&base->lock);
+       raw_spin_lock(&base->lock);
        nextevt = __next_timer_interrupt(base);
        is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
        base->next_expiry = nextevt;
@@ -1502,7 +1502,7 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
                if ((expires - basem) > TICK_NSEC)
                        base->is_idle = true;
        }
-       spin_unlock(&base->lock);
+       raw_spin_unlock(&base->lock);
 
        return cmp_next_hrtimer_event(basem, expires);
 }
@@ -1590,7 +1590,7 @@ static inline void __run_timers(struct timer_base *base)
        if (!time_after_eq(jiffies, base->clk))
                return;
 
-       spin_lock_irq(&base->lock);
+       raw_spin_lock_irq(&base->lock);
 
        while (time_after_eq(jiffies, base->clk)) {
 
@@ -1601,7 +1601,7 @@ static inline void __run_timers(struct timer_base *base)
                        expire_timers(base, heads + levels);
        }
        base->running_timer = NULL;
-       spin_unlock_irq(&base->lock);
+       raw_spin_unlock_irq(&base->lock);
 }
 
 /*
@@ -1786,16 +1786,16 @@ int timers_dead_cpu(unsigned int cpu)
                 * The caller is globally serialized and nobody else
                 * takes two locks at once, deadlock is not possible.
                 */
-               spin_lock_irq(&new_base->lock);
-               spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
+               raw_spin_lock_irq(&new_base->lock);
+               raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
 
                BUG_ON(old_base->running_timer);
 
                for (i = 0; i < WHEEL_SIZE; i++)
                        migrate_timer_list(new_base, old_base->vectors + i);
 
-               spin_unlock(&old_base->lock);
-               spin_unlock_irq(&new_base->lock);
+               raw_spin_unlock(&old_base->lock);
+               raw_spin_unlock_irq(&new_base->lock);
                put_cpu_ptr(&timer_bases);
        }
        return 0;
@@ -1811,7 +1811,7 @@ static void __init init_timer_cpu(int cpu)
        for (i = 0; i < NR_BASES; i++) {
                base = per_cpu_ptr(&timer_bases[i], cpu);
                base->cpu = cpu;
-               spin_lock_init(&base->lock);
+               raw_spin_lock_init(&base->lock);
                base->clk = jiffies;
        }
 }