cpu-hotplug: refcount based cpu hotplug

This patch implements a Refcount + Waitqueue based model for
cpu-hotplug.

Now, a thread which wants to prevent cpu-hotplug, will bump up a global
refcount and the thread which wants to perform a cpu-hotplug operation
will block till the global refcount goes to zero.

The readers, if any, during an ongoing cpu-hotplug operation are blocked
until the cpu-hotplug operation is over.

Signed-off-by: Gautham R Shenoy <ego@in.ibm.com>
Signed-off-by: Paul Jackson <pj@sgi.com> [For !CONFIG_HOTPLUG_CPU ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index 92f2029a34f33522c255f08f961416e1e22647a7..a40247e4d462e271c2017709b725e22510bb09c7 100644 (file)
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -83,6 +83,9 @@ static inline void unregister_cpu_notifier(struct notifier_block *nb)
 
 #endif /* CONFIG_SMP */
 extern struct sysdev_class cpu_sysdev_class;
+extern void cpu_hotplug_init(void);
+extern void cpu_maps_update_begin(void);
+extern void cpu_maps_update_done(void);
 
 #ifdef CONFIG_HOTPLUG_CPU
 /* Stop CPUs going up and down. */

diff --git a/init/main.c b/init/main.c
index 80b04b6c5157c85151669fdfe2864c16f876680e..f287ca5862b9c7e9a8d0f1b1b4ba689a4af7e4e4 100644 (file)
--- a/init/main.c
+++ b/init/main.c
@@ -607,6 +607,7 @@ asmlinkage void __init start_kernel(void)
        vfs_caches_init_early();
        cpuset_init_early();
        mem_init();
+       cpu_hotplug_init();
        kmem_cache_init();
        setup_per_cpu_pageset();
        numa_policy_init();

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 6b3a0c15144f3404294dde422348e17242990e2b..656dc3fcbbaeb0601581474f14a498d4ca514d7a 100644 (file)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -15,9 +15,8 @@
 #include <linux/stop_machine.h>
 #include <linux/mutex.h>
 
-/* This protects CPUs going up and down... */
+/* Serializes the updates to cpu_online_map, cpu_present_map */
 static DEFINE_MUTEX(cpu_add_remove_lock);
-static DEFINE_MUTEX(cpu_bitmask_lock);
 
 static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
 
@@ -26,52 +25,123 @@ static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
  */
 static int cpu_hotplug_disabled;
 
-#ifdef CONFIG_HOTPLUG_CPU
+static struct {
+       struct task_struct *active_writer;
+       struct mutex lock; /* Synchronizes accesses to refcount, */
+       /*
+        * Also blocks the new readers during
+        * an ongoing cpu hotplug operation.
+        */
+       int refcount;
+       wait_queue_head_t writer_queue;
+} cpu_hotplug;
 
-/* Crappy recursive lock-takers in cpufreq! Complain loudly about idiots */
-static struct task_struct *recursive;
-static int recursive_depth;
+#define writer_exists() (cpu_hotplug.active_writer != NULL)
+
+void __init cpu_hotplug_init(void)
+{
+       cpu_hotplug.active_writer = NULL;
+       mutex_init(&cpu_hotplug.lock);
+       cpu_hotplug.refcount = 0;
+       init_waitqueue_head(&cpu_hotplug.writer_queue);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
 
 void lock_cpu_hotplug(void)
 {
-       struct task_struct *tsk = current;
-
-       if (tsk == recursive) {
-               static int warnings = 10;
-               if (warnings) {
-                       printk(KERN_ERR "Lukewarm IQ detected in hotplug locking\n");
-                       WARN_ON(1);
-                       warnings--;
-               }
-               recursive_depth++;
+       might_sleep();
+       if (cpu_hotplug.active_writer == current)
                return;
-       }
-       mutex_lock(&cpu_bitmask_lock);
-       recursive = tsk;
+       mutex_lock(&cpu_hotplug.lock);
+       cpu_hotplug.refcount++;
+       mutex_unlock(&cpu_hotplug.lock);
+
 }
 EXPORT_SYMBOL_GPL(lock_cpu_hotplug);
 
 void unlock_cpu_hotplug(void)
 {
-       WARN_ON(recursive != current);
-       if (recursive_depth) {
-               recursive_depth--;
+       if (cpu_hotplug.active_writer == current)
                return;
-       }
-       recursive = NULL;
-       mutex_unlock(&cpu_bitmask_lock);
+       mutex_lock(&cpu_hotplug.lock);
+       cpu_hotplug.refcount--;
+
+       if (unlikely(writer_exists()) && !cpu_hotplug.refcount)
+               wake_up(&cpu_hotplug.writer_queue);
+
+       mutex_unlock(&cpu_hotplug.lock);
+
 }
 EXPORT_SYMBOL_GPL(unlock_cpu_hotplug);
 
 #endif /* CONFIG_HOTPLUG_CPU */
 
+/*
+ * The following two API's must be used when attempting
+ * to serialize the updates to cpu_online_map, cpu_present_map.
+ */
+void cpu_maps_update_begin(void)
+{
+       mutex_lock(&cpu_add_remove_lock);
+}
+
+void cpu_maps_update_done(void)
+{
+       mutex_unlock(&cpu_add_remove_lock);
+}
+
+/*
+ * This ensures that the hotplug operation can begin only when the
+ * refcount goes to zero.
+ *
+ * Note that during a cpu-hotplug operation, the new readers, if any,
+ * will be blocked by the cpu_hotplug.lock
+ *
+ * Since cpu_maps_update_begin is always called after invoking
+ * cpu_maps_update_begin, we can be sure that only one writer is active.
+ *
+ * Note that theoretically, there is a possibility of a livelock:
+ * - Refcount goes to zero, last reader wakes up the sleeping
+ *   writer.
+ * - Last reader unlocks the cpu_hotplug.lock.
+ * - A new reader arrives at this moment, bumps up the refcount.
+ * - The writer acquires the cpu_hotplug.lock finds the refcount
+ *   non zero and goes to sleep again.
+ *
+ * However, this is very difficult to achieve in practice since
+ * lock_cpu_hotplug() not an api which is called all that often.
+ *
+ */
+static void cpu_hotplug_begin(void)
+{
+       DECLARE_WAITQUEUE(wait, current);
+
+       mutex_lock(&cpu_hotplug.lock);
+
+       cpu_hotplug.active_writer = current;
+       add_wait_queue_exclusive(&cpu_hotplug.writer_queue, &wait);
+       while (cpu_hotplug.refcount) {
+               set_current_state(TASK_UNINTERRUPTIBLE);
+               mutex_unlock(&cpu_hotplug.lock);
+               schedule();
+               mutex_lock(&cpu_hotplug.lock);
+       }
+       remove_wait_queue_locked(&cpu_hotplug.writer_queue, &wait);
+}
+
+static void cpu_hotplug_done(void)
+{
+       cpu_hotplug.active_writer = NULL;
+       mutex_unlock(&cpu_hotplug.lock);
+}
 /* Need to know about CPUs going up/down? */
 int __cpuinit register_cpu_notifier(struct notifier_block *nb)
 {
        int ret;
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        ret = raw_notifier_chain_register(&cpu_chain, nb);
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
        return ret;
 }
 
@@ -81,9 +151,9 @@ EXPORT_SYMBOL(register_cpu_notifier);
 
 void unregister_cpu_notifier(struct notifier_block *nb)
 {
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        raw_notifier_chain_unregister(&cpu_chain, nb);
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
 }
 EXPORT_SYMBOL(unregister_cpu_notifier);
 
@@ -147,6 +217,7 @@ static int _cpu_down(unsigned int cpu, int tasks_frozen)
        if (!cpu_online(cpu))
                return -EINVAL;
 
+       cpu_hotplug_begin();
        raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu);
        err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
                                        hcpu, -1, &nr_calls);
@@ -166,9 +237,7 @@ static int _cpu_down(unsigned int cpu, int tasks_frozen)
        cpu_clear(cpu, tmp);
        set_cpus_allowed(current, tmp);
 
-       mutex_lock(&cpu_bitmask_lock);
        p = __stop_machine_run(take_cpu_down, &tcd_param, cpu);
-       mutex_unlock(&cpu_bitmask_lock);
 
        if (IS_ERR(p) || cpu_online(cpu)) {
                /* CPU didn't die: tell everyone.  Can't complain. */
@@ -203,6 +272,7 @@ out_allowed:
        set_cpus_allowed(current, old_allowed);
 out_release:
        raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu);
+       cpu_hotplug_done();
        return err;
 }
 
@@ -210,13 +280,13 @@ int cpu_down(unsigned int cpu)
 {
        int err = 0;
 
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        if (cpu_hotplug_disabled)
                err = -EBUSY;
        else
                err = _cpu_down(cpu, 0);
 
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
        return err;
 }
 #endif /*CONFIG_HOTPLUG_CPU*/
@@ -231,6 +301,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
        if (cpu_online(cpu) || !cpu_present(cpu))
                return -EINVAL;
 
+       cpu_hotplug_begin();
        raw_notifier_call_chain(&cpu_chain, CPU_LOCK_ACQUIRE, hcpu);
        ret = __raw_notifier_call_chain(&cpu_chain, CPU_UP_PREPARE | mod, hcpu,
                                                        -1, &nr_calls);
@@ -243,9 +314,7 @@ static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
        }
 
        /* Arch-specific enabling code. */
-       mutex_lock(&cpu_bitmask_lock);
        ret = __cpu_up(cpu);
-       mutex_unlock(&cpu_bitmask_lock);
        if (ret != 0)
                goto out_notify;
        BUG_ON(!cpu_online(cpu));
@@ -258,6 +327,7 @@ out_notify:
                __raw_notifier_call_chain(&cpu_chain,
                                CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
        raw_notifier_call_chain(&cpu_chain, CPU_LOCK_RELEASE, hcpu);
+       cpu_hotplug_done();
 
        return ret;
 }
@@ -275,13 +345,13 @@ int __cpuinit cpu_up(unsigned int cpu)
                return -EINVAL;
        }
 
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        if (cpu_hotplug_disabled)
                err = -EBUSY;
        else
                err = _cpu_up(cpu, 0);
 
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
        return err;
 }
 
@@ -292,7 +362,7 @@ int disable_nonboot_cpus(void)
 {
        int cpu, first_cpu, error = 0;
 
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        first_cpu = first_cpu(cpu_online_map);
        /* We take down all of the non-boot CPUs in one shot to avoid races
         * with the userspace trying to use the CPU hotplug at the same time
@@ -319,7 +389,7 @@ int disable_nonboot_cpus(void)
        } else {
                printk(KERN_ERR "Non-boot CPUs are not disabled\n");
        }
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
        return error;
 }
 
@@ -328,7 +398,7 @@ void enable_nonboot_cpus(void)
        int cpu, error;
 
        /* Allow everyone to use the CPU hotplug again */
-       mutex_lock(&cpu_add_remove_lock);
+       cpu_maps_update_begin();
        cpu_hotplug_disabled = 0;
        if (cpus_empty(frozen_cpus))
                goto out;
@@ -344,6 +414,6 @@ void enable_nonboot_cpus(void)
        }
        cpus_clear(frozen_cpus);
 out:
-       mutex_unlock(&cpu_add_remove_lock);
+       cpu_maps_update_done();
 }
 #endif /* CONFIG_PM_SLEEP_SMP */