kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched.h>
  11 #include <linux/unistd.h>
  12 #include <linux/cpu.h>
  13 #include <linux/oom.h>
  14 #include <linux/rcupdate.h>
  15 #include <linux/export.h>
  16 #include <linux/bug.h>
  17 #include <linux/kthread.h>
  18 #include <linux/stop_machine.h>
  19 #include <linux/mutex.h>
  20 #include <linux/gfp.h>
  21 #include <linux/suspend.h>
  22 /*******************************************************************************
  23 * 20131225 marc.huang                                                          *
  24 * CPU Hotplug debug mechanism                                                  *
  25 *******************************************************************************/
  26 #include <linux/kallsyms.h>
  27 /******************************************************************************/
  28 #ifdef CONFIG_MT_LOAD_BALANCE_PROFILER
  29 #include <mtlbprof/mtlbprof.h>
  30 #endif
  31
  32 #include "smpboot.h"
  33
  34 /*******************************************************************************
  35 * 20131225 marc.huang                                                          *
  36 * CPU Hotplug and idle integration                                             *
  37 *******************************************************************************/
  38 atomic_t is_in_hotplug = ATOMIC_INIT(0);
  39 void __attribute__((weak)) spm_mcdi_wakeup_all_cores(void) {}
  40 /******************************************************************************/
  41
  42 #ifdef CONFIG_SMP
  43 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
  44 static DEFINE_MUTEX(cpu_add_remove_lock);
  45
  46 /*
  47  * The following two API's must be used when attempting
  48  * to serialize the updates to cpu_online_mask, cpu_present_mask.
  49  */
  50 void cpu_maps_update_begin(void)
  51 {
  52         mutex_lock(&cpu_add_remove_lock);
  53 }
  54
  55 void cpu_maps_update_done(void)
  56 {
  57         mutex_unlock(&cpu_add_remove_lock);
  58 }
  59
  60 /*******************************************************************************
  61 * 20131225 marc.huang                                                          *
  62 * CPU Hotplug debug mechanism                                                  *
  63 *******************************************************************************/
  64 #if defined(MTK_CPU_HOTPLUG_DEBUG_1) || defined(MTK_CPU_HOTPLUG_DEBUG_2)
  65 RAW_NOTIFIER_HEAD(cpu_chain);
  66 #else
  67 static RAW_NOTIFIER_HEAD(cpu_chain);
  68 #endif
  69 /******************************************************************************/
  70
  71 /* If set, cpu_up and cpu_down will return -EBUSY and do nothing.
  72  * Should always be manipulated under cpu_add_remove_lock
  73  */
  74 static int cpu_hotplug_disabled;
  75
  76 #ifdef CONFIG_HOTPLUG_CPU
  77
  78 static struct {
  79         struct task_struct *active_writer;
  80         struct mutex lock; /* Synchronizes accesses to refcount, */
  81         /*
  82          * Also blocks the new readers during
  83          * an ongoing cpu hotplug operation.
  84          */
  85         int refcount;
  86 } cpu_hotplug = {
  87         .active_writer = NULL,
  88         .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
  89         .refcount = 0,
  90 };
  91
  92 void get_online_cpus(void)
  93 {
  94         might_sleep();
  95         if (cpu_hotplug.active_writer == current)
  96                 return;
  97         mutex_lock(&cpu_hotplug.lock);
  98         cpu_hotplug.refcount++;
  99         mutex_unlock(&cpu_hotplug.lock);
 100
 101 }
 102 EXPORT_SYMBOL_GPL(get_online_cpus);
 103
 104 void put_online_cpus(void)
 105 {
 106         if (cpu_hotplug.active_writer == current)
 107                 return;
 108         mutex_lock(&cpu_hotplug.lock);
 109
 110         if (WARN_ON(!cpu_hotplug.refcount))
 111                 cpu_hotplug.refcount++; /* try to fix things up */
 112
 113         if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
 114                 wake_up_process(cpu_hotplug.active_writer);
 115         mutex_unlock(&cpu_hotplug.lock);
 116
 117 }
 118 EXPORT_SYMBOL_GPL(put_online_cpus);
 119
 120 /*
 121  * This ensures that the hotplug operation can begin only when the
 122  * refcount goes to zero.
 123  *
 124  * Note that during a cpu-hotplug operation, the new readers, if any,
 125  * will be blocked by the cpu_hotplug.lock
 126  *
 127  * Since cpu_hotplug_begin() is always called after invoking
 128  * cpu_maps_update_begin(), we can be sure that only one writer is active.
 129  *
 130  * Note that theoretically, there is a possibility of a livelock:
 131  * - Refcount goes to zero, last reader wakes up the sleeping
 132  *   writer.
 133  * - Last reader unlocks the cpu_hotplug.lock.
 134  * - A new reader arrives at this moment, bumps up the refcount.
 135  * - The writer acquires the cpu_hotplug.lock finds the refcount
 136  *   non zero and goes to sleep again.
 137  *
 138  * However, this is very difficult to achieve in practice since
 139  * get_online_cpus() not an api which is called all that often.
 140  *
 141  */
 142 static void cpu_hotplug_begin(void)
 143 {
 144         cpu_hotplug.active_writer = current;
 145
 146         for (;;) {
 147                 mutex_lock(&cpu_hotplug.lock);
 148                 if (likely(!cpu_hotplug.refcount))
 149                         break;
 150                 __set_current_state(TASK_UNINTERRUPTIBLE);
 151                 mutex_unlock(&cpu_hotplug.lock);
 152                 schedule();
 153         }
 154
 155 /*******************************************************************************
 156 * 20131225 marc.huang                                                          *
 157 * CPU Hotplug and idle integration                                             *
 158 *******************************************************************************/
 159         atomic_inc(&is_in_hotplug);
 160         spm_mcdi_wakeup_all_cores();
 161 /******************************************************************************/
 162 }
 163
 164 static void cpu_hotplug_done(void)
 165 {
 166 /*******************************************************************************
 167 * 20131225 marc.huang                                                          *
 168 * CPU Hotplug and idle integration                                             *
 169 *******************************************************************************/
 170         atomic_dec(&is_in_hotplug);
 171 /******************************************************************************/
 172
 173         cpu_hotplug.active_writer = NULL;
 174         mutex_unlock(&cpu_hotplug.lock);
 175 }
 176
 177 /*
 178  * Wait for currently running CPU hotplug operations to complete (if any) and
 179  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 180  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 181  * hotplug path before performing hotplug operations. So acquiring that lock
 182  * guarantees mutual exclusion from any currently running hotplug operations.
 183  */
 184 void cpu_hotplug_disable(void)
 185 {
 186         cpu_maps_update_begin();
 187         cpu_hotplug_disabled = 1;
 188         cpu_maps_update_done();
 189 }
 190
 191 void cpu_hotplug_enable(void)
 192 {
 193         cpu_maps_update_begin();
 194         cpu_hotplug_disabled = 0;
 195         cpu_maps_update_done();
 196 }
 197
 198 #else /* #if CONFIG_HOTPLUG_CPU */
 199 static void cpu_hotplug_begin(void) {}
 200 static void cpu_hotplug_done(void) {}
 201 #endif  /* #else #if CONFIG_HOTPLUG_CPU */
 202
 203 /* Need to know about CPUs going up/down? */
 204 int __ref register_cpu_notifier(struct notifier_block *nb)
 205 {
 206         int ret;
 207
 208 /*******************************************************************************
 209 * 20131225 marc.huang                                                          *
 210 * CPU Hotplug debug mechanism                                             *
 211 *******************************************************************************/
 212 #ifdef MTK_CPU_HOTPLUG_DEBUG_0
 213         static int index = 0;
 214  #ifdef CONFIG_KALLSYMS
 215         char namebuf[128] = {0};
 216         const char *symname;
 217
 218         symname = kallsyms_lookup((unsigned long)nb->notifier_call, NULL, NULL, NULL, namebuf);
 219         if (symname)
 220                 printk("[cpu_ntf] <%02d>%08lx (%s)\n", index++, (unsigned long)nb->notifier_call, symname);
 221         else
 222                 printk("[cpu_ntf] <%02d>%08lx\n", index++, (unsigned long)nb->notifier_call);
 223  #else //#ifdef CONFIG_KALLSYMS
 224         printk("[cpu_ntf] <%02d>%08lx\n", index++, (unsigned long)nb->notifier_call);
 225  #endif //#ifdef CONFIG_KALLSYMS
 226 #endif //#ifdef MTK_CPU_HOTPLUG_DEBUG_0
 227 /******************************************************************************/
 228
 229         cpu_maps_update_begin();
 230         ret = raw_notifier_chain_register(&cpu_chain, nb);
 231         cpu_maps_update_done();
 232         return ret;
 233 }
 234
 235 static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
 236                         int *nr_calls)
 237 {
 238         int ret;
 239
 240         ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
 241                                         nr_calls);
 242
 243         return notifier_to_errno(ret);
 244 }
 245
 246 static int cpu_notify(unsigned long val, void *v)
 247 {
 248         return __cpu_notify(val, v, -1, NULL);
 249 }
 250
 251 static void cpu_notify_nofail(unsigned long val, void *v)
 252 {
 253         BUG_ON(cpu_notify(val, v));
 254 }
 255 EXPORT_SYMBOL(register_cpu_notifier);
 256
 257 void __ref unregister_cpu_notifier(struct notifier_block *nb)
 258 {
 259         cpu_maps_update_begin();
 260         raw_notifier_chain_unregister(&cpu_chain, nb);
 261         cpu_maps_update_done();
 262 }
 263 EXPORT_SYMBOL(unregister_cpu_notifier);
 264
 265 #ifdef CONFIG_HOTPLUG_CPU
 266 /**
 267  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 268  * @cpu: a CPU id
 269  *
 270  * This function walks all processes, finds a valid mm struct for each one and
 271  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 272  * trivial, there are various non-obvious corner cases, which this function
 273  * tries to solve in a safe manner.
 274  *
 275  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 276  * be called only for an already offlined CPU.
 277  */
 278 void clear_tasks_mm_cpumask(int cpu)
 279 {
 280         struct task_struct *p;
 281
 282         /*
 283          * This function is called after the cpu is taken down and marked
 284          * offline, so its not like new tasks will ever get this cpu set in
 285          * their mm mask. -- Peter Zijlstra
 286          * Thus, we may use rcu_read_lock() here, instead of grabbing
 287          * full-fledged tasklist_lock.
 288          */
 289         WARN_ON(cpu_online(cpu));
 290         rcu_read_lock();
 291         for_each_process(p) {
 292                 struct task_struct *t;
 293
 294                 /*
 295                  * Main thread might exit, but other threads may still have
 296                  * a valid mm. Find one.
 297                  */
 298                 t = find_lock_task_mm(p);
 299                 if (!t)
 300                         continue;
 301                 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 302                 task_unlock(t);
 303         }
 304         rcu_read_unlock();
 305 }
 306
 307 static inline void check_for_tasks(int cpu)
 308 {
 309         struct task_struct *p;
 310         cputime_t utime, stime;
 311
 312         write_lock_irq(&tasklist_lock);
 313         for_each_process(p) {
 314                 task_cputime(p, &utime, &stime);
 315                 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
 316                     (utime || stime))
 317                         printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
 318                                 "(state = %ld, flags = %x)\n",
 319                                 p->comm, task_pid_nr(p), cpu,
 320                                 p->state, p->flags);
 321         }
 322         write_unlock_irq(&tasklist_lock);
 323 }
 324
 325 struct take_cpu_down_param {
 326         unsigned long mod;
 327         void *hcpu;
 328 };
 329
 330 /* Take this CPU down. */
 331 static int __ref take_cpu_down(void *_param)
 332 {
 333         struct take_cpu_down_param *param = _param;
 334         int err;
 335
 336         /* Ensure this CPU doesn't handle any more interrupts. */
 337         err = __cpu_disable();
 338         if (err < 0)
 339                 return err;
 340
 341         cpu_notify(CPU_DYING | param->mod, param->hcpu);
 342         /* Park the stopper thread */
 343         kthread_park(current);
 344         return 0;
 345 }
 346
 347 /* Requires cpu_add_remove_lock to be held */
 348 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
 349 {
 350         int err, nr_calls = 0;
 351         void *hcpu = (void *)(long)cpu;
 352         unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
 353         struct take_cpu_down_param tcd_param = {
 354                 .mod = mod,
 355                 .hcpu = hcpu,
 356         };
 357
 358         if (num_online_cpus() == 1)
 359                 return -EBUSY;
 360
 361         if (!cpu_online(cpu))
 362                 return -EINVAL;
 363
 364         cpu_hotplug_begin();
 365
 366         err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
 367         if (err) {
 368                 nr_calls--;
 369                 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
 370                 printk("%s: attempt to take down CPU %u failed\n",
 371                                 __func__, cpu);
 372                 goto out_release;
 373         }
 374         smpboot_park_threads(cpu);
 375
 376         err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
 377         if (err) {
 378                 /* CPU didn't die: tell everyone.  Can't complain. */
 379                 smpboot_unpark_threads(cpu);
 380                 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
 381                 goto out_release;
 382         }
 383         BUG_ON(cpu_online(cpu));
 384
 385         /*
 386          * The migration_call() CPU_DYING callback will have removed all
 387          * runnable tasks from the cpu, there's only the idle task left now
 388          * that the migration thread is done doing the stop_machine thing.
 389          *
 390          * Wait for the stop thread to go away.
 391          */
 392         while (!idle_cpu(cpu))
 393                 cpu_relax();
 394
 395 #ifdef CONFIG_MT_LOAD_BALANCE_PROFILER
 396         mt_lbprof_update_state(cpu, MT_LBPROF_HOTPLUG_STATE);
 397 #endif
 398
 399         /* This actually kills the CPU. */
 400         __cpu_die(cpu);
 401
 402         /* CPU is completely dead: tell everyone.  Too late to complain. */
 403         cpu_notify_nofail(CPU_DEAD | mod, hcpu);
 404
 405         check_for_tasks(cpu);
 406
 407 out_release:
 408         cpu_hotplug_done();
 409         if (!err)
 410                 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
 411         return err;
 412 }
 413
 414 int __ref cpu_down(unsigned int cpu)
 415 {
 416         int err;
 417
 418         cpu_maps_update_begin();
 419
 420         if (cpu_hotplug_disabled) {
 421                 err = -EBUSY;
 422                 goto out;
 423         }
 424
 425         err = _cpu_down(cpu, 0);
 426
 427 out:
 428         cpu_maps_update_done();
 429         return err;
 430 }
 431 EXPORT_SYMBOL(cpu_down);
 432 #endif /*CONFIG_HOTPLUG_CPU*/
 433
 434 /* Requires cpu_add_remove_lock to be held */
 435 static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
 436 {
 437         int ret, nr_calls = 0;
 438         void *hcpu = (void *)(long)cpu;
 439         unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
 440         struct task_struct *idle;
 441
 442         cpu_hotplug_begin();
 443
 444         if (cpu_online(cpu) || !cpu_present(cpu)) {
 445                 ret = -EINVAL;
 446                 goto out;
 447         }
 448
 449         idle = idle_thread_get(cpu);
 450         if (IS_ERR(idle)) {
 451                 ret = PTR_ERR(idle);
 452                 goto out;
 453         }
 454
 455         ret = smpboot_create_threads(cpu);
 456         if (ret)
 457                 goto out;
 458
 459         ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
 460         if (ret) {
 461                 nr_calls--;
 462                 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
 463                                 __func__, cpu);
 464                 goto out_notify;
 465         }
 466
 467         /* Arch-specific enabling code. */
 468         ret = __cpu_up(cpu, idle);
 469         if (ret != 0)
 470                 goto out_notify;
 471         BUG_ON(!cpu_online(cpu));
 472
 473         /* Wake the per cpu threads */
 474         smpboot_unpark_threads(cpu);
 475
 476         /* Now call notifier in preparation. */
 477         cpu_notify(CPU_ONLINE | mod, hcpu);
 478
 479 out_notify:
 480         if (ret != 0)
 481                 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
 482 out:
 483         cpu_hotplug_done();
 484
 485         return ret;
 486 }
 487
 488 int __cpuinit cpu_up(unsigned int cpu)
 489 {
 490         int err = 0;
 491
 492 #ifdef  CONFIG_MEMORY_HOTPLUG
 493         int nid;
 494         pg_data_t       *pgdat;
 495 #endif
 496
 497         if (!cpu_possible(cpu)) {
 498                 printk(KERN_ERR "can't online cpu %d because it is not "
 499                         "configured as may-hotadd at boot time\n", cpu);
 500 #if defined(CONFIG_IA64)
 501                 printk(KERN_ERR "please check additional_cpus= boot "
 502                                 "parameter\n");
 503 #endif
 504                 return -EINVAL;
 505         }
 506
 507 #ifdef  CONFIG_MEMORY_HOTPLUG
 508         nid = cpu_to_node(cpu);
 509         if (!node_online(nid)) {
 510                 err = mem_online_node(nid);
 511                 if (err)
 512                         return err;
 513         }
 514
 515         pgdat = NODE_DATA(nid);
 516         if (!pgdat) {
 517                 printk(KERN_ERR
 518                         "Can't online cpu %d due to NULL pgdat\n", cpu);
 519                 return -ENOMEM;
 520         }
 521
 522         if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
 523                 mutex_lock(&zonelists_mutex);
 524                 build_all_zonelists(NULL, NULL);
 525                 mutex_unlock(&zonelists_mutex);
 526         }
 527 #endif
 528
 529         cpu_maps_update_begin();
 530
 531         if (cpu_hotplug_disabled) {
 532                 err = -EBUSY;
 533                 goto out;
 534         }
 535
 536         err = _cpu_up(cpu, 0);
 537
 538 out:
 539         cpu_maps_update_done();
 540         return err;
 541 }
 542 EXPORT_SYMBOL_GPL(cpu_up);
 543
 544 #ifdef CONFIG_PM_SLEEP_SMP
 545 static cpumask_var_t frozen_cpus;
 546
 547 int disable_nonboot_cpus(void)
 548 {
 549         int cpu, first_cpu, error = 0;
 550
 551         cpu_maps_update_begin();
 552         first_cpu = cpumask_first(cpu_online_mask);
 553         /*
 554          * We take down all of the non-boot CPUs in one shot to avoid races
 555          * with the userspace trying to use the CPU hotplug at the same time
 556          */
 557         cpumask_clear(frozen_cpus);
 558
 559         printk("Disabling non-boot CPUs ...\n");
 560         for_each_online_cpu(cpu) {
 561                 if (cpu == first_cpu)
 562                         continue;
 563                 error = _cpu_down(cpu, 1);
 564                 if (!error)
 565                         cpumask_set_cpu(cpu, frozen_cpus);
 566                 else {
 567                         printk(KERN_ERR "Error taking CPU%d down: %d\n",
 568                                 cpu, error);
 569                         break;
 570                 }
 571         }
 572
 573         if (!error) {
 574                 BUG_ON(num_online_cpus() > 1);
 575                 /* Make sure the CPUs won't be enabled by someone else */
 576                 cpu_hotplug_disabled = 1;
 577         } else {
 578                 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
 579         }
 580         cpu_maps_update_done();
 581         return error;
 582 }
 583 EXPORT_SYMBOL_GPL(disable_nonboot_cpus);
 584
 585 void __weak arch_enable_nonboot_cpus_begin(void)
 586 {
 587 }
 588
 589 void __weak arch_enable_nonboot_cpus_end(void)
 590 {
 591 }
 592
 593 void __ref enable_nonboot_cpus(void)
 594 {
 595         int cpu, error;
 596
 597         /* Allow everyone to use the CPU hotplug again */
 598         cpu_maps_update_begin();
 599         cpu_hotplug_disabled = 0;
 600         if (cpumask_empty(frozen_cpus))
 601                 goto out;
 602
 603         printk(KERN_INFO "Enabling non-boot CPUs ...\n");
 604
 605         arch_enable_nonboot_cpus_begin();
 606
 607         for_each_cpu(cpu, frozen_cpus) {
 608                 error = _cpu_up(cpu, 1);
 609                 if (!error) {
 610                         printk(KERN_INFO "CPU%d is up\n", cpu);
 611                         continue;
 612                 }
 613                 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
 614         }
 615
 616         arch_enable_nonboot_cpus_end();
 617
 618         cpumask_clear(frozen_cpus);
 619 out:
 620         cpu_maps_update_done();
 621 }
 622 EXPORT_SYMBOL_GPL(enable_nonboot_cpus);
 623
 624 static int __init alloc_frozen_cpus(void)
 625 {
 626         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
 627                 return -ENOMEM;
 628         return 0;
 629 }
 630 core_initcall(alloc_frozen_cpus);
 631
 632 /*
 633  * When callbacks for CPU hotplug notifications are being executed, we must
 634  * ensure that the state of the system with respect to the tasks being frozen
 635  * or not, as reported by the notification, remains unchanged *throughout the
 636  * duration* of the execution of the callbacks.
 637  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
 638  *
 639  * This synchronization is implemented by mutually excluding regular CPU
 640  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
 641  * Hibernate notifications.
 642  */
 643 static int
 644 cpu_hotplug_pm_callback(struct notifier_block *nb,
 645                         unsigned long action, void *ptr)
 646 {
 647         switch (action) {
 648
 649         case PM_SUSPEND_PREPARE:
 650         case PM_HIBERNATION_PREPARE:
 651                 cpu_hotplug_disable();
 652                 break;
 653
 654         case PM_POST_SUSPEND:
 655         case PM_POST_HIBERNATION:
 656                 cpu_hotplug_enable();
 657                 break;
 658
 659         default:
 660                 return NOTIFY_DONE;
 661         }
 662
 663         return NOTIFY_OK;
 664 }
 665
 666
 667 static int __init cpu_hotplug_pm_sync_init(void)
 668 {
 669         /*
 670          * cpu_hotplug_pm_callback has higher priority than x86
 671          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
 672          * to disable cpu hotplug to avoid cpu hotplug race.
 673          */
 674         pm_notifier(cpu_hotplug_pm_callback, 0);
 675         return 0;
 676 }
 677 core_initcall(cpu_hotplug_pm_sync_init);
 678
 679 #endif /* CONFIG_PM_SLEEP_SMP */
 680
 681 /**
 682  * notify_cpu_starting(cpu) - call the CPU_STARTING notifiers
 683  * @cpu: cpu that just started
 684  *
 685  * This function calls the cpu_chain notifiers with CPU_STARTING.
 686  * It must be called by the arch code on the new cpu, before the new cpu
 687  * enables interrupts and before the "boot" cpu returns from __cpu_up().
 688  */
 689 void __cpuinit notify_cpu_starting(unsigned int cpu)
 690 {
 691         unsigned long val = CPU_STARTING;
 692
 693 #ifdef CONFIG_PM_SLEEP_SMP
 694         if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
 695                 val = CPU_STARTING_FROZEN;
 696 #endif /* CONFIG_PM_SLEEP_SMP */
 697         cpu_notify(val, (void *)(long)cpu);
 698 }
 699
 700 #endif /* CONFIG_SMP */
 701
 702 /*
 703  * cpu_bit_bitmap[] is a special, "compressed" data structure that
 704  * represents all NR_CPUS bits binary values of 1<<nr.
 705  *
 706  * It is used by cpumask_of() to get a constant address to a CPU
 707  * mask value that has a single bit set only.
 708  */
 709
 710 /* cpu_bit_bitmap[0] is empty - so we can back into it */
 711 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
 712 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
 713 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
 714 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
 715
 716 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
 717
 718         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
 719         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
 720 #if BITS_PER_LONG > 32
 721         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
 722         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
 723 #endif
 724 };
 725 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
 726
 727 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
 728 EXPORT_SYMBOL(cpu_all_bits);
 729
 730 #ifdef CONFIG_INIT_ALL_POSSIBLE
 731 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
 732         = CPU_BITS_ALL;
 733 #else
 734 static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
 735 #endif
 736 const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
 737 EXPORT_SYMBOL(cpu_possible_mask);
 738
 739 static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
 740 const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
 741 EXPORT_SYMBOL(cpu_online_mask);
 742
 743 static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
 744 const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
 745 EXPORT_SYMBOL(cpu_present_mask);
 746
 747 static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
 748 const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
 749 EXPORT_SYMBOL(cpu_active_mask);
 750
 751 void set_cpu_possible(unsigned int cpu, bool possible)
 752 {
 753         if (possible)
 754                 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
 755         else
 756                 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
 757 }
 758
 759 void set_cpu_present(unsigned int cpu, bool present)
 760 {
 761         if (present)
 762                 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
 763         else
 764                 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
 765 }
 766
 767 void set_cpu_online(unsigned int cpu, bool online)
 768 {
 769         if (online) {
 770                 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
 771                 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
 772         } else {
 773                 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
 774         }
 775 }
 776
 777 void set_cpu_active(unsigned int cpu, bool active)
 778 {
 779         if (active)
 780                 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
 781         else
 782                 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
 783 }
 784
 785 void init_cpu_present(const struct cpumask *src)
 786 {
 787         cpumask_copy(to_cpumask(cpu_present_bits), src);
 788 }
 789
 790 void init_cpu_possible(const struct cpumask *src)
 791 {
 792         cpumask_copy(to_cpumask(cpu_possible_bits), src);
 793 }
 794
 795 void init_cpu_online(const struct cpumask *src)
 796 {
 797         cpumask_copy(to_cpumask(cpu_online_bits), src);
 798 }
 799
 800 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
 801
 802 void idle_notifier_register(struct notifier_block *n)
 803 {
 804         atomic_notifier_chain_register(&idle_notifier, n);
 805 }
 806 EXPORT_SYMBOL_GPL(idle_notifier_register);
 807
 808 void idle_notifier_unregister(struct notifier_block *n)
 809 {
 810         atomic_notifier_chain_unregister(&idle_notifier, n);
 811 }
 812 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
 813
 814 void idle_notifier_call_chain(unsigned long val)
 815 {
 816         atomic_notifier_call_chain(&idle_notifier, val, NULL);
 817 }
 818 EXPORT_SYMBOL_GPL(idle_notifier_call_chain);