percpu: Always align percpu output section to PAGE_SIZE

Percpu allocator honors alignment request upto PAGE_SIZE and both the
percpu addresses in the percpu address space and the translated kernel
addresses should be aligned accordingly.  The calculation of the
former depends on the alignment of percpu output section in the kernel
image.

The linker script macros PERCPU_VADDR() and PERCPU() are used to
define this output section and the latter takes @align parameter.
Several architectures are using @align smaller than PAGE_SIZE breaking
percpu memory alignment.

This patch removes @align parameter from PERCPU(), renames it to
PERCPU_SECTION() and makes it always align to PAGE_SIZE.  While at it,
add PCPU_SETUP_BUG_ON() checks such that alignment problems are
reliably detected and remove percpu alignment comment recently added
in workqueue.c as the condition would trigger BUG way before reaching
there.

For um, this patch raises the alignment of percpu area.  As the area
is in .init, there shouldn't be any noticeable difference.

This problem was discovered by David Howells while debugging boot
failure on mn10300.

Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Mike Frysinger <vapier@gentoo.org>
Cc: uclinux-dist-devel@blackfin.uclinux.org
Cc: David Howells <dhowells@redhat.com>
Cc: Jeff Dike <jdike@addtoit.com>
Cc: user-mode-linux-devel@lists.sourceforge.net

diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S
index 433be2a24f3149261a03600ca19aa6a33992bbc8..8d57948c0aefad9700ab45542f7610edd3e0b93e 100644 (file)
--- a/arch/alpha/kernel/vmlinux.lds.S
+++ b/arch/alpha/kernel/vmlinux.lds.S
@@ -39,7 +39,7 @@ SECTIONS
        __init_begin = ALIGN(PAGE_SIZE);
        INIT_TEXT_SECTION(PAGE_SIZE)
        INIT_DATA_SECTION(16)
-       PERCPU(L1_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(L1_CACHE_BYTES)
        /* Align to THREAD_SIZE rather than PAGE_SIZE here so any padding page
           needed for the THREAD_SIZE aligned init_task gets freed after init */
        . = ALIGN(THREAD_SIZE);

diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index b4348e62ef06e7903d07561701dd78c462ff1aaf..e5287f21badc7e87c3c2e7a2d8953d5fb270ec86 100644 (file)
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -82,7 +82,7 @@ SECTIONS
 #endif
        }
 
-       PERCPU(32, PAGE_SIZE)
+       PERCPU_SECTION(32)
 
 #ifndef CONFIG_XIP_KERNEL
        . = ALIGN(PAGE_SIZE);

diff --git a/arch/blackfin/kernel/vmlinux.lds.S b/arch/blackfin/kernel/vmlinux.lds.S
index 854fa49f1c3ec964f7f105fd9c0019d133cca420..8d85c8c6f857c5dd985bd4f2bbd07b4668b157fb 100644 (file)
--- a/arch/blackfin/kernel/vmlinux.lds.S
+++ b/arch/blackfin/kernel/vmlinux.lds.S
@@ -136,7 +136,7 @@ SECTIONS
 
        . = ALIGN(16);
        INIT_DATA_SECTION(16)
-       PERCPU(32, PAGE_SIZE)
+       PERCPU_SECTION(32)
 
        .exit.data :
        {

diff --git a/arch/cris/kernel/vmlinux.lds.S b/arch/cris/kernel/vmlinux.lds.S
index 728bbd9e7d4c1105b802cbc53558e54c350db93a..a6990cb0f098772c8739b8e7e738789414af7dc8 100644 (file)
--- a/arch/cris/kernel/vmlinux.lds.S
+++ b/arch/cris/kernel/vmlinux.lds.S
@@ -102,7 +102,7 @@ SECTIONS
 #endif
        __vmlinux_end = .;              /* Last address of the physical file. */
 #ifdef CONFIG_ETRAX_ARCH_V32
-       PERCPU(32, PAGE_SIZE)
+       PERCPU_SECTION(32)
 
        .init.ramfs : {
                INIT_RAM_FS

diff --git a/arch/frv/kernel/vmlinux.lds.S b/arch/frv/kernel/vmlinux.lds.S
index 0daae8af5787bfd44fe467689b2fdbd2e7b73289..7e958d829ec9810020c75eca7beef1cbfd44a3c2 100644 (file)
--- a/arch/frv/kernel/vmlinux.lds.S
+++ b/arch/frv/kernel/vmlinux.lds.S
@@ -37,7 +37,7 @@ SECTIONS
   _einittext = .;
 
   INIT_DATA_SECTION(8)
-  PERCPU(L1_CACHE_BYTES, 4096)
+  PERCPU_SECTION(L1_CACHE_BYTES)
 
   . = ALIGN(PAGE_SIZE);
   __init_end = .;

diff --git a/arch/m32r/kernel/vmlinux.lds.S b/arch/m32r/kernel/vmlinux.lds.S
index c194d64cdbb9173912ba2962c4a185561a1e2ad4..2e7ccf7299a048731bb9ce31417a501a2b4c1271 100644 (file)
--- a/arch/m32r/kernel/vmlinux.lds.S
+++ b/arch/m32r/kernel/vmlinux.lds.S
@@ -53,7 +53,7 @@ SECTIONS
   __init_begin = .;
   INIT_TEXT_SECTION(PAGE_SIZE)
   INIT_DATA_SECTION(16)
-  PERCPU(32, PAGE_SIZE)
+  PERCPU_SECTION(32)
   . = ALIGN(PAGE_SIZE);
   __init_end = .;
   /* freed after init ends here */

diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S
index 832afbb87588bedcf2b8265495fd44b6874ccabb..8616709452b160c481a2109c5723034dcbafdd54 100644 (file)
--- a/arch/mips/kernel/vmlinux.lds.S
+++ b/arch/mips/kernel/vmlinux.lds.S
@@ -115,7 +115,7 @@ SECTIONS
                EXIT_DATA
        }
 
-       PERCPU(1 << CONFIG_MIPS_L1_CACHE_SHIFT, PAGE_SIZE)
+       PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT)
        . = ALIGN(PAGE_SIZE);
        __init_end = .;
        /* freed after init ends here */

diff --git a/arch/mn10300/kernel/vmlinux.lds.S b/arch/mn10300/kernel/vmlinux.lds.S
index 968bcd2cb0226c325cc37837518e25dafe4ed6b9..6f702a6ab3955531ff31f65fb7ace7c37f01aa1f 100644 (file)
--- a/arch/mn10300/kernel/vmlinux.lds.S
+++ b/arch/mn10300/kernel/vmlinux.lds.S
@@ -70,7 +70,7 @@ SECTIONS
        .exit.text : { EXIT_TEXT; }
        .exit.data : { EXIT_DATA; }
 
-  PERCPU(32, PAGE_SIZE)
+  PERCPU_SECTION(32)
   . = ALIGN(PAGE_SIZE);
   __init_end = .;
   /* freed after init ends here */

diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S
index 8f1e4efd143e0f32e40ec48a6afb09247b75f0bc..85b86617fe0beb953d43cff4b76426ca50040a75 100644 (file)
--- a/arch/parisc/kernel/vmlinux.lds.S
+++ b/arch/parisc/kernel/vmlinux.lds.S
@@ -145,7 +145,7 @@ SECTIONS
                EXIT_DATA
        }
 
-       PERCPU(L1_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(L1_CACHE_BYTES)
        . = ALIGN(PAGE_SIZE);
        __init_end = .;
        /* freed after init ends here */

diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index b9150f07d2664a9a763ebeda3564b1c08f18fb17..920276c0f6a1d47396451a5a3e9ac0d71d8b3c4c 100644 (file)
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -160,7 +160,7 @@ SECTIONS
                INIT_RAM_FS
        }
 
-       PERCPU(L1_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(L1_CACHE_BYTES)
 
        . = ALIGN(8);
        .machine.desc : AT(ADDR(.machine.desc) - LOAD_OFFSET) {

diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 1bc18cdb525b3d838d9b47f7b98fc3474101355f..56fe6bc81fee45804a61c37fb189ccf73a0b72e8 100644 (file)
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -77,7 +77,7 @@ SECTIONS
        . = ALIGN(PAGE_SIZE);
        INIT_DATA_SECTION(0x100)
 
-       PERCPU(0x100, PAGE_SIZE)
+       PERCPU_SECTION(0x100)
        . = ALIGN(PAGE_SIZE);
        __init_end = .;         /* freed after init ends here */
 

diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S
index af4d46187a79f18bb8ae350294ccb5923f5410fb..731c10ce67b56dd482e95c2ecdb239de023ea831 100644 (file)
--- a/arch/sh/kernel/vmlinux.lds.S
+++ b/arch/sh/kernel/vmlinux.lds.S
@@ -66,7 +66,7 @@ SECTIONS
                __machvec_end = .;
        }
 
-       PERCPU(L1_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(L1_CACHE_BYTES)
 
        /*
         * .exit.text is discarded at runtime, not link time, to deal with

diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index 92b557afe535a59e95d8ab185d5d51c254d4d081..c0220759003e13d302a52ebd1ec5e75dd641c531 100644 (file)
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -108,7 +108,7 @@ SECTIONS
                __sun4v_2insn_patch_end = .;
        }
 
-       PERCPU(SMP_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(SMP_CACHE_BYTES)
 
        . = ALIGN(PAGE_SIZE);
        __init_end = .;

diff --git a/arch/tile/kernel/vmlinux.lds.S b/arch/tile/kernel/vmlinux.lds.S
index 38f64fafdc10a9d8a3271a50d333b9787074ce19..631f10de12feee423e5c3376f56813cc35fb6053 100644 (file)
--- a/arch/tile/kernel/vmlinux.lds.S
+++ b/arch/tile/kernel/vmlinux.lds.S
@@ -60,7 +60,7 @@ SECTIONS
   . = ALIGN(PAGE_SIZE);
   VMLINUX_SYMBOL(_sinitdata) = .;
   INIT_DATA_SECTION(16) :data =0
-  PERCPU(L2_CACHE_BYTES, PAGE_SIZE)
+  PERCPU_SECTION(L2_CACHE_BYTES)
   . = ALIGN(PAGE_SIZE);
   VMLINUX_SYMBOL(_einitdata) = .;
 

diff --git a/arch/um/include/asm/common.lds.S b/arch/um/include/asm/common.lds.S
index 34bede8aad4a9ca9c23df9261be3a7c3d8ac20ec..4938de5512d20add327d2865e9c60ce6fb21181a 100644 (file)
--- a/arch/um/include/asm/common.lds.S
+++ b/arch/um/include/asm/common.lds.S
@@ -42,7 +42,7 @@
        INIT_SETUP(0)
   }
 
-  PERCPU(32, 32)
+  PERCPU_SECTION(32)
        
   .initcall.init : {
        INIT_CALLS

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 624a2016198ef9317992069aaf676da19e5bd6b9..3e9fb5d54f96d98dc9c0463b9f9751724f19ce57 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -319,7 +319,7 @@ SECTIONS
        }
 
 #if !defined(CONFIG_X86_64) || !defined(CONFIG_SMP)
-       PERCPU(INTERNODE_CACHE_BYTES, PAGE_SIZE)
+       PERCPU_SECTION(INTERNODE_CACHE_BYTES)
 #endif
 
        . = ALIGN(PAGE_SIZE);

diff --git a/arch/xtensa/kernel/vmlinux.lds.S b/arch/xtensa/kernel/vmlinux.lds.S
index a2820065927eaa642224af0d748491506fdd473d..88ecea3facb4e8c81f507a667ed55feda6cfcb1a 100644 (file)
--- a/arch/xtensa/kernel/vmlinux.lds.S
+++ b/arch/xtensa/kernel/vmlinux.lds.S
@@ -155,7 +155,7 @@ SECTIONS
     INIT_RAM_FS
   }
 
-  PERCPU(XCHAL_ICACHE_LINESIZE, PAGE_SIZE)
+  PERCPU_SECTION(XCHAL_ICACHE_LINESIZE)
 
   /* We need this dummy segment here */
 

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 32c45e5fe0abfa4516fb3b56103b68b134b1d70c..f301cea5ca2d7cab0b52de3f96e6a927a4859173 100644 (file)
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -15,7 +15,7 @@
  *     HEAD_TEXT_SECTION
  *     INIT_TEXT_SECTION(PAGE_SIZE)
  *     INIT_DATA_SECTION(...)
- *     PERCPU(CACHELINE_SIZE, PAGE_SIZE)
+ *     PERCPU_SECTION(CACHELINE_SIZE)
  *     __init_end = .;
  *
  *     _stext = .;
@@ -709,7 +709,7 @@
  *
  * Note that this macros defines __per_cpu_load as an absolute symbol.
  * If there is no need to put the percpu section at a predetermined
- * address, use PERCPU().
+ * address, use PERCPU_SECTION.
  */
 #define PERCPU_VADDR(cacheline, vaddr, phdr)                           \
        VMLINUX_SYMBOL(__per_cpu_load) = .;                             \
@@ -729,20 +729,19 @@
        . = VMLINUX_SYMBOL(__per_cpu_load) + SIZEOF(.data..percpu);
 
 /**
- * PERCPU - define output section for percpu area, simple version
+ * PERCPU_SECTION - define output section for percpu area, simple version
  * @cacheline: cacheline size
- * @align: required alignment
  *
- * Align to @align and outputs output section for percpu area.  This macro
- * doesn't manipulate @vaddr or @phdr and __per_cpu_load and
+ * Align to PAGE_SIZE and outputs output section for percpu area.  This
+ * macro doesn't manipulate @vaddr or @phdr and __per_cpu_load and
  * __per_cpu_start will be identical.
  *
- * This macro is equivalent to ALIGN(@align); PERCPU_VADDR(@cacheline,,)
+ * This macro is equivalent to ALIGN(PAGE_SIZE); PERCPU_VADDR(@cacheline,,)
  * except that __per_cpu_load is defined as a relative symbol against
  * .data..percpu which is required for relocatable x86_32 configuration.
  */
-#define PERCPU(cacheline, align)                                       \
-       . = ALIGN(align);                                               \
+#define PERCPU_SECTION(cacheline)                                      \
+       . = ALIGN(PAGE_SIZE);                                           \
        .data..percpu   : AT(ADDR(.data..percpu) - LOAD_OFFSET) {       \
                VMLINUX_SYMBOL(__per_cpu_load) = .;                     \
                VMLINUX_SYMBOL(__per_cpu_start) = .;                    \

diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 04ef830690eceea974d90add3de55b57bc4048f6..d30a502e8c6d345289817b4f16c7d72bb04bc109 100644 (file)
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -2860,9 +2860,7 @@ static int alloc_cwqs(struct workqueue_struct *wq)
                }
        }
 
-       /* just in case, make sure it's actually aligned
-        * - this is affected by PERCPU() alignment in vmlinux.lds.S
-        */
+       /* just in case, make sure it's actually aligned */
        BUG_ON(!IS_ALIGNED(wq->cpu_wq.v, align));
        return wq->cpu_wq.v ? 0 : -ENOMEM;
 }

diff --git a/mm/percpu.c b/mm/percpu.c
index 3f930018aa60dc075bc49f7283c57b2b3b6f0035..c5feb79f5995d375f37b17ebe599f3fc407b422a 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1216,8 +1216,10 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
        PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
 #ifdef CONFIG_SMP
        PCPU_SETUP_BUG_ON(!ai->static_size);
+       PCPU_SETUP_BUG_ON((unsigned long)__per_cpu_start & ~PAGE_MASK);
 #endif
        PCPU_SETUP_BUG_ON(!base_addr);
+       PCPU_SETUP_BUG_ON((unsigned long)base_addr & ~PAGE_MASK);
        PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
        PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
        PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);