arm64: alternative: put secondary CPUs into polling loop during patch

When patching the kernel text with alternatives, we may end up patching
parts of the stop_machine state machine (e.g. atomic_dec_and_test in
ack_state) and consequently corrupt the instruction stream of any
secondary CPUs.

This patch passes the cpu_online_mask to stop_machine, forcing all of
the CPUs into our own callback which can place the secondary cores into
a dumb (but safe!) polling loop whilst the patching is carried out.

Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h
index b474e9106bc277c9309702182c3fe886a08252f2..d56ec07151570e6e498e9bf35b2fd9ccaed335fe 100644 (file)
--- a/arch/arm64/include/asm/alternative.h
+++ b/arch/arm64/include/asm/alternative.h
@@ -3,6 +3,7 @@
 
 #ifndef __ASSEMBLY__
 
+#include <linux/init.h>
 #include <linux/kconfig.h>
 #include <linux/types.h>
 #include <linux/stddef.h>
@@ -16,7 +17,7 @@ struct alt_instr {
        u8  alt_len;            /* size of new instruction(s), <= orig_len */
 };
 
-void apply_alternatives_all(void);
+void __init apply_alternatives_all(void);
 void apply_alternatives(void *start, size_t length);
 void free_alternatives_memory(void);
 

diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 221b98312f0c6a7fa7977ec61cef316c89a529e7..fa1d575ab2c2f5b59cbc616093b95dffe3cea9af 100644 (file)
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -85,7 +85,7 @@ static u32 get_alt_insn(struct alt_instr *alt, u32 *insnptr, u32 *altinsnptr)
        return insn;
 }
 
-static int __apply_alternatives(void *alt_region)
+static void __apply_alternatives(void *alt_region)
 {
        struct alt_instr *alt;
        struct alt_region *region = alt_region;
@@ -114,19 +114,38 @@ static int __apply_alternatives(void *alt_region)
                flush_icache_range((uintptr_t)origptr,
                                   (uintptr_t)(origptr + nr_inst));
        }
-
-       return 0;
 }
 
-void apply_alternatives_all(void)
+/*
+ * We might be patching the stop_machine state machine, so implement a
+ * really simple polling protocol here.
+ */
+static int __apply_alternatives_multi_stop(void *unused)
 {
+       static int patched = 0;
        struct alt_region region = {
                .begin  = __alt_instructions,
                .end    = __alt_instructions_end,
        };
 
+       /* We always have a CPU 0 at this point (__init) */
+       if (smp_processor_id()) {
+               while (!READ_ONCE(patched))
+                       cpu_relax();
+       } else {
+               BUG_ON(patched);
+               __apply_alternatives(&region);
+               /* Barriers provided by the cache flushing */
+               WRITE_ONCE(patched, 1);
+       }
+
+       return 0;
+}
+
+void __init apply_alternatives_all(void)
+{
        /* better not try code patching on a live SMP system */
-       stop_machine(__apply_alternatives, &region, NULL);
+       stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
 }
 
 void apply_alternatives(void *start, size_t length)