[PATCH] md: RAID6: clean up CPUID and FPU enter/exit code

- Use kernel_fpu_begin() and kernel_fpu_end()
- Use boot_cpu_has() for feature testing even in userspace

Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Signed-off-by: Neil Brown <neilb@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/drivers/md/raid6mmx.c b/drivers/md/raid6mmx.c
index 359157aaf9e0c2479dc58b12c99b7ebee2ece238..6181a5a3365a198d3aad83b52f6c7e09a4d5aad3 100644 (file)
--- a/drivers/md/raid6mmx.c
+++ b/drivers/md/raid6mmx.c
@@ -30,14 +30,8 @@ const struct raid6_mmx_constants {
 
 static int raid6_have_mmx(void)
 {
-#ifdef __KERNEL__
        /* Not really "boot_cpu" but "all_cpus" */
        return boot_cpu_has(X86_FEATURE_MMX);
-#else
-       /* User space test code */
-       u32 features = cpuid_features();
-       return ( (features & (1<<23)) == (1<<23) );
-#endif
 }
 
 /*
@@ -48,13 +42,12 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_mmx_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_mmx(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
@@ -78,7 +71,7 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("pxor %mm4,%mm4");
        }
 
-       raid6_after_mmx(&sa);
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_mmxx1 = {
@@ -96,13 +89,12 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_mmx_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_mmx(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
@@ -137,7 +129,7 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
        }
 
-       raid6_after_mmx(&sa);
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_mmxx2 = {

diff --git a/drivers/md/raid6sse1.c b/drivers/md/raid6sse1.c
index f7e7859f71aa33acc7cd74a2b4fe91638ef438cd..f0a1ba8f40ba90c6b59fdc69073c4bb997b425b4 100644 (file)
--- a/drivers/md/raid6sse1.c
+++ b/drivers/md/raid6sse1.c
@@ -33,16 +33,10 @@ extern const struct raid6_mmx_constants {
 
 static int raid6_have_sse1_or_mmxext(void)
 {
-#ifdef __KERNEL__
        /* Not really boot_cpu but "all_cpus" */
        return boot_cpu_has(X86_FEATURE_MMX) &&
                (boot_cpu_has(X86_FEATURE_XMM) ||
                 boot_cpu_has(X86_FEATURE_MMXEXT));
-#else
-       /* User space test code - this incorrectly breaks on some Athlons */
-       u32 features = cpuid_features();
-       return ( (features & (5<<23)) == (5<<23) );
-#endif
 }
 
 /*
@@ -53,14 +47,12 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_mmx_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       /* This is really MMX code, not SSE */
-       raid6_before_mmx(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
@@ -94,8 +86,8 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
        }
 
-       raid6_after_mmx(&sa);
        asm volatile("sfence" : : : "memory");
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_sse1x1 = {
@@ -113,13 +105,12 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_mmx_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_mmx(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
@@ -157,8 +148,8 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("movntq %%mm6,%0" : "=m" (q[d+8]));
        }
 
-       raid6_after_mmx(&sa);
        asm volatile("sfence" : :: "memory");
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_sse1x2 = {

diff --git a/drivers/md/raid6sse2.c b/drivers/md/raid6sse2.c
index b3aa7fe0877e8d8a94a9923b0885e4aa9eeb9b11..0f019762a7c3cba42cba1ea68a99459465f17223 100644 (file)
--- a/drivers/md/raid6sse2.c
+++ b/drivers/md/raid6sse2.c
@@ -30,17 +30,11 @@ static const struct raid6_sse_constants {
 
 static int raid6_have_sse2(void)
 {
-#ifdef __KERNEL__
        /* Not really boot_cpu but "all_cpus" */
        return boot_cpu_has(X86_FEATURE_MMX) &&
                boot_cpu_has(X86_FEATURE_FXSR) &&
                boot_cpu_has(X86_FEATURE_XMM) &&
                boot_cpu_has(X86_FEATURE_XMM2);
-#else
-       /* User space test code */
-       u32 features = cpuid_features();
-       return ( (features & (15<<23)) == (15<<23) );
-#endif
 }
 
 /*
@@ -51,13 +45,12 @@ static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_sse_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_sse2(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
        asm volatile("pxor %xmm5,%xmm5");       /* Zero temp */
@@ -93,8 +86,8 @@ static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("pxor %xmm4,%xmm4");
        }
 
-       raid6_after_sse2(&sa);
        asm volatile("sfence" : : : "memory");
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_sse2x1 = {
@@ -112,13 +105,12 @@ static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_sse_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_sse2(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
        asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
@@ -156,8 +148,8 @@ static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
        }
 
-       raid6_after_sse2(&sa);
        asm volatile("sfence" : : : "memory");
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_sse2x2 = {
@@ -179,13 +171,12 @@ static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
        u8 **dptr = (u8 **)ptrs;
        u8 *p, *q;
        int d, z, z0;
-       raid6_sse16_save_t sa;
 
        z0 = disks - 3;         /* Highest data disk */
        p = dptr[z0+1];         /* XOR parity */
        q = dptr[z0+2];         /* RS syndrome */
 
-       raid6_before_sse16(&sa);
+       kernel_fpu_begin();
 
        asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
        asm volatile("pxor %xmm2,%xmm2");       /* P[0] */
@@ -256,8 +247,9 @@ static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
                asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
                asm volatile("pxor %xmm14,%xmm14");
        }
+
        asm volatile("sfence" : : : "memory");
-       raid6_after_sse16(&sa);
+       kernel_fpu_end();
 }
 
 const struct raid6_calls raid6_sse2x4 = {

diff --git a/drivers/md/raid6x86.h b/drivers/md/raid6x86.h
index 4cf20534fe442ea750b219f9f0de9c4520ff3f48..9111950414ff2d71c003591f2ad4c3503303de5c 100644 (file)
--- a/drivers/md/raid6x86.h
+++ b/drivers/md/raid6x86.h
@@ -21,224 +21,40 @@
 
 #if defined(__i386__) || defined(__x86_64__)
 
-#ifdef __x86_64__
-
-typedef struct {
-       unsigned int fsave[27];
-       unsigned long cr0;
-} raid6_mmx_save_t __attribute__((aligned(16)));
-
-/* N.B.: For SSE we only save %xmm0-%xmm7 even for x86-64, since
-   the code doesn't know about the additional x86-64 registers */
-typedef struct {
-       unsigned int sarea[8*4+2];
-       unsigned long cr0;
-} raid6_sse_save_t __attribute__((aligned(16)));
-
-/* This is for x86-64-specific code which uses all 16 XMM registers */
-typedef struct {
-       unsigned int sarea[16*4+2];
-       unsigned long cr0;
-} raid6_sse16_save_t __attribute__((aligned(16)));
-
-/* On x86-64 the stack *SHOULD* be 16-byte aligned, but currently this
-   is buggy in the kernel and it's only 8-byte aligned in places, so
-   we need to do this anyway.  Sigh. */
-#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
-
-#else /* __i386__ */
-
-typedef struct {
-       unsigned int fsave[27];
-       unsigned long cr0;
-} raid6_mmx_save_t;
-
-/* On i386, the stack is only 8-byte aligned, but SSE requires 16-byte
-   alignment.  The +3 is so we have the slack space to manually align
-   a properly-sized area correctly.  */
-typedef struct {
-       unsigned int sarea[8*4+3];
-       unsigned long cr0;
-} raid6_sse_save_t;
-
-/* Find the 16-byte aligned save area */
-#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
-
-#endif
-
 #ifdef __KERNEL__ /* Real code */
 
-/* Note: %cr0 is 32 bits on i386 and 64 bits on x86-64 */
-
-static inline unsigned long raid6_get_fpu(void)
-{
-       unsigned long cr0;
-
-       preempt_disable();
-       asm volatile("mov %%cr0,%0 ; clts" : "=r" (cr0));
-       return cr0;
-}
-
-static inline void raid6_put_fpu(unsigned long cr0)
-{
-       asm volatile("mov %0,%%cr0" : : "r" (cr0));
-       preempt_enable();
-}
+#include <asm/i387.h>
 
 #else /* Dummy code for user space testing */
 
-static inline unsigned long raid6_get_fpu(void)
-{
-       return 0xf00ba6;
-}
-
-static inline void raid6_put_fpu(unsigned long cr0)
-{
-       (void)cr0;
-}
-
-#endif
-
-static inline void raid6_before_mmx(raid6_mmx_save_t *s)
-{
-       s->cr0 = raid6_get_fpu();
-       asm volatile("fsave %0 ; fwait" : "=m" (s->fsave[0]));
-}
-
-static inline void raid6_after_mmx(raid6_mmx_save_t *s)
-{
-       asm volatile("frstor %0" : : "m" (s->fsave[0]));
-       raid6_put_fpu(s->cr0);
-}
-
-static inline void raid6_before_sse(raid6_sse_save_t *s)
-{
-       unsigned int *rsa = SAREA(s);
-
-       s->cr0 = raid6_get_fpu();
-
-       asm volatile("movaps %%xmm0,%0" : "=m" (rsa[0]));
-       asm volatile("movaps %%xmm1,%0" : "=m" (rsa[4]));
-       asm volatile("movaps %%xmm2,%0" : "=m" (rsa[8]));
-       asm volatile("movaps %%xmm3,%0" : "=m" (rsa[12]));
-       asm volatile("movaps %%xmm4,%0" : "=m" (rsa[16]));
-       asm volatile("movaps %%xmm5,%0" : "=m" (rsa[20]));
-       asm volatile("movaps %%xmm6,%0" : "=m" (rsa[24]));
-       asm volatile("movaps %%xmm7,%0" : "=m" (rsa[28]));
-}
-
-static inline void raid6_after_sse(raid6_sse_save_t *s)
-{
-       unsigned int *rsa = SAREA(s);
-
-       asm volatile("movaps %0,%%xmm0" : : "m" (rsa[0]));
-       asm volatile("movaps %0,%%xmm1" : : "m" (rsa[4]));
-       asm volatile("movaps %0,%%xmm2" : : "m" (rsa[8]));
-       asm volatile("movaps %0,%%xmm3" : : "m" (rsa[12]));
-       asm volatile("movaps %0,%%xmm4" : : "m" (rsa[16]));
-       asm volatile("movaps %0,%%xmm5" : : "m" (rsa[20]));
-       asm volatile("movaps %0,%%xmm6" : : "m" (rsa[24]));
-       asm volatile("movaps %0,%%xmm7" : : "m" (rsa[28]));
-
-       raid6_put_fpu(s->cr0);
-}
-
-static inline void raid6_before_sse2(raid6_sse_save_t *s)
+static inline void kernel_fpu_begin(void)
 {
-       unsigned int *rsa = SAREA(s);
-
-       s->cr0 = raid6_get_fpu();
-
-       asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
-       asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
-       asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
-       asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
-       asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
-       asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
-       asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
-       asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
 }
 
-static inline void raid6_after_sse2(raid6_sse_save_t *s)
+static inline void kernel_fpu_end(void)
 {
-       unsigned int *rsa = SAREA(s);
-
-       asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
-       asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
-       asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
-       asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
-       asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
-       asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
-       asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
-       asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
-
-       raid6_put_fpu(s->cr0);
 }
 
-#ifdef __x86_64__
-
-static inline void raid6_before_sse16(raid6_sse16_save_t *s)
-{
-       unsigned int *rsa = SAREA(s);
-
-       s->cr0 = raid6_get_fpu();
+#define X86_FEATURE_MMX                (0*32+23) /* Multimedia Extensions */
+#define X86_FEATURE_FXSR       (0*32+24) /* FXSAVE and FXRSTOR instructions
+                                          * (fast save and restore) */
+#define X86_FEATURE_XMM                (0*32+25) /* Streaming SIMD Extensions */
+#define X86_FEATURE_XMM2       (0*32+26) /* Streaming SIMD Extensions-2 */
+#define X86_FEATURE_MMXEXT     (1*32+22) /* AMD MMX extensions */
 
-       asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
-       asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
-       asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
-       asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
-       asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
-       asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
-       asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
-       asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
-       asm volatile("movdqa %%xmm8,%0" : "=m" (rsa[32]));
-       asm volatile("movdqa %%xmm9,%0" : "=m" (rsa[36]));
-       asm volatile("movdqa %%xmm10,%0" : "=m" (rsa[40]));
-       asm volatile("movdqa %%xmm11,%0" : "=m" (rsa[44]));
-       asm volatile("movdqa %%xmm12,%0" : "=m" (rsa[48]));
-       asm volatile("movdqa %%xmm13,%0" : "=m" (rsa[52]));
-       asm volatile("movdqa %%xmm14,%0" : "=m" (rsa[56]));
-       asm volatile("movdqa %%xmm15,%0" : "=m" (rsa[60]));
-}
-
-static inline void raid6_after_sse16(raid6_sse16_save_t *s)
+/* Should work well enough on modern CPUs for testing */
+static inline int boot_cpu_has(int flag)
 {
-       unsigned int *rsa = SAREA(s);
+       u32 eax = (flag >> 5) ? 0x80000001 : 1;
+       u32 edx;
 
-       asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
-       asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
-       asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
-       asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
-       asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
-       asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
-       asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
-       asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
-       asm volatile("movdqa %0,%%xmm8" : : "m" (rsa[32]));
-       asm volatile("movdqa %0,%%xmm9" : : "m" (rsa[36]));
-       asm volatile("movdqa %0,%%xmm10" : : "m" (rsa[40]));
-       asm volatile("movdqa %0,%%xmm11" : : "m" (rsa[44]));
-       asm volatile("movdqa %0,%%xmm12" : : "m" (rsa[48]));
-       asm volatile("movdqa %0,%%xmm13" : : "m" (rsa[52]));
-       asm volatile("movdqa %0,%%xmm14" : : "m" (rsa[56]));
-       asm volatile("movdqa %0,%%xmm15" : : "m" (rsa[60]));
+       asm volatile("cpuid"
+                    : "+a" (eax), "=d" (edx)
+                    : : "ecx", "ebx");
 
-       raid6_put_fpu(s->cr0);
+       return (edx >> (flag & 31)) & 1;
 }
 
-#endif /* __x86_64__ */
-
-/* User space test hack */
-#ifndef __KERNEL__
-static inline int cpuid_features(void)
-{
-       u32 eax = 1;
-       u32 ebx, ecx, edx;
-
-       asm volatile("cpuid" :
-                    "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx));
-
-       return edx;
-}
 #endif /* ndef __KERNEL__ */
 
 #endif