crypto: chacha20 - Fix chacha20_block() keystream alignment (again)

[ Upstream commit a5e9f557098e54af44ade5d501379be18435bfbf ]

In commit 9f480faec58c ("crypto: chacha20 - Fix keystream alignment for
chacha20_block()"), I had missed that chacha20_block() can be called
directly on the buffer passed to get_random_bytes(), which can have any
alignment.  So, while my commit didn't break anything, it didn't fully
solve the alignment problems.

Revert my solution and just update chacha20_block() to use
put_unaligned_le32(), so the output buffer need not be aligned.
This is simpler, and on many CPUs it's the same speed.

But, I kept the 'tmp' buffers in extract_crng_user() and
_get_random_bytes() 4-byte aligned, since that alignment is actually
needed for _crng_backtrack_protect() too.

Reported-by: Stephan Müller <smueller@chronox.de>
Cc: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c
index 80b8aa2d68b0f1458643327b6a938dcc0c0e989b..24827d00fb679e38b93847ca09246025bda10e25 100644 (file)
--- a/crypto/chacha20_generic.c
+++ b/crypto/chacha20_generic.c
@@ -22,20 +22,21 @@ static inline u32 le32_to_cpuvp(const void *p)
 static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src,
                             unsigned int bytes)
 {
-       u32 stream[CHACHA20_BLOCK_WORDS];
+       /* aligned to potentially speed up crypto_xor() */
+       u8 stream[CHACHA20_BLOCK_SIZE] __aligned(sizeof(long));
 
        if (dst != src)
                memcpy(dst, src, bytes);
 
        while (bytes >= CHACHA20_BLOCK_SIZE) {
                chacha20_block(state, stream);
-               crypto_xor(dst, (const u8 *)stream, CHACHA20_BLOCK_SIZE);
+               crypto_xor(dst, stream, CHACHA20_BLOCK_SIZE);
                bytes -= CHACHA20_BLOCK_SIZE;
                dst += CHACHA20_BLOCK_SIZE;
        }
        if (bytes) {
                chacha20_block(state, stream);
-               crypto_xor(dst, (const u8 *)stream, bytes);
+               crypto_xor(dst, stream, bytes);
        }
 }
 

diff --git a/drivers/char/random.c b/drivers/char/random.c
index 69ec8aea3463c02f7b8c98f852f84044615ee413..5d2710441b65fe86ef88c6025c25c02fa37a08cb 100644 (file)
--- a/drivers/char/random.c
+++ b/drivers/char/random.c
@@ -485,9 +485,9 @@ static int crng_init_cnt = 0;
 static unsigned long crng_global_init_time = 0;
 #define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
 static void _extract_crng(struct crng_state *crng,
-                         __u32 out[CHACHA20_BLOCK_WORDS]);
+                         __u8 out[CHACHA20_BLOCK_SIZE]);
 static void _crng_backtrack_protect(struct crng_state *crng,
-                                   __u32 tmp[CHACHA20_BLOCK_WORDS], int used);
+                                   __u8 tmp[CHACHA20_BLOCK_SIZE], int used);
 static void process_random_ready_list(void);
 static void _get_random_bytes(void *buf, int nbytes);
 
@@ -986,7 +986,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
        unsigned long   flags;
        int             i, num;
        union {
-               __u32   block[CHACHA20_BLOCK_WORDS];
+               __u8    block[CHACHA20_BLOCK_SIZE];
                __u32   key[8];
        } buf;
 
@@ -1014,7 +1014,7 @@ static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
 }
 
 static void _extract_crng(struct crng_state *crng,
-                         __u32 out[CHACHA20_BLOCK_WORDS])
+                         __u8 out[CHACHA20_BLOCK_SIZE])
 {
        unsigned long flags, init_time;
 
@@ -1032,7 +1032,7 @@ static void _extract_crng(struct crng_state *crng,
        spin_unlock_irqrestore(&crng->lock, flags);
 }
 
-static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
+static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
 {
        _extract_crng(select_crng(), out);
 }
@@ -1042,7 +1042,7 @@ static void extract_crng(__u32 out[CHACHA20_BLOCK_WORDS])
  * enough) to mutate the CRNG key to provide backtracking protection.
  */
 static void _crng_backtrack_protect(struct crng_state *crng,
-                                   __u32 tmp[CHACHA20_BLOCK_WORDS], int used)
+                                   __u8 tmp[CHACHA20_BLOCK_SIZE], int used)
 {
        unsigned long   flags;
        __u32           *s, *d;
@@ -1054,14 +1054,14 @@ static void _crng_backtrack_protect(struct crng_state *crng,
                used = 0;
        }
        spin_lock_irqsave(&crng->lock, flags);
-       s = &tmp[used / sizeof(__u32)];
+       s = (__u32 *) &tmp[used];
        d = &crng->state[4];
        for (i=0; i < 8; i++)
                *d++ ^= *s++;
        spin_unlock_irqrestore(&crng->lock, flags);
 }
 
-static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
+static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used)
 {
        _crng_backtrack_protect(select_crng(), tmp, used);
 }
@@ -1069,7 +1069,7 @@ static void crng_backtrack_protect(__u32 tmp[CHACHA20_BLOCK_WORDS], int used)
 static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
 {
        ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE;
-       __u32 tmp[CHACHA20_BLOCK_WORDS];
+       __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4);
        int large_request = (nbytes > 256);
 
        while (nbytes) {
@@ -1529,7 +1529,7 @@ static void _warn_unseeded_randomness(const char *func_name, void *caller,
  */
 static void _get_random_bytes(void *buf, int nbytes)
 {
-       __u32 tmp[CHACHA20_BLOCK_WORDS];
+       __u8 tmp[CHACHA20_BLOCK_SIZE] __aligned(4);
 
        trace_get_random_bytes(nbytes, _RET_IP_);
 
@@ -2116,7 +2116,7 @@ u64 get_random_u64(void)
        batch = raw_cpu_ptr(&batched_entropy_u64);
        spin_lock_irqsave(&batch->batch_lock, flags);
        if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
-               extract_crng((__u32 *)batch->entropy_u64);
+               extract_crng((u8 *)batch->entropy_u64);
                batch->position = 0;
        }
        ret = batch->entropy_u64[batch->position++];
@@ -2140,7 +2140,7 @@ u32 get_random_u32(void)
        batch = raw_cpu_ptr(&batched_entropy_u32);
        spin_lock_irqsave(&batch->batch_lock, flags);
        if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
-               extract_crng(batch->entropy_u32);
+               extract_crng((u8 *)batch->entropy_u32);
                batch->position = 0;
        }
        ret = batch->entropy_u32[batch->position++];

diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h
index 04bdf4bf197d59f18d7e249c65a36f00c55b367f..3dd5ab189543fe3a414a5f922e118e15950cd337 100644 (file)
--- a/include/crypto/chacha20.h
+++ b/include/crypto/chacha20.h
@@ -13,13 +13,12 @@
 #define CHACHA20_IV_SIZE       16
 #define CHACHA20_KEY_SIZE      32
 #define CHACHA20_BLOCK_SIZE    64
-#define CHACHA20_BLOCK_WORDS   (CHACHA20_BLOCK_SIZE / sizeof(u32))
 
 struct chacha20_ctx {
        u32 key[8];
 };
 
-void chacha20_block(u32 *state, u32 *stream);
+void chacha20_block(u32 *state, u8 *stream);
 void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv);
 int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
                           unsigned int keysize);

diff --git a/lib/chacha20.c b/lib/chacha20.c
index 29d3801dee24f5a3b1a6290a578d1ef5baf710c0..5f6c222e939a9ce3563802d9a90b4345ea7815c0 100644 (file)
--- a/lib/chacha20.c
+++ b/lib/chacha20.c
@@ -21,9 +21,9 @@ static inline u32 rotl32(u32 v, u8 n)
        return (v << n) | (v >> (sizeof(v) * 8 - n));
 }
 
-void chacha20_block(u32 *state, u32 *stream)
+void chacha20_block(u32 *state, u8 *stream)
 {
-       u32 x[16], *out = stream;
+       u32 x[16];
        int i;
 
        for (i = 0; i < ARRAY_SIZE(x); i++)
@@ -72,7 +72,7 @@ void chacha20_block(u32 *state, u32 *stream)
        }
 
        for (i = 0; i < ARRAY_SIZE(x); i++)
-               out[i] = cpu_to_le32(x[i] + state[i]);
+               put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]);
 
        state[12]++;
 }