Commit | Line | Data |
---|---|---|
6fa3eb70 S |
1 | /* |
2 | * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions | |
3 | * | |
4 | * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org> | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License version 2 as | |
8 | * published by the Free Software Foundation. | |
9 | */ | |
10 | ||
11 | #include <asm/neon.h> | |
12 | #include <crypto/aes.h> | |
13 | #include <linux/cpufeature.h> | |
14 | #include <linux/crypto.h> | |
15 | #include <linux/module.h> | |
16 | ||
17 | MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions"); | |
18 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); | |
19 | MODULE_LICENSE("GPL v2"); | |
20 | ||
21 | struct aes_block { | |
22 | u8 b[AES_BLOCK_SIZE]; | |
23 | }; | |
24 | ||
25 | static int num_rounds(struct crypto_aes_ctx *ctx) | |
26 | { | |
27 | /* | |
28 | * # of rounds specified by AES: | |
29 | * 128 bit key 10 rounds | |
30 | * 192 bit key 12 rounds | |
31 | * 256 bit key 14 rounds | |
32 | * => n byte key => 6 + (n/4) rounds | |
33 | */ | |
34 | return 6 + ctx->key_length / 4; | |
35 | } | |
36 | ||
37 | static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) | |
38 | { | |
39 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
40 | struct aes_block *out = (struct aes_block *)dst; | |
41 | struct aes_block const *in = (struct aes_block *)src; | |
42 | void *dummy0; | |
43 | int dummy1; | |
44 | ||
45 | kernel_neon_begin_partial(4); | |
46 | ||
47 | __asm__(" ld1 {v0.16b}, %[in] ;" | |
48 | " ld1 {v1.2d}, [%[key]], #16 ;" | |
49 | " cmp %w[rounds], #10 ;" | |
50 | " bmi 0f ;" | |
51 | " bne 3f ;" | |
52 | " mov v3.16b, v1.16b ;" | |
53 | " b 2f ;" | |
54 | "0: mov v2.16b, v1.16b ;" | |
55 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
56 | "1: aese v0.16b, v2.16b ;" | |
57 | " aesmc v0.16b, v0.16b ;" | |
58 | "2: ld1 {v1.2d}, [%[key]], #16 ;" | |
59 | " aese v0.16b, v3.16b ;" | |
60 | " aesmc v0.16b, v0.16b ;" | |
61 | "3: ld1 {v2.2d}, [%[key]], #16 ;" | |
62 | " subs %w[rounds], %w[rounds], #3 ;" | |
63 | " aese v0.16b, v1.16b ;" | |
64 | " aesmc v0.16b, v0.16b ;" | |
65 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
66 | " bpl 1b ;" | |
67 | " aese v0.16b, v2.16b ;" | |
68 | " eor v0.16b, v0.16b, v3.16b ;" | |
69 | " st1 {v0.16b}, %[out] ;" | |
70 | ||
71 | : [out] "=Q"(*out), | |
72 | [key] "=r"(dummy0), | |
73 | [rounds] "=r"(dummy1) | |
74 | : [in] "Q"(*in), | |
75 | "1"(ctx->key_enc), | |
76 | "2"(num_rounds(ctx) - 2) | |
77 | : "cc"); | |
78 | ||
79 | kernel_neon_end(); | |
80 | } | |
81 | ||
82 | static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) | |
83 | { | |
84 | struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); | |
85 | struct aes_block *out = (struct aes_block *)dst; | |
86 | struct aes_block const *in = (struct aes_block *)src; | |
87 | void *dummy0; | |
88 | int dummy1; | |
89 | ||
90 | kernel_neon_begin_partial(4); | |
91 | ||
92 | __asm__(" ld1 {v0.16b}, %[in] ;" | |
93 | " ld1 {v1.2d}, [%[key]], #16 ;" | |
94 | " cmp %w[rounds], #10 ;" | |
95 | " bmi 0f ;" | |
96 | " bne 3f ;" | |
97 | " mov v3.16b, v1.16b ;" | |
98 | " b 2f ;" | |
99 | "0: mov v2.16b, v1.16b ;" | |
100 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
101 | "1: aesd v0.16b, v2.16b ;" | |
102 | " aesimc v0.16b, v0.16b ;" | |
103 | "2: ld1 {v1.2d}, [%[key]], #16 ;" | |
104 | " aesd v0.16b, v3.16b ;" | |
105 | " aesimc v0.16b, v0.16b ;" | |
106 | "3: ld1 {v2.2d}, [%[key]], #16 ;" | |
107 | " subs %w[rounds], %w[rounds], #3 ;" | |
108 | " aesd v0.16b, v1.16b ;" | |
109 | " aesimc v0.16b, v0.16b ;" | |
110 | " ld1 {v3.2d}, [%[key]], #16 ;" | |
111 | " bpl 1b ;" | |
112 | " aesd v0.16b, v2.16b ;" | |
113 | " eor v0.16b, v0.16b, v3.16b ;" | |
114 | " st1 {v0.16b}, %[out] ;" | |
115 | ||
116 | : [out] "=Q"(*out), | |
117 | [key] "=r"(dummy0), | |
118 | [rounds] "=r"(dummy1) | |
119 | : [in] "Q"(*in), | |
120 | "1"(ctx->key_dec), | |
121 | "2"(num_rounds(ctx) - 2) | |
122 | : "cc"); | |
123 | ||
124 | kernel_neon_end(); | |
125 | } | |
126 | ||
127 | static struct crypto_alg aes_alg = { | |
128 | .cra_name = "aes", | |
129 | .cra_driver_name = "aes-ce", | |
130 | .cra_priority = 300, | |
131 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, | |
132 | .cra_blocksize = AES_BLOCK_SIZE, | |
133 | .cra_ctxsize = sizeof(struct crypto_aes_ctx), | |
134 | .cra_module = THIS_MODULE, | |
135 | .cra_cipher = { | |
136 | .cia_min_keysize = AES_MIN_KEY_SIZE, | |
137 | .cia_max_keysize = AES_MAX_KEY_SIZE, | |
138 | .cia_setkey = crypto_aes_set_key, | |
139 | .cia_encrypt = aes_cipher_encrypt, | |
140 | .cia_decrypt = aes_cipher_decrypt | |
141 | } | |
142 | }; | |
143 | ||
144 | static int __init aes_mod_init(void) | |
145 | { | |
146 | return crypto_register_alg(&aes_alg); | |
147 | } | |
148 | ||
149 | static void __exit aes_mod_exit(void) | |
150 | { | |
151 | crypto_unregister_alg(&aes_alg); | |
152 | } | |
153 | ||
154 | module_cpu_feature_match(AES, aes_mod_init); | |
155 | module_exit(aes_mod_exit); |