2 * GCM: Galois/Counter Mode.
4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
11 #include <crypto/algapi.h>
12 #include <crypto/gf128mul.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/err.h>
15 #include <linux/init.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/slab.h>
22 struct gcm_instance_ctx
{
23 struct crypto_spawn ctr
;
26 struct crypto_gcm_ctx
{
27 struct crypto_ablkcipher
*ctr
;
28 struct gf128mul_4k
*gf128
;
31 struct crypto_gcm_ghash_ctx
{
34 struct gf128mul_4k
*gf128
;
38 struct crypto_gcm_req_priv_ctx
{
42 struct crypto_gcm_ghash_ctx ghash
;
43 struct ablkcipher_request abreq
;
46 static inline struct crypto_gcm_req_priv_ctx
*crypto_gcm_reqctx(
47 struct aead_request
*req
)
49 unsigned long align
= crypto_aead_alignmask(crypto_aead_reqtfm(req
));
51 return (void *)PTR_ALIGN((u8
*)aead_request_ctx(req
), align
+ 1);
54 static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx
*ctx
, u32 flags
,
55 struct gf128mul_4k
*gf128
)
60 memset(ctx
->buffer
, 0, 16);
63 static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx
*ctx
,
64 const u8
*src
, unsigned int srclen
)
66 u8
*dst
= ctx
->buffer
;
69 int n
= min(srclen
, ctx
->bytes
);
70 u8
*pos
= dst
+ (16 - ctx
->bytes
);
79 gf128mul_4k_lle((be128
*)dst
, ctx
->gf128
);
82 while (srclen
>= 16) {
83 crypto_xor(dst
, src
, 16);
84 gf128mul_4k_lle((be128
*)dst
, ctx
->gf128
);
90 ctx
->bytes
= 16 - srclen
;
96 static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx
*ctx
,
97 struct scatterlist
*sg
, int len
)
99 struct scatter_walk walk
;
106 scatterwalk_start(&walk
, sg
);
109 n
= scatterwalk_clamp(&walk
, len
);
112 scatterwalk_start(&walk
, scatterwalk_sg_next(walk
.sg
));
113 n
= scatterwalk_clamp(&walk
, len
);
116 src
= scatterwalk_map(&walk
, 0);
118 crypto_gcm_ghash_update(ctx
, src
, n
);
121 scatterwalk_unmap(src
, 0);
122 scatterwalk_advance(&walk
, n
);
123 scatterwalk_done(&walk
, 0, len
);
125 crypto_yield(ctx
->flags
);
129 static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx
*ctx
)
131 u8
*dst
= ctx
->buffer
;
134 u8
*tmp
= dst
+ (16 - ctx
->bytes
);
139 gf128mul_4k_lle((be128
*)dst
, ctx
->gf128
);
145 static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx
*ctx
,
146 unsigned int authlen
,
147 unsigned int cryptlen
, u8
*dst
)
149 u8
*buf
= ctx
->buffer
;
152 lengths
.a
= cpu_to_be64(authlen
* 8);
153 lengths
.b
= cpu_to_be64(cryptlen
* 8);
155 crypto_gcm_ghash_flush(ctx
);
156 crypto_xor(buf
, (u8
*)&lengths
, 16);
157 gf128mul_4k_lle((be128
*)buf
, ctx
->gf128
);
158 crypto_xor(dst
, buf
, 16);
161 static inline void crypto_gcm_set_counter(u8
*counterblock
, u32 value
)
163 *((u32
*)&counterblock
[12]) = cpu_to_be32(value
+ 1);
166 static int crypto_gcm_encrypt_counter(struct crypto_aead
*aead
, u8
*block
,
167 u32 value
, const u8
*iv
)
169 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(aead
);
170 struct crypto_ablkcipher
*ctr
= ctx
->ctr
;
171 struct ablkcipher_request req
;
172 struct scatterlist sg
;
176 memset(counterblock
, 0, 12);
178 memcpy(counterblock
, iv
, 12);
180 crypto_gcm_set_counter(counterblock
, value
);
182 sg_init_one(&sg
, block
, 16);
183 ablkcipher_request_set_tfm(&req
, ctr
);
184 ablkcipher_request_set_crypt(&req
, &sg
, &sg
, 16, counterblock
);
185 ablkcipher_request_set_callback(&req
, 0, NULL
, NULL
);
186 memset(block
, 0, 16);
187 return crypto_ablkcipher_encrypt(&req
);
190 static int crypto_gcm_setkey(struct crypto_aead
*aead
, const u8
*key
,
193 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(aead
);
194 struct crypto_ablkcipher
*ctr
= ctx
->ctr
;
195 int alignmask
= crypto_ablkcipher_alignmask(ctr
);
196 u8 alignbuf
[16+alignmask
];
197 u8
*hash
= (u8
*)ALIGN((unsigned long)alignbuf
, alignmask
+1);
200 crypto_ablkcipher_clear_flags(ctr
, CRYPTO_TFM_REQ_MASK
);
201 crypto_ablkcipher_set_flags(ctr
, crypto_aead_get_flags(aead
) &
202 CRYPTO_TFM_REQ_MASK
);
204 err
= crypto_ablkcipher_setkey(ctr
, key
, keylen
);
208 crypto_aead_set_flags(aead
, crypto_ablkcipher_get_flags(ctr
) &
209 CRYPTO_TFM_RES_MASK
);
211 err
= crypto_gcm_encrypt_counter(aead
, hash
, -1, NULL
);
215 if (ctx
->gf128
!= NULL
)
216 gf128mul_free_4k(ctx
->gf128
);
218 ctx
->gf128
= gf128mul_init_4k_lle((be128
*)hash
);
220 if (ctx
->gf128
== NULL
)
227 static int crypto_gcm_init_crypt(struct ablkcipher_request
*ablk_req
,
228 struct aead_request
*req
,
229 unsigned int cryptlen
,
230 void (*done
)(struct crypto_async_request
*,
233 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
234 struct crypto_gcm_ctx
*ctx
= crypto_aead_ctx(aead
);
235 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
236 u32 flags
= req
->base
.tfm
->crt_flags
;
237 u8
*auth_tag
= pctx
->auth_tag
;
238 u8
*counter
= pctx
->counter
;
239 struct crypto_gcm_ghash_ctx
*ghash
= &pctx
->ghash
;
242 ablkcipher_request_set_tfm(ablk_req
, ctx
->ctr
);
243 ablkcipher_request_set_callback(ablk_req
, aead_request_flags(req
),
245 ablkcipher_request_set_crypt(ablk_req
, req
->src
, req
->dst
,
248 err
= crypto_gcm_encrypt_counter(aead
, auth_tag
, 0, req
->iv
);
252 memcpy(counter
, req
->iv
, 12);
253 crypto_gcm_set_counter(counter
, 1);
255 crypto_gcm_ghash_init(ghash
, flags
, ctx
->gf128
);
257 crypto_gcm_ghash_update_sg(ghash
, req
->assoc
, req
->assoclen
);
258 crypto_gcm_ghash_flush(ghash
);
264 static int crypto_gcm_hash(struct aead_request
*req
)
266 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
267 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
268 u8
*auth_tag
= pctx
->auth_tag
;
269 struct crypto_gcm_ghash_ctx
*ghash
= &pctx
->ghash
;
271 crypto_gcm_ghash_update_sg(ghash
, req
->dst
, req
->cryptlen
);
272 crypto_gcm_ghash_final_xor(ghash
, req
->assoclen
, req
->cryptlen
,
275 scatterwalk_map_and_copy(auth_tag
, req
->dst
, req
->cryptlen
,
276 crypto_aead_authsize(aead
), 1);
280 static void crypto_gcm_encrypt_done(struct crypto_async_request
*areq
, int err
)
282 struct aead_request
*req
= areq
->data
;
285 err
= crypto_gcm_hash(req
);
287 aead_request_complete(req
, err
);
290 static int crypto_gcm_encrypt(struct aead_request
*req
)
292 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
293 struct ablkcipher_request
*abreq
= &pctx
->abreq
;
296 err
= crypto_gcm_init_crypt(abreq
, req
, req
->cryptlen
,
297 crypto_gcm_encrypt_done
);
302 err
= crypto_ablkcipher_encrypt(abreq
);
307 return crypto_gcm_hash(req
);
310 static void crypto_gcm_decrypt_done(struct crypto_async_request
*areq
, int err
)
312 aead_request_complete(areq
->data
, err
);
315 static int crypto_gcm_decrypt(struct aead_request
*req
)
317 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
318 struct crypto_gcm_req_priv_ctx
*pctx
= crypto_gcm_reqctx(req
);
319 struct ablkcipher_request
*abreq
= &pctx
->abreq
;
320 u8
*auth_tag
= pctx
->auth_tag
;
321 u8
*iauth_tag
= pctx
->iauth_tag
;
322 struct crypto_gcm_ghash_ctx
*ghash
= &pctx
->ghash
;
323 unsigned int cryptlen
= req
->cryptlen
;
324 unsigned int authsize
= crypto_aead_authsize(aead
);
327 if (cryptlen
< authsize
)
329 cryptlen
-= authsize
;
331 err
= crypto_gcm_init_crypt(abreq
, req
, cryptlen
,
332 crypto_gcm_decrypt_done
);
336 crypto_gcm_ghash_update_sg(ghash
, req
->src
, cryptlen
);
337 crypto_gcm_ghash_final_xor(ghash
, req
->assoclen
, cryptlen
, auth_tag
);
339 scatterwalk_map_and_copy(iauth_tag
, req
->src
, cryptlen
, authsize
, 0);
340 if (memcmp(iauth_tag
, auth_tag
, authsize
))
343 return crypto_ablkcipher_decrypt(abreq
);
346 static int crypto_gcm_init_tfm(struct crypto_tfm
*tfm
)
348 struct crypto_instance
*inst
= (void *)tfm
->__crt_alg
;
349 struct gcm_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
350 struct crypto_gcm_ctx
*ctx
= crypto_tfm_ctx(tfm
);
351 struct crypto_ablkcipher
*ctr
;
355 ctr
= crypto_spawn_ablkcipher(&ictx
->ctr
);
363 align
= crypto_tfm_alg_alignmask(tfm
);
364 align
&= ~(crypto_tfm_ctx_alignment() - 1);
365 tfm
->crt_aead
.reqsize
= align
+
366 sizeof(struct crypto_gcm_req_priv_ctx
) +
367 crypto_ablkcipher_reqsize(ctr
);
372 static void crypto_gcm_exit_tfm(struct crypto_tfm
*tfm
)
374 struct crypto_gcm_ctx
*ctx
= crypto_tfm_ctx(tfm
);
376 if (ctx
->gf128
!= NULL
)
377 gf128mul_free_4k(ctx
->gf128
);
379 crypto_free_ablkcipher(ctx
->ctr
);
382 static struct crypto_instance
*crypto_gcm_alloc(struct rtattr
**tb
)
384 struct crypto_instance
*inst
;
385 struct crypto_alg
*ctr
;
386 struct crypto_alg
*cipher
;
387 struct gcm_instance_ctx
*ctx
;
389 char ctr_name
[CRYPTO_MAX_ALG_NAME
];
391 err
= crypto_check_attr_type(tb
, CRYPTO_ALG_TYPE_AEAD
);
395 cipher
= crypto_attr_alg(tb
[1], CRYPTO_ALG_TYPE_CIPHER
,
396 CRYPTO_ALG_TYPE_MASK
);
398 inst
= ERR_PTR(PTR_ERR(cipher
));
402 inst
= ERR_PTR(ENAMETOOLONG
);
403 if (snprintf(ctr_name
, CRYPTO_MAX_ALG_NAME
, "ctr(%s)",
404 cipher
->cra_name
) >= CRYPTO_MAX_ALG_NAME
)
407 ctr
= crypto_alg_mod_lookup(ctr_name
, CRYPTO_ALG_TYPE_BLKCIPHER
,
408 CRYPTO_ALG_TYPE_MASK
);
411 return ERR_PTR(PTR_ERR(ctr
));
413 if (cipher
->cra_blocksize
!= 16)
416 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
422 if (snprintf(inst
->alg
.cra_name
, CRYPTO_MAX_ALG_NAME
,
423 "gcm(%s)", cipher
->cra_name
) >= CRYPTO_MAX_ALG_NAME
||
424 snprintf(inst
->alg
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
425 "gcm(%s)", cipher
->cra_driver_name
) >= CRYPTO_MAX_ALG_NAME
)
429 ctx
= crypto_instance_ctx(inst
);
430 err
= crypto_init_spawn(&ctx
->ctr
, ctr
, inst
, CRYPTO_ALG_TYPE_MASK
);
434 inst
->alg
.cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
;
435 inst
->alg
.cra_priority
= ctr
->cra_priority
;
436 inst
->alg
.cra_blocksize
= 16;
437 inst
->alg
.cra_alignmask
= ctr
->cra_alignmask
| (__alignof__(u64
) - 1);
438 inst
->alg
.cra_type
= &crypto_aead_type
;
439 inst
->alg
.cra_aead
.ivsize
= 12;
440 inst
->alg
.cra_aead
.maxauthsize
= 16;
441 inst
->alg
.cra_ctxsize
= sizeof(struct crypto_gcm_ctx
);
442 inst
->alg
.cra_init
= crypto_gcm_init_tfm
;
443 inst
->alg
.cra_exit
= crypto_gcm_exit_tfm
;
444 inst
->alg
.cra_aead
.setkey
= crypto_gcm_setkey
;
445 inst
->alg
.cra_aead
.encrypt
= crypto_gcm_encrypt
;
446 inst
->alg
.cra_aead
.decrypt
= crypto_gcm_decrypt
;
458 static void crypto_gcm_free(struct crypto_instance
*inst
)
460 struct gcm_instance_ctx
*ctx
= crypto_instance_ctx(inst
);
462 crypto_drop_spawn(&ctx
->ctr
);
466 static struct crypto_template crypto_gcm_tmpl
= {
468 .alloc
= crypto_gcm_alloc
,
469 .free
= crypto_gcm_free
,
470 .module
= THIS_MODULE
,
473 static int __init
crypto_gcm_module_init(void)
475 return crypto_register_template(&crypto_gcm_tmpl
);
478 static void __exit
crypto_gcm_module_exit(void)
480 crypto_unregister_template(&crypto_gcm_tmpl
);
483 module_init(crypto_gcm_module_init
);
484 module_exit(crypto_gcm_module_exit
);
486 MODULE_LICENSE("GPL");
487 MODULE_DESCRIPTION("Galois/Counter Mode");
488 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");