2 * Software multibuffer async crypto daemon.
4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com>
6 * Adapted from crypto daemon.
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
15 #include <crypto/algapi.h>
16 #include <crypto/internal/hash.h>
17 #include <crypto/internal/aead.h>
18 #include <crypto/mcryptd.h>
19 #include <crypto/crypto_wq.h>
20 #include <linux/err.h>
21 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <linux/sched.h>
27 #include <linux/slab.h>
28 #include <linux/hardirq.h>
30 #define MCRYPTD_MAX_CPU_QLEN 100
31 #define MCRYPTD_BATCH 9
33 static void *mcryptd_alloc_instance(struct crypto_alg
*alg
, unsigned int head
,
36 struct mcryptd_flush_list
{
37 struct list_head list
;
41 static struct mcryptd_flush_list __percpu
*mcryptd_flist
;
43 struct hashd_instance_ctx
{
44 struct crypto_shash_spawn spawn
;
45 struct mcryptd_queue
*queue
;
48 static void mcryptd_queue_worker(struct work_struct
*work
);
50 void mcryptd_arm_flusher(struct mcryptd_alg_cstate
*cstate
, unsigned long delay
)
52 struct mcryptd_flush_list
*flist
;
54 if (!cstate
->flusher_engaged
) {
55 /* put the flusher on the flush list */
56 flist
= per_cpu_ptr(mcryptd_flist
, smp_processor_id());
57 mutex_lock(&flist
->lock
);
58 list_add_tail(&cstate
->flush_list
, &flist
->list
);
59 cstate
->flusher_engaged
= true;
60 cstate
->next_flush
= jiffies
+ delay
;
61 queue_delayed_work_on(smp_processor_id(), kcrypto_wq
,
62 &cstate
->flush
, delay
);
63 mutex_unlock(&flist
->lock
);
66 EXPORT_SYMBOL(mcryptd_arm_flusher
);
68 static int mcryptd_init_queue(struct mcryptd_queue
*queue
,
69 unsigned int max_cpu_qlen
)
72 struct mcryptd_cpu_queue
*cpu_queue
;
74 queue
->cpu_queue
= alloc_percpu(struct mcryptd_cpu_queue
);
75 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue
, queue
->cpu_queue
);
76 if (!queue
->cpu_queue
)
78 for_each_possible_cpu(cpu
) {
79 cpu_queue
= per_cpu_ptr(queue
->cpu_queue
, cpu
);
80 pr_debug("cpu_queue #%d %p\n", cpu
, queue
->cpu_queue
);
81 crypto_init_queue(&cpu_queue
->queue
, max_cpu_qlen
);
82 INIT_WORK(&cpu_queue
->work
, mcryptd_queue_worker
);
87 static void mcryptd_fini_queue(struct mcryptd_queue
*queue
)
90 struct mcryptd_cpu_queue
*cpu_queue
;
92 for_each_possible_cpu(cpu
) {
93 cpu_queue
= per_cpu_ptr(queue
->cpu_queue
, cpu
);
94 BUG_ON(cpu_queue
->queue
.qlen
);
96 free_percpu(queue
->cpu_queue
);
99 static int mcryptd_enqueue_request(struct mcryptd_queue
*queue
,
100 struct crypto_async_request
*request
,
101 struct mcryptd_hash_request_ctx
*rctx
)
104 struct mcryptd_cpu_queue
*cpu_queue
;
107 cpu_queue
= this_cpu_ptr(queue
->cpu_queue
);
110 err
= crypto_enqueue_request(&cpu_queue
->queue
, request
);
111 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n",
112 cpu
, cpu_queue
, request
);
113 queue_work_on(cpu
, kcrypto_wq
, &cpu_queue
->work
);
120 * Try to opportunisticlly flush the partially completed jobs if
121 * crypto daemon is the only task running.
123 static void mcryptd_opportunistic_flush(void)
125 struct mcryptd_flush_list
*flist
;
126 struct mcryptd_alg_cstate
*cstate
;
128 flist
= per_cpu_ptr(mcryptd_flist
, smp_processor_id());
129 while (single_task_running()) {
130 mutex_lock(&flist
->lock
);
131 if (list_empty(&flist
->list
)) {
132 mutex_unlock(&flist
->lock
);
135 cstate
= list_entry(flist
->list
.next
,
136 struct mcryptd_alg_cstate
, flush_list
);
137 if (!cstate
->flusher_engaged
) {
138 mutex_unlock(&flist
->lock
);
141 list_del(&cstate
->flush_list
);
142 cstate
->flusher_engaged
= false;
143 mutex_unlock(&flist
->lock
);
144 cstate
->alg_state
->flusher(cstate
);
149 * Called in workqueue context, do one real cryption work (via
150 * req->complete) and reschedule itself if there are more work to
153 static void mcryptd_queue_worker(struct work_struct
*work
)
155 struct mcryptd_cpu_queue
*cpu_queue
;
156 struct crypto_async_request
*req
, *backlog
;
160 * Need to loop through more than once for multi-buffer to
164 cpu_queue
= container_of(work
, struct mcryptd_cpu_queue
, work
);
166 while (i
< MCRYPTD_BATCH
|| single_task_running()) {
168 * preempt_disable/enable is used to prevent
169 * being preempted by mcryptd_enqueue_request()
173 backlog
= crypto_get_backlog(&cpu_queue
->queue
);
174 req
= crypto_dequeue_request(&cpu_queue
->queue
);
179 mcryptd_opportunistic_flush();
184 backlog
->complete(backlog
, -EINPROGRESS
);
185 req
->complete(req
, 0);
186 if (!cpu_queue
->queue
.qlen
)
190 if (cpu_queue
->queue
.qlen
)
191 queue_work(kcrypto_wq
, &cpu_queue
->work
);
194 void mcryptd_flusher(struct work_struct
*__work
)
196 struct mcryptd_alg_cstate
*alg_cpu_state
;
197 struct mcryptd_alg_state
*alg_state
;
198 struct mcryptd_flush_list
*flist
;
201 cpu
= smp_processor_id();
202 alg_cpu_state
= container_of(to_delayed_work(__work
),
203 struct mcryptd_alg_cstate
, flush
);
204 alg_state
= alg_cpu_state
->alg_state
;
205 if (alg_cpu_state
->cpu
!= cpu
)
206 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n",
207 cpu
, alg_cpu_state
->cpu
);
209 if (alg_cpu_state
->flusher_engaged
) {
210 flist
= per_cpu_ptr(mcryptd_flist
, cpu
);
211 mutex_lock(&flist
->lock
);
212 list_del(&alg_cpu_state
->flush_list
);
213 alg_cpu_state
->flusher_engaged
= false;
214 mutex_unlock(&flist
->lock
);
215 alg_state
->flusher(alg_cpu_state
);
218 EXPORT_SYMBOL_GPL(mcryptd_flusher
);
220 static inline struct mcryptd_queue
*mcryptd_get_queue(struct crypto_tfm
*tfm
)
222 struct crypto_instance
*inst
= crypto_tfm_alg_instance(tfm
);
223 struct mcryptd_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
228 static void *mcryptd_alloc_instance(struct crypto_alg
*alg
, unsigned int head
,
232 struct crypto_instance
*inst
;
235 p
= kzalloc(head
+ sizeof(*inst
) + tail
, GFP_KERNEL
);
237 return ERR_PTR(-ENOMEM
);
239 inst
= (void *)(p
+ head
);
242 if (snprintf(inst
->alg
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
243 "mcryptd(%s)", alg
->cra_driver_name
) >= CRYPTO_MAX_ALG_NAME
)
246 memcpy(inst
->alg
.cra_name
, alg
->cra_name
, CRYPTO_MAX_ALG_NAME
);
248 inst
->alg
.cra_priority
= alg
->cra_priority
+ 50;
249 inst
->alg
.cra_blocksize
= alg
->cra_blocksize
;
250 inst
->alg
.cra_alignmask
= alg
->cra_alignmask
;
261 static inline bool mcryptd_check_internal(struct rtattr
**tb
, u32
*type
,
264 struct crypto_attr_type
*algt
;
266 algt
= crypto_get_attr_type(tb
);
270 *type
|= algt
->type
& CRYPTO_ALG_INTERNAL
;
271 *mask
|= algt
->mask
& CRYPTO_ALG_INTERNAL
;
273 if (*type
& *mask
& CRYPTO_ALG_INTERNAL
)
279 static int mcryptd_hash_init_tfm(struct crypto_tfm
*tfm
)
281 struct crypto_instance
*inst
= crypto_tfm_alg_instance(tfm
);
282 struct hashd_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
283 struct crypto_shash_spawn
*spawn
= &ictx
->spawn
;
284 struct mcryptd_hash_ctx
*ctx
= crypto_tfm_ctx(tfm
);
285 struct crypto_shash
*hash
;
287 hash
= crypto_spawn_shash(spawn
);
289 return PTR_ERR(hash
);
292 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
293 sizeof(struct mcryptd_hash_request_ctx
) +
294 crypto_shash_descsize(hash
));
298 static void mcryptd_hash_exit_tfm(struct crypto_tfm
*tfm
)
300 struct mcryptd_hash_ctx
*ctx
= crypto_tfm_ctx(tfm
);
302 crypto_free_shash(ctx
->child
);
305 static int mcryptd_hash_setkey(struct crypto_ahash
*parent
,
306 const u8
*key
, unsigned int keylen
)
308 struct mcryptd_hash_ctx
*ctx
= crypto_ahash_ctx(parent
);
309 struct crypto_shash
*child
= ctx
->child
;
312 crypto_shash_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
313 crypto_shash_set_flags(child
, crypto_ahash_get_flags(parent
) &
314 CRYPTO_TFM_REQ_MASK
);
315 err
= crypto_shash_setkey(child
, key
, keylen
);
316 crypto_ahash_set_flags(parent
, crypto_shash_get_flags(child
) &
317 CRYPTO_TFM_RES_MASK
);
321 static int mcryptd_hash_enqueue(struct ahash_request
*req
,
322 crypto_completion_t complete
)
326 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
327 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
328 struct mcryptd_queue
*queue
=
329 mcryptd_get_queue(crypto_ahash_tfm(tfm
));
331 rctx
->complete
= req
->base
.complete
;
332 req
->base
.complete
= complete
;
334 ret
= mcryptd_enqueue_request(queue
, &req
->base
, rctx
);
339 static void mcryptd_hash_init(struct crypto_async_request
*req_async
, int err
)
341 struct mcryptd_hash_ctx
*ctx
= crypto_tfm_ctx(req_async
->tfm
);
342 struct crypto_shash
*child
= ctx
->child
;
343 struct ahash_request
*req
= ahash_request_cast(req_async
);
344 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
345 struct shash_desc
*desc
= &rctx
->desc
;
347 if (unlikely(err
== -EINPROGRESS
))
351 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
;
353 err
= crypto_shash_init(desc
);
355 req
->base
.complete
= rctx
->complete
;
359 rctx
->complete(&req
->base
, err
);
363 static int mcryptd_hash_init_enqueue(struct ahash_request
*req
)
365 return mcryptd_hash_enqueue(req
, mcryptd_hash_init
);
368 static void mcryptd_hash_update(struct crypto_async_request
*req_async
, int err
)
370 struct ahash_request
*req
= ahash_request_cast(req_async
);
371 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
373 if (unlikely(err
== -EINPROGRESS
))
376 err
= shash_ahash_mcryptd_update(req
, &rctx
->desc
);
378 req
->base
.complete
= rctx
->complete
;
385 rctx
->complete(&req
->base
, err
);
389 static int mcryptd_hash_update_enqueue(struct ahash_request
*req
)
391 return mcryptd_hash_enqueue(req
, mcryptd_hash_update
);
394 static void mcryptd_hash_final(struct crypto_async_request
*req_async
, int err
)
396 struct ahash_request
*req
= ahash_request_cast(req_async
);
397 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
399 if (unlikely(err
== -EINPROGRESS
))
402 err
= shash_ahash_mcryptd_final(req
, &rctx
->desc
);
404 req
->base
.complete
= rctx
->complete
;
411 rctx
->complete(&req
->base
, err
);
415 static int mcryptd_hash_final_enqueue(struct ahash_request
*req
)
417 return mcryptd_hash_enqueue(req
, mcryptd_hash_final
);
420 static void mcryptd_hash_finup(struct crypto_async_request
*req_async
, int err
)
422 struct ahash_request
*req
= ahash_request_cast(req_async
);
423 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
425 if (unlikely(err
== -EINPROGRESS
))
428 err
= shash_ahash_mcryptd_finup(req
, &rctx
->desc
);
431 req
->base
.complete
= rctx
->complete
;
438 rctx
->complete(&req
->base
, err
);
442 static int mcryptd_hash_finup_enqueue(struct ahash_request
*req
)
444 return mcryptd_hash_enqueue(req
, mcryptd_hash_finup
);
447 static void mcryptd_hash_digest(struct crypto_async_request
*req_async
, int err
)
449 struct mcryptd_hash_ctx
*ctx
= crypto_tfm_ctx(req_async
->tfm
);
450 struct crypto_shash
*child
= ctx
->child
;
451 struct ahash_request
*req
= ahash_request_cast(req_async
);
452 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
453 struct shash_desc
*desc
= &rctx
->desc
;
455 if (unlikely(err
== -EINPROGRESS
))
459 desc
->flags
= CRYPTO_TFM_REQ_MAY_SLEEP
; /* check this again */
461 err
= shash_ahash_mcryptd_digest(req
, desc
);
464 req
->base
.complete
= rctx
->complete
;
471 rctx
->complete(&req
->base
, err
);
475 static int mcryptd_hash_digest_enqueue(struct ahash_request
*req
)
477 return mcryptd_hash_enqueue(req
, mcryptd_hash_digest
);
480 static int mcryptd_hash_export(struct ahash_request
*req
, void *out
)
482 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
484 return crypto_shash_export(&rctx
->desc
, out
);
487 static int mcryptd_hash_import(struct ahash_request
*req
, const void *in
)
489 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
491 return crypto_shash_import(&rctx
->desc
, in
);
494 static int mcryptd_create_hash(struct crypto_template
*tmpl
, struct rtattr
**tb
,
495 struct mcryptd_queue
*queue
)
497 struct hashd_instance_ctx
*ctx
;
498 struct ahash_instance
*inst
;
499 struct shash_alg
*salg
;
500 struct crypto_alg
*alg
;
505 if (!mcryptd_check_internal(tb
, &type
, &mask
))
508 salg
= shash_attr_alg(tb
[1], type
, mask
);
510 return PTR_ERR(salg
);
513 pr_debug("crypto: mcryptd hash alg: %s\n", alg
->cra_name
);
514 inst
= mcryptd_alloc_instance(alg
, ahash_instance_headroom(),
520 ctx
= ahash_instance_ctx(inst
);
523 err
= crypto_init_shash_spawn(&ctx
->spawn
, salg
,
524 ahash_crypto_instance(inst
));
528 type
= CRYPTO_ALG_ASYNC
;
529 if (alg
->cra_flags
& CRYPTO_ALG_INTERNAL
)
530 type
|= CRYPTO_ALG_INTERNAL
;
531 inst
->alg
.halg
.base
.cra_flags
= type
;
533 inst
->alg
.halg
.digestsize
= salg
->digestsize
;
534 inst
->alg
.halg
.statesize
= salg
->statesize
;
535 inst
->alg
.halg
.base
.cra_ctxsize
= sizeof(struct mcryptd_hash_ctx
);
537 inst
->alg
.halg
.base
.cra_init
= mcryptd_hash_init_tfm
;
538 inst
->alg
.halg
.base
.cra_exit
= mcryptd_hash_exit_tfm
;
540 inst
->alg
.init
= mcryptd_hash_init_enqueue
;
541 inst
->alg
.update
= mcryptd_hash_update_enqueue
;
542 inst
->alg
.final
= mcryptd_hash_final_enqueue
;
543 inst
->alg
.finup
= mcryptd_hash_finup_enqueue
;
544 inst
->alg
.export
= mcryptd_hash_export
;
545 inst
->alg
.import
= mcryptd_hash_import
;
546 inst
->alg
.setkey
= mcryptd_hash_setkey
;
547 inst
->alg
.digest
= mcryptd_hash_digest_enqueue
;
549 err
= ahash_register_instance(tmpl
, inst
);
551 crypto_drop_shash(&ctx
->spawn
);
561 static struct mcryptd_queue mqueue
;
563 static int mcryptd_create(struct crypto_template
*tmpl
, struct rtattr
**tb
)
565 struct crypto_attr_type
*algt
;
567 algt
= crypto_get_attr_type(tb
);
569 return PTR_ERR(algt
);
571 switch (algt
->type
& algt
->mask
& CRYPTO_ALG_TYPE_MASK
) {
572 case CRYPTO_ALG_TYPE_DIGEST
:
573 return mcryptd_create_hash(tmpl
, tb
, &mqueue
);
580 static void mcryptd_free(struct crypto_instance
*inst
)
582 struct mcryptd_instance_ctx
*ctx
= crypto_instance_ctx(inst
);
583 struct hashd_instance_ctx
*hctx
= crypto_instance_ctx(inst
);
585 switch (inst
->alg
.cra_flags
& CRYPTO_ALG_TYPE_MASK
) {
586 case CRYPTO_ALG_TYPE_AHASH
:
587 crypto_drop_shash(&hctx
->spawn
);
588 kfree(ahash_instance(inst
));
591 crypto_drop_spawn(&ctx
->spawn
);
596 static struct crypto_template mcryptd_tmpl
= {
598 .create
= mcryptd_create
,
599 .free
= mcryptd_free
,
600 .module
= THIS_MODULE
,
603 struct mcryptd_ahash
*mcryptd_alloc_ahash(const char *alg_name
,
606 char mcryptd_alg_name
[CRYPTO_MAX_ALG_NAME
];
607 struct crypto_ahash
*tfm
;
609 if (snprintf(mcryptd_alg_name
, CRYPTO_MAX_ALG_NAME
,
610 "mcryptd(%s)", alg_name
) >= CRYPTO_MAX_ALG_NAME
)
611 return ERR_PTR(-EINVAL
);
612 tfm
= crypto_alloc_ahash(mcryptd_alg_name
, type
, mask
);
614 return ERR_CAST(tfm
);
615 if (tfm
->base
.__crt_alg
->cra_module
!= THIS_MODULE
) {
616 crypto_free_ahash(tfm
);
617 return ERR_PTR(-EINVAL
);
620 return __mcryptd_ahash_cast(tfm
);
622 EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash
);
624 int shash_ahash_mcryptd_digest(struct ahash_request
*req
,
625 struct shash_desc
*desc
)
629 err
= crypto_shash_init(desc
) ?:
630 shash_ahash_mcryptd_finup(req
, desc
);
634 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest
);
636 int shash_ahash_mcryptd_update(struct ahash_request
*req
,
637 struct shash_desc
*desc
)
639 struct crypto_shash
*tfm
= desc
->tfm
;
640 struct shash_alg
*shash
= crypto_shash_alg(tfm
);
642 /* alignment is to be done by multi-buffer crypto algorithm if needed */
644 return shash
->update(desc
, NULL
, 0);
646 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update
);
648 int shash_ahash_mcryptd_finup(struct ahash_request
*req
,
649 struct shash_desc
*desc
)
651 struct crypto_shash
*tfm
= desc
->tfm
;
652 struct shash_alg
*shash
= crypto_shash_alg(tfm
);
654 /* alignment is to be done by multi-buffer crypto algorithm if needed */
656 return shash
->finup(desc
, NULL
, 0, req
->result
);
658 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup
);
660 int shash_ahash_mcryptd_final(struct ahash_request
*req
,
661 struct shash_desc
*desc
)
663 struct crypto_shash
*tfm
= desc
->tfm
;
664 struct shash_alg
*shash
= crypto_shash_alg(tfm
);
666 /* alignment is to be done by multi-buffer crypto algorithm if needed */
668 return shash
->final(desc
, req
->result
);
670 EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final
);
672 struct crypto_shash
*mcryptd_ahash_child(struct mcryptd_ahash
*tfm
)
674 struct mcryptd_hash_ctx
*ctx
= crypto_ahash_ctx(&tfm
->base
);
678 EXPORT_SYMBOL_GPL(mcryptd_ahash_child
);
680 struct shash_desc
*mcryptd_shash_desc(struct ahash_request
*req
)
682 struct mcryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
685 EXPORT_SYMBOL_GPL(mcryptd_shash_desc
);
687 void mcryptd_free_ahash(struct mcryptd_ahash
*tfm
)
689 crypto_free_ahash(&tfm
->base
);
691 EXPORT_SYMBOL_GPL(mcryptd_free_ahash
);
694 static int __init
mcryptd_init(void)
697 struct mcryptd_flush_list
*flist
;
699 mcryptd_flist
= alloc_percpu(struct mcryptd_flush_list
);
700 for_each_possible_cpu(cpu
) {
701 flist
= per_cpu_ptr(mcryptd_flist
, cpu
);
702 INIT_LIST_HEAD(&flist
->list
);
703 mutex_init(&flist
->lock
);
706 err
= mcryptd_init_queue(&mqueue
, MCRYPTD_MAX_CPU_QLEN
);
708 free_percpu(mcryptd_flist
);
712 err
= crypto_register_template(&mcryptd_tmpl
);
714 mcryptd_fini_queue(&mqueue
);
715 free_percpu(mcryptd_flist
);
721 static void __exit
mcryptd_exit(void)
723 mcryptd_fini_queue(&mqueue
);
724 crypto_unregister_template(&mcryptd_tmpl
);
725 free_percpu(mcryptd_flist
);
728 subsys_initcall(mcryptd_init
);
729 module_exit(mcryptd_exit
);
731 MODULE_LICENSE("GPL");
732 MODULE_DESCRIPTION("Software async multibuffer crypto daemon");
733 MODULE_ALIAS_CRYPTO("mcryptd");