[GitHub/mt8127/android_kernel_alcatel_ttab.git] / crypto / pcrypt.c

/*
 * pcrypt - Parallel crypto wrapper.
 *
 * Copyright (C) 2009 secunet Security Networks AG
 * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 */

#include <crypto/algapi.h>
#include <crypto/internal/aead.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/notifier.h>
#include <linux/kobject.h>
#include <crypto/pcrypt.h>

struct padata_pcrypt {
	struct padata_instance *pinst;
	struct workqueue_struct *wq;

	/*
	 * Cpumask for callback CPUs. It should be
	 * equal to serial cpumask of corresponding padata instance,
	 * so it is updated when padata notifies us about serial
	 * cpumask change.
	 *
	 * cb_cpumask is protected by RCU. This fact prevents us from
	 * using cpumask_var_t directly because the actual type of
	 * cpumsak_var_t depends on kernel configuration(particularly on
	 * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
	 * cpumask_var_t may be either a pointer to the struct cpumask
	 * or a variable allocated on the stack. Thus we can not safely use
	 * cpumask_var_t with RCU operations such as rcu_assign_pointer or
	 * rcu_dereference. So cpumask_var_t is wrapped with struct
	 * pcrypt_cpumask which makes possible to use it with RCU.
	 */
	struct pcrypt_cpumask {
		cpumask_var_t mask;
	} *cb_cpumask;
	struct notifier_block nblock;
};

static struct padata_pcrypt pencrypt;
static struct padata_pcrypt pdecrypt;
static struct kset           *pcrypt_kset;

struct pcrypt_instance_ctx {
	struct crypto_spawn spawn;
	unsigned int tfm_count;
};

struct pcrypt_aead_ctx {
	struct crypto_aead *child;
	unsigned int cb_cpu;
};

static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu,
			      struct padata_pcrypt *pcrypt)
{
	unsigned int cpu_index, cpu, i;
	struct pcrypt_cpumask *cpumask;

	cpu = *cb_cpu;

	rcu_read_lock_bh();
	cpumask = rcu_dereference(pcrypt->cb_cpumask);
	if (cpumask_test_cpu(cpu, cpumask->mask))
			goto out;

	if (!cpumask_weight(cpumask->mask))
			goto out;

	cpu_index = cpu % cpumask_weight(cpumask->mask);

	cpu = cpumask_first(cpumask->mask);
	for (i = 0; i < cpu_index; i++)
		cpu = cpumask_next(cpu, cpumask->mask);

	*cb_cpu = cpu;

out:
	rcu_read_unlock_bh();
	return padata_do_parallel(pcrypt->pinst, padata, cpu);
}

static int pcrypt_aead_setkey(struct crypto_aead *parent,
			      const u8 *key, unsigned int keylen)
{
	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);

	return crypto_aead_setkey(ctx->child, key, keylen);
}

static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
				   unsigned int authsize)
{
	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);

	return crypto_aead_setauthsize(ctx->child, authsize);
}

static void pcrypt_aead_serial(struct padata_priv *padata)
{
	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	struct aead_request *req = pcrypt_request_ctx(preq);

	aead_request_complete(req->base.data, padata->info);
}

static void pcrypt_aead_giv_serial(struct padata_priv *padata)
{
	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);

	aead_request_complete(req->areq.base.data, padata->info);
}

static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;
	struct pcrypt_request *preq = aead_request_ctx(req);
	struct padata_priv *padata = pcrypt_request_padata(preq);

	padata->info = err;
	req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;

	padata_do_serial(padata);
}

static void pcrypt_aead_enc(struct padata_priv *padata)
{
	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	struct aead_request *req = pcrypt_request_ctx(preq);

	padata->info = crypto_aead_encrypt(req);

	if (padata->info == -EINPROGRESS)
		return;

	padata_do_serial(padata);
}

static int pcrypt_aead_encrypt(struct aead_request *req)
{
	int err;
	struct pcrypt_request *preq = aead_request_ctx(req);
	struct aead_request *creq = pcrypt_request_ctx(preq);
	struct padata_priv *padata = pcrypt_request_padata(preq);
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	u32 flags = aead_request_flags(req);

	memset(padata, 0, sizeof(struct padata_priv));

	padata->parallel = pcrypt_aead_enc;
	padata->serial = pcrypt_aead_serial;

	aead_request_set_tfm(creq, ctx->child);
	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
				  pcrypt_aead_done, req);
	aead_request_set_crypt(creq, req->src, req->dst,
			       req->cryptlen, req->iv);
	aead_request_set_assoc(creq, req->assoc, req->assoclen);

	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
	if (!err)
		return -EINPROGRESS;

	return err;
}

static void pcrypt_aead_dec(struct padata_priv *padata)
{
	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	struct aead_request *req = pcrypt_request_ctx(preq);

	padata->info = crypto_aead_decrypt(req);

	if (padata->info == -EINPROGRESS)
		return;

	padata_do_serial(padata);
}

static int pcrypt_aead_decrypt(struct aead_request *req)
{
	int err;
	struct pcrypt_request *preq = aead_request_ctx(req);
	struct aead_request *creq = pcrypt_request_ctx(preq);
	struct padata_priv *padata = pcrypt_request_padata(preq);
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	u32 flags = aead_request_flags(req);

	memset(padata, 0, sizeof(struct padata_priv));

	padata->parallel = pcrypt_aead_dec;
	padata->serial = pcrypt_aead_serial;

	aead_request_set_tfm(creq, ctx->child);
	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
				  pcrypt_aead_done, req);
	aead_request_set_crypt(creq, req->src, req->dst,
			       req->cryptlen, req->iv);
	aead_request_set_assoc(creq, req->assoc, req->assoclen);

	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt);
	if (!err)
		return -EINPROGRESS;

	return err;
}

static void pcrypt_aead_givenc(struct padata_priv *padata)
{
	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);

	padata->info = crypto_aead_givencrypt(req);

	if (padata->info == -EINPROGRESS)
		return;

	padata_do_serial(padata);
}

static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
{
	int err;
	struct aead_request *areq = &req->areq;
	struct pcrypt_request *preq = aead_request_ctx(areq);
	struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
	struct padata_priv *padata = pcrypt_request_padata(preq);
	struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	u32 flags = aead_request_flags(areq);

	memset(padata, 0, sizeof(struct padata_priv));

	padata->parallel = pcrypt_aead_givenc;
	padata->serial = pcrypt_aead_giv_serial;

	aead_givcrypt_set_tfm(creq, ctx->child);
	aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
				   pcrypt_aead_done, areq);
	aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
				areq->cryptlen, areq->iv);
	aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
	aead_givcrypt_set_giv(creq, req->giv, req->seq);

	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
	if (!err)
		return -EINPROGRESS;

	return err;
}

static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
{
	int cpu, cpu_index;
	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
	struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
	struct crypto_aead *cipher;

	ictx->tfm_count++;

	cpu_index = ictx->tfm_count % cpumask_weight(cpu_active_mask);

	ctx->cb_cpu = cpumask_first(cpu_active_mask);
	for (cpu = 0; cpu < cpu_index; cpu++)
		ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_active_mask);

	cipher = crypto_spawn_aead(crypto_instance_ctx(inst));

	if (IS_ERR(cipher))
		return PTR_ERR(cipher);

	ctx->child = cipher;
	tfm->crt_aead.reqsize = sizeof(struct pcrypt_request)
		+ sizeof(struct aead_givcrypt_request)
		+ crypto_aead_reqsize(cipher);

	return 0;
}

static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
{
	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);

	crypto_free_aead(ctx->child);
}

static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg)
{
	struct crypto_instance *inst;
	struct pcrypt_instance_ctx *ctx;
	int err;

	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
	if (!inst) {
		inst = ERR_PTR(-ENOMEM);
		goto out;
	}

	err = -ENAMETOOLONG;
	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
		goto out_free_inst;

	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);

	ctx = crypto_instance_ctx(inst);
	err = crypto_init_spawn(&ctx->spawn, alg, inst,
				CRYPTO_ALG_TYPE_MASK);
	if (err)
		goto out_free_inst;

	inst->alg.cra_priority = alg->cra_priority + 100;
	inst->alg.cra_blocksize = alg->cra_blocksize;
	inst->alg.cra_alignmask = alg->cra_alignmask;

out:
	return inst;

out_free_inst:
	kfree(inst);
	inst = ERR_PTR(err);
	goto out;
}

static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb,
						 u32 type, u32 mask)
{
	struct crypto_instance *inst;
	struct crypto_alg *alg;

	alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK));
	if (IS_ERR(alg))
		return ERR_CAST(alg);

	inst = pcrypt_alloc_instance(alg);
	if (IS_ERR(inst))
		goto out_put_alg;

	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC;
	inst->alg.cra_type = &crypto_aead_type;

	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;

	inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);

	inst->alg.cra_init = pcrypt_aead_init_tfm;
	inst->alg.cra_exit = pcrypt_aead_exit_tfm;

	inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
	inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
	inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
	inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
	inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;

out_put_alg:
	crypto_mod_put(alg);
	return inst;
}

static struct crypto_instance *pcrypt_alloc(struct rtattr **tb)
{
	struct crypto_attr_type *algt;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
		return ERR_CAST(algt);

	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
	case CRYPTO_ALG_TYPE_AEAD:
		return pcrypt_alloc_aead(tb, algt->type, algt->mask);
	}

	return ERR_PTR(-EINVAL);
}

static void pcrypt_free(struct crypto_instance *inst)
{
	struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst);

	crypto_drop_spawn(&ctx->spawn);
	kfree(inst);
}

static int pcrypt_cpumask_change_notify(struct notifier_block *self,
					unsigned long val, void *data)
{
	struct padata_pcrypt *pcrypt;
	struct pcrypt_cpumask *new_mask, *old_mask;

	if (!(val & PADATA_CPU_SERIAL))
		return 0;

	pcrypt = container_of(self, struct padata_pcrypt, nblock);
	new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL);
	if (!new_mask)
		return -ENOMEM;
	if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) {
		kfree(new_mask);
		return -ENOMEM;
	}

	old_mask = pcrypt->cb_cpumask;

	padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask);
	rcu_assign_pointer(pcrypt->cb_cpumask, new_mask);
	synchronize_rcu_bh();

	free_cpumask_var(old_mask->mask);
	kfree(old_mask);
	return 0;
}

static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
{
	int ret;

	pinst->kobj.kset = pcrypt_kset;
	ret = kobject_add(&pinst->kobj, NULL, name);
	if (!ret)
		kobject_uevent(&pinst->kobj, KOBJ_ADD);

	return ret;
}

static int pcrypt_init_padata(struct padata_pcrypt *pcrypt,
			      const char *name)
{
	int ret = -ENOMEM;
	struct pcrypt_cpumask *mask;

	pcrypt->wq = create_workqueue(name);
	if (!pcrypt->wq)
		goto err;

	pcrypt->pinst = padata_alloc_possible(pcrypt->wq);
	if (!pcrypt->pinst)
		goto err_destroy_workqueue;

	mask = kmalloc(sizeof(*mask), GFP_KERNEL);
	if (!mask)
		goto err_free_padata;
	if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) {
		kfree(mask);
		goto err_free_padata;
	}

	padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask);
	rcu_assign_pointer(pcrypt->cb_cpumask, mask);

	pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify;
	ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
	if (ret)
		goto err_free_cpumask;

	ret = pcrypt_sysfs_add(pcrypt->pinst, name);
	if (ret)
		goto err_unregister_notifier;

	return ret;
err_unregister_notifier:
	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
err_free_cpumask:
	free_cpumask_var(mask->mask);
	kfree(mask);
err_free_padata:
	padata_free(pcrypt->pinst);
err_destroy_workqueue:
	destroy_workqueue(pcrypt->wq);
err:
	return ret;
}

static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt)
{
	kobject_put(&pcrypt->pinst->kobj);
	free_cpumask_var(pcrypt->cb_cpumask->mask);
	kfree(pcrypt->cb_cpumask);

	padata_stop(pcrypt->pinst);
	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
	destroy_workqueue(pcrypt->wq);
	padata_free(pcrypt->pinst);
}

static struct crypto_template pcrypt_tmpl = {
	.name = "pcrypt",
	.alloc = pcrypt_alloc,
	.free = pcrypt_free,
	.module = THIS_MODULE,
};

static int __init pcrypt_init(void)
{
	int err = -ENOMEM;

	pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
	if (!pcrypt_kset)
		goto err;

	err = pcrypt_init_padata(&pencrypt, "pencrypt");
	if (err)
		goto err_unreg_kset;

	err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
	if (err)
		goto err_deinit_pencrypt;

	padata_start(pencrypt.pinst);
	padata_start(pdecrypt.pinst);

	return crypto_register_template(&pcrypt_tmpl);

err_deinit_pencrypt:
	pcrypt_fini_padata(&pencrypt);
err_unreg_kset:
	kset_unregister(pcrypt_kset);
err:
	return err;
}

static void __exit pcrypt_exit(void)
{
	pcrypt_fini_padata(&pencrypt);
	pcrypt_fini_padata(&pdecrypt);

	kset_unregister(pcrypt_kset);
	crypto_unregister_template(&pcrypt_tmpl);
}

module_init(pcrypt_init);
module_exit(pcrypt_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
MODULE_DESCRIPTION("Parallel crypto wrapper");
Commit	Line	Data
5068c7a8 SK	1	/*
	2	* pcrypt - Parallel crypto wrapper.
	3	*
	4	* Copyright (C) 2009 secunet Security Networks AG
	5	* Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
	6	*
	7	* This program is free software; you can redistribute it and/or modify it
	8	* under the terms and conditions of the GNU General Public License,
	9	* version 2, as published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope it will be useful, but WITHOUT
	12	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	13	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	14	* more details.
	15	*
	16	* You should have received a copy of the GNU General Public License along with
	17	* this program; if not, write to the Free Software Foundation, Inc.,
	18	* 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
	19	*/
	20
	21	#include <crypto/algapi.h>
	22	#include <crypto/internal/aead.h>
	23	#include <linux/err.h>
	24	#include <linux/init.h>
	25	#include <linux/module.h>
	26	#include <linux/slab.h>
e15bacbe	27	#include <linux/notifier.h>
a3fb1e33	28	#include <linux/kobject.h>
5068c7a8 SK	29	#include <crypto/pcrypt.h>
5068c7a8 SK	30
c57e842e	31	struct padata_pcrypt {
e15bacbe DK	32	struct padata_instance *pinst;
	33	struct workqueue_struct *wq;
	34
	35	/*
	36	* Cpumask for callback CPUs. It should be
	37	* equal to serial cpumask of corresponding padata instance,
	38	* so it is updated when padata notifies us about serial
	39	* cpumask change.
	40	*
	41	* cb_cpumask is protected by RCU. This fact prevents us from
	42	* using cpumask_var_t directly because the actual type of
	43	* cpumsak_var_t depends on kernel configuration(particularly on
	44	* CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration
	45	* cpumask_var_t may be either a pointer to the struct cpumask
	46	* or a variable allocated on the stack. Thus we can not safely use
	47	* cpumask_var_t with RCU operations such as rcu_assign_pointer or
	48	* rcu_dereference. So cpumask_var_t is wrapped with struct
	49	* pcrypt_cpumask which makes possible to use it with RCU.
	50	*/
	51	struct pcrypt_cpumask {
	52	cpumask_var_t mask;
	53	} *cb_cpumask;
	54	struct notifier_block nblock;
	55	};
	56
c57e842e SK	57	static struct padata_pcrypt pencrypt;
c57e842e SK	58	static struct padata_pcrypt pdecrypt;
a3fb1e33	59	static struct kset *pcrypt_kset;
5068c7a8 SK	60
	61	struct pcrypt_instance_ctx {
	62	struct crypto_spawn spawn;
	63	unsigned int tfm_count;
	64	};
	65
	66	struct pcrypt_aead_ctx {
	67	struct crypto_aead *child;
	68	unsigned int cb_cpu;
	69	};
	70
	71	static int pcrypt_do_parallel(struct padata_priv padata, unsigned int cb_cpu,
c57e842e	72	struct padata_pcrypt *pcrypt)
5068c7a8 SK	73	{
5068c7a8 SK	74	unsigned int cpu_index, cpu, i;
e15bacbe	75	struct pcrypt_cpumask *cpumask;
5068c7a8 SK	76
	77	cpu = *cb_cpu;
	78
e15bacbe DK	79	rcu_read_lock_bh();
	80	cpumask = rcu_dereference(pcrypt->cb_cpumask);
	81	if (cpumask_test_cpu(cpu, cpumask->mask))
5068c7a8 SK	82	goto out;
5068c7a8 SK	83
cc74f4bc SK	84	if (!cpumask_weight(cpumask->mask))
	85	goto out;
	86
e15bacbe	87	cpu_index = cpu % cpumask_weight(cpumask->mask);
5068c7a8	88
e15bacbe	89	cpu = cpumask_first(cpumask->mask);
5068c7a8	90	for (i = 0; i < cpu_index; i++)
e15bacbe	91	cpu = cpumask_next(cpu, cpumask->mask);
5068c7a8 SK	92
	93	*cb_cpu = cpu;
	94
	95	out:
e15bacbe DK	96	rcu_read_unlock_bh();
e15bacbe DK	97	return padata_do_parallel(pcrypt->pinst, padata, cpu);
5068c7a8 SK	98	}
	99
	100	static int pcrypt_aead_setkey(struct crypto_aead *parent,
	101	const u8 *key, unsigned int keylen)
	102	{
	103	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
	104
	105	return crypto_aead_setkey(ctx->child, key, keylen);
	106	}
	107
	108	static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
	109	unsigned int authsize)
	110	{
	111	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
	112
	113	return crypto_aead_setauthsize(ctx->child, authsize);
	114	}
	115
	116	static void pcrypt_aead_serial(struct padata_priv *padata)
	117	{
	118	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	119	struct aead_request *req = pcrypt_request_ctx(preq);
	120
	121	aead_request_complete(req->base.data, padata->info);
	122	}
	123
	124	static void pcrypt_aead_giv_serial(struct padata_priv *padata)
	125	{
	126	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	127	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
	128
	129	aead_request_complete(req->areq.base.data, padata->info);
	130	}
	131
	132	static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
	133	{
	134	struct aead_request *req = areq->data;
	135	struct pcrypt_request *preq = aead_request_ctx(req);
	136	struct padata_priv *padata = pcrypt_request_padata(preq);
	137
	138	padata->info = err;
	139	req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
	140
	141	padata_do_serial(padata);
	142	}
	143
	144	static void pcrypt_aead_enc(struct padata_priv *padata)
	145	{
	146	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	147	struct aead_request *req = pcrypt_request_ctx(preq);
	148
	149	padata->info = crypto_aead_encrypt(req);
	150
5a1436be	151	if (padata->info == -EINPROGRESS)
5068c7a8 SK	152	return;
	153
	154	padata_do_serial(padata);
	155	}
	156
	157	static int pcrypt_aead_encrypt(struct aead_request *req)
	158	{
	159	int err;
	160	struct pcrypt_request *preq = aead_request_ctx(req);
	161	struct aead_request *creq = pcrypt_request_ctx(preq);
	162	struct padata_priv *padata = pcrypt_request_padata(preq);
	163	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	164	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	165	u32 flags = aead_request_flags(req);
	166
	167	memset(padata, 0, sizeof(struct padata_priv));
	168
	169	padata->parallel = pcrypt_aead_enc;
	170	padata->serial = pcrypt_aead_serial;
	171
	172	aead_request_set_tfm(creq, ctx->child);
	173	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
	174	pcrypt_aead_done, req);
	175	aead_request_set_crypt(creq, req->src, req->dst,
	176	req->cryptlen, req->iv);
	177	aead_request_set_assoc(creq, req->assoc, req->assoclen);
	178
e15bacbe	179	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
83f619f3 SK	180	if (!err)
83f619f3 SK	181	return -EINPROGRESS;
5068c7a8 SK	182
	183	return err;
	184	}
	185
	186	static void pcrypt_aead_dec(struct padata_priv *padata)
	187	{
	188	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	189	struct aead_request *req = pcrypt_request_ctx(preq);
	190
	191	padata->info = crypto_aead_decrypt(req);
	192
5a1436be	193	if (padata->info == -EINPROGRESS)
5068c7a8 SK	194	return;
	195
	196	padata_do_serial(padata);
	197	}
	198
	199	static int pcrypt_aead_decrypt(struct aead_request *req)
	200	{
	201	int err;
	202	struct pcrypt_request *preq = aead_request_ctx(req);
	203	struct aead_request *creq = pcrypt_request_ctx(preq);
	204	struct padata_priv *padata = pcrypt_request_padata(preq);
	205	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	206	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	207	u32 flags = aead_request_flags(req);
	208
	209	memset(padata, 0, sizeof(struct padata_priv));
	210
	211	padata->parallel = pcrypt_aead_dec;
	212	padata->serial = pcrypt_aead_serial;
	213
	214	aead_request_set_tfm(creq, ctx->child);
	215	aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
	216	pcrypt_aead_done, req);
	217	aead_request_set_crypt(creq, req->src, req->dst,
	218	req->cryptlen, req->iv);
	219	aead_request_set_assoc(creq, req->assoc, req->assoclen);
	220
e15bacbe	221	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt);
83f619f3 SK	222	if (!err)
83f619f3 SK	223	return -EINPROGRESS;
5068c7a8 SK	224
	225	return err;
	226	}
	227
	228	static void pcrypt_aead_givenc(struct padata_priv *padata)
	229	{
	230	struct pcrypt_request *preq = pcrypt_padata_request(padata);
	231	struct aead_givcrypt_request *req = pcrypt_request_ctx(preq);
	232
	233	padata->info = crypto_aead_givencrypt(req);
	234
5a1436be	235	if (padata->info == -EINPROGRESS)
5068c7a8 SK	236	return;
	237
	238	padata_do_serial(padata);
	239	}
	240
	241	static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req)
	242	{
	243	int err;
	244	struct aead_request *areq = &req->areq;
	245	struct pcrypt_request *preq = aead_request_ctx(areq);
	246	struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq);
	247	struct padata_priv *padata = pcrypt_request_padata(preq);
	248	struct crypto_aead *aead = aead_givcrypt_reqtfm(req);
	249	struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
	250	u32 flags = aead_request_flags(areq);
	251
	252	memset(padata, 0, sizeof(struct padata_priv));
	253
	254	padata->parallel = pcrypt_aead_givenc;
	255	padata->serial = pcrypt_aead_giv_serial;
	256
	257	aead_givcrypt_set_tfm(creq, ctx->child);
	258	aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
	259	pcrypt_aead_done, areq);
	260	aead_givcrypt_set_crypt(creq, areq->src, areq->dst,
	261	areq->cryptlen, areq->iv);
	262	aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen);
	263	aead_givcrypt_set_giv(creq, req->giv, req->seq);
	264
e15bacbe	265	err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt);
83f619f3 SK	266	if (!err)
83f619f3 SK	267	return -EINPROGRESS;
5068c7a8 SK	268
	269	return err;
	270	}
	271
	272	static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
	273	{
	274	int cpu, cpu_index;
	275	struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
	276	struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst);
	277	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
	278	struct crypto_aead *cipher;
	279
	280	ictx->tfm_count++;
	281
	282	cpu_index = ictx->tfm_count % cpumask_weight(cpu_active_mask);
	283
	284	ctx->cb_cpu = cpumask_first(cpu_active_mask);
	285	for (cpu = 0; cpu < cpu_index; cpu++)
	286	ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_active_mask);
	287
	288	cipher = crypto_spawn_aead(crypto_instance_ctx(inst));
	289
	290	if (IS_ERR(cipher))
	291	return PTR_ERR(cipher);
	292
	293	ctx->child = cipher;
	294	tfm->crt_aead.reqsize = sizeof(struct pcrypt_request)
	295	+ sizeof(struct aead_givcrypt_request)
	296	+ crypto_aead_reqsize(cipher);
	297
	298	return 0;
	299	}
	300
	301	static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm)
	302	{
	303	struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm);
	304
	305	crypto_free_aead(ctx->child);
	306	}
	307
	308	static struct crypto_instance pcrypt_alloc_instance(struct crypto_alg alg)
	309	{
	310	struct crypto_instance *inst;
	311	struct pcrypt_instance_ctx *ctx;
	312	int err;
	313
	314	inst = kzalloc(sizeof(inst) + sizeof(ctx), GFP_KERNEL);
	315	if (!inst) {
	316	inst = ERR_PTR(-ENOMEM);
	317	goto out;
	318	}
	319
	320	err = -ENAMETOOLONG;
	321	if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
	322	"pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
	323	goto out_free_inst;
	324
	325	memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
	326
	327	ctx = crypto_instance_ctx(inst);
	328	err = crypto_init_spawn(&ctx->spawn, alg, inst,
	329	CRYPTO_ALG_TYPE_MASK);
	330	if (err)
	331	goto out_free_inst;
332
333	inst->alg.cra_priority = alg->cra_priority + 100;
334	inst->alg.cra_blocksize = alg->cra_blocksize;
335	inst->alg.cra_alignmask = alg->cra_alignmask;
336
337	out:
338	return inst;
339
340	out_free_inst:
341	kfree(inst);
342	inst = ERR_PTR(err);
343	goto out;
344	}
345
80a6d7db DC	346	static struct crypto_instance pcrypt_alloc_aead(struct rtattr *tb,
80a6d7db DC	347	u32 type, u32 mask)
5068c7a8 SK	348	{
	349	struct crypto_instance *inst;
	350	struct crypto_alg *alg;
5068c7a8	351
80a6d7db	352	alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK));
5068c7a8 SK	353	if (IS_ERR(alg))
	354	return ERR_CAST(alg);
	355
	356	inst = pcrypt_alloc_instance(alg);
	357	if (IS_ERR(inst))
	358	goto out_put_alg;
	359
	360	inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD \| CRYPTO_ALG_ASYNC;
	361	inst->alg.cra_type = &crypto_aead_type;
	362
	363	inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize;
	364	inst->alg.cra_aead.geniv = alg->cra_aead.geniv;
	365	inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize;
	366
	367	inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
	368
	369	inst->alg.cra_init = pcrypt_aead_init_tfm;
	370	inst->alg.cra_exit = pcrypt_aead_exit_tfm;
	371
	372	inst->alg.cra_aead.setkey = pcrypt_aead_setkey;
	373	inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize;
	374	inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt;
	375	inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt;
	376	inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt;
	377
	378	out_put_alg:
	379	crypto_mod_put(alg);
	380	return inst;
	381	}
	382
	383	static struct crypto_instance pcrypt_alloc(struct rtattr *tb)
	384	{
	385	struct crypto_attr_type *algt;
	386
	387	algt = crypto_get_attr_type(tb);
	388	if (IS_ERR(algt))
	389	return ERR_CAST(algt);
	390
	391	switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
	392	case CRYPTO_ALG_TYPE_AEAD:
80a6d7db	393	return pcrypt_alloc_aead(tb, algt->type, algt->mask);
5068c7a8 SK	394	}
	395
	396	return ERR_PTR(-EINVAL);
	397	}
	398
	399	static void pcrypt_free(struct crypto_instance *inst)
	400	{
	401	struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst);
	402
	403	crypto_drop_spawn(&ctx->spawn);
	404	kfree(inst);
	405	}
	406
e15bacbe DK	407	static int pcrypt_cpumask_change_notify(struct notifier_block *self,
	408	unsigned long val, void *data)
	409	{
c57e842e	410	struct padata_pcrypt *pcrypt;
e15bacbe DK	411	struct pcrypt_cpumask new_mask, old_mask;
	412
	413	if (!(val & PADATA_CPU_SERIAL))
	414	return 0;
	415
c57e842e	416	pcrypt = container_of(self, struct padata_pcrypt, nblock);
e15bacbe DK	417	new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL);
	418	if (!new_mask)
	419	return -ENOMEM;
	420	if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) {
	421	kfree(new_mask);
	422	return -ENOMEM;
	423	}
	424
	425	old_mask = pcrypt->cb_cpumask;
	426
	427	padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, new_mask->mask);
	428	rcu_assign_pointer(pcrypt->cb_cpumask, new_mask);
	429	synchronize_rcu_bh();
	430
	431	free_cpumask_var(old_mask->mask);
	432	kfree(old_mask);
	433	return 0;
	434	}
	435
a3fb1e33 DK	436	static int pcrypt_sysfs_add(struct padata_instance pinst, const char name)
	437	{
	438	int ret;
	439
	440	pinst->kobj.kset = pcrypt_kset;
	441	ret = kobject_add(&pinst->kobj, NULL, name);
	442	if (!ret)
	443	kobject_uevent(&pinst->kobj, KOBJ_ADD);
	444
	445	return ret;
	446	}
	447
c57e842e SK	448	static int pcrypt_init_padata(struct padata_pcrypt *pcrypt,
c57e842e SK	449	const char *name)
e15bacbe DK	450	{
	451	int ret = -ENOMEM;
	452	struct pcrypt_cpumask *mask;
	453
	454	pcrypt->wq = create_workqueue(name);
	455	if (!pcrypt->wq)
	456	goto err;
	457
e6cc1170	458	pcrypt->pinst = padata_alloc_possible(pcrypt->wq);
e15bacbe DK	459	if (!pcrypt->pinst)
	460	goto err_destroy_workqueue;
	461
	462	mask = kmalloc(sizeof(*mask), GFP_KERNEL);
	463	if (!mask)
	464	goto err_free_padata;
	465	if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) {
	466	kfree(mask);
	467	goto err_free_padata;
	468	}
	469
	470	padata_get_cpumask(pcrypt->pinst, PADATA_CPU_SERIAL, mask->mask);
	471	rcu_assign_pointer(pcrypt->cb_cpumask, mask);
	472
	473	pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify;
	474	ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
	475	if (ret)
	476	goto err_free_cpumask;
	477
a3fb1e33 DK	478	ret = pcrypt_sysfs_add(pcrypt->pinst, name);
	479	if (ret)
	480	goto err_unregister_notifier;
	481
e15bacbe	482	return ret;
a3fb1e33 DK	483	err_unregister_notifier:
a3fb1e33 DK	484	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
e15bacbe DK	485	err_free_cpumask:
	486	free_cpumask_var(mask->mask);
	487	kfree(mask);
	488	err_free_padata:
	489	padata_free(pcrypt->pinst);
	490	err_destroy_workqueue:
	491	destroy_workqueue(pcrypt->wq);
	492	err:
	493	return ret;
	494	}
	495
c57e842e	496	static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt)
e15bacbe	497	{
a3fb1e33	498	kobject_put(&pcrypt->pinst->kobj);
e15bacbe DK	499	free_cpumask_var(pcrypt->cb_cpumask->mask);
	500	kfree(pcrypt->cb_cpumask);
	501
	502	padata_stop(pcrypt->pinst);
	503	padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock);
	504	destroy_workqueue(pcrypt->wq);
	505	padata_free(pcrypt->pinst);
	506	}
	507
5068c7a8 SK	508	static struct crypto_template pcrypt_tmpl = {
	509	.name = "pcrypt",
	510	.alloc = pcrypt_alloc,
	511	.free = pcrypt_free,
	512	.module = THIS_MODULE,
	513	};
	514
	515	static int __init pcrypt_init(void)
	516	{
a3fb1e33 DK	517	int err = -ENOMEM;
	518
	519	pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
	520	if (!pcrypt_kset)
	521	goto err;
5068c7a8	522
c57e842e	523	err = pcrypt_init_padata(&pencrypt, "pencrypt");
4c879170	524	if (err)
a3fb1e33	525	goto err_unreg_kset;
4c879170	526
c57e842e	527	err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
4c879170	528	if (err)
e15bacbe	529	goto err_deinit_pencrypt;
4c879170	530
e15bacbe DK	531	padata_start(pencrypt.pinst);
e15bacbe DK	532	padata_start(pdecrypt.pinst);
5068c7a8	533
e15bacbe	534	return crypto_register_template(&pcrypt_tmpl);
5068c7a8	535
e15bacbe	536	err_deinit_pencrypt:
c57e842e	537	pcrypt_fini_padata(&pencrypt);
a3fb1e33 DK	538	err_unreg_kset:
a3fb1e33 DK	539	kset_unregister(pcrypt_kset);
5068c7a8	540	err:
4c879170	541	return err;
5068c7a8 SK	542	}
	543
	544	static void __exit pcrypt_exit(void)
	545	{
c57e842e SK	546	pcrypt_fini_padata(&pencrypt);
c57e842e SK	547	pcrypt_fini_padata(&pdecrypt);
5068c7a8	548
a3fb1e33	549	kset_unregister(pcrypt_kset);
5068c7a8 SK	550	crypto_unregister_template(&pcrypt_tmpl);
	551	}
	552
	553	module_init(pcrypt_init);
	554	module_exit(pcrypt_exit);
	555
	556	MODULE_LICENSE("GPL");
	557	MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
	558	MODULE_DESCRIPTION("Parallel crypto wrapper");