[GitHub/moto-9609/android_kernel_motorola_exynos9610.git] / crypto / diskcipher.c

/*
 * Copyright (C) 2017 Samsung Electronics Co., Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/blkdev.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/seq_file.h>
#include <linux/string.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/diskcipher.h>
#include <linux/delay.h>

#include "internal.h"

#ifdef CONFIG_CRYPTO_DISKCIPHER_DEBUG
#include <crypto/fmp.h>
#include <linux/mm_types.h>
#include <linux/fs.h>
#include <linux/fscrypt.h>

#define DUMP_MAX 20

struct dump_err {
	struct page *page;
	struct bio bio;
	struct fmp_crypto_info ci;
	enum diskcipher_dbg api;
};

struct diskc_debug_info {
	struct dump_err dump[DUMP_MAX];
	int err;
	int cnt[DISKC_USER_MAX][2];
};

static struct diskc_debug_info diskc_dbg;

void crypto_diskcipher_debug(enum diskcipher_dbg api, int bi_opf)
{
	int idx = 0;
	struct diskc_debug_info *dbg = &diskc_dbg;

	if (api <= DISKC_API_MAX)
		dbg->cnt[api][bi_opf]++;
	else {
		if (bi_opf)
			idx = 1;
		dbg->cnt[api][idx]++;
	}
}

static void print_err(void)
{
	int i, j;
	struct bio_vec *bv;	/* bio page list */
	struct bio *bio;
	struct fmp_crypto_info *ci;
	struct diskc_debug_info *dbg = &diskc_dbg;

	for (j = 0; j < dbg->err; j++) {
		bio = &dbg->dump[j].bio;
		ci = &dbg->dump[j].ci;

		if (bio) {
			pr_info
			    ("%s(%d/%d): bio:%p ci:%p page:%p flag:%x, opf:%x, crypt:%p\n",
			     __func__, j, dbg->err, bio, ci, &dbg->dump[j].page,
			     bio->bi_flags, bio->bi_opf, bio->bi_aux_private);
			print_hex_dump(KERN_CONT, "bio:", DUMP_PREFIX_OFFSET,
				16, 1, bio, sizeof(struct bio), false);
			for (i = 0; i < bio->bi_max_vecs; i++) {
				bv = &bio->bi_io_vec[i];
				pr_info("bv[%d] page:%p len:%d offset:%d\n",
					i, bv->bv_page, bv->bv_len, bv->bv_offset);
			}
		}

		if (ci) {
			pr_info("[ci] key_size:%d algo_mode:%d\n",
				ci->key_size, ci->algo_mode);
			print_hex_dump(KERN_CONT, "key:", DUMP_PREFIX_OFFSET,
				       16, 1, ci->key, sizeof(ci->key), false);
		}
	}
}

static void dump_err(struct crypto_diskcipher *ci, enum diskcipher_dbg api,
	      struct bio *bio, struct page *page)
{
	struct diskc_debug_info *dbg = &diskc_dbg;

	if ((dbg->err < DUMP_MAX) && ci) {
		struct crypto_tfm *tfm = crypto_diskcipher_tfm(ci);

		dbg->dump[dbg->err].api = api;
		memcpy(&dbg->dump[dbg->err].ci, crypto_tfm_ctx(tfm),
		       sizeof(struct fmp_crypto_info));

		if (page)
			dbg->dump[dbg->err].page = page;
		if (bio)
			memcpy(&dbg->dump[dbg->err].bio, bio,
				sizeof(struct bio));
	}
	dbg->err++;
}

static void disckipher_log_show(struct seq_file *m)
{
	int i;
	struct diskc_debug_info *dbg = &diskc_dbg;
	char name[DISKC_USER_MAX][32] = {
		"ALLOC", "FREE", "FREEREQ", "SETKEY", "SET", "GET", "CRYPT", "CLEAR",
		"DISKC_API_MAX", "FS_PAGEIO", "FS_READP", "FS_DIO", "FS_BLOCK_WRITE",
		"FS_ZEROPAGE", "BLK_BH", "DMCRYPT", "DISKC_MERGE", "DISKC_MERGE_ERR_INODE", "DISKC_MERGE_ERR_DISK",
		"FS_DEC_WARN", "FS_ENC_WARN", "DISKC_MERGE_DIO", "DISKC_FREE_REQ_WARN",
		"DISKC_FREE_WQ_WARN", "DISKC_CRYPT_WARN",
		"DM_CRYPT_NONENCRYPT", "DM_CRYPT_CTR", "DM_CRYPT_DTR", "DM_CRYPT_OVER",
		"F2FS_gc", "F2FS_gc_data_page", "F2FS_gc_data_page_no_key", "F2FS_gc_data_page_no_key_FC",
		"F2FS_gc_data_page_FC",	"F2FS_gc_data_block", "F2FS_gc_data_block_key",
		"F2FS_gc_data_block_err1", "F2FS_gc_data_block_err2", "F2FS_gc_data_block_err3", "F2FS_gc_skip",
		"DISKC_ERR", "DISKC_NO_KEY_ERR", "DISKC_NO_SYNC_ERR", "DISKC_NO_CRYPT_ERR", "DISKC_NO_DISKC_ERR"};

	for (i = 0; i < DISKC_USER_MAX; i++)
		if (dbg->cnt[i][0] || dbg->cnt[i][1])
			seq_printf(m, "%s\t: %6u(err:%u)\n",
				name[i], dbg->cnt[i][0], dbg->cnt[i][1]);

	if (dbg->err)
		print_err();
}

/* check diskcipher for FBE */
#define DISKC_FS_ENCRYPT_DEBUG
#ifdef DISKC_FS_ENCRYPT_DEBUG
static bool crypto_diskcipher_check(struct bio *bio)
{
	int ret = 0;
	struct crypto_diskcipher *ci = NULL;
	struct inode *inode = NULL;
	struct page *page = NULL;

	if (!bio) {
		pr_err("%s: doesn't exist bio\n", __func__);
		goto out;
	}

	page = bio->bi_io_vec[0].bv_page;
	if (page && !PageAnon(page) && bio)
		if (page->mapping)
			if (page->mapping->host) {
				if (page->mapping->host->i_crypt_info) {
					inode = page->mapping->host;
					ci = fscrypt_get_diskcipher(inode);
					if (ci && (bio->bi_aux_private != ci)
					    && (!(bio->bi_flags & REQ_OP_DISCARD))) {
						pr_err("%s: no sync err\n", __func__);
						dump_err(ci, DISKC_API_GET, bio, page);
						crypto_diskcipher_debug(DISKC_NO_SYNC_ERR, 0);
						ret = -EINVAL;
					}
					if (!ci) {
						crypto_diskcipher_debug(DISKC_NO_DISKC_ERR, 1);
						pr_err("%s: no crypt err\n", __func__);
						ret = -EINVAL;
					}
				} else {
					crypto_diskcipher_debug(DISKC_NO_KEY_ERR, 1);
					ret = -EINVAL;
				}
			}
out:
	crypto_diskcipher_debug(DISKC_API_GET, ret);
	return ret;
}
#else
#define crypto_diskcipher_check(a) ((void)0)
#endif
#else
#define crypto_diskcipher_check(a) (0)
#define disckipher_log_show(a) do { } while (0)
#endif

struct crypto_diskcipher *crypto_diskcipher_get(struct bio *bio)
{
	if (!bio || !virt_addr_valid(bio)) {
		pr_err("%s: Invalid bio:%p\n", __func__, bio);
		return NULL;
	}

	if (bio->bi_opf & REQ_CRYPT) {
		if (bio->bi_aux_private) {
			if (!crypto_diskcipher_check(bio))
				return bio->bi_aux_private;
			else
				return ERR_PTR(-EINVAL);
		} else {
			crypto_diskcipher_debug(DISKC_NO_CRYPT_ERR, 0);
			return ERR_PTR(-EINVAL);
		}
	}

	return NULL;
}

struct inode *crypto_diskcipher_get_inode(struct bio *bio)
{
	struct crypto_diskcipher *tfm;

	if (bio->bi_opf & REQ_CRYPT) {
		tfm = bio->bi_aux_private;
		return tfm->inode;
	} else {
		return NULL;
	}
}

void crypto_diskcipher_set(struct bio *bio, struct crypto_diskcipher *tfm,
			const struct inode *inode, u64 dun)
{
	if (bio && tfm) {
		bio->bi_opf |= REQ_CRYPT;
		bio->bi_aux_private = tfm;
		tfm->inode = (struct inode *)inode;
#ifdef CONFIG_CRYPTO_DISKCIPHER_DUN
		if (dun)
			bio->bi_iter.bi_dun = dun;
#endif
	}
	crypto_diskcipher_debug(DISKC_API_SET, 0);
}

/* debug freerq */
enum diskc_status {
	DISKC_ST_INIT,
	DISKC_ST_FREE_REQ,
	DISKC_ST_FREE,
};

int crypto_diskcipher_setkey(struct crypto_diskcipher *tfm, const char *in_key,
			     unsigned int key_len, bool persistent)
{
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);

	if (!cra) {
		pr_err("%s: doesn't exist cra. base:%p", __func__, base);
		return -EINVAL;
	}

	crypto_diskcipher_debug(DISKC_API_SETKEY, 0);
	return cra->setkey(base, in_key, key_len, persistent);
}

int crypto_diskcipher_clearkey(struct crypto_diskcipher *tfm)
{
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);

	if (!cra) {
		pr_err("%s: doesn't exist cra. base:%p", __func__, base);
		return -EINVAL;
	}
	return cra->clearkey(base);
}

int crypto_diskcipher_set_crypt(struct crypto_diskcipher *tfm, void *req)
{
	int ret = 0;
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = NULL;

	if (!base || (base && !virt_addr_valid(base))) {
		pr_err("%s: doesn't exist base, tfm:%p, base:%p(vaild:%d)\n",
			__func__, tfm, base, virt_addr_valid(base));
		ret = -EINVAL;
		goto out;
	}

	cra = crypto_diskcipher_alg(base->__crt_alg);
	if (!cra) {
		pr_err("%s: doesn't exist cra:%p base:%p\n",__func__, cra, base);
		ret = -EINVAL;
		goto out;
	}

	if (atomic_read(&tfm->status) == DISKC_ST_FREE) {
		pr_err("%s: tfm is free\n", __func__);
		crypto_diskcipher_debug(DISKC_CRYPT_WARN, 0);
		return -EINVAL;
	}

	ret = cra->crypt(base, req);
#ifdef USE_FREE_REQ
	if (!list_empty(&cra->freectrl.freelist)) {
		if (!atomic_read(&cra->freectrl.freewq_active)) {
			atomic_set(&cra->freectrl.freewq_active, 1);
			schedule_delayed_work(&cra->freectrl.freewq, 0);
		}
	}
#endif
out:
	if (ret)
		pr_err("%s fails ret:%d, cra:%p\n", __func__, ret, cra);
	crypto_diskcipher_debug(DISKC_API_CRYPT, ret);
	return ret;
}

int crypto_diskcipher_clear_crypt(struct crypto_diskcipher *tfm, void *req)
{
	int ret = 0;
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = NULL;

	if (!base || (base && !virt_addr_valid(base))) {
		pr_err("%s: doesn't exist base, tfm:%p, base:%p(vaild:%d)\n",
			__func__, tfm, base, virt_addr_valid(base));
		ret = -EINVAL;
		goto out;
	}

	cra = crypto_diskcipher_alg(base->__crt_alg);
	if (!cra) {
		pr_err("%s: doesn't exist cra:%p base:%p\n",__func__, cra, base);
		ret = -EINVAL;
		goto out;
	}

	if (atomic_read(&tfm->status) == DISKC_ST_FREE) {
		pr_warn("%s: tfm is free\n", __func__);
		return -EINVAL;
	}

	ret = cra->clear(base, req);
	if (ret)
		pr_err("%s fails", __func__);

out:
	crypto_diskcipher_debug(DISKC_API_CLEAR, ret);
	return ret;
}

#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
int diskcipher_do_crypt(struct crypto_diskcipher *tfm,
			struct diskcipher_test_request *req)
{
	int ret;
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);

	if (!cra) {
		pr_err("%s: doesn't exist cra. base:%p\n", __func__, base);
		ret = -EINVAL;
		goto out;
	}

	if (cra->do_crypt)
		ret = cra->do_crypt(base, req);
	else
		ret = -EINVAL;
	if (ret)
		pr_err("%s fails ret:%d", __func__, ret);

out:
	return ret;
}
#endif

static int crypto_diskcipher_init_tfm(struct crypto_tfm *base)
{
	struct crypto_diskcipher *tfm = __crypto_diskcipher_cast(base);

	atomic_set(&tfm->status, DISKC_ST_INIT);
	return 0;
}

#ifdef USE_FREE_REQ
static void free_workq_func(struct work_struct *work)
{
	struct diskcipher_alg *cra =
		container_of(work, struct diskcipher_alg, freectrl.freewq.work);
	struct diskcipher_freectrl *fctrl = &cra->freectrl;
	struct crypto_diskcipher *_tfm, *tmp;
	unsigned long cur_jiffies = jiffies;
	struct list_head poss_free_list;
	unsigned long flags;

	INIT_LIST_HEAD(&poss_free_list);

	/* pickup freelist */
	spin_lock_irqsave(&fctrl->freelist_lock, flags);
	list_for_each_entry_safe(_tfm, tmp, &fctrl->freelist, node) {
		if (jiffies_to_msecs(cur_jiffies - _tfm->req_jiffies) > fctrl->max_io_ms)
			list_move_tail(&_tfm->node, &poss_free_list);
	}
	spin_unlock_irqrestore(&fctrl->freelist_lock, flags);

	list_for_each_entry_safe(_tfm, tmp, &poss_free_list, node) {
		if (atomic_read (&_tfm->status) != DISKC_ST_FREE_REQ)
			crypto_diskcipher_debug(DISKC_FREE_WQ_WARN, 0);
		crypto_free_diskcipher(_tfm);
	}

	if (!list_empty(&fctrl->freelist))
		schedule_delayed_work(&fctrl->freewq, msecs_to_jiffies(fctrl->max_io_ms));
	else
		atomic_set(&fctrl->freewq_active, 0);
}
#endif

void crypto_free_req_diskcipher(struct crypto_diskcipher *tfm)
{
#ifdef USE_FREE_REQ
	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);
	struct diskcipher_freectrl *fctrl = &cra->freectrl;
	unsigned long flags;

	if (atomic_read(&tfm->status) != DISKC_ST_INIT) {
		crypto_diskcipher_debug(DISKC_FREE_REQ_WARN, 0);
		pr_warn("%s: already submit status:%d\n", __func__, atomic_read(&tfm->status));
		return;
	}

	atomic_set(&tfm->status, DISKC_ST_FREE_REQ);
	INIT_LIST_HEAD(&tfm->node);
	tfm->req_jiffies = jiffies;
	spin_lock_irqsave(&fctrl->freelist_lock, flags);
	list_move_tail(&tfm->node, &fctrl->freelist);
	spin_unlock_irqrestore(&fctrl->freelist_lock, flags);
	crypto_diskcipher_debug(DISKC_API_FREEREQ, 0);
#else
	crypto_free_diskcipher(tfm);
#endif
}

unsigned int crypto_diskcipher_extsize(struct crypto_alg *alg)
{
	return alg->cra_ctxsize +
	    (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
}

static void crypto_diskcipher_show(struct seq_file *m, struct crypto_alg *alg)
{
	seq_printf(m, "type         : diskcipher\n");
	disckipher_log_show(m);
}

static const struct crypto_type crypto_diskcipher_type = {
	.extsize = crypto_diskcipher_extsize,
	.init_tfm = crypto_diskcipher_init_tfm,
#ifdef CONFIG_PROC_FS
	.show = crypto_diskcipher_show,
#endif
	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
	.maskset = CRYPTO_ALG_TYPE_MASK,
	.type = CRYPTO_ALG_TYPE_DISKCIPHER,
	.tfmsize = offsetof(struct crypto_diskcipher, base),
};

#define DISKC_NAME "-disk"
#define DISKC_NAME_SIZE (5)
#define DISKCIPHER_MAX_IO_MS (1000)
struct crypto_diskcipher *crypto_alloc_diskcipher(const char *alg_name,
			u32 type, u32 mask, bool force)
{
	crypto_diskcipher_debug(DISKC_API_ALLOC, 0);
	if (force) {
		if (strlen(alg_name) + DISKC_NAME_SIZE < CRYPTO_MAX_ALG_NAME) {
			char diskc_name[CRYPTO_MAX_ALG_NAME];

			strcpy(diskc_name, alg_name);
			strcat(diskc_name, DISKC_NAME);
			return crypto_alloc_tfm(diskc_name,
				&crypto_diskcipher_type, type, mask);
		}
	} else {
		return crypto_alloc_tfm(alg_name, &crypto_diskcipher_type, type, mask);
	}
	return NULL;
}

void crypto_free_diskcipher(struct crypto_diskcipher *tfm)
{
	crypto_diskcipher_debug(DISKC_API_FREE, 0);
	atomic_set(&tfm->status, DISKC_ST_FREE);
	if (tfm && virt_addr_valid(tfm))
		crypto_destroy_tfm(tfm, crypto_diskcipher_tfm(tfm));
	else
		pr_warn("%s: invalid tfm:%p(valid:%d)\n", __func__, tfm, virt_addr_valid(tfm));
}

int crypto_register_diskcipher(struct diskcipher_alg *alg)
{
	struct crypto_alg *base = &alg->base;

#ifdef USE_FREE_REQ
	struct diskcipher_freectrl *fctrl = &alg->freectrl;

	INIT_LIST_HEAD(&fctrl->freelist);
	INIT_DELAYED_WORK(&fctrl->freewq, free_workq_func);
	spin_lock_init(&fctrl->freelist_lock);
	if (!fctrl->max_io_ms)
		fctrl->max_io_ms = DISKCIPHER_MAX_IO_MS;
#endif
	base->cra_type = &crypto_diskcipher_type;
	base->cra_flags = CRYPTO_ALG_TYPE_DISKCIPHER;
	return crypto_register_alg(base);
}

void crypto_unregister_diskcipher(struct diskcipher_alg *alg)
{
	crypto_unregister_alg(&alg->base);
}

int crypto_register_diskciphers(struct diskcipher_alg *algs, int count)
{
	int i, ret;

	for (i = 0; i < count; i++) {
		ret = crypto_register_diskcipher(algs + i);
		if (ret)
			goto err;
	}
	return 0;

err:
	for (--i; i >= 0; --i)
		crypto_unregister_diskcipher(algs + i);
	return ret;
}

void crypto_unregister_diskciphers(struct diskcipher_alg *algs, int count)
{
	int i;

	for (i = count - 1; i >= 0; --i)
		crypto_unregister_diskcipher(algs + i);
}
Commit	Line	Data
2ad5939e BK	1	/*
	2	* Copyright (C) 2017 Samsung Electronics Co., Ltd.
	3	*
	4	* This program is free software; you can redistribute it and/or modify
	5	* it under the terms of the GNU General Public License as published by
	6	* the Free Software Foundation; either version 2 of the License, or
	7	* (at your option) any later version.
	8	*/
	9
	10	#include <linux/kernel.h>
	11	#include <linux/blkdev.h>
	12	#include <linux/errno.h>
	13	#include <linux/module.h>
	14	#include <linux/seq_file.h>
	15	#include <linux/string.h>
	16	#include <linux/crypto.h>
	17	#include <crypto/algapi.h>
	18	#include <crypto/diskcipher.h>
37cd97e0	19	#include <linux/delay.h>
2ad5939e BK	20
	21	#include "internal.h"
	22
37cd97e0	23	#ifdef CONFIG_CRYPTO_DISKCIPHER_DEBUG
2ad5939e BK	24	#include <crypto/fmp.h>
2ad5939e BK	25	#include <linux/mm_types.h>
37cd97e0 BK	26	#include <linux/fs.h>
37cd97e0 BK	27	#include <linux/fscrypt.h>
2ad5939e BK	28
	29	#define DUMP_MAX 20
	30
2ad5939e BK	31	struct dump_err {
	32	struct page *page;
	33	struct bio bio;
	34	struct fmp_crypto_info ci;
37cd97e0	35	enum diskcipher_dbg api;
2ad5939e BK	36	};
	37
	38	struct diskc_debug_info {
	39	struct dump_err dump[DUMP_MAX];
	40	int err;
37cd97e0	41	int cnt[DISKC_USER_MAX][2];
2ad5939e BK	42	};
	43
	44	static struct diskc_debug_info diskc_dbg;
	45
37cd97e0 BK	46	void crypto_diskcipher_debug(enum diskcipher_dbg api, int bi_opf)
	47	{
	48	int idx = 0;
	49	struct diskc_debug_info *dbg = &diskc_dbg;
	50
	51	if (api <= DISKC_API_MAX)
	52	dbg->cnt[api][bi_opf]++;
	53	else {
49e3bbe3	54	if (bi_opf)
37cd97e0 BK	55	idx = 1;
	56	dbg->cnt[api][idx]++;
	57	}
	58	}
	59
2ad5939e BK	60	static void print_err(void)
	61	{
	62	int i, j;
	63	struct bio_vec bv; / bio page list */
	64	struct bio *bio;
	65	struct fmp_crypto_info *ci;
	66	struct diskc_debug_info *dbg = &diskc_dbg;
	67
	68	for (j = 0; j < dbg->err; j++) {
	69	bio = &dbg->dump[j].bio;
	70	ci = &dbg->dump[j].ci;
	71
	72	if (bio) {
	73	pr_info
	74	("%s(%d/%d): bio:%p ci:%p page:%p flag:%x, opf:%x, crypt:%p\n",
	75	__func__, j, dbg->err, bio, ci, &dbg->dump[j].page,
	76	bio->bi_flags, bio->bi_opf, bio->bi_aux_private);
	77	print_hex_dump(KERN_CONT, "bio:", DUMP_PREFIX_OFFSET,
	78	16, 1, bio, sizeof(struct bio), false);
	79	for (i = 0; i < bio->bi_max_vecs; i++) {
	80	bv = &bio->bi_io_vec[i];
	81	pr_info("bv[%d] page:%p len:%d offset:%d\n",
	82	i, bv->bv_page, bv->bv_len, bv->bv_offset);
	83	}
	84	}
	85
	86	if (ci) {
	87	pr_info("[ci] key_size:%d algo_mode:%d\n",
	88	ci->key_size, ci->algo_mode);
	89	print_hex_dump(KERN_CONT, "key:", DUMP_PREFIX_OFFSET,
	90	16, 1, ci->key, sizeof(ci->key), false);
	91	}
	92	}
	93	}
	94
37cd97e0	95	static void dump_err(struct crypto_diskcipher *ci, enum diskcipher_dbg api,
2ad5939e BK	96	struct bio bio, struct page page)
	97	{
	98	struct diskc_debug_info *dbg = &diskc_dbg;
	99
	100	if ((dbg->err < DUMP_MAX) && ci) {
	101	struct crypto_tfm *tfm = crypto_diskcipher_tfm(ci);
	102
	103	dbg->dump[dbg->err].api = api;
	104	memcpy(&dbg->dump[dbg->err].ci, crypto_tfm_ctx(tfm),
	105	sizeof(struct fmp_crypto_info));
	106
	107	if (page)
	108	dbg->dump[dbg->err].page = page;
	109	if (bio)
	110	memcpy(&dbg->dump[dbg->err].bio, bio,
	111	sizeof(struct bio));
	112	}
	113	dbg->err++;
	114	}
	115
2ad5939e BK	116	static void disckipher_log_show(struct seq_file *m)
	117	{
	118	int i;
2ad5939e	119	struct diskc_debug_info *dbg = &diskc_dbg;
49e3bbe3 BK	120	char name[DISKC_USER_MAX][32] = {
	121	"ALLOC", "FREE", "FREEREQ", "SETKEY", "SET", "GET", "CRYPT", "CLEAR",
	122	"DISKC_API_MAX", "FS_PAGEIO", "FS_READP", "FS_DIO", "FS_BLOCK_WRITE",
	123	"FS_ZEROPAGE", "BLK_BH", "DMCRYPT", "DISKC_MERGE", "DISKC_MERGE_ERR_INODE", "DISKC_MERGE_ERR_DISK",
	124	"FS_DEC_WARN", "FS_ENC_WARN", "DISKC_MERGE_DIO", "DISKC_FREE_REQ_WARN",
	125	"DISKC_FREE_WQ_WARN", "DISKC_CRYPT_WARN",
	126	"DM_CRYPT_NONENCRYPT", "DM_CRYPT_CTR", "DM_CRYPT_DTR", "DM_CRYPT_OVER",
	127	"F2FS_gc", "F2FS_gc_data_page", "F2FS_gc_data_page_no_key", "F2FS_gc_data_page_no_key_FC",
	128	"F2FS_gc_data_page_FC", "F2FS_gc_data_block", "F2FS_gc_data_block_key",
	129	"F2FS_gc_data_block_err1", "F2FS_gc_data_block_err2", "F2FS_gc_data_block_err3", "F2FS_gc_skip",
	130	"DISKC_ERR", "DISKC_NO_KEY_ERR", "DISKC_NO_SYNC_ERR", "DISKC_NO_CRYPT_ERR", "DISKC_NO_DISKC_ERR"};
37cd97e0 BK	131
	132	for (i = 0; i < DISKC_USER_MAX; i++)
	133	if (dbg->cnt[i][0] \|\| dbg->cnt[i][1])
	134	seq_printf(m, "%s\t: %6u(err:%u)\n",
	135	name[i], dbg->cnt[i][0], dbg->cnt[i][1]);
2ad5939e BK	136
	137	if (dbg->err)
	138	print_err();
	139	}
	140
	141	/* check diskcipher for FBE */
37cd97e0	142	#define DISKC_FS_ENCRYPT_DEBUG
37cd97e0	143	#ifdef DISKC_FS_ENCRYPT_DEBUG
49e3bbe3 BK	144	static bool crypto_diskcipher_check(struct bio *bio)
49e3bbe3 BK	145	{
2ad5939e BK	146	int ret = 0;
2ad5939e BK	147	struct crypto_diskcipher *ci = NULL;
37cd97e0	148	struct inode *inode = NULL;
c99b5690	149	struct page *page = NULL;
2ad5939e	150
c99b5690	151	if (!bio) {
	152	pr_err("%s: doesn't exist bio\n", __func__);
	153	goto out;
	154	}
	155
	156	page = bio->bi_io_vec[0].bv_page;
2ad5939e BK	157	if (page && !PageAnon(page) && bio)
2ad5939e BK	158	if (page->mapping)
49e3bbe3	159	if (page->mapping->host) {
2ad5939e	160	if (page->mapping->host->i_crypt_info) {
37cd97e0	161	inode = page->mapping->host;
49e3bbe3	162	ci = fscrypt_get_diskcipher(inode);
2ad5939e BK	163	if (ci && (bio->bi_aux_private != ci)
2ad5939e BK	164	&& (!(bio->bi_flags & REQ_OP_DISCARD))) {
49e3bbe3	165	pr_err("%s: no sync err\n", __func__);
2ad5939e	166	dump_err(ci, DISKC_API_GET, bio, page);
49e3bbe3 BK	167	crypto_diskcipher_debug(DISKC_NO_SYNC_ERR, 0);
49e3bbe3 BK	168	ret = -EINVAL;
37cd97e0	169	}
49e3bbe3 BK	170	if (!ci) {
	171	crypto_diskcipher_debug(DISKC_NO_DISKC_ERR, 1);
	172	pr_err("%s: no crypt err\n", __func__);
	173	ret = -EINVAL;
2ad5939e	174	}
49e3bbe3 BK	175	} else {
	176	crypto_diskcipher_debug(DISKC_NO_KEY_ERR, 1);
	177	ret = -EINVAL;
2ad5939e	178	}
49e3bbe3	179	}
c99b5690	180	out:
37cd97e0	181	crypto_diskcipher_debug(DISKC_API_GET, ret);
49e3bbe3	182	return ret;
2ad5939e BK	183	}
2ad5939e BK	184	#else
49e3bbe3 BK	185	#define crypto_diskcipher_check(a) ((void)0)
	186	#endif
	187	#else
	188	#define crypto_diskcipher_check(a) (0)
2ad5939e	189	#define disckipher_log_show(a) do { } while (0)
2ad5939e BK	190	#endif
2ad5939e BK	191
2ad5939e BK	192	struct crypto_diskcipher crypto_diskcipher_get(struct bio bio)
	193	{
	194	if (!bio \|\| !virt_addr_valid(bio)) {
	195	pr_err("%s: Invalid bio:%p\n", __func__, bio);
	196	return NULL;
	197	}
49e3bbe3 BK	198
	199	if (bio->bi_opf & REQ_CRYPT) {
	200	if (bio->bi_aux_private) {
	201	if (!crypto_diskcipher_check(bio))
	202	return bio->bi_aux_private;
	203	else
	204	return ERR_PTR(-EINVAL);
	205	} else {
	206	crypto_diskcipher_debug(DISKC_NO_CRYPT_ERR, 0);
	207	return ERR_PTR(-EINVAL);
	208	}
	209	}
	210
	211	return NULL;
	212	}
	213
	214	struct inode crypto_diskcipher_get_inode(struct bio bio)
	215	{
	216	struct crypto_diskcipher *tfm;
	217
	218	if (bio->bi_opf & REQ_CRYPT) {
	219	tfm = bio->bi_aux_private;
	220	return tfm->inode;
	221	} else {
2ad5939e	222	return NULL;
49e3bbe3	223	}
2ad5939e BK	224	}
2ad5939e BK	225
49e3bbe3 BK	226	void crypto_diskcipher_set(struct bio bio, struct crypto_diskcipher tfm,
49e3bbe3 BK	227	const struct inode *inode, u64 dun)
2ad5939e	228	{
37cd97e0	229	if (bio && tfm) {
e4bdfe04	230	bio->bi_opf \|= REQ_CRYPT;
37cd97e0	231	bio->bi_aux_private = tfm;
49e3bbe3 BK	232	tfm->inode = (struct inode *)inode;
	233	#ifdef CONFIG_CRYPTO_DISKCIPHER_DUN
	234	if (dun)
	235	bio->bi_iter.bi_dun = dun;
	236	#endif
2ad5939e	237	}
37cd97e0	238	crypto_diskcipher_debug(DISKC_API_SET, 0);
2ad5939e BK	239	}
2ad5939e BK	240
37cd97e0 BK	241	/* debug freerq */
	242	enum diskc_status {
	243	DISKC_ST_INIT,
	244	DISKC_ST_FREE_REQ,
	245	DISKC_ST_FREE,
	246	};
49e3bbe3	247
2ad5939e BK	248	int crypto_diskcipher_setkey(struct crypto_diskcipher tfm, const char in_key,
	249	unsigned int key_len, bool persistent)
	250	{
	251	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	252	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);
	253
	254	if (!cra) {
c99b5690	255	pr_err("%s: doesn't exist cra. base:%p", __func__, base);
2ad5939e BK	256	return -EINVAL;
2ad5939e BK	257	}
37cd97e0 BK	258
37cd97e0 BK	259	crypto_diskcipher_debug(DISKC_API_SETKEY, 0);
2ad5939e BK	260	return cra->setkey(base, in_key, key_len, persistent);
	261	}
	262
	263	int crypto_diskcipher_clearkey(struct crypto_diskcipher *tfm)
	264	{
	265	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	266	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);
	267
	268	if (!cra) {
c99b5690	269	pr_err("%s: doesn't exist cra. base:%p", __func__, base);
2ad5939e BK	270	return -EINVAL;
	271	}
	272	return cra->clearkey(base);
	273	}
	274
	275	int crypto_diskcipher_set_crypt(struct crypto_diskcipher tfm, void req)
	276	{
	277	int ret = 0;
	278	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
c99b5690	279	struct diskcipher_alg *cra = NULL;
c99b5690	280
a87c4fbf BK	281	if (!base \|\| (base && !virt_addr_valid(base))) {
	282	pr_err("%s: doesn't exist base, tfm:%p, base:%p(vaild:%d)\n",
	283	__func__, tfm, base, virt_addr_valid(base));
c99b5690	284	ret = -EINVAL;
	285	goto out;
	286	}
	287
	288	cra = crypto_diskcipher_alg(base->__crt_alg);
2ad5939e	289	if (!cra) {
a87c4fbf	290	pr_err("%s: doesn't exist cra:%p base:%p\n",__func__, cra, base);
c99b5690	291	ret = -EINVAL;
2ad5939e BK	292	goto out;
	293	}
	294
37cd97e0 BK	295	if (atomic_read(&tfm->status) == DISKC_ST_FREE) {
	296	pr_err("%s: tfm is free\n", __func__);
	297	crypto_diskcipher_debug(DISKC_CRYPT_WARN, 0);
	298	return -EINVAL;
	299	}
	300
2ad5939e	301	ret = cra->crypt(base, req);
91b23ddd BK	302	#ifdef USE_FREE_REQ
	303	if (!list_empty(&cra->freectrl.freelist)) {
	304	if (!atomic_read(&cra->freectrl.freewq_active)) {
	305	atomic_set(&cra->freectrl.freewq_active, 1);
	306	schedule_delayed_work(&cra->freectrl.freewq, 0);
37cd97e0 BK	307	}
37cd97e0 BK	308	}
91b23ddd	309	#endif
2ad5939e BK	310	out:
	311	if (ret)
	312	pr_err("%s fails ret:%d, cra:%p\n", __func__, ret, cra);
37cd97e0	313	crypto_diskcipher_debug(DISKC_API_CRYPT, ret);
2ad5939e BK	314	return ret;
	315	}
	316
	317	int crypto_diskcipher_clear_crypt(struct crypto_diskcipher tfm, void req)
	318	{
	319	int ret = 0;
	320	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
c99b5690	321	struct diskcipher_alg *cra = NULL;
c99b5690	322
a87c4fbf BK	323	if (!base \|\| (base && !virt_addr_valid(base))) {
	324	pr_err("%s: doesn't exist base, tfm:%p, base:%p(vaild:%d)\n",
	325	__func__, tfm, base, virt_addr_valid(base));
c99b5690	326	ret = -EINVAL;
	327	goto out;
	328	}
	329
	330	cra = crypto_diskcipher_alg(base->__crt_alg);
2ad5939e	331	if (!cra) {
a87c4fbf	332	pr_err("%s: doesn't exist cra:%p base:%p\n",__func__, cra, base);
c99b5690	333	ret = -EINVAL;
2ad5939e BK	334	goto out;
	335	}
	336
c99b5690	337	if (atomic_read(&tfm->status) == DISKC_ST_FREE) {
	338	pr_warn("%s: tfm is free\n", __func__);
	339	return -EINVAL;
	340	}
	341
2ad5939e BK	342	ret = cra->clear(base, req);
	343	if (ret)
	344	pr_err("%s fails", __func__);
	345
	346	out:
37cd97e0	347	crypto_diskcipher_debug(DISKC_API_CLEAR, ret);
2ad5939e BK	348	return ret;
	349	}
	350
	351	#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS
	352	int diskcipher_do_crypt(struct crypto_diskcipher *tfm,
	353	struct diskcipher_test_request *req)
	354	{
	355	int ret;
	356	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	357	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);
	358
	359	if (!cra) {
c99b5690	360	pr_err("%s: doesn't exist cra. base:%p\n", __func__, base);
c99b5690	361	ret = -EINVAL;
2ad5939e BK	362	goto out;
	363	}
	364
	365	if (cra->do_crypt)
	366	ret = cra->do_crypt(base, req);
	367	else
	368	ret = -EINVAL;
	369	if (ret)
	370	pr_err("%s fails ret:%d", __func__, ret);
	371
	372	out:
2ad5939e BK	373	return ret;
	374	}
	375	#endif
	376
37cd97e0	377	static int crypto_diskcipher_init_tfm(struct crypto_tfm *base)
2ad5939e	378	{
37cd97e0 BK	379	struct crypto_diskcipher *tfm = __crypto_diskcipher_cast(base);
37cd97e0 BK	380
37cd97e0	381	atomic_set(&tfm->status, DISKC_ST_INIT);
2ad5939e BK	382	return 0;
	383	}
	384
91b23ddd	385	#ifdef USE_FREE_REQ
37cd97e0 BK	386	static void free_workq_func(struct work_struct *work)
	387	{
	388	struct diskcipher_alg *cra =
	389	container_of(work, struct diskcipher_alg, freectrl.freewq.work);
	390	struct diskcipher_freectrl *fctrl = &cra->freectrl;
	391	struct crypto_diskcipher _tfm, tmp;
	392	unsigned long cur_jiffies = jiffies;
	393	struct list_head poss_free_list;
	394	unsigned long flags;
	395
	396	INIT_LIST_HEAD(&poss_free_list);
	397
	398	/* pickup freelist */
	399	spin_lock_irqsave(&fctrl->freelist_lock, flags);
	400	list_for_each_entry_safe(_tfm, tmp, &fctrl->freelist, node) {
	401	if (jiffies_to_msecs(cur_jiffies - _tfm->req_jiffies) > fctrl->max_io_ms)
	402	list_move_tail(&_tfm->node, &poss_free_list);
	403	}
	404	spin_unlock_irqrestore(&fctrl->freelist_lock, flags);
	405
	406	list_for_each_entry_safe(_tfm, tmp, &poss_free_list, node) {
	407	if (atomic_read (&_tfm->status) != DISKC_ST_FREE_REQ)
	408	crypto_diskcipher_debug(DISKC_FREE_WQ_WARN, 0);
	409	crypto_free_diskcipher(_tfm);
	410	}
	411
	412	if (!list_empty(&fctrl->freelist))
	413	schedule_delayed_work(&fctrl->freewq, msecs_to_jiffies(fctrl->max_io_ms));
	414	else
	415	atomic_set(&fctrl->freewq_active, 0);
	416	}
91b23ddd	417	#endif
37cd97e0 BK	418
	419	void crypto_free_req_diskcipher(struct crypto_diskcipher *tfm)
	420	{
91b23ddd	421	#ifdef USE_FREE_REQ
37cd97e0 BK	422	struct crypto_tfm *base = crypto_diskcipher_tfm(tfm);
	423	struct diskcipher_alg *cra = crypto_diskcipher_alg(base->__crt_alg);
	424	struct diskcipher_freectrl *fctrl = &cra->freectrl;
	425	unsigned long flags;
	426
	427	if (atomic_read(&tfm->status) != DISKC_ST_INIT) {
	428	crypto_diskcipher_debug(DISKC_FREE_REQ_WARN, 0);
	429	pr_warn("%s: already submit status:%d\n", __func__, atomic_read(&tfm->status));
	430	return;
	431	}
	432
	433	atomic_set(&tfm->status, DISKC_ST_FREE_REQ);
	434	INIT_LIST_HEAD(&tfm->node);
	435	tfm->req_jiffies = jiffies;
	436	spin_lock_irqsave(&fctrl->freelist_lock, flags);
	437	list_move_tail(&tfm->node, &fctrl->freelist);
	438	spin_unlock_irqrestore(&fctrl->freelist_lock, flags);
	439	crypto_diskcipher_debug(DISKC_API_FREEREQ, 0);
91b23ddd BK	440	#else
	441	crypto_free_diskcipher(tfm);
	442	#endif
37cd97e0 BK	443	}
37cd97e0 BK	444
2ad5939e BK	445	unsigned int crypto_diskcipher_extsize(struct crypto_alg *alg)
	446	{
	447	return alg->cra_ctxsize +
	448	(alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
	449	}
	450
	451	static void crypto_diskcipher_show(struct seq_file m, struct crypto_alg alg)
	452	{
	453	seq_printf(m, "type : diskcipher\n");
	454	disckipher_log_show(m);
	455	}
	456
	457	static const struct crypto_type crypto_diskcipher_type = {
	458	.extsize = crypto_diskcipher_extsize,
	459	.init_tfm = crypto_diskcipher_init_tfm,
	460	#ifdef CONFIG_PROC_FS
	461	.show = crypto_diskcipher_show,
	462	#endif
	463	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
	464	.maskset = CRYPTO_ALG_TYPE_MASK,
	465	.type = CRYPTO_ALG_TYPE_DISKCIPHER,
	466	.tfmsize = offsetof(struct crypto_diskcipher, base),
	467	};
	468
37cd97e0 BK	469	#define DISKC_NAME "-disk"
	470	#define DISKC_NAME_SIZE (5)
	471	#define DISKCIPHER_MAX_IO_MS (1000)
2ad5939e BK	472	struct crypto_diskcipher crypto_alloc_diskcipher(const char alg_name,
	473	u32 type, u32 mask, bool force)
	474	{
37cd97e0	475	crypto_diskcipher_debug(DISKC_API_ALLOC, 0);
2ad5939e BK	476	if (force) {
	477	if (strlen(alg_name) + DISKC_NAME_SIZE < CRYPTO_MAX_ALG_NAME) {
	478	char diskc_name[CRYPTO_MAX_ALG_NAME];
	479
	480	strcpy(diskc_name, alg_name);
	481	strcat(diskc_name, DISKC_NAME);
	482	return crypto_alloc_tfm(diskc_name,
	483	&crypto_diskcipher_type, type, mask);
	484	}
	485	} else {
	486	return crypto_alloc_tfm(alg_name, &crypto_diskcipher_type, type, mask);
	487	}
2ad5939e BK	488	return NULL;
	489	}
	490
	491	void crypto_free_diskcipher(struct crypto_diskcipher *tfm)
	492	{
37cd97e0 BK	493	crypto_diskcipher_debug(DISKC_API_FREE, 0);
37cd97e0 BK	494	atomic_set(&tfm->status, DISKC_ST_FREE);
a87c4fbf BK	495	if (tfm && virt_addr_valid(tfm))
	496	crypto_destroy_tfm(tfm, crypto_diskcipher_tfm(tfm));
	497	else
	498	pr_warn("%s: invalid tfm:%p(valid:%d)\n", __func__, tfm, virt_addr_valid(tfm));
2ad5939e BK	499	}
	500
	501	int crypto_register_diskcipher(struct diskcipher_alg *alg)
	502	{
	503	struct crypto_alg *base = &alg->base;
91b23ddd BK	504
91b23ddd BK	505	#ifdef USE_FREE_REQ
37cd97e0	506	struct diskcipher_freectrl *fctrl = &alg->freectrl;
2ad5939e	507
37cd97e0 BK	508	INIT_LIST_HEAD(&fctrl->freelist);
	509	INIT_DELAYED_WORK(&fctrl->freewq, free_workq_func);
	510	spin_lock_init(&fctrl->freelist_lock);
	511	if (!fctrl->max_io_ms)
	512	fctrl->max_io_ms = DISKCIPHER_MAX_IO_MS;
91b23ddd	513	#endif
2ad5939e BK	514	base->cra_type = &crypto_diskcipher_type;
	515	base->cra_flags = CRYPTO_ALG_TYPE_DISKCIPHER;
	516	return crypto_register_alg(base);
	517	}
	518
	519	void crypto_unregister_diskcipher(struct diskcipher_alg *alg)
	520	{
	521	crypto_unregister_alg(&alg->base);
	522	}
	523
	524	int crypto_register_diskciphers(struct diskcipher_alg *algs, int count)
	525	{
	526	int i, ret;
	527
	528	for (i = 0; i < count; i++) {
	529	ret = crypto_register_diskcipher(algs + i);
	530	if (ret)
	531	goto err;
	532	}
	533	return 0;
	534
	535	err:
	536	for (--i; i >= 0; --i)
	537	crypto_unregister_diskcipher(algs + i);
	538	return ret;
	539	}
	540
	541	void crypto_unregister_diskciphers(struct diskcipher_alg *algs, int count)
	542	{
	543	int i;
	544
	545	for (i = count - 1; i >= 0; --i)
	546	crypto_unregister_diskcipher(algs + i);
	547	}