#include <crypto/hash.h>
#include <crypto/internal/hash.h>
#include <linux/amlogic/iomap.h>
+#include <linux/of_platform.h>
+#include <crypto/skcipher.h>
#include "aml-crypto-dma.h"
/* AES flags */
u32 key[AES_KEYSIZE_256 / sizeof(u32)];
u16 block_size;
+ struct crypto_skcipher *fallback;
};
struct aml_aes_reqctx {
spinlock_t lock;
};
+struct aml_aes_info {
+ struct crypto_alg *algs;
+ uint32_t num_algs;
+};
+
static struct aml_aes_drv aml_aes = {
.dev_list = LIST_HEAD_INIT(aml_aes.dev_list),
.lock = __SPIN_LOCK_UNLOCKED(aml_aes.lock),
uint32_t *nents, size_t total)
{
size_t count = 0;
+ size_t process = 0;
size_t count_total = 0;
size_t count_sg = 0;
uint32_t i = 0;
dma_addr_t addr_in, addr_out;
while (total && in_sg && out_sg && (in_sg->length == out_sg->length)
+ && IS_ALIGNED(in_sg->length, AES_BLOCK_SIZE)
&& *nents < MAX_NUM_TABLES) {
- count += min_t(unsigned int, total, in_sg->length);
+ process = min_t(unsigned int, total, in_sg->length);
+ count += process;
*nents += 1;
- total -= count;
+ total -= process;
in_sg = sg_next(in_sg);
out_sg = sg_next(out_sg);
}
static void aml_aes_finish_req(struct aml_aes_dev *dd, int32_t err)
{
struct ablkcipher_request *req = dd->req;
+ unsigned long flags;
dd->flags &= ~AES_FLAGS_BUSY;
+ spin_lock_irqsave(&dd->dma->dma_lock, flags);
dd->dma->dma_busy = 0;
+ spin_unlock_irqrestore(&dd->dma->dma_lock, flags);
req->base.complete(&req->base, err);
}
PAGE_SIZE, DMA_TO_DEVICE);
aml_dma_debug(dsc, nents, __func__, dd->thread, dd->status);
+
+ /* Start DMA transfer */
aml_write_crypto_reg(dd->thread, dd->dma_descript_tab | 2);
- return 0;
+ return -EINPROGRESS;
}
static int aml_aes_crypt_dma_start(struct aml_aes_dev *dd)
dd->flags |= AES_FLAGS_FAST;
nents = dd->fast_nents;
dd->fast_total = count;
+ dbgp(1, "use fast dma: n:%u, t:%zd\n",
+ dd->fast_nents, dd->fast_total);
} else {
/* slow dma */
/* use cache buffers */
count = aml_aes_sg_copy(&dd->in_sg, &dd->in_offset,
- dd->buf_in, dd->buflen,
- ((dd->total + dd->ctx->block_size - 1)
- / dd->ctx->block_size)
- * dd->ctx->block_size, 0);
+ dd->buf_in, dd->buflen, dd->total, 0);
addr_in = dd->dma_addr_in;
addr_out = dd->dma_addr_out;
dd->dma_size = count;
- dma_sync_single_for_device(dd->dev, addr_in, dd->dma_size,
+ dma_sync_single_for_device(dd->dev, addr_in,
+ ((dd->dma_size + AES_BLOCK_SIZE - 1)
+ / AES_BLOCK_SIZE) * AES_BLOCK_SIZE,
DMA_TO_DEVICE);
dsc->src_addr = (uint32_t)addr_in;
dsc->tgt_addr = (uint32_t)addr_out;
dsc->dsc_cfg.d32 = 0;
- dsc->dsc_cfg.b.length = ((count + dd->ctx->block_size - 1)
- / dd->ctx->block_size) * dd->ctx->block_size;
+ /* We align data to AES_BLOCK_SIZE for old aml-dma devices */
+ dsc->dsc_cfg.b.length = ((count + AES_BLOCK_SIZE - 1)
+ / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
nents = 1;
dd->flags &= ~AES_FLAGS_FAST;
+ dbgp(1, "use slow dma: cnt:%zd, len:%d\n",
+ count, dsc->dsc_cfg.b.length);
}
dd->total -= count;
err = aml_aes_crypt_dma(dd, dsc, nents);
- if (err && (dd->flags & AES_FLAGS_FAST)) {
- dma_unmap_sg(dd->dev, dd->in_sg,
- dd->fast_nents, DMA_TO_DEVICE);
- dma_unmap_sg(dd->dev, dd->out_sg,
- dd->fast_nents, DMA_TO_DEVICE);
- }
-
return err;
}
err = -EINVAL;
}
}
- if (err) {
+ if (err != -EINPROGRESS) {
/* aes_task will not finish it, so do it here */
aml_aes_finish_req(dd, err);
tasklet_schedule(&dd->queue_task);
16, 0);
} else {
dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out,
- dd->dma_size, DMA_FROM_DEVICE);
+ ((dd->dma_size + AES_BLOCK_SIZE - 1)
+ / AES_BLOCK_SIZE) * AES_BLOCK_SIZE,
+ DMA_FROM_DEVICE);
/* copy data */
count = aml_aes_sg_copy(&dd->out_sg, &dd->out_offset,
err = -EINVAL;
pr_err("not all data converted: %zu\n", count);
}
+ /* install IV for CBC */
if (dd->flags & AES_FLAGS_CBC) {
memcpy(dd->req->info, dd->buf_out +
dd->dma_size - 16, 16);
crypto_ablkcipher_reqtfm(req));
struct aml_aes_reqctx *rctx = ablkcipher_request_ctx(req);
struct aml_aes_dev *dd;
+ int ret;
if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE) &&
!(mode & AES_FLAGS_CTR)) {
}
ctx->block_size = AES_BLOCK_SIZE;
+ if (ctx->fallback && ctx->keylen == AES_KEYSIZE_192) {
+ char *__subreq_desc = kzalloc(sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(ctx->fallback),
+ GFP_ATOMIC);
+ struct skcipher_request *subreq = (void *)__subreq_desc;
+
+ if (!subreq)
+ return -ENOMEM;
+
+ skcipher_request_set_tfm(subreq, ctx->fallback);
+ skcipher_request_set_callback(subreq, req->base.flags, NULL,
+ NULL);
+ skcipher_request_set_crypt(subreq, req->src, req->dst,
+ req->nbytes, req->info);
+
+ if (mode & AES_FLAGS_ENCRYPT)
+ ret = crypto_skcipher_encrypt(subreq);
+ else
+ ret = crypto_skcipher_decrypt(subreq);
+
+ skcipher_request_free(subreq);
+ return ret;
+ }
+
dd = aml_aes_find_dev(ctx);
if (!dd)
return -ENODEV;
if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
keylen != AES_KEYSIZE_256) {
crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ pr_err("aml-aes:invalid keysize: %d\n", keylen);
return -EINVAL;
}
return 0;
}
+static int aml_aes_lite_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct aml_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ int ret = 0;
+
+ if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256) {
+ crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ pr_err("aml-aes-lite:invalid keysize: %d\n", keylen);
+ return -EINVAL;
+ }
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ if (keylen == AES_KEYSIZE_192) {
+ crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
+ CRYPTO_TFM_REQ_MASK);
+ ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
+ }
+
+ return ret;
+}
+
static int aml_aes_ecb_encrypt(struct ablkcipher_request *req)
{
return aml_aes_crypt(req,
static int aml_aes_cra_init(struct crypto_tfm *tfm)
{
+ struct aml_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
tfm->crt_ablkcipher.reqsize = sizeof(struct aml_aes_reqctx);
+ ctx->fallback = NULL;
return 0;
}
{
.cra_name = "ecb(aes)",
.cra_driver_name = "ecb-aes-aml",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_aes_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = 0xf,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = aml_aes_cra_init,
{
.cra_name = "cbc(aes)",
.cra_driver_name = "cbc-aes-aml",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_aes_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = 0xf,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = aml_aes_cra_init,
{
.cra_name = "ctr(aes)",
.cra_driver_name = "ctr-aes-aml",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
- .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_blocksize = 1,
.cra_ctxsize = sizeof(struct aml_aes_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = 0xf,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
.cra_init = aml_aes_cra_init,
}
};
+static int aml_aes_lite_cra_init(struct crypto_tfm *tfm)
+{
+ struct aml_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ const char *alg_name = crypto_tfm_alg_name(tfm);
+ const u32 flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
+
+ tfm->crt_ablkcipher.reqsize = sizeof(struct aml_aes_reqctx);
+
+ /* Allocate a fallback and abort if it failed. */
+ ctx->fallback = crypto_alloc_skcipher(alg_name, 0,
+ flags);
+ if (IS_ERR(ctx->fallback)) {
+ pr_err("aml-aes: fallback '%s' could not be loaded.\n",
+ alg_name);
+ return PTR_ERR(ctx->fallback);
+ }
+
+ return 0;
+}
+
+static void aml_aes_lite_cra_exit(struct crypto_tfm *tfm)
+{
+ struct aml_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (ctx->fallback)
+ crypto_free_skcipher(ctx->fallback);
+
+ ctx->fallback = NULL;
+}
+
+static struct crypto_alg aes_lite_algs[] = {
+ {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-lite-aml",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aml_aes_ctx),
+ .cra_alignmask = 0xf,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = aml_aes_lite_cra_init,
+ .cra_exit = aml_aes_lite_cra_exit,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aml_aes_lite_setkey,
+ .encrypt = aml_aes_ecb_encrypt,
+ .decrypt = aml_aes_ecb_decrypt,
+ }
+ },
+ {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-lite-aml",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aml_aes_ctx),
+ .cra_alignmask = 0xf,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = aml_aes_lite_cra_init,
+ .cra_exit = aml_aes_lite_cra_exit,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aml_aes_lite_setkey,
+ .encrypt = aml_aes_cbc_encrypt,
+ .decrypt = aml_aes_cbc_decrypt,
+ }
+ },
+ {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "ctr-aes-lite-aml",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct aml_aes_ctx),
+ .cra_alignmask = 0xf,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = aml_aes_lite_cra_init,
+ .cra_exit = aml_aes_lite_cra_exit,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aml_aes_lite_setkey,
+ .encrypt = aml_aes_ctr_encrypt,
+ .decrypt = aml_aes_ctr_decrypt,
+ }
+ },
+};
+
+struct aml_aes_info aml_gxl_aes = {
+ .algs = aes_algs,
+ .num_algs = ARRAY_SIZE(aes_algs),
+};
+
+struct aml_aes_info aml_aes_lite = {
+ .algs = aes_lite_algs,
+ .num_algs = ARRAY_SIZE(aes_lite_algs),
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id aml_aes_dt_match[] = {
+ { .compatible = "amlogic,aes_dma",
+ .data = &aml_gxl_aes,
+ },
+ { .compatible = "amlogic,aes_g12a_dma",
+ .data = &aml_aes_lite,
+ },
+ {},
+};
+#else
+#define aml_aes_dt_match NULL
+#endif
+
static void aml_aes_queue_task(unsigned long data)
{
struct aml_aes_dev *dd = (struct aml_aes_dev *)data;
for (i = 0; i < dd->fast_nents; i++) {
dd->in_sg = sg_next(dd->in_sg);
dd->out_sg = sg_next(dd->out_sg);
- if (!dd->in_sg || !dd->out_sg)
+ if (!dd->in_sg || !dd->out_sg) {
+ pr_err("aml-aes: sg invalid\n");
err = -EINVAL;
+ }
}
}
if (!err)
err = aml_aes_crypt_dma_start(dd);
- if (!err)
+ if (err == -EINPROGRESS)
return; /* DMA started. Not fininishing. */
}
return IRQ_NONE;
}
-static void aml_aes_unregister_algs(struct aml_aes_dev *dd)
+static void aml_aes_unregister_algs(struct aml_aes_dev *dd,
+ const struct aml_aes_info *aes_info)
{
int i;
- for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
- crypto_unregister_alg(&aes_algs[i]);
+ for (i = 0; i < aes_info->num_algs; i++)
+ crypto_unregister_alg(&(aes_info->algs[i]));
}
-static int aml_aes_register_algs(struct aml_aes_dev *dd)
+static int aml_aes_register_algs(struct aml_aes_dev *dd,
+ const struct aml_aes_info *aes_info)
{
int err, i, j;
- for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
- err = crypto_register_alg(&aes_algs[i]);
+ for (i = 0; i < aes_info->num_algs; i++) {
+ err = crypto_register_alg(&(aes_info->algs[i]));
if (err)
goto err_aes_algs;
}
{
struct aml_aes_dev *aes_dd;
struct device *dev = &pdev->dev;
+ const struct of_device_id *match;
int err = -EPERM;
+ const struct aml_aes_info *aes_info = NULL;
aes_dd = kzalloc(sizeof(struct aml_aes_dev), GFP_KERNEL);
if (aes_dd == NULL) {
goto aes_dd_err;
}
+ match = of_match_device(aml_aes_dt_match, &pdev->dev);
+ if (!match) {
+ pr_err("%s: cannot find match dt\n", __func__);
+ err = -EINVAL;
+ kfree(aes_dd);
+ goto aes_dd_err;
+ }
+ aes_info = match->data;
aes_dd->dev = dev;
aes_dd->dma = dev_get_drvdata(dev->parent);
aes_dd->thread = aes_dd->dma->thread;
list_add_tail(&aes_dd->list, &aml_aes.dev_list);
spin_unlock(&aml_aes.lock);
- err = aml_aes_register_algs(aes_dd);
+ err = aml_aes_register_algs(aes_dd, aes_info);
if (err)
goto err_algs;
static int aml_aes_remove(struct platform_device *pdev)
{
static struct aml_aes_dev *aes_dd;
+ const struct of_device_id *match;
+ const struct aml_aes_info *aes_info = NULL;
aes_dd = platform_get_drvdata(pdev);
if (!aes_dd)
return -ENODEV;
+ match = of_match_device(aml_aes_dt_match, &pdev->dev);
+ if (!match) {
+ pr_err("%s: cannot find match dt\n", __func__);
+ return -EINVAL;
+ }
+ aes_info = match->data;
spin_lock(&aml_aes.lock);
list_del(&aes_dd->list);
spin_unlock(&aml_aes.lock);
- aml_aes_unregister_algs(aes_dd);
+ aml_aes_unregister_algs(aes_dd, aes_info);
tasklet_kill(&aes_dd->done_task);
tasklet_kill(&aes_dd->queue_task);
return 0;
}
-#ifdef CONFIG_OF
-static const struct of_device_id aml_aes_dt_match[] = {
- { .compatible = "amlogic,aes_dma",
- },
- {},
-};
-#else
-#define aml_aes_dt_match NULL
-#endif
-
static struct platform_driver aml_aes_driver = {
.probe = aml_aes_probe,
.remove = aml_aes_remove,
/* SHA flags */
#define SHA_FLAGS_BUSY BIT(0)
-#define SHA_FLAGS_FINAL BIT(1)
-#define SHA_FLAGS_DMA_ACTIVE BIT(2)
-#define SHA_FLAGS_OUTPUT_READY BIT(3)
-#define SHA_FLAGS_INIT BIT(4)
-#define SHA_FLAGS_DMA_READY BIT(5)
-#define SHA_FLAGS_DMA_FAST BIT(6)
-
-#define SHA_FLAGS_FINUP BIT(16)
-#define SHA_FLAGS_SHA1 BIT(17)
-#define SHA_FLAGS_SHA224 BIT(18)
-#define SHA_FLAGS_SHA256 BIT(19)
-#define SHA_FLAGS_HMAC BIT(20)
-#define SHA_FLAGS_ERROR BIT(21)
+#define SHA_FLAGS_DMA_ACTIVE BIT(1)
+#define SHA_FLAGS_OUTPUT_READY BIT(2)
+#define SHA_FLAGS_INIT BIT(3)
+#define SHA_FLAGS_NEED_KEY BIT(4)
+
+#define SHA_FLAGS_FINAL BIT(16)
+#define SHA_FLAGS_SHA1 BIT(18)
+#define SHA_FLAGS_SHA224 BIT(19)
+#define SHA_FLAGS_SHA256 BIT(20)
+#define SHA_FLAGS_ERROR BIT(21)
+#define SHA_FLAGS_FAST BIT(22)
#define SHA_OP_UPDATE 1
#define SHA_OP_FINAL 2
-#define SHA_BUFFER_LEN PAGE_SIZE
+#define SHA_BUFFER_LEN (SHA512_BLOCK_SIZE)
+
+#define AML_DIGEST_BUFSZ (SHA256_DIGEST_SIZE + 16)
struct aml_sha_dev;
void *descriptor;
dma_addr_t dma_descript_tab;
- u8 *digest;
- u8 buffer[0] __aligned(sizeof(u32));
+ u8 *digest;
+ u8 buffer[0] __aligned(sizeof(u32));
};
struct aml_sha_ctx {
struct aml_sha_dev *dd;
- u8 key[SHA256_BLOCK_SIZE];
- /* 8 bytes bit length and 8 bytes garbage */
- u8 state[SHA256_DIGEST_SIZE + 16];
+ u8 key[SHA512_BLOCK_SIZE];
u32 keylen;
- unsigned long flags;
+ /* 8 bytes bit length and 8 bytes garbage */
+ u32 is_hmac;
+ struct crypto_shash *shash;
};
#define AML_SHA_QUEUE_LENGTH 50
unsigned long flags;
struct crypto_queue queue;
struct ahash_request *req;
-
};
struct aml_sha_drv {
if (ctx->offset == ctx->sg->length) {
ctx->sg = sg_next(ctx->sg);
- if (ctx->sg)
- ctx->offset = 0;
- else
- ctx->total = 0;
+ ctx->offset = 0;
}
}
ctx->flags = 0;
- pr_info("init: digest size: %d\n", crypto_ahash_digestsize(tfm));
+ dbgp(1, "init: tfm: %p, ctx: %p, digest size: %d\n",
+ tfm, ctx, crypto_ahash_digestsize(tfm));
switch (crypto_ahash_digestsize(tfm)) {
case SHA1_DIGEST_SIZE:
return -EINVAL;
}
- ctx->digest = 0;
+ ctx->digest = kzalloc(AML_DIGEST_BUFSZ, GFP_ATOMIC|GFP_DMA);
+ if (!ctx->digest)
+ return -ENOMEM;
+
+ if (tctx->is_hmac)
+ ctx->flags |= SHA_FLAGS_NEED_KEY;
+
ctx->bufcnt = 0;
ctx->digcnt[0] = 0;
ctx->digcnt[1] = 0;
ctx->fast_nents = 0;
ctx->buflen = SHA_BUFFER_LEN;
- ctx->descriptor = (void *)ctx->buffer + SHA_BUFFER_LEN;
+ ctx->descriptor = (void *)__get_free_pages(GFP_ATOMIC | GFP_DMA, 0);
+ if (!ctx->descriptor) {
+ kfree(ctx->digest);
+ return -ENOMEM;
+ }
return 0;
}
-static int aml_sha_xmit_dma(struct aml_sha_dev *dd, struct dma_dsc *dsc,
- uint32_t nents, int final)
+static int aml_sha_xmit_dma(struct aml_sha_dev *dd,
+ struct ahash_request *req, struct dma_dsc *dsc,
+ uint32_t nents, int final)
{
int i = 0;
u32 mode;
- struct aml_sha_ctx *tctx = crypto_tfm_ctx(dd->req->base.tfm);
- struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req);
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
+ struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
size_t length = 0;
- pr_info("xmit_dma: digcnt: 0x%llx 0x%llx, nents: %u, final: %d\n",
- ctx->digcnt[1], ctx->digcnt[0], nents, final);
+ dbgp(1, "xmit_dma:ctx:%p,digcnt:0x%llx 0x%llx,nents: %u,final:%d\n",
+ ctx, ctx->digcnt[1], ctx->digcnt[0], nents, final);
mode = MODE_SHA1;
else if (ctx->flags & SHA_FLAGS_SHA256)
mode = MODE_SHA256;
- if (final) {
- kfree(ctx->digest);
- ctx->digest = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL);
- ctx->hash_addr = dma_map_single(dd->dev, ctx->digest,
- SHA256_DIGEST_SIZE, DMA_FROM_DEVICE);
- if (dma_mapping_error(dd->dev, ctx->hash_addr)) {
- dev_err(dd->dev, "hash %d bytes error\n",
- SHA256_DIGEST_SIZE);
- return -EINVAL;
- }
- } else {
- ctx->hash_addr = dma_map_single(dd->dev, tctx->state,
- sizeof(tctx->state), DMA_FROM_DEVICE);
- if (dma_mapping_error(dd->dev, ctx->hash_addr)) {
- dev_err(dd->dev, "hash %lu bytes error\n",
- sizeof(tctx->state));
- return -EINVAL;
- }
+ ctx->hash_addr = dma_map_single(dd->dev, ctx->digest,
+ AML_DIGEST_BUFSZ, DMA_FROM_DEVICE);
+ if (dma_mapping_error(dd->dev, ctx->hash_addr)) {
+ dev_err(dd->dev, "hash %d bytes error\n",
+ AML_DIGEST_BUFSZ);
+ return -EINVAL;
}
for (i = 0; i < nents; i++) {
dsc[i].dsc_cfg.b.mode = mode;
dsc[i].dsc_cfg.b.begin =
(i == 0 && !(ctx->digcnt[0] || ctx->digcnt[1]) &&
- !(tctx->flags & SHA_FLAGS_HMAC));
+ !(tctx->is_hmac));
dsc[i].dsc_cfg.b.end = final;
dsc[i].dsc_cfg.b.op_mode = OP_MODE_SHA;
dsc[i].dsc_cfg.b.eoc = (i == (nents - 1));
if (dma_mapping_error(dd->dev, ctx->dma_descript_tab)) {
dev_err(dd->dev, "mapping descriptor failed\n");
dma_unmap_single(dd->dev, ctx->hash_addr,
- SHA256_DIGEST_SIZE, DMA_FROM_DEVICE);
+ AML_DIGEST_BUFSZ, DMA_FROM_DEVICE);
return -EINVAL;
}
aml_dma_debug(dsc, nents, __func__, dd->thread, dd->status);
}
if (final)
- dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
+ ctx->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
- dd->flags |= SHA_FLAGS_DMA_ACTIVE;
+ dd->req = req;
- pr_info("xmit before : digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n",
- ctx->digcnt[1], ctx->digcnt[0], length, final);
+ dbgp(1, "xmit before:ctx:%p,digcnt:0x%llx 0x%llx,length:%zd,final:%d\n",
+ ctx, ctx->digcnt[1], ctx->digcnt[0], length, final);
/* Start DMA transfer */
+ dd->flags |= SHA_FLAGS_DMA_ACTIVE;
aml_write_crypto_reg(dd->thread,
(uintptr_t) ctx->dma_descript_tab | 2);
return -EINPROGRESS;
}
-static int aml_sha_xmit_start(struct aml_sha_dev *dd, struct dma_dsc *dsc,
+static int aml_sha_xmit_start(struct aml_sha_dev *dd,
+ struct ahash_request *req, struct dma_dsc *dsc,
uint32_t nents, int final)
{
- return aml_sha_xmit_dma(dd, dsc, nents, final);
+ return aml_sha_xmit_dma(dd, req, dsc, nents, final);
}
static int aml_sha_xmit_dma_map(struct aml_sha_dev *dd,
+ struct ahash_request *req,
struct aml_sha_reqctx *ctx,
size_t length, int final)
{
dsc->dsc_cfg.b.length = length;
/* next call does not fail... so no unmap in the case of error */
- return aml_sha_xmit_start(dd, dsc, 1, final);
+ return aml_sha_xmit_start(dd, req, dsc, 1, final);
}
-static int aml_sha_update_dma_slow(struct aml_sha_dev *dd)
+static int aml_sha_update_dma_slow(struct aml_sha_dev *dd,
+ struct ahash_request *req)
{
- struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req);
+ struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
unsigned int final;
size_t count;
- ctx->flags &= ~SHA_FLAGS_DMA_FAST;
+ ctx->flags &= ~SHA_FLAGS_FAST;
aml_sha_append_sg(ctx);
- final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
+ final = (ctx->flags & SHA_FLAGS_FINAL) && !ctx->total;
- pr_info("slow: bufcnt: %zd, digcnt: 0x%llx 0x%llx, final: %d, total: %u\n",
- ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final, ctx->total);
+ dbgp(1, "slow:ctx:%p,bc:%zd,dc:0x%llx 0x%llx,final:%d,total:%u\n",
+ ctx, ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final, ctx->total);
if (IS_ALIGNED(ctx->bufcnt, ctx->block_size) || final) {
count = ctx->bufcnt;
ctx->bufcnt = 0;
- return aml_sha_xmit_dma_map(dd, ctx, count, final);
+ return aml_sha_xmit_dma_map(dd, req, ctx, count, final);
}
return 0;
}
-static int aml_sha_update_dma_start(struct aml_sha_dev *dd)
+static int aml_sha_update_dma_start(struct aml_sha_dev *dd,
+ struct ahash_request *req)
{
- struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req);
+ struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
unsigned int length = 0, final = 0, tail = 0;
struct scatterlist *sg;
struct dma_dsc *dsc = ctx->descriptor;
- pr_info("start: total: %u, fast_nents: %u offset: %u\n",
- ctx->total, ctx->fast_nents, ctx->offset);
+ dbgp(1, "start: ctx: %p, total: %u, fast_nents: %u offset: %u,",
+ ctx, ctx->total, ctx->fast_nents, ctx->offset);
+ dbgp(1, " block size: %zd, flag: %lx\n",
+ ctx->block_size, ctx->flags);
if (!ctx->total)
return 0;
ctx->fast_nents = 0;
- if (ctx->bufcnt || ctx->offset || ctx->total < ctx->block_size)
- return aml_sha_update_dma_slow(dd);
+ if (ctx->bufcnt || ctx->offset ||
+ ctx->total < ctx->block_size ||
+ ctx->sg->length < ctx->block_size)
+ return aml_sha_update_dma_slow(dd, req);
sg = ctx->sg;
while (ctx->total && ctx->fast_nents < MAX_NUM_TABLES && sg) {
- pr_info("fast: dig: 0x%llx 0x%llx, bufcnt: %zd, total: %u, sglen: %u\n",
- ctx->digcnt[1], ctx->digcnt[0],
+ dbgp(1, "fast:ctx:%p,dig:0x%llx 0x%llx,bc:%zd,tot:%u,sgl: %u\n",
+ ctx, ctx->digcnt[1], ctx->digcnt[0],
ctx->bufcnt, ctx->total, ctx->sg->length);
length = min(ctx->total, sg->length);
- if (!(ctx->flags & SHA_FLAGS_FINUP)) {
+ if (!(ctx->flags & SHA_FLAGS_FINAL && sg_next(sg) == 0)) {
/* not last sg must be ctx->block_size aligned */
tail = length & (ctx->block_size - 1);
length -= tail;
ctx->total -= length;
ctx->offset = length; /* offset where to start slow */
- final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
+ final = (ctx->flags & SHA_FLAGS_FINAL) && !ctx->total;
dma_map_sg(dd->dev, sg, 1, DMA_TO_DEVICE);
sg = sg_next(sg);
ctx->fast_nents++;
- pr_info("fast: total: %u, offset: %u, tail: %u\n",
- ctx->total, ctx->offset, tail);
+ dbgp(1, "fast: ctx: %p, total: %u, offset: %u, tail: %u\n",
+ ctx, ctx->total, ctx->offset, tail);
if (tail)
break;
}
if (ctx->fast_nents) {
- ctx->flags |= SHA_FLAGS_DMA_FAST;
- return aml_sha_xmit_start(dd, dsc, ctx->fast_nents, final);
+ ctx->flags |= SHA_FLAGS_FAST;
+ return aml_sha_xmit_start(dd, req, dsc, ctx->fast_nents, final);
}
return 0;
{
struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req);
- if (ctx->flags & SHA_FLAGS_DMA_FAST)
+ dma_unmap_single(dd->dev, ctx->dma_descript_tab, PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ aml_dma_debug(ctx->descriptor, ctx->fast_nents ?
+ ctx->fast_nents : 1, __func__, dd->thread, dd->status);
+
+ if (ctx->flags & SHA_FLAGS_FAST)
dma_unmap_sg(dd->dev, ctx->sg, ctx->fast_nents, DMA_TO_DEVICE);
else
dma_unmap_single(dd->dev, ctx->dma_addr,
ctx->buflen, DMA_TO_DEVICE);
+
if (ctx->hash_addr)
dma_unmap_single(dd->dev, ctx->hash_addr,
- SHA256_DIGEST_SIZE, DMA_FROM_DEVICE);
+ AML_DIGEST_BUFSZ, DMA_FROM_DEVICE);
return 0;
}
-static int aml_sha_update_req(struct aml_sha_dev *dd)
+static int aml_sha_update_req(struct aml_sha_dev *dd, struct ahash_request *req)
{
int err;
- struct ahash_request *req = dd->req;
struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
- pr_info("update_req: total: %u, digcnt: 0x%llx 0x%llx\n",
- ctx->total, ctx->digcnt[1], ctx->digcnt[0]);
+ dbgp(1, "update_req: ctx: %p, total: %u, digcnt: 0x%llx 0x%llx\n",
+ ctx, ctx->total, ctx->digcnt[1], ctx->digcnt[0]);
- err = aml_sha_update_dma_start(dd);
+ err = aml_sha_update_dma_start(dd, req);
return err;
}
-static int aml_sha_final_req(struct aml_sha_dev *dd)
+static int aml_sha_final_req(struct aml_sha_dev *dd, struct ahash_request *req)
{
int err = 0;
+ struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
- err = aml_sha_update_dma_slow(dd);
+ err = aml_sha_update_dma_slow(dd, req);
- pr_info("final_req: err: %d\n", err);
+ dbgp(1, "final_req: ctx: %p, err: %d\n", ctx, err);
return err;
}
if (!req->result)
return;
- if (!ctx->digest)
- return;
-
if (ctx->flags & SHA_FLAGS_SHA1)
memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
else if (ctx->flags & SHA_FLAGS_SHA224)
memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
else if (ctx->flags & SHA_FLAGS_SHA256)
memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
-
- kfree(ctx->digest);
}
static int aml_sha_finish_hmac(struct ahash_request *req)
struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
struct aml_sha_dev *dd = ctx->dd;
struct dma_dsc *dsc = ctx->descriptor;
+ dma_addr_t hmac_digest = 0;
+ u8 *digest = 0;
u32 mode;
u32 ds = 0;
u8 *key;
dma_addr_t dma_key = 0;
- if (!ctx->digest)
- return -1;
+ digest = dmam_alloc_coherent(dd->dev, AML_DIGEST_BUFSZ,
+ &hmac_digest, GFP_ATOMIC | GFP_DMA);
+ if (!digest)
+ return -ENOMEM;
- ctx->hash_addr = dma_map_single(dd->dev, ctx->digest,
- SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL);
- if (dma_mapping_error(dd->dev, ctx->hash_addr)) {
- dev_err(dd->dev, "mapping hash addr failed: %d\n",
- SHA256_DIGEST_SIZE);
- return -EINVAL;
- }
+ memcpy(digest, ctx->digest, AML_DIGEST_BUFSZ);
+
+ key = kzalloc(tctx->keylen, GFP_ATOMIC | GFP_DMA);
+ if (!key)
+ return -ENOMEM;
- key = kmalloc(tctx->keylen, GFP_ATOMIC);
memcpy(key, tctx->key, tctx->keylen);
dma_key = dma_map_single(dd->dev, key,
tctx->keylen, DMA_TO_DEVICE);
if (dma_mapping_error(dd->dev, dma_key)) {
dev_err(dd->dev, "mapping key addr failed: %d\n", tctx->keylen);
- dma_unmap_single(dd->dev, ctx->hash_addr,
- SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL);
+ dmam_free_coherent(dd->dev, AML_DIGEST_BUFSZ,
+ digest, hmac_digest);
return -EINVAL;
}
dsc[0].dsc_cfg.b.owner = 1;
/* 2nd stage hash */
- dsc[1].src_addr = (uintptr_t)ctx->hash_addr;
- dsc[1].tgt_addr = (uintptr_t)ctx->hash_addr;
+ dsc[1].src_addr = (uintptr_t)hmac_digest;
+ dsc[1].tgt_addr = (uintptr_t)hmac_digest;
dsc[1].dsc_cfg.d32 = 0;
dsc[1].dsc_cfg.b.length = ds;
dsc[1].dsc_cfg.b.mode = mode;
dev_err(dd->dev, "mapping descriptor failed\n");
dma_unmap_single(dd->dev, dma_key,
tctx->keylen, DMA_TO_DEVICE);
- dma_unmap_single(dd->dev, ctx->hash_addr,
- SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL);
+ dmam_free_coherent(dd->dev, AML_DIGEST_BUFSZ,
+ digest, hmac_digest);
return -EINVAL;
}
aml_dma_debug(dsc, 2, __func__, dd->thread, dd->status);
+
aml_write_crypto_reg(dd->thread,
(uintptr_t) ctx->dma_descript_tab | 2);
while (aml_read_crypto_reg(dd->status) == 0)
;
aml_write_crypto_reg(dd->status, 0xf);
+
dma_unmap_single(dd->dev, dma_key,
tctx->keylen, DMA_TO_DEVICE);
- dma_unmap_single(dd->dev, ctx->hash_addr,
- SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL);
dma_unmap_single(dd->dev, ctx->dma_descript_tab, PAGE_SIZE,
DMA_FROM_DEVICE);
+ memcpy(ctx->digest, digest, AML_DIGEST_BUFSZ);
+ dmam_free_coherent(dd->dev, AML_DIGEST_BUFSZ, digest, hmac_digest);
return 0;
}
struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
int err = 0;
- if (ctx->digcnt[0] || ctx->digcnt[1]) {
- if (tctx->flags & SHA_FLAGS_HMAC)
- err = aml_sha_finish_hmac(req);
- aml_sha_copy_ready_hash(req);
- }
+ if (tctx->is_hmac)
+ err = aml_sha_finish_hmac(req);
+ aml_sha_copy_ready_hash(req);
- pr_info("finish digcnt: 0x%llx 0x%llx, bufcnt: %zd\n",
- ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt);
+ kfree(ctx->digest);
+ free_page((uintptr_t)ctx->descriptor);
+ dbgp(1, "finish digcnt: ctx: %p, 0x%llx 0x%llx, bufcnt: %zd, %x %x\n",
+ ctx, ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt,
+ req->result[0], req->result[1]);
return err;
}
{
struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
struct aml_sha_dev *dd = ctx->dd;
+ unsigned long flags;
if (!err) {
- if (SHA_FLAGS_FINAL & dd->flags)
+ if (SHA_FLAGS_FINAL & ctx->flags)
err = aml_sha_finish(req);
} else {
ctx->flags |= SHA_FLAGS_ERROR;
}
- /* atomic operation is not needed here */
- if (dd->flags & SHA_FLAGS_BUSY) {
- dd->flags &= ~SHA_FLAGS_BUSY;
- dd->dma->dma_busy = 0;
- }
- dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL |
- SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
+ spin_lock_irqsave(&dd->dma->dma_lock, flags);
+ dd->dma->dma_busy = 0;
+ dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_OUTPUT_READY);
+ spin_unlock_irqrestore(&dd->dma->dma_lock, flags);
if (req->base.complete)
req->base.complete(&req->base, err);
-
- /* handle new request */
- tasklet_schedule(&dd->done_task);
}
static int aml_sha_hw_init(struct aml_sha_dev *dd)
dma_addr_t dma_ctx;
struct dma_dsc *dsc = ctx->descriptor;
uint32_t i = 0;
- int32_t len = sizeof(tctx->state);
+ int32_t len = AML_DIGEST_BUFSZ;
- if (!ctx->digcnt[0] && !ctx->digcnt[1])
+ if (!ctx->digcnt[0] && !ctx->digcnt[1] && !tctx->is_hmac)
return;
- dma_ctx = dma_map_single(dd->dev, tctx->state,
- sizeof(tctx->state), DMA_TO_DEVICE);
+ dma_ctx = dma_map_single(dd->dev, ctx->digest,
+ AML_DIGEST_BUFSZ, DMA_TO_DEVICE);
if (dma_mapping_error(dd->dev, dma_ctx)) {
dev_err(dd->dev, "mapping dma_ctx failed\n");
return;
if (dma_mapping_error(dd->dev, dma_ctx)) {
dev_err(dd->dev, "mapping descript tab failed\n");
dma_unmap_single(dd->dev, dma_ctx,
- sizeof(tctx->state), DMA_TO_DEVICE);
+ AML_DIGEST_BUFSZ, DMA_TO_DEVICE);
return;
}
+
aml_write_crypto_reg(dd->thread,
(uintptr_t) ctx->dma_descript_tab | 2);
aml_dma_debug(dsc, 1, __func__, dd->thread, dd->status);
while (aml_read_crypto_reg(dd->status) == 0)
;
aml_write_crypto_reg(dd->status, 0xf);
+
dma_unmap_single(dd->dev, dma_ctx,
- sizeof(tctx->state), DMA_TO_DEVICE);
+ AML_DIGEST_BUFSZ, DMA_TO_DEVICE);
dma_unmap_single(dd->dev, ctx->dma_descript_tab, PAGE_SIZE,
DMA_FROM_DEVICE);
}
+static int aml_hmac_install_key(struct aml_sha_dev *dd,
+ struct ahash_request *req)
+{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
+ struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct dma_dsc *dsc = 0;
+ uint32_t bs = 0;
+ int err = 0;
+ dma_addr_t dma_key = 0;
+ dma_addr_t dma_descript_tab = 0;
+ dma_addr_t partial = 0;
+ uint8_t *key = 0;
+ uint8_t *digest = 0;
+ uint32_t i;
+ uint32_t mode = MODE_SHA1;
+
+ switch (crypto_ahash_digestsize(tfm)) {
+ case SHA1_DIGEST_SIZE:
+ bs = SHA1_BLOCK_SIZE;
+ break;
+ case SHA224_DIGEST_SIZE:
+ bs = SHA224_BLOCK_SIZE;
+ break;
+ case SHA256_DIGEST_SIZE:
+ bs = SHA256_BLOCK_SIZE;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ mode = MODE_SHA1;
+ if (ctx->flags & SHA_FLAGS_SHA224)
+ mode = MODE_SHA224;
+ else if (ctx->flags & SHA_FLAGS_SHA256)
+ mode = MODE_SHA256;
+
+ dsc = dmam_alloc_coherent(dd->dev, sizeof(struct dma_dsc),
+ &dma_descript_tab, GFP_ATOMIC | GFP_DMA);
+ if (!dsc) {
+ dd->flags &= ~SHA_FLAGS_BUSY;
+ return -ENOMEM;
+ }
+
+ key = dmam_alloc_coherent(dd->dev, bs,
+ &dma_key, GFP_ATOMIC | GFP_DMA);
+ if (!key) {
+ dmam_free_coherent(dd->dev, sizeof(struct dma_dsc),
+ dsc, dma_descript_tab);
+ return -ENOMEM;
+ }
+ memcpy(key, tctx->key, tctx->keylen);
+ memset(key + tctx->keylen, 0, bs - tctx->keylen);
+ /* Do ipad */
+ for (i = 0; i < bs; i++)
+ *(uint8_t *)(key + i) ^= 0x36;
+
+ digest = dmam_alloc_coherent(dd->dev, AML_DIGEST_BUFSZ,
+ &partial, GFP_ATOMIC | GFP_DMA);
+ if (!digest) {
+ dmam_free_coherent(dd->dev, bs, key, dma_key);
+ dmam_free_coherent(dd->dev, sizeof(struct dma_dsc),
+ dsc, dma_descript_tab);
+ return -ENOMEM;
+ }
+
+ /* start first round hash */
+ dsc[0].src_addr = (uintptr_t)dma_key;
+ dsc[0].tgt_addr = (uintptr_t)partial;
+ dsc[0].dsc_cfg.d32 = 0;
+ dsc[0].dsc_cfg.b.length = bs;
+ dsc[0].dsc_cfg.b.mode = mode;
+ dsc[0].dsc_cfg.b.enc_sha_only = 1;
+ dsc[0].dsc_cfg.b.op_mode = OP_MODE_SHA;
+ dsc[0].dsc_cfg.b.begin = 1;
+ dsc[0].dsc_cfg.b.end = 0;
+ dsc[0].dsc_cfg.b.eoc = 1;
+ dsc[0].dsc_cfg.b.owner = 1;
+
+ aml_dma_debug(dsc, 1, __func__, dd->thread, dd->status);
+
+ aml_write_crypto_reg(dd->thread,
+ (uintptr_t) dma_descript_tab | 2);
+ while (aml_read_crypto_reg(dd->status) == 0)
+ ;
+ aml_write_crypto_reg(dd->status, 0xf);
+
+ memcpy(ctx->digest, digest, AML_DIGEST_BUFSZ);
+ dmam_free_coherent(dd->dev, bs, key, dma_key);
+ dmam_free_coherent(dd->dev, sizeof(struct dma_dsc),
+ dsc, dma_descript_tab);
+ dmam_free_coherent(dd->dev, AML_DIGEST_BUFSZ, digest, partial);
+
+ return err;
+}
+
static int aml_sha_handle_queue(struct aml_sha_dev *dd,
struct ahash_request *req)
{
dd->flags |= SHA_FLAGS_BUSY;
dd->dma->dma_busy = 1;
}
-
spin_unlock_irqrestore(&dd->dma->dma_lock, flags);
if (!async_req)
backlog->complete(backlog, -EINPROGRESS);
req = ahash_request_cast(async_req);
- dd->req = req;
ctx = ahash_request_ctx(req);
- pr_info("handling new req, op: %lu, nbytes: %d\n",
- ctx->op, req->nbytes);
+ dbgp(1, "handling new req, ctx: %p, op: %lu, nbytes: %d\n",
+ ctx, ctx->op, req->nbytes);
err = aml_sha_hw_init(dd);
if (err)
goto err1;
+ if (ctx->flags & SHA_FLAGS_NEED_KEY) {
+ err = aml_hmac_install_key(dd, req);
+ if (err)
+ goto err1;
+ ctx->flags &= ~SHA_FLAGS_NEED_KEY;
+ }
+
aml_sha_state_restore(req);
if (ctx->op == SHA_OP_UPDATE) {
- err = aml_sha_update_req(dd);
+ err = aml_sha_update_req(dd, req);
/* no final() after finup() */
- if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
- err = aml_sha_final_req(dd);
+ if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINAL))
+ err = aml_sha_final_req(dd, req);
} else if (ctx->op == SHA_OP_FINAL) {
- err = aml_sha_final_req(dd);
+ err = aml_sha_final_req(dd, req);
}
err1:
{
struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
- ctx->flags |= SHA_FLAGS_FINUP;
+ ctx->flags |= SHA_FLAGS_FINAL;
+ ctx->total = 0;
+ ctx->sg = 0;
+ ctx->offset = 0;
if (ctx->flags & SHA_FLAGS_ERROR)
return 0; /* uncompleted hash is not needed */
- if (ctx->bufcnt)
- return aml_sha_enqueue(req, SHA_OP_FINAL);
- else
- return aml_sha_finish(req); /* copy ready hash */
+ return aml_sha_enqueue(req, SHA_OP_FINAL);
}
static int aml_sha_finup(struct ahash_request *req)
struct aml_sha_reqctx *ctx = ahash_request_ctx(req);
int err1, err2;
- ctx->flags |= SHA_FLAGS_FINUP;
+ ctx->flags |= SHA_FLAGS_FINAL;
err1 = aml_sha_update(req);
if (err1 == -EINPROGRESS || err1 == -EBUSY)
static int aml_sha_import(struct ahash_request *req, const void *in)
{
- struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
- const struct aml_sha_ctx *ictx = in;
+ struct aml_sha_reqctx *rctx = ahash_request_ctx(req);
+ const struct aml_sha_reqctx *ctx_in = in;
- memcpy(tctx->state, ictx->state, sizeof(tctx->state));
+ memcpy(rctx, in, sizeof(*rctx) + ctx_in->bufcnt);
return 0;
}
static int aml_sha_export(struct ahash_request *req, void *out)
{
- struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
- struct aml_sha_ctx *octx = out;
+ struct aml_sha_reqctx *rctx = ahash_request_ctx(req);
- memcpy(octx->state, tctx->state, sizeof(tctx->state));
+ memcpy(out, rctx, sizeof(*rctx) + rctx->bufcnt);
return 0;
}
+static int aml_shash_digest(struct crypto_shash *tfm, u32 flags,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ char *__shash_desc = kzalloc(sizeof(struct shash_desc) +
+ crypto_shash_descsize(tfm), GFP_KERNEL | GFP_ATOMIC);
+ struct shash_desc *shash = (struct shash_desc *)__shash_desc;
+ int err;
+
+ if (!shash)
+ return -ENOMEM;
+
+ shash->tfm = tfm;
+ shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ err = crypto_shash_digest(shash, data, len, out);
+
+ kfree(shash);
+ return err;
+}
+
+
static int aml_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
unsigned int keylen)
{
struct aml_sha_ctx *tctx = crypto_ahash_ctx(tfm);
- struct aml_sha_dev *dd = 0;
- struct aml_sha_dev *tmp = 0;
- struct dma_dsc *dsc = 0;
- uint32_t bs = 0;
- uint32_t ds = 0;
+ int bs = crypto_shash_blocksize(tctx->shash);
+ int ds = crypto_shash_digestsize(tctx->shash);
+ struct aml_sha_dev *dd = NULL, *tmp;
int err = 0;
- dma_addr_t dma_key = 0;
- dma_addr_t dma_descript_tab = 0;
- uint8_t *key_raw = 0;
- uint32_t mode = MODE_SHA1;
- uint32_t map_len = 0;
- uint32_t ipad = 0;
spin_lock_bh(&aml_sha.lock);
if (!tctx->dd) {
} else {
dd = tctx->dd;
}
-
- dsc = kmalloc(sizeof(struct dma_dsc) * 2, GFP_KERNEL);
- if (!dsc)
- return -ENOMEM;
-
spin_unlock_bh(&aml_sha.lock);
- switch (crypto_ahash_digestsize(tfm)) {
- case SHA1_DIGEST_SIZE:
- bs = SHA1_BLOCK_SIZE;
- ds = SHA1_DIGEST_SIZE;
- mode = MODE_SHA1;
- break;
- case SHA224_DIGEST_SIZE:
- bs = SHA224_BLOCK_SIZE;
- ds = SHA224_DIGEST_SIZE;
- mode = MODE_SHA224;
- break;
- case SHA256_DIGEST_SIZE:
- bs = SHA256_BLOCK_SIZE;
- ds = SHA256_DIGEST_SIZE;
- mode = MODE_SHA256;
- break;
- default:
- return -EINVAL;
- }
-
- if (keylen > bs)
- key_raw = kzalloc(keylen, GFP_KERNEL);
- else
- key_raw = kzalloc(bs, GFP_KERNEL);
-
- memcpy(key_raw, key, keylen);
- map_len = keylen > bs ? keylen : bs;
- dma_key = dma_map_single(dd->dev, key_raw,
- map_len, DMA_BIDIRECTIONAL);
- if (dma_mapping_error(dd->dev, dma_key)) {
- dev_err(dd->dev, "mapping dma_key failed\n");
- return -EINVAL;
- }
-
if (keylen > bs) {
- dsc[0].src_addr = (uintptr_t)dma_key;
- dsc[0].tgt_addr = (uintptr_t)dma_key;
- dsc[0].dsc_cfg.d32 = 0;
- dsc[0].dsc_cfg.b.length = keylen;
- dsc[0].dsc_cfg.b.mode = mode;
- dsc[0].dsc_cfg.b.enc_sha_only = 1;
- dsc[0].dsc_cfg.b.begin = 1;
- dsc[0].dsc_cfg.b.end = 1;
- dsc[0].dsc_cfg.b.op_mode = OP_MODE_SHA;
- dsc[0].dsc_cfg.b.eoc = 0;
- dsc[0].dsc_cfg.b.owner = 1;
- keylen = ds;
- ipad = 1;
- }
-
- /* ipad */
- dsc[ipad].src_addr = (uintptr_t)dma_key;
- dsc[ipad].tgt_addr = 0;
- dsc[ipad].dsc_cfg.d32 = 0;
- dsc[ipad].dsc_cfg.b.length = keylen;
- dsc[ipad].dsc_cfg.b.mode = mode;
- dsc[ipad].dsc_cfg.b.enc_sha_only = 1;
- dsc[ipad].dsc_cfg.b.op_mode = OP_MODE_HMAC_I;
- dsc[ipad].dsc_cfg.b.begin = 1;
- dsc[ipad].dsc_cfg.b.end = 0;
- dsc[ipad].dsc_cfg.b.eoc = 1;
- dsc[ipad].dsc_cfg.b.owner = 1;
-
- dma_descript_tab = dma_map_single(dd->dev, dsc,
- sizeof(struct dma_dsc) * 2, DMA_TO_DEVICE);
- if (dma_mapping_error(dd->dev, dma_descript_tab)) {
- dev_err(dd->dev, "mapping descript_tab failed\n");
- dma_unmap_single(dd->dev, dma_key,
- map_len, DMA_BIDIRECTIONAL);
- return -EINVAL;
+ err = aml_shash_digest(tctx->shash,
+ crypto_shash_get_flags(tctx->shash),
+ key, keylen, tctx->key);
+ if (err)
+ return err;
+ tctx->keylen = ds;
+ } else {
+ tctx->keylen = keylen;
+ memcpy(tctx->key, key, tctx->keylen);
}
- aml_dma_debug(dsc, ipad + 1, __func__, dd->thread, dd->status);
- aml_write_crypto_reg(dd->thread,
- (uintptr_t) dma_descript_tab | 2);
- while (aml_read_crypto_reg(dd->status) == 0)
- ;
- aml_write_crypto_reg(dd->status, 0xf);
- dma_unmap_single(dd->dev, dma_key,
- map_len, DMA_BIDIRECTIONAL);
- dma_unmap_single(dd->dev, dma_descript_tab, PAGE_SIZE,
- DMA_FROM_DEVICE);
- tctx->keylen = keylen;
- memcpy(tctx->key, key_raw, keylen);
-
- kfree(key_raw);
return err;
}
+
static int aml_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
+
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct aml_sha_reqctx) +
- SHA_BUFFER_LEN + PAGE_SIZE);
+ SHA_BUFFER_LEN);
+
+ if (alg_base) {
+ tctx->shash = crypto_alloc_shash(alg_base, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(tctx->shash)) {
+ pr_err("aml-sha: could not load base driver '%s' c ",
+ alg_base);
+ return PTR_ERR(tctx->shash);
+ }
+ } else {
+ tctx->shash = NULL;
+ }
+
return 0;
}
static int aml_sha_cra_init(struct crypto_tfm *tfm)
{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->is_hmac = 0;
return aml_sha_cra_init_alg(tfm, NULL);
}
static void aml_sha_cra_exit(struct crypto_tfm *tfm)
{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ if (tctx->is_hmac && tctx->shash != NULL)
+ crypto_free_shash(tctx->shash);
}
-static int aml_hmac_cra_init(struct crypto_tfm *tfm)
+static int aml_hmac_sha1_cra_init(struct crypto_tfm *tfm)
{
struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
- tctx->flags |= SHA_FLAGS_HMAC;
- return aml_sha_cra_init_alg(tfm, NULL);
+ tctx->is_hmac = 1;
+ return aml_sha_cra_init_alg(tfm, "sha1");
+}
+
+static int aml_hmac_sha224_cra_init(struct crypto_tfm *tfm)
+{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->is_hmac = 1;
+ return aml_sha_cra_init_alg(tfm, "sha224");
+}
+
+static int aml_hmac_sha256_cra_init(struct crypto_tfm *tfm)
+{
+ struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ tctx->is_hmac = 1;
+ return aml_sha_cra_init_alg(tfm, "sha256");
}
static void aml_hmac_cra_exit(struct crypto_tfm *tfm)
.import = aml_sha_import,
.halg = {
.digestsize = SHA1_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "sha1",
.cra_driver_name = "aml-sha1",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.import = aml_sha_import,
.halg = {
.digestsize = SHA256_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "sha256",
.cra_driver_name = "aml-sha256",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.import = aml_sha_import,
.halg = {
.digestsize = SHA224_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "sha224",
.cra_driver_name = "aml-sha224",
- .cra_priority = 100,
+ .cra_priority = 200,
.cra_flags = CRYPTO_ALG_ASYNC,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.setkey = aml_sha_setkey,
.halg = {
.digestsize = SHA1_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "hmac(sha1)",
.cra_driver_name = "aml-hmac-sha1",
- .cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
- .cra_init = aml_hmac_cra_init,
+ .cra_init = aml_hmac_sha1_cra_init,
.cra_exit = aml_hmac_cra_exit,
}
}
.setkey = aml_sha_setkey,
.halg = {
.digestsize = SHA224_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "hmac(sha224)",
.cra_driver_name = "aml-hmac-sha224",
- .cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA224_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
- .cra_init = aml_hmac_cra_init,
+ .cra_init = aml_hmac_sha224_cra_init,
.cra_exit = aml_hmac_cra_exit,
}
}
.setkey = aml_sha_setkey,
.halg = {
.digestsize = SHA256_DIGEST_SIZE,
- .statesize = sizeof(struct aml_sha_ctx),
+ .statesize =
+ sizeof(struct aml_sha_reqctx) + SHA_BUFFER_LEN,
.base = {
.cra_name = "hmac(sha256)",
.cra_driver_name = "aml-hmac-sha256",
- .cra_priority = 100,
- .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA256_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aml_sha_ctx),
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
- .cra_init = aml_hmac_cra_init,
+ .cra_init = aml_hmac_sha256_cra_init,
.cra_exit = aml_hmac_cra_exit,
}
}
static void aml_sha_done_task(unsigned long data)
{
struct aml_sha_dev *dd = (struct aml_sha_dev *)data;
- struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req);
int err = 0;
if (!(SHA_FLAGS_BUSY & dd->flags)) {
return;
}
- dma_unmap_single(dd->dev, ctx->dma_descript_tab, PAGE_SIZE,
- DMA_FROM_DEVICE);
- aml_dma_debug(ctx->descriptor, ctx->fast_nents ?
- ctx->fast_nents : 1, __func__, dd->thread, dd->status);
-
- if (SHA_FLAGS_DMA_READY & dd->flags) {
+ if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
if (SHA_FLAGS_DMA_ACTIVE & dd->flags) {
dd->flags &= ~SHA_FLAGS_DMA_ACTIVE;
aml_sha_update_dma_stop(dd);
err = dd->err;
goto finish;
}
- }
- if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
/* hash or semi-hash ready */
- dd->flags &= ~(SHA_FLAGS_DMA_READY |
- SHA_FLAGS_OUTPUT_READY);
- err = aml_sha_update_dma_start(dd);
+ dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
+ err = aml_sha_update_dma_start(dd, dd->req);
if (err != -EINPROGRESS)
goto finish;
}
finish:
/* finish curent request */
aml_sha_finish_req(dd->req, err);
+
+ if (!(SHA_FLAGS_BUSY & dd->flags)) {
+ aml_sha_handle_queue(dd, NULL);
+ return;
+ }
}
static irqreturn_t aml_sha_irq(int irq, void *dev_id)
pr_err("irq overwrite\n");
if (SHA_FLAGS_DMA_ACTIVE & sha_dd->flags) {
sha_dd->flags |= SHA_FLAGS_OUTPUT_READY;
- sha_dd->flags |= SHA_FLAGS_DMA_READY;
aml_write_crypto_reg(sha_dd->status, 0xf);
tasklet_schedule(&sha_dd->done_task);
return IRQ_HANDLED;
projects / GitHub / LineageOS / G12 / android_kernel_amlogic_linux-4.9.git / commitdiff