projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: talitos - fix bad kfree
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / crypto / talitos.c
CommitLineData
9c4a7965
KP		 1/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
5228f0f7 4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
9c4a7965
KP		 5 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
35#include <linux/of_platform.h>
36#include <linux/dma-mapping.h>
37#include <linux/io.h>
38#include <linux/spinlock.h>
39#include <linux/rtnetlink.h>
5a0e3ad6 40#include <linux/slab.h>
9c4a7965
KP		 41
42#include <crypto/algapi.h>
43#include <crypto/aes.h>
3952f17e 44#include <crypto/des.h>
9c4a7965 45#include <crypto/sha.h>
497f2e6b 46#include <crypto/md5.h>
9c4a7965
KP		 47#include <crypto/aead.h>
48#include <crypto/authenc.h>
4de9d0b5 49#include <crypto/skcipher.h>
acbf7c62
LN		 50#include <crypto/hash.h>
51#include <crypto/internal/hash.h>
4de9d0b5 52#include <crypto/scatterwalk.h>
9c4a7965
KP		 53
54#include "talitos.h"
55
56#define TALITOS_TIMEOUT 100000
57#define TALITOS_MAX_DATA_LEN 65535
58
59#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
60#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
61#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
62
63/* descriptor pointer entry */
64struct talitos_ptr {
65 __be16 len; /* length */
66 u8 j_extent; /* jump to sg link table and/or extent */
67 u8 eptr; /* extended address */
68 __be32 ptr; /* address */
69};
70
497f2e6b
LN		 71static const struct talitos_ptr zero_entry = {
72 .len = 0,
73 .j_extent = 0,
74 .eptr = 0,
75 .ptr = 0
76};
77
9c4a7965
KP		 78/* descriptor */
79struct talitos_desc {
80 __be32 hdr; /* header high bits */
81 __be32 hdr_lo; /* header low bits */
82 struct talitos_ptr ptr[7]; /* ptr/len pair array */
83};
84
85/**
86 * talitos_request - descriptor submission request
87 * @desc: descriptor pointer (kernel virtual)
88 * @dma_desc: descriptor's physical bus address
89 * @callback: whom to call when descriptor processing is done
90 * @context: caller context (optional)
91 */
92struct talitos_request {
93 struct talitos_desc *desc;
94 dma_addr_t dma_desc;
95 void (*callback) (struct device *dev, struct talitos_desc *desc,
96 void *context, int error);
97 void *context;
98};
99
4b992628
KP		 100/* per-channel fifo management */
101struct talitos_channel {
ad42d5fc
KP		 102 void __iomem *reg;
103
4b992628
KP		 104 /* request fifo */
105 struct talitos_request *fifo;
106
107 /* number of requests pending in channel h/w fifo */
108 atomic_t submit_count ____cacheline_aligned;
109
110 /* request submission (head) lock */
111 spinlock_t head_lock ____cacheline_aligned;
112 /* index to next free descriptor request */
113 int head;
114
115 /* request release (tail) lock */
116 spinlock_t tail_lock ____cacheline_aligned;
117 /* index to next in-progress/done descriptor request */
118 int tail;
119};
120
9c4a7965
KP		 121struct talitos_private {
122 struct device *dev;
2dc11581 123 struct platform_device *ofdev;
9c4a7965 124 void __iomem *reg;
c3e337f8 125 int irq[2];
9c4a7965
KP		 126
127 /* SEC version geometry (from device tree node) */
128 unsigned int num_channels;
129 unsigned int chfifo_len;
130 unsigned int exec_units;
131 unsigned int desc_types;
132
f3c85bc1
LN		 133 /* SEC Compatibility info */
134 unsigned long features;
135
9c4a7965
KP		 136 /*
137 * length of the request fifo
138 * fifo_len is chfifo_len rounded up to next power of 2
139 * so we can use bitwise ops to wrap
140 */
141 unsigned int fifo_len;
142
4b992628 143 struct talitos_channel *chan;
9c4a7965 144
4b992628
KP		 145 /* next channel to be assigned next incoming descriptor */
146 atomic_t last_chan ____cacheline_aligned;
9c4a7965
KP		 147
148 /* request callback tasklet */
c3e337f8 149 struct tasklet_struct done_task[2];
9c4a7965
KP		 150
151 /* list of registered algorithms */
152 struct list_head alg_list;
153
154 /* hwrng device */
155 struct hwrng rng;
156};
157
f3c85bc1
LN		 158/* .features flag */
159#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
fe5720e2 160#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
60f208d7 161#define TALITOS_FTR_SHA224_HWINIT 0x00000004
79b3a418 162#define TALITOS_FTR_HMAC_OK 0x00000008
f3c85bc1 163
81eb024c
KP		 164static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr)
165{
166 talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
a752447a 167 talitos_ptr->eptr = upper_32_bits(dma_addr);
81eb024c
KP		 168}
169
9c4a7965
KP		 170/*
171 * map virtual single (contiguous) pointer to h/w descriptor pointer
172 */
173static void map_single_talitos_ptr(struct device *dev,
174 struct talitos_ptr *talitos_ptr,
175 unsigned short len, void *data,
176 unsigned char extent,
177 enum dma_data_direction dir)
178{
81eb024c
KP		 179 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
180
9c4a7965 181 talitos_ptr->len = cpu_to_be16(len);
81eb024c 182 to_talitos_ptr(talitos_ptr, dma_addr);
9c4a7965
KP		 183 talitos_ptr->j_extent = extent;
184}
185
186/*
187 * unmap bus single (contiguous) h/w descriptor pointer
188 */
189static void unmap_single_talitos_ptr(struct device *dev,
190 struct talitos_ptr *talitos_ptr,
191 enum dma_data_direction dir)
192{
193 dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
194 be16_to_cpu(talitos_ptr->len), dir);
195}
196
197static int reset_channel(struct device *dev, int ch)
198{
199 struct talitos_private *priv = dev_get_drvdata(dev);
200 unsigned int timeout = TALITOS_TIMEOUT;
201
ad42d5fc 202 setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);
9c4a7965 203
ad42d5fc 204 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)
9c4a7965
KP		 205 && --timeout)
206 cpu_relax();
207
208 if (timeout == 0) {
209 dev_err(dev, "failed to reset channel %d\n", ch);
210 return -EIO;
211 }
212
81eb024c 213 /* set 36-bit addressing, done writeback enable and done IRQ enable */
ad42d5fc 214 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
81eb024c 215 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
9c4a7965 216
fe5720e2
KP		 217 /* and ICCR writeback, if available */
218 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
ad42d5fc 219 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
fe5720e2
KP		 220 TALITOS_CCCR_LO_IWSE);
221
9c4a7965
KP		 222 return 0;
223}
224
225static int reset_device(struct device *dev)
226{
227 struct talitos_private *priv = dev_get_drvdata(dev);
228 unsigned int timeout = TALITOS_TIMEOUT;
c3e337f8 229 u32 mcr = TALITOS_MCR_SWR;
9c4a7965 230
c3e337f8 231 setbits32(priv->reg + TALITOS_MCR, mcr);
9c4a7965
KP		 232
233 while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
234 && --timeout)
235 cpu_relax();
236
c3e337f8
KP		 237 if (priv->irq[1] != NO_IRQ) {
238 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
239 setbits32(priv->reg + TALITOS_MCR, mcr);
240 }
241
9c4a7965
KP		 242 if (timeout == 0) {
243 dev_err(dev, "failed to reset device\n");
244 return -EIO;
245 }
246
247 return 0;
248}
249
250/*
251 * Reset and initialize the device
252 */
253static int init_device(struct device *dev)
254{
255 struct talitos_private *priv = dev_get_drvdata(dev);
256 int ch, err;
257
258 /*
259 * Master reset
260 * errata documentation: warning: certain SEC interrupts
261 * are not fully cleared by writing the MCR:SWR bit,
262 * set bit twice to completely reset
263 */
264 err = reset_device(dev);
265 if (err)
266 return err;
267
268 err = reset_device(dev);
269 if (err)
270 return err;
271
272 /* reset channels */
273 for (ch = 0; ch < priv->num_channels; ch++) {
274 err = reset_channel(dev, ch);
275 if (err)
276 return err;
277 }
278
279 /* enable channel done and error interrupts */
280 setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
281 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
282
fe5720e2
KP		 283 /* disable integrity check error interrupts (use writeback instead) */
284 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
285 setbits32(priv->reg + TALITOS_MDEUICR_LO,
286 TALITOS_MDEUICR_LO_ICE);
287
9c4a7965
KP		 288 return 0;
289}
290
291/**
292 * talitos_submit - submits a descriptor to the device for processing
293 * @dev: the SEC device to be used
5228f0f7 294 * @ch: the SEC device channel to be used
9c4a7965
KP		 295 * @desc: the descriptor to be processed by the device
296 * @callback: whom to call when processing is complete
297 * @context: a handle for use by caller (optional)
298 *
299 * desc must contain valid dma-mapped (bus physical) address pointers.
300 * callback must check err and feedback in descriptor header
301 * for device processing status.
302 */
5228f0f7 303static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
9c4a7965
KP		 304 void (*callback)(struct device *dev,
305 struct talitos_desc *desc,
306 void *context, int error),
307 void *context)
308{
309 struct talitos_private *priv = dev_get_drvdata(dev);
310 struct talitos_request *request;
5228f0f7 311 unsigned long flags;
9c4a7965
KP		 312 int head;
313
4b992628 314 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
9c4a7965 315
4b992628 316 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
ec6644d6 317 /* h/w fifo is full */
4b992628 318 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 319 return -EAGAIN;
320 }
321
4b992628
KP		 322 head = priv->chan[ch].head;
323 request = &priv->chan[ch].fifo[head];
ec6644d6 324
9c4a7965
KP		 325 /* map descriptor and save caller data */
326 request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
327 DMA_BIDIRECTIONAL);
328 request->callback = callback;
329 request->context = context;
330
331 /* increment fifo head */
4b992628 332 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
9c4a7965
KP		 333
334 smp_wmb();
335 request->desc = desc;
336
337 /* GO! */
338 wmb();
ad42d5fc
KP		 339 out_be32(priv->chan[ch].reg + TALITOS_FF,
340 upper_32_bits(request->dma_desc));
341 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
a752447a 342 lower_32_bits(request->dma_desc));
9c4a7965 343
4b992628 344 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
9c4a7965
KP		 345
346 return -EINPROGRESS;
347}
348
349/*
350 * process what was done, notify callback of error if not
351 */
352static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
353{
354 struct talitos_private *priv = dev_get_drvdata(dev);
355 struct talitos_request *request, saved_req;
356 unsigned long flags;
357 int tail, status;
358
4b992628 359 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
9c4a7965 360
4b992628
KP		 361 tail = priv->chan[ch].tail;
362 while (priv->chan[ch].fifo[tail].desc) {
363 request = &priv->chan[ch].fifo[tail];
9c4a7965
KP		 364
365 /* descriptors with their done bits set don't get the error */
366 rmb();
ca38a814 367 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
9c4a7965 368 status = 0;
ca38a814 369 else
9c4a7965
KP		 370 if (!error)
371 break;
372 else
373 status = error;
374
375 dma_unmap_single(dev, request->dma_desc,
e938e465
KP		 376 sizeof(struct talitos_desc),
377 DMA_BIDIRECTIONAL);
9c4a7965
KP		 378
379 /* copy entries so we can call callback outside lock */
380 saved_req.desc = request->desc;
381 saved_req.callback = request->callback;
382 saved_req.context = request->context;
383
384 /* release request entry in fifo */
385 smp_wmb();
386 request->desc = NULL;
387
388 /* increment fifo tail */
4b992628 389 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
9c4a7965 390
4b992628 391 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
ec6644d6 392
4b992628 393 atomic_dec(&priv->chan[ch].submit_count);
ec6644d6 394
9c4a7965
KP		 395 saved_req.callback(dev, saved_req.desc, saved_req.context,
396 status);
397 /* channel may resume processing in single desc error case */
398 if (error && !reset_ch && status == error)
399 return;
4b992628
KP		 400 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
401 tail = priv->chan[ch].tail;
9c4a7965
KP		 402 }
403
4b992628 404 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
9c4a7965
KP		 405}
406
407/*
408 * process completed requests for channels that have done status
409 */
c3e337f8
KP		 410#define DEF_TALITOS_DONE(name, ch_done_mask) \
411static void talitos_done_##name(unsigned long data) \
412{ \
413 struct device *dev = (struct device *)data; \
414 struct talitos_private *priv = dev_get_drvdata(dev); \
415 \
416 if (ch_done_mask & 1) \
417 flush_channel(dev, 0, 0, 0); \
418 if (priv->num_channels == 1) \
419 goto out; \
420 if (ch_done_mask & (1 << 2)) \
421 flush_channel(dev, 1, 0, 0); \
422 if (ch_done_mask & (1 << 4)) \
423 flush_channel(dev, 2, 0, 0); \
424 if (ch_done_mask & (1 << 6)) \
425 flush_channel(dev, 3, 0, 0); \
426 \
427out: \
428 /* At this point, all completed channels have been processed */ \
429 /* Unmask done interrupts for channels completed later on. */ \
430 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
431 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \
9c4a7965 432}
c3e337f8
KP		 433DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
434DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE)
435DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE)
9c4a7965
KP		 436
437/*
438 * locate current (offending) descriptor
439 */
3e721aeb 440static u32 current_desc_hdr(struct device *dev, int ch)
9c4a7965
KP		 441{
442 struct talitos_private *priv = dev_get_drvdata(dev);
4b992628 443 int tail = priv->chan[ch].tail;
9c4a7965
KP		 444 dma_addr_t cur_desc;
445
ad42d5fc 446 cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
9c4a7965 447
4b992628 448 while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
9c4a7965 449 tail = (tail + 1) & (priv->fifo_len - 1);
4b992628 450 if (tail == priv->chan[ch].tail) {
9c4a7965 451 dev_err(dev, "couldn't locate current descriptor\n");
3e721aeb 452 return 0;
9c4a7965
KP		 453 }
454 }
455
3e721aeb 456 return priv->chan[ch].fifo[tail].desc->hdr;
9c4a7965
KP		 457}
458
459/*
460 * user diagnostics; report root cause of error based on execution unit status
461 */
3e721aeb 462static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
9c4a7965
KP		 463{
464 struct talitos_private *priv = dev_get_drvdata(dev);
465 int i;
466
3e721aeb 467 if (!desc_hdr)
ad42d5fc 468 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
3e721aeb
KP		 469
470 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
9c4a7965
KP		 471 case DESC_HDR_SEL0_AFEU:
472 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
473 in_be32(priv->reg + TALITOS_AFEUISR),
474 in_be32(priv->reg + TALITOS_AFEUISR_LO));
475 break;
476 case DESC_HDR_SEL0_DEU:
477 dev_err(dev, "DEUISR 0x%08x_%08x\n",
478 in_be32(priv->reg + TALITOS_DEUISR),
479 in_be32(priv->reg + TALITOS_DEUISR_LO));
480 break;
481 case DESC_HDR_SEL0_MDEUA:
482 case DESC_HDR_SEL0_MDEUB:
483 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
484 in_be32(priv->reg + TALITOS_MDEUISR),
485 in_be32(priv->reg + TALITOS_MDEUISR_LO));
486 break;
487 case DESC_HDR_SEL0_RNG:
488 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
489 in_be32(priv->reg + TALITOS_RNGUISR),
490 in_be32(priv->reg + TALITOS_RNGUISR_LO));
491 break;
492 case DESC_HDR_SEL0_PKEU:
493 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
494 in_be32(priv->reg + TALITOS_PKEUISR),
495 in_be32(priv->reg + TALITOS_PKEUISR_LO));
496 break;
497 case DESC_HDR_SEL0_AESU:
498 dev_err(dev, "AESUISR 0x%08x_%08x\n",
499 in_be32(priv->reg + TALITOS_AESUISR),
500 in_be32(priv->reg + TALITOS_AESUISR_LO));
501 break;
502 case DESC_HDR_SEL0_CRCU:
503 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
504 in_be32(priv->reg + TALITOS_CRCUISR),
505 in_be32(priv->reg + TALITOS_CRCUISR_LO));
506 break;
507 case DESC_HDR_SEL0_KEU:
508 dev_err(dev, "KEUISR 0x%08x_%08x\n",
509 in_be32(priv->reg + TALITOS_KEUISR),
510 in_be32(priv->reg + TALITOS_KEUISR_LO));
511 break;
512 }
513
3e721aeb 514 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
9c4a7965
KP		 515 case DESC_HDR_SEL1_MDEUA:
516 case DESC_HDR_SEL1_MDEUB:
517 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
518 in_be32(priv->reg + TALITOS_MDEUISR),
519 in_be32(priv->reg + TALITOS_MDEUISR_LO));
520 break;
521 case DESC_HDR_SEL1_CRCU:
522 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
523 in_be32(priv->reg + TALITOS_CRCUISR),
524 in_be32(priv->reg + TALITOS_CRCUISR_LO));
525 break;
526 }
527
528 for (i = 0; i < 8; i++)
529 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
ad42d5fc
KP		 530 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
531 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
9c4a7965
KP		 532}
533
534/*
535 * recover from error interrupts
536 */
40405f10 537static void talitos_error(unsigned long data, u32 isr, u32 isr_lo)
9c4a7965
KP		 538{
539 struct device *dev = (struct device *)data;
540 struct talitos_private *priv = dev_get_drvdata(dev);
541 unsigned int timeout = TALITOS_TIMEOUT;
542 int ch, error, reset_dev = 0, reset_ch = 0;
40405f10 543 u32 v, v_lo;
9c4a7965
KP		 544
545 for (ch = 0; ch < priv->num_channels; ch++) {
546 /* skip channels without errors */
547 if (!(isr & (1 << (ch * 2 + 1))))
548 continue;
549
550 error = -EINVAL;
551
ad42d5fc
KP		 552 v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);
553 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
9c4a7965
KP		 554
555 if (v_lo & TALITOS_CCPSR_LO_DOF) {
556 dev_err(dev, "double fetch fifo overflow error\n");
557 error = -EAGAIN;
558 reset_ch = 1;
559 }
560 if (v_lo & TALITOS_CCPSR_LO_SOF) {
561 /* h/w dropped descriptor */
562 dev_err(dev, "single fetch fifo overflow error\n");
563 error = -EAGAIN;
564 }
565 if (v_lo & TALITOS_CCPSR_LO_MDTE)
566 dev_err(dev, "master data transfer error\n");
567 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
568 dev_err(dev, "s/g data length zero error\n");
569 if (v_lo & TALITOS_CCPSR_LO_FPZ)
570 dev_err(dev, "fetch pointer zero error\n");
571 if (v_lo & TALITOS_CCPSR_LO_IDH)
572 dev_err(dev, "illegal descriptor header error\n");
573 if (v_lo & TALITOS_CCPSR_LO_IEU)
574 dev_err(dev, "invalid execution unit error\n");
575 if (v_lo & TALITOS_CCPSR_LO_EU)
3e721aeb 576 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
9c4a7965
KP		 577 if (v_lo & TALITOS_CCPSR_LO_GB)
578 dev_err(dev, "gather boundary error\n");
579 if (v_lo & TALITOS_CCPSR_LO_GRL)
580 dev_err(dev, "gather return/length error\n");
581 if (v_lo & TALITOS_CCPSR_LO_SB)
582 dev_err(dev, "scatter boundary error\n");
583 if (v_lo & TALITOS_CCPSR_LO_SRL)
584 dev_err(dev, "scatter return/length error\n");
585
586 flush_channel(dev, ch, error, reset_ch);
587
588 if (reset_ch) {
589 reset_channel(dev, ch);
590 } else {
ad42d5fc 591 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
9c4a7965 592 TALITOS_CCCR_CONT);
ad42d5fc
KP		 593 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
594 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
9c4a7965
KP		 595 TALITOS_CCCR_CONT) && --timeout)
596 cpu_relax();
597 if (timeout == 0) {
598 dev_err(dev, "failed to restart channel %d\n",
599 ch);
600 reset_dev = 1;
601 }
602 }
603 }
c3e337f8 604 if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) {
9c4a7965
KP		 605 dev_err(dev, "done overflow, internal time out, or rngu error: "
606 "ISR 0x%08x_%08x\n", isr, isr_lo);
607
608 /* purge request queues */
609 for (ch = 0; ch < priv->num_channels; ch++)
610 flush_channel(dev, ch, -EIO, 1);
611
612 /* reset and reinitialize the device */
613 init_device(dev);
614 }
615}
616
c3e337f8
KP		 617#define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
618static irqreturn_t talitos_interrupt_##name(int irq, void *data) \
619{ \
620 struct device *dev = data; \
621 struct talitos_private *priv = dev_get_drvdata(dev); \
622 u32 isr, isr_lo; \
623 \
624 isr = in_be32(priv->reg + TALITOS_ISR); \
625 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
626 /* Acknowledge interrupt */ \
627 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
628 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
629 \
630 if (unlikely((isr & ~TALITOS_ISR_4CHDONE) & ch_err_mask || isr_lo)) \
631 talitos_error((unsigned long)data, isr, isr_lo); \
632 else \
633 if (likely(isr & ch_done_mask)) { \
634 /* mask further done interrupts. */ \
635 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
636 /* done_task will unmask done interrupts at exit */ \
637 tasklet_schedule(&priv->done_task[tlet]); \
638 } \
639 \
640 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
641 IRQ_NONE; \
9c4a7965 642}
c3e337f8
KP		 643DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
644DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0)
645DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1)
9c4a7965
KP		 646
647/*
648 * hwrng
649 */
650static int talitos_rng_data_present(struct hwrng *rng, int wait)
651{
652 struct device *dev = (struct device *)rng->priv;
653 struct talitos_private *priv = dev_get_drvdata(dev);
654 u32 ofl;
655 int i;
656
657 for (i = 0; i < 20; i++) {
658 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
659 TALITOS_RNGUSR_LO_OFL;
660 if (ofl || !wait)
661 break;
662 udelay(10);
663 }
664
665 return !!ofl;
666}
667
668static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
669{
670 struct device *dev = (struct device *)rng->priv;
671 struct talitos_private *priv = dev_get_drvdata(dev);
672
673 /* rng fifo requires 64-bit accesses */
674 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
675 *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
676
677 return sizeof(u32);
678}
679
680static int talitos_rng_init(struct hwrng *rng)
681{
682 struct device *dev = (struct device *)rng->priv;
683 struct talitos_private *priv = dev_get_drvdata(dev);
684 unsigned int timeout = TALITOS_TIMEOUT;
685
686 setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
687 while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
688 && --timeout)
689 cpu_relax();
690 if (timeout == 0) {
691 dev_err(dev, "failed to reset rng hw\n");
692 return -ENODEV;
693 }
694
695 /* start generating */
696 setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
697
698 return 0;
699}
700
701static int talitos_register_rng(struct device *dev)
702{
703 struct talitos_private *priv = dev_get_drvdata(dev);
704
705 priv->rng.name = dev_driver_string(dev),
706 priv->rng.init = talitos_rng_init,
707 priv->rng.data_present = talitos_rng_data_present,
708 priv->rng.data_read = talitos_rng_data_read,
709 priv->rng.priv = (unsigned long)dev;
710
711 return hwrng_register(&priv->rng);
712}
713
714static void talitos_unregister_rng(struct device *dev)
715{
716 struct talitos_private *priv = dev_get_drvdata(dev);
717
718 hwrng_unregister(&priv->rng);
719}
720
721/*
722 * crypto alg
723 */
724#define TALITOS_CRA_PRIORITY 3000
725#define TALITOS_MAX_KEY_SIZE 64
3952f17e 726#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
70bcaca7 727
497f2e6b 728#define MD5_BLOCK_SIZE 64
9c4a7965
KP		 729
730struct talitos_ctx {
731 struct device *dev;
5228f0f7 732 int ch;
9c4a7965
KP		 733 __be32 desc_hdr_template;
734 u8 key[TALITOS_MAX_KEY_SIZE];
70bcaca7 735 u8 iv[TALITOS_MAX_IV_LENGTH];
9c4a7965
KP		 736 unsigned int keylen;
737 unsigned int enckeylen;
738 unsigned int authkeylen;
739 unsigned int authsize;
740};
741
497f2e6b
LN		 742#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
743#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
744
745struct talitos_ahash_req_ctx {
60f208d7 746 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
497f2e6b
LN		 747 unsigned int hw_context_size;
748 u8 buf[HASH_MAX_BLOCK_SIZE];
749 u8 bufnext[HASH_MAX_BLOCK_SIZE];
60f208d7 750 unsigned int swinit;
497f2e6b
LN		 751 unsigned int first;
752 unsigned int last;
753 unsigned int to_hash_later;
5e833bc4 754 u64 nbuf;
497f2e6b
LN		 755 struct scatterlist bufsl[2];
756 struct scatterlist *psrc;
757};
758
56af8cd4
LN		 759static int aead_setauthsize(struct crypto_aead *authenc,
760 unsigned int authsize)
9c4a7965
KP		 761{
762 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
763
764 ctx->authsize = authsize;
765
766 return 0;
767}
768
56af8cd4
LN		 769static int aead_setkey(struct crypto_aead *authenc,
770 const u8 *key, unsigned int keylen)
9c4a7965
KP		 771{
772 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
773 struct rtattr *rta = (void *)key;
774 struct crypto_authenc_key_param *param;
775 unsigned int authkeylen;
776 unsigned int enckeylen;
777
778 if (!RTA_OK(rta, keylen))
779 goto badkey;
780
781 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
782 goto badkey;
783
784 if (RTA_PAYLOAD(rta) < sizeof(*param))
785 goto badkey;
786
787 param = RTA_DATA(rta);
788 enckeylen = be32_to_cpu(param->enckeylen);
789
790 key += RTA_ALIGN(rta->rta_len);
791 keylen -= RTA_ALIGN(rta->rta_len);
792
793 if (keylen < enckeylen)
794 goto badkey;
795
796 authkeylen = keylen - enckeylen;
797
798 if (keylen > TALITOS_MAX_KEY_SIZE)
799 goto badkey;
800
801 memcpy(&ctx->key, key, keylen);
802
803 ctx->keylen = keylen;
804 ctx->enckeylen = enckeylen;
805 ctx->authkeylen = authkeylen;
806
807 return 0;
808
809badkey:
810 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
811 return -EINVAL;
812}
813
814/*
56af8cd4 815 * talitos_edesc - s/w-extended descriptor
9c4a7965
KP		 816 * @src_nents: number of segments in input scatterlist
817 * @dst_nents: number of segments in output scatterlist
818 * @dma_len: length of dma mapped link_tbl space
819 * @dma_link_tbl: bus physical address of link_tbl
820 * @desc: h/w descriptor
821 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
822 *
823 * if decrypting (with authcheck), or either one of src_nents or dst_nents
824 * is greater than 1, an integrity check value is concatenated to the end
825 * of link_tbl data
826 */
56af8cd4 827struct talitos_edesc {
9c4a7965
KP		 828 int src_nents;
829 int dst_nents;
4de9d0b5
LN		 830 int src_is_chained;
831 int dst_is_chained;
9c4a7965
KP		 832 int dma_len;
833 dma_addr_t dma_link_tbl;
834 struct talitos_desc desc;
835 struct talitos_ptr link_tbl[0];
836};
837
4de9d0b5
LN		 838static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
839 unsigned int nents, enum dma_data_direction dir,
840 int chained)
841{
842 if (unlikely(chained))
843 while (sg) {
844 dma_map_sg(dev, sg, 1, dir);
845 sg = scatterwalk_sg_next(sg);
846 }
847 else
848 dma_map_sg(dev, sg, nents, dir);
849 return nents;
850}
851
852static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
853 enum dma_data_direction dir)
854{
855 while (sg) {
856 dma_unmap_sg(dev, sg, 1, dir);
857 sg = scatterwalk_sg_next(sg);
858 }
859}
860
861static void talitos_sg_unmap(struct device *dev,
862 struct talitos_edesc *edesc,
863 struct scatterlist *src,
864 struct scatterlist *dst)
865{
866 unsigned int src_nents = edesc->src_nents ? : 1;
867 unsigned int dst_nents = edesc->dst_nents ? : 1;
868
869 if (src != dst) {
870 if (edesc->src_is_chained)
871 talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
872 else
873 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
874
497f2e6b
LN		 875 if (dst) {
876 if (edesc->dst_is_chained)
877 talitos_unmap_sg_chain(dev, dst,
878 DMA_FROM_DEVICE);
879 else
880 dma_unmap_sg(dev, dst, dst_nents,
881 DMA_FROM_DEVICE);
882 }
4de9d0b5
LN		 883 } else
884 if (edesc->src_is_chained)
885 talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
886 else
887 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
888}
889
9c4a7965 890static void ipsec_esp_unmap(struct device *dev,
56af8cd4 891 struct talitos_edesc *edesc,
9c4a7965
KP		 892 struct aead_request *areq)
893{
894 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
895 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
896 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
897 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
898
899 dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
900
4de9d0b5 901 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
9c4a7965
KP		 902
903 if (edesc->dma_len)
904 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
905 DMA_BIDIRECTIONAL);
906}
907
908/*
909 * ipsec_esp descriptor callbacks
910 */
911static void ipsec_esp_encrypt_done(struct device *dev,
912 struct talitos_desc *desc, void *context,
913 int err)
914{
915 struct aead_request *areq = context;
9c4a7965
KP		 916 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
917 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
19bbbc63 918 struct talitos_edesc *edesc;
9c4a7965
KP		 919 struct scatterlist *sg;
920 void *icvdata;
921
19bbbc63
KP		 922 edesc = container_of(desc, struct talitos_edesc, desc);
923
9c4a7965
KP		 924 ipsec_esp_unmap(dev, edesc, areq);
925
926 /* copy the generated ICV to dst */
927 if (edesc->dma_len) {
928 icvdata = &edesc->link_tbl[edesc->src_nents +
f3c85bc1 929 edesc->dst_nents + 2];
9c4a7965
KP		 930 sg = sg_last(areq->dst, edesc->dst_nents);
931 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
932 icvdata, ctx->authsize);
933 }
934
935 kfree(edesc);
936
937 aead_request_complete(areq, err);
938}
939
fe5720e2 940static void ipsec_esp_decrypt_swauth_done(struct device *dev,
e938e465
KP		 941 struct talitos_desc *desc,
942 void *context, int err)
9c4a7965
KP		 943{
944 struct aead_request *req = context;
9c4a7965
KP		 945 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
946 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
19bbbc63 947 struct talitos_edesc *edesc;
9c4a7965
KP		 948 struct scatterlist *sg;
949 void *icvdata;
950
19bbbc63
KP		 951 edesc = container_of(desc, struct talitos_edesc, desc);
952
9c4a7965
KP		 953 ipsec_esp_unmap(dev, edesc, req);
954
955 if (!err) {
956 /* auth check */
957 if (edesc->dma_len)
958 icvdata = &edesc->link_tbl[edesc->src_nents +
f3c85bc1 959 edesc->dst_nents + 2];
9c4a7965
KP		 960 else
961 icvdata = &edesc->link_tbl[0];
962
963 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
964 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
965 ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
966 }
967
968 kfree(edesc);
969
970 aead_request_complete(req, err);
971}
972
fe5720e2 973static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
e938e465
KP		 974 struct talitos_desc *desc,
975 void *context, int err)
fe5720e2
KP		 976{
977 struct aead_request *req = context;
19bbbc63
KP		 978 struct talitos_edesc *edesc;
979
980 edesc = container_of(desc, struct talitos_edesc, desc);
fe5720e2
KP		 981
982 ipsec_esp_unmap(dev, edesc, req);
983
984 /* check ICV auth status */
e938e465
KP		 985 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
986 DESC_HDR_LO_ICCR1_PASS))
987 err = -EBADMSG;
fe5720e2
KP		 988
989 kfree(edesc);
990
991 aead_request_complete(req, err);
992}
993
9c4a7965
KP		 994/*
995 * convert scatterlist to SEC h/w link table format
996 * stop at cryptlen bytes
997 */
70bcaca7 998static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
9c4a7965
KP		 999 int cryptlen, struct talitos_ptr *link_tbl_ptr)
1000{
70bcaca7
LN		 1001 int n_sg = sg_count;
1002
1003 while (n_sg--) {
81eb024c 1004 to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg));
9c4a7965
KP		 1005 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
1006 link_tbl_ptr->j_extent = 0;
1007 link_tbl_ptr++;
1008 cryptlen -= sg_dma_len(sg);
4de9d0b5 1009 sg = scatterwalk_sg_next(sg);
9c4a7965
KP		 1010 }
1011
70bcaca7 1012 /* adjust (decrease) last one (or two) entry's len to cryptlen */
9c4a7965 1013 link_tbl_ptr--;
c0e741d4 1014 while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
70bcaca7
LN		 1015 /* Empty this entry, and move to previous one */
1016 cryptlen += be16_to_cpu(link_tbl_ptr->len);
1017 link_tbl_ptr->len = 0;
1018 sg_count--;
1019 link_tbl_ptr--;
1020 }
9c4a7965
KP		 1021 link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
1022 + cryptlen);
1023
1024 /* tag end of link table */
1025 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
70bcaca7
LN		 1026
1027 return sg_count;
9c4a7965
KP		 1028}
1029
1030/*
1031 * fill in and submit ipsec_esp descriptor
1032 */
56af8cd4 1033static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
9c4a7965
KP		 1034 u8 *giv, u64 seq,
1035 void (*callback) (struct device *dev,
1036 struct talitos_desc *desc,
1037 void *context, int error))
1038{
1039 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1040 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1041 struct device *dev = ctx->dev;
1042 struct talitos_desc *desc = &edesc->desc;
1043 unsigned int cryptlen = areq->cryptlen;
1044 unsigned int authsize = ctx->authsize;
e41256f1 1045 unsigned int ivsize = crypto_aead_ivsize(aead);
fa86a267 1046 int sg_count, ret;
fe5720e2 1047 int sg_link_tbl_len;
9c4a7965
KP		 1048
1049 /* hmac key */
1050 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1051 0, DMA_TO_DEVICE);
1052 /* hmac data */
e41256f1
KP		 1053 map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
1054 sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
9c4a7965 1055 /* cipher iv */
9c4a7965
KP		 1056 map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
1057 DMA_TO_DEVICE);
1058
1059 /* cipher key */
1060 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1061 (char *)&ctx->key + ctx->authkeylen, 0,
1062 DMA_TO_DEVICE);
1063
1064 /*
1065 * cipher in
1066 * map and adjust cipher len to aead request cryptlen.
1067 * extent is bytes of HMAC postpended to ciphertext,
1068 * typically 12 for ipsec
1069 */
1070 desc->ptr[4].len = cpu_to_be16(cryptlen);
1071 desc->ptr[4].j_extent = authsize;
1072
e938e465
KP		 1073 sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1074 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1075 : DMA_TO_DEVICE,
4de9d0b5 1076 edesc->src_is_chained);
9c4a7965
KP		 1077
1078 if (sg_count == 1) {
81eb024c 1079 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));
9c4a7965 1080 } else {
fe5720e2
KP		 1081 sg_link_tbl_len = cryptlen;
1082
962a9c99 1083 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
fe5720e2 1084 sg_link_tbl_len = cryptlen + authsize;
e938e465 1085
fe5720e2 1086 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
70bcaca7
LN		 1087 &edesc->link_tbl[0]);
1088 if (sg_count > 1) {
1089 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
81eb024c 1090 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl);
e938e465
KP		 1091 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1092 edesc->dma_len,
1093 DMA_BIDIRECTIONAL);
70bcaca7
LN		 1094 } else {
1095 /* Only one segment now, so no link tbl needed */
81eb024c
KP		 1096 to_talitos_ptr(&desc->ptr[4],
1097 sg_dma_address(areq->src));
70bcaca7 1098 }
9c4a7965
KP		 1099 }
1100
1101 /* cipher out */
1102 desc->ptr[5].len = cpu_to_be16(cryptlen);
1103 desc->ptr[5].j_extent = authsize;
1104
e938e465 1105 if (areq->src != areq->dst)
4de9d0b5
LN		 1106 sg_count = talitos_map_sg(dev, areq->dst,
1107 edesc->dst_nents ? : 1,
1108 DMA_FROM_DEVICE,
1109 edesc->dst_is_chained);
9c4a7965
KP		 1110
1111 if (sg_count == 1) {
81eb024c 1112 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));
9c4a7965
KP		 1113 } else {
1114 struct talitos_ptr *link_tbl_ptr =
f3c85bc1 1115 &edesc->link_tbl[edesc->src_nents + 1];
9c4a7965 1116
81eb024c
KP		 1117 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1118 (edesc->src_nents + 1) *
1119 sizeof(struct talitos_ptr));
fe5720e2
KP		 1120 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1121 link_tbl_ptr);
1122
f3c85bc1 1123 /* Add an entry to the link table for ICV data */
9c4a7965 1124 link_tbl_ptr += sg_count - 1;
9c4a7965 1125 link_tbl_ptr->j_extent = 0;
f3c85bc1 1126 sg_count++;
9c4a7965
KP		 1127 link_tbl_ptr++;
1128 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1129 link_tbl_ptr->len = cpu_to_be16(authsize);
1130
1131 /* icv data follows link tables */
81eb024c
KP		 1132 to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl +
1133 (edesc->src_nents + edesc->dst_nents + 2) *
1134 sizeof(struct talitos_ptr));
9c4a7965
KP		 1135 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1136 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1137 edesc->dma_len, DMA_BIDIRECTIONAL);
1138 }
1139
1140 /* iv out */
1141 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1142 DMA_FROM_DEVICE);
1143
5228f0f7 1144 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
fa86a267
KP		 1145 if (ret != -EINPROGRESS) {
1146 ipsec_esp_unmap(dev, edesc, areq);
1147 kfree(edesc);
1148 }
1149 return ret;
9c4a7965
KP		 1150}
1151
9c4a7965
KP		 1152/*
1153 * derive number of elements in scatterlist
1154 */
4de9d0b5 1155static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
9c4a7965
KP		 1156{
1157 struct scatterlist *sg = sg_list;
1158 int sg_nents = 0;
1159
4de9d0b5
LN		 1160 *chained = 0;
1161 while (nbytes > 0) {
9c4a7965
KP		 1162 sg_nents++;
1163 nbytes -= sg->length;
4de9d0b5
LN		 1164 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1165 *chained = 1;
1166 sg = scatterwalk_sg_next(sg);
9c4a7965
KP		 1167 }
1168
1169 return sg_nents;
1170}
1171
497f2e6b
LN		 1172/**
1173 * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
1174 * @sgl: The SG list
1175 * @nents: Number of SG entries
1176 * @buf: Where to copy to
1177 * @buflen: The number of bytes to copy
1178 * @skip: The number of bytes to skip before copying.
1179 * Note: skip + buflen should equal SG total size.
1180 *
1181 * Returns the number of copied bytes.
1182 *
1183 **/
1184static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
1185 void *buf, size_t buflen, unsigned int skip)
1186{
1187 unsigned int offset = 0;
1188 unsigned int boffset = 0;
1189 struct sg_mapping_iter miter;
1190 unsigned long flags;
1191 unsigned int sg_flags = SG_MITER_ATOMIC;
1192 size_t total_buffer = buflen + skip;
1193
1194 sg_flags |= SG_MITER_FROM_SG;
1195
1196 sg_miter_start(&miter, sgl, nents, sg_flags);
1197
1198 local_irq_save(flags);
1199
1200 while (sg_miter_next(&miter) && offset < total_buffer) {
1201 unsigned int len;
1202 unsigned int ignore;
1203
1204 if ((offset + miter.length) > skip) {
1205 if (offset < skip) {
1206 /* Copy part of this segment */
1207 ignore = skip - offset;
1208 len = miter.length - ignore;
7260042b
LN		 1209 if (boffset + len > buflen)
1210 len = buflen - boffset;
497f2e6b
LN		 1211 memcpy(buf + boffset, miter.addr + ignore, len);
1212 } else {
7260042b 1213 /* Copy all of this segment (up to buflen) */
497f2e6b 1214 len = miter.length;
7260042b
LN		 1215 if (boffset + len > buflen)
1216 len = buflen - boffset;
497f2e6b
LN		 1217 memcpy(buf + boffset, miter.addr, len);
1218 }
1219 boffset += len;
1220 }
1221 offset += miter.length;
1222 }
1223
1224 sg_miter_stop(&miter);
1225
1226 local_irq_restore(flags);
1227 return boffset;
1228}
1229
9c4a7965 1230/*
56af8cd4 1231 * allocate and map the extended descriptor
9c4a7965 1232 */
4de9d0b5
LN		 1233static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1234 struct scatterlist *src,
1235 struct scatterlist *dst,
497f2e6b 1236 int hash_result,
4de9d0b5
LN		 1237 unsigned int cryptlen,
1238 unsigned int authsize,
1239 int icv_stashing,
1240 u32 cryptoflags)
9c4a7965 1241{
56af8cd4 1242 struct talitos_edesc *edesc;
9c4a7965 1243 int src_nents, dst_nents, alloc_len, dma_len;
4de9d0b5
LN		 1244 int src_chained, dst_chained = 0;
1245 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
586725f8 1246 GFP_ATOMIC;
9c4a7965 1247
4de9d0b5
LN		 1248 if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
1249 dev_err(dev, "length exceeds h/w max limit\n");
9c4a7965
KP		 1250 return ERR_PTR(-EINVAL);
1251 }
1252
4de9d0b5 1253 src_nents = sg_count(src, cryptlen + authsize, &src_chained);
9c4a7965
KP		 1254 src_nents = (src_nents == 1) ? 0 : src_nents;
1255
497f2e6b
LN		 1256 if (hash_result) {
1257 dst_nents = 0;
9c4a7965 1258 } else {
497f2e6b
LN		 1259 if (dst == src) {
1260 dst_nents = src_nents;
1261 } else {
1262 dst_nents = sg_count(dst, cryptlen + authsize,
1263 &dst_chained);
1264 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1265 }
9c4a7965
KP		 1266 }
1267
1268 /*
1269 * allocate space for base edesc plus the link tables,
f3c85bc1 1270 * allowing for two separate entries for ICV and generated ICV (+ 2),
9c4a7965
KP		 1271 * and the ICV data itself
1272 */
56af8cd4 1273 alloc_len = sizeof(struct talitos_edesc);
9c4a7965 1274 if (src_nents || dst_nents) {
f3c85bc1 1275 dma_len = (src_nents + dst_nents + 2) *
4de9d0b5 1276 sizeof(struct talitos_ptr) + authsize;
9c4a7965
KP		 1277 alloc_len += dma_len;
1278 } else {
1279 dma_len = 0;
4de9d0b5 1280 alloc_len += icv_stashing ? authsize : 0;
9c4a7965
KP		 1281 }
1282
586725f8 1283 edesc = kmalloc(alloc_len, GFP_DMA | flags);
9c4a7965 1284 if (!edesc) {
4de9d0b5 1285 dev_err(dev, "could not allocate edescriptor\n");
9c4a7965
KP		 1286 return ERR_PTR(-ENOMEM);
1287 }
1288
1289 edesc->src_nents = src_nents;
1290 edesc->dst_nents = dst_nents;
4de9d0b5
LN		 1291 edesc->src_is_chained = src_chained;
1292 edesc->dst_is_chained = dst_chained;
9c4a7965 1293 edesc->dma_len = dma_len;
497f2e6b
LN		 1294 if (dma_len)
1295 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1296 edesc->dma_len,
1297 DMA_BIDIRECTIONAL);
9c4a7965
KP		 1298
1299 return edesc;
1300}
1301
4de9d0b5
LN		 1302static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
1303 int icv_stashing)
1304{
1305 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1306 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1307
497f2e6b 1308 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
4de9d0b5
LN		 1309 areq->cryptlen, ctx->authsize, icv_stashing,
1310 areq->base.flags);
1311}
1312
56af8cd4 1313static int aead_encrypt(struct aead_request *req)
9c4a7965
KP		 1314{
1315 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1316 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
56af8cd4 1317 struct talitos_edesc *edesc;
9c4a7965
KP		 1318
1319 /* allocate extended descriptor */
4de9d0b5 1320 edesc = aead_edesc_alloc(req, 0);
9c4a7965
KP		 1321 if (IS_ERR(edesc))
1322 return PTR_ERR(edesc);
1323
1324 /* set encrypt */
70bcaca7 1325 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965
KP		 1326
1327 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1328}
1329
56af8cd4 1330static int aead_decrypt(struct aead_request *req)
9c4a7965
KP		 1331{
1332 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1333 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1334 unsigned int authsize = ctx->authsize;
fe5720e2 1335 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
56af8cd4 1336 struct talitos_edesc *edesc;
9c4a7965
KP		 1337 struct scatterlist *sg;
1338 void *icvdata;
1339
1340 req->cryptlen -= authsize;
1341
1342 /* allocate extended descriptor */
4de9d0b5 1343 edesc = aead_edesc_alloc(req, 1);
9c4a7965
KP		 1344 if (IS_ERR(edesc))
1345 return PTR_ERR(edesc);
1346
fe5720e2 1347 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
e938e465
KP		 1348 ((!edesc->src_nents && !edesc->dst_nents) ||
1349 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
9c4a7965 1350
fe5720e2 1351 /* decrypt and check the ICV */
e938e465
KP		 1352 edesc->desc.hdr = ctx->desc_hdr_template |
1353 DESC_HDR_DIR_INBOUND |
fe5720e2 1354 DESC_HDR_MODE1_MDEU_CICV;
9c4a7965 1355
fe5720e2
KP		 1356 /* reset integrity check result bits */
1357 edesc->desc.hdr_lo = 0;
9c4a7965 1358
e938e465
KP		 1359 return ipsec_esp(edesc, req, NULL, 0,
1360 ipsec_esp_decrypt_hwauth_done);
fe5720e2 1361
e938e465 1362 }
fe5720e2 1363
e938e465
KP		 1364 /* Have to check the ICV with software */
1365 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
fe5720e2 1366
e938e465
KP		 1367 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1368 if (edesc->dma_len)
1369 icvdata = &edesc->link_tbl[edesc->src_nents +
1370 edesc->dst_nents + 2];
1371 else
1372 icvdata = &edesc->link_tbl[0];
fe5720e2 1373
e938e465 1374 sg = sg_last(req->src, edesc->src_nents ? : 1);
fe5720e2 1375
e938e465
KP		 1376 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1377 ctx->authsize);
fe5720e2 1378
e938e465 1379 return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
9c4a7965
KP		 1380}
1381
56af8cd4 1382static int aead_givencrypt(struct aead_givcrypt_request *req)
9c4a7965
KP		 1383{
1384 struct aead_request *areq = &req->areq;
1385 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1386 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
56af8cd4 1387 struct talitos_edesc *edesc;
9c4a7965
KP		 1388
1389 /* allocate extended descriptor */
4de9d0b5 1390 edesc = aead_edesc_alloc(areq, 0);
9c4a7965
KP		 1391 if (IS_ERR(edesc))
1392 return PTR_ERR(edesc);
1393
1394 /* set encrypt */
70bcaca7 1395 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
9c4a7965
KP		 1396
1397 memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
ba95487d
KP		 1398 /* avoid consecutive packets going out with same IV */
1399 *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
9c4a7965
KP		 1400
1401 return ipsec_esp(edesc, areq, req->giv, req->seq,
1402 ipsec_esp_encrypt_done);
1403}
1404
4de9d0b5
LN		 1405static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1406 const u8 *key, unsigned int keylen)
1407{
1408 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
4de9d0b5
LN		 1409
1410 memcpy(&ctx->key, key, keylen);
1411 ctx->keylen = keylen;
1412
1413 return 0;
4de9d0b5
LN		 1414}
1415
1416static void common_nonsnoop_unmap(struct device *dev,
1417 struct talitos_edesc *edesc,
1418 struct ablkcipher_request *areq)
1419{
1420 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1421 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1422 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1423
1424 talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
1425
1426 if (edesc->dma_len)
1427 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1428 DMA_BIDIRECTIONAL);
1429}
1430
1431static void ablkcipher_done(struct device *dev,
1432 struct talitos_desc *desc, void *context,
1433 int err)
1434{
1435 struct ablkcipher_request *areq = context;
19bbbc63
KP		 1436 struct talitos_edesc *edesc;
1437
1438 edesc = container_of(desc, struct talitos_edesc, desc);
4de9d0b5
LN		 1439
1440 common_nonsnoop_unmap(dev, edesc, areq);
1441
1442 kfree(edesc);
1443
1444 areq->base.complete(&areq->base, err);
1445}
1446
1447static int common_nonsnoop(struct talitos_edesc *edesc,
1448 struct ablkcipher_request *areq,
4de9d0b5
LN		 1449 void (*callback) (struct device *dev,
1450 struct talitos_desc *desc,
1451 void *context, int error))
1452{
1453 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1454 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1455 struct device *dev = ctx->dev;
1456 struct talitos_desc *desc = &edesc->desc;
1457 unsigned int cryptlen = areq->nbytes;
1458 unsigned int ivsize;
1459 int sg_count, ret;
1460
1461 /* first DWORD empty */
1462 desc->ptr[0].len = 0;
81eb024c 1463 to_talitos_ptr(&desc->ptr[0], 0);
4de9d0b5
LN		 1464 desc->ptr[0].j_extent = 0;
1465
1466 /* cipher iv */
1467 ivsize = crypto_ablkcipher_ivsize(cipher);
febec542 1468 map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
4de9d0b5
LN		 1469 DMA_TO_DEVICE);
1470
1471 /* cipher key */
1472 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1473 (char *)&ctx->key, 0, DMA_TO_DEVICE);
1474
1475 /*
1476 * cipher in
1477 */
1478 desc->ptr[3].len = cpu_to_be16(cryptlen);
1479 desc->ptr[3].j_extent = 0;
1480
1481 sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1482 (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1483 : DMA_TO_DEVICE,
1484 edesc->src_is_chained);
1485
1486 if (sg_count == 1) {
81eb024c 1487 to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));
4de9d0b5
LN		 1488 } else {
1489 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
1490 &edesc->link_tbl[0]);
1491 if (sg_count > 1) {
81eb024c 1492 to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
4de9d0b5 1493 desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
e938e465
KP		 1494 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1495 edesc->dma_len,
1496 DMA_BIDIRECTIONAL);
4de9d0b5
LN		 1497 } else {
1498 /* Only one segment now, so no link tbl needed */
81eb024c
KP		 1499 to_talitos_ptr(&desc->ptr[3],
1500 sg_dma_address(areq->src));
4de9d0b5
LN		 1501 }
1502 }
1503
1504 /* cipher out */
1505 desc->ptr[4].len = cpu_to_be16(cryptlen);
1506 desc->ptr[4].j_extent = 0;
1507
1508 if (areq->src != areq->dst)
1509 sg_count = talitos_map_sg(dev, areq->dst,
1510 edesc->dst_nents ? : 1,
1511 DMA_FROM_DEVICE,
1512 edesc->dst_is_chained);
1513
1514 if (sg_count == 1) {
81eb024c 1515 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));
4de9d0b5
LN		 1516 } else {
1517 struct talitos_ptr *link_tbl_ptr =
1518 &edesc->link_tbl[edesc->src_nents + 1];
1519
81eb024c
KP		 1520 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1521 (edesc->src_nents + 1) *
1522 sizeof(struct talitos_ptr));
4de9d0b5 1523 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
4de9d0b5
LN		 1524 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1525 link_tbl_ptr);
1526 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1527 edesc->dma_len, DMA_BIDIRECTIONAL);
1528 }
1529
1530 /* iv out */
1531 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0,
1532 DMA_FROM_DEVICE);
1533
1534 /* last DWORD empty */
1535 desc->ptr[6].len = 0;
81eb024c 1536 to_talitos_ptr(&desc->ptr[6], 0);
4de9d0b5
LN		 1537 desc->ptr[6].j_extent = 0;
1538
5228f0f7 1539 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
4de9d0b5
LN		 1540 if (ret != -EINPROGRESS) {
1541 common_nonsnoop_unmap(dev, edesc, areq);
1542 kfree(edesc);
1543 }
1544 return ret;
1545}
1546
e938e465
KP		 1547static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1548 areq)
4de9d0b5
LN		 1549{
1550 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1551 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1552
497f2e6b
LN		 1553 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1554 areq->nbytes, 0, 0, areq->base.flags);
4de9d0b5
LN		 1555}
1556
1557static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1558{
1559 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1560 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1561 struct talitos_edesc *edesc;
1562
1563 /* allocate extended descriptor */
1564 edesc = ablkcipher_edesc_alloc(areq);
1565 if (IS_ERR(edesc))
1566 return PTR_ERR(edesc);
1567
1568 /* set encrypt */
1569 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1570
febec542 1571 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1572}
1573
1574static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1575{
1576 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1577 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1578 struct talitos_edesc *edesc;
1579
1580 /* allocate extended descriptor */
1581 edesc = ablkcipher_edesc_alloc(areq);
1582 if (IS_ERR(edesc))
1583 return PTR_ERR(edesc);
1584
1585 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1586
febec542 1587 return common_nonsnoop(edesc, areq, ablkcipher_done);
4de9d0b5
LN		 1588}
1589
497f2e6b
LN		 1590static void common_nonsnoop_hash_unmap(struct device *dev,
1591 struct talitos_edesc *edesc,
1592 struct ahash_request *areq)
1593{
1594 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1595
1596 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1597
1598 /* When using hashctx-in, must unmap it. */
1599 if (edesc->desc.ptr[1].len)
1600 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1601 DMA_TO_DEVICE);
1602
1603 if (edesc->desc.ptr[2].len)
1604 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1605 DMA_TO_DEVICE);
1606
1607 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);
1608
1609 if (edesc->dma_len)
1610 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1611 DMA_BIDIRECTIONAL);
1612
1613}
1614
1615static void ahash_done(struct device *dev,
1616 struct talitos_desc *desc, void *context,
1617 int err)
1618{
1619 struct ahash_request *areq = context;
1620 struct talitos_edesc *edesc =
1621 container_of(desc, struct talitos_edesc, desc);
1622 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1623
1624 if (!req_ctx->last && req_ctx->to_hash_later) {
1625 /* Position any partial block for next update/final/finup */
1626 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
5e833bc4 1627 req_ctx->nbuf = req_ctx->to_hash_later;
497f2e6b
LN		 1628 }
1629 common_nonsnoop_hash_unmap(dev, edesc, areq);
1630
1631 kfree(edesc);
1632
1633 areq->base.complete(&areq->base, err);
1634}
1635
1636static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1637 struct ahash_request *areq, unsigned int length,
1638 void (*callback) (struct device *dev,
1639 struct talitos_desc *desc,
1640 void *context, int error))
1641{
1642 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1643 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1644 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1645 struct device *dev = ctx->dev;
1646 struct talitos_desc *desc = &edesc->desc;
1647 int sg_count, ret;
1648
1649 /* first DWORD empty */
1650 desc->ptr[0] = zero_entry;
1651
60f208d7
KP		 1652 /* hash context in */
1653 if (!req_ctx->first || req_ctx->swinit) {
497f2e6b
LN		 1654 map_single_talitos_ptr(dev, &desc->ptr[1],
1655 req_ctx->hw_context_size,
1656 (char *)req_ctx->hw_context, 0,
1657 DMA_TO_DEVICE);
60f208d7 1658 req_ctx->swinit = 0;
497f2e6b
LN		 1659 } else {
1660 desc->ptr[1] = zero_entry;
1661 /* Indicate next op is not the first. */
1662 req_ctx->first = 0;
1663 }
1664
1665 /* HMAC key */
1666 if (ctx->keylen)
1667 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1668 (char *)&ctx->key, 0, DMA_TO_DEVICE);
1669 else
1670 desc->ptr[2] = zero_entry;
1671
1672 /*
1673 * data in
1674 */
1675 desc->ptr[3].len = cpu_to_be16(length);
1676 desc->ptr[3].j_extent = 0;
1677
1678 sg_count = talitos_map_sg(dev, req_ctx->psrc,
1679 edesc->src_nents ? : 1,
1680 DMA_TO_DEVICE,
1681 edesc->src_is_chained);
1682
1683 if (sg_count == 1) {
1684 to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
1685 } else {
1686 sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length,
1687 &edesc->link_tbl[0]);
1688 if (sg_count > 1) {
1689 desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1690 to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
1691 dma_sync_single_for_device(ctx->dev,
1692 edesc->dma_link_tbl,
1693 edesc->dma_len,
1694 DMA_BIDIRECTIONAL);
1695 } else {
1696 /* Only one segment now, so no link tbl needed */
1697 to_talitos_ptr(&desc->ptr[3],
1698 sg_dma_address(req_ctx->psrc));
1699 }
1700 }
1701
1702 /* fifth DWORD empty */
1703 desc->ptr[4] = zero_entry;
1704
1705 /* hash/HMAC out -or- hash context out */
1706 if (req_ctx->last)
1707 map_single_talitos_ptr(dev, &desc->ptr[5],
1708 crypto_ahash_digestsize(tfm),
1709 areq->result, 0, DMA_FROM_DEVICE);
1710 else
1711 map_single_talitos_ptr(dev, &desc->ptr[5],
1712 req_ctx->hw_context_size,
1713 req_ctx->hw_context, 0, DMA_FROM_DEVICE);
1714
1715 /* last DWORD empty */
1716 desc->ptr[6] = zero_entry;
1717
5228f0f7 1718 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
497f2e6b
LN		 1719 if (ret != -EINPROGRESS) {
1720 common_nonsnoop_hash_unmap(dev, edesc, areq);
1721 kfree(edesc);
1722 }
1723 return ret;
1724}
1725
1726static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1727 unsigned int nbytes)
1728{
1729 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1730 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1731 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1732
1733 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
1734 nbytes, 0, 0, areq->base.flags);
1735}
1736
1737static int ahash_init(struct ahash_request *areq)
1738{
1739 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1740 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1741
1742 /* Initialize the context */
5e833bc4 1743 req_ctx->nbuf = 0;
60f208d7
KP		 1744 req_ctx->first = 1; /* first indicates h/w must init its context */
1745 req_ctx->swinit = 0; /* assume h/w init of context */
497f2e6b
LN		 1746 req_ctx->hw_context_size =
1747 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1748 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1749 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1750
1751 return 0;
1752}
1753
60f208d7
KP		 1754/*
1755 * on h/w without explicit sha224 support, we initialize h/w context
1756 * manually with sha224 constants, and tell it to run sha256.
1757 */
1758static int ahash_init_sha224_swinit(struct ahash_request *areq)
1759{
1760 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1761
1762 ahash_init(areq);
1763 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1764
a752447a
KP		 1765 req_ctx->hw_context[0] = SHA224_H0;
1766 req_ctx->hw_context[1] = SHA224_H1;
1767 req_ctx->hw_context[2] = SHA224_H2;
1768 req_ctx->hw_context[3] = SHA224_H3;
1769 req_ctx->hw_context[4] = SHA224_H4;
1770 req_ctx->hw_context[5] = SHA224_H5;
1771 req_ctx->hw_context[6] = SHA224_H6;
1772 req_ctx->hw_context[7] = SHA224_H7;
60f208d7
KP		 1773
1774 /* init 64-bit count */
1775 req_ctx->hw_context[8] = 0;
1776 req_ctx->hw_context[9] = 0;
1777
1778 return 0;
1779}
1780
497f2e6b
LN		 1781static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1782{
1783 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1784 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1785 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1786 struct talitos_edesc *edesc;
1787 unsigned int blocksize =
1788 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1789 unsigned int nbytes_to_hash;
1790 unsigned int to_hash_later;
5e833bc4 1791 unsigned int nsg;
497f2e6b
LN		 1792 int chained;
1793
5e833bc4
LN		 1794 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1795 /* Buffer up to one whole block */
497f2e6b
LN		 1796 sg_copy_to_buffer(areq->src,
1797 sg_count(areq->src, nbytes, &chained),
5e833bc4
LN		 1798 req_ctx->buf + req_ctx->nbuf, nbytes);
1799 req_ctx->nbuf += nbytes;
497f2e6b
LN		 1800 return 0;
1801 }
1802
5e833bc4
LN		 1803 /* At least (blocksize + 1) bytes are available to hash */
1804 nbytes_to_hash = nbytes + req_ctx->nbuf;
1805 to_hash_later = nbytes_to_hash & (blocksize - 1);
1806
1807 if (req_ctx->last)
1808 to_hash_later = 0;
1809 else if (to_hash_later)
1810 /* There is a partial block. Hash the full block(s) now */
1811 nbytes_to_hash -= to_hash_later;
1812 else {
1813 /* Keep one block buffered */
1814 nbytes_to_hash -= blocksize;
1815 to_hash_later = blocksize;
1816 }
1817
1818 /* Chain in any previously buffered data */
1819 if (req_ctx->nbuf) {
1820 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1821 sg_init_table(req_ctx->bufsl, nsg);
1822 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1823 if (nsg > 1)
1824 scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
497f2e6b 1825 req_ctx->psrc = req_ctx->bufsl;
5e833bc4 1826 } else
497f2e6b 1827 req_ctx->psrc = areq->src;
5e833bc4
LN		 1828
1829 if (to_hash_later) {
1830 int nents = sg_count(areq->src, nbytes, &chained);
1831 sg_copy_end_to_buffer(areq->src, nents,
1832 req_ctx->bufnext,
1833 to_hash_later,
1834 nbytes - to_hash_later);
497f2e6b 1835 }
5e833bc4 1836 req_ctx->to_hash_later = to_hash_later;
497f2e6b 1837
5e833bc4 1838 /* Allocate extended descriptor */
497f2e6b
LN		 1839 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1840 if (IS_ERR(edesc))
1841 return PTR_ERR(edesc);
1842
1843 edesc->desc.hdr = ctx->desc_hdr_template;
1844
1845 /* On last one, request SEC to pad; otherwise continue */
1846 if (req_ctx->last)
1847 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1848 else
1849 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1850
60f208d7
KP		 1851 /* request SEC to INIT hash. */
1852 if (req_ctx->first && !req_ctx->swinit)
497f2e6b
LN		 1853 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1854
1855 /* When the tfm context has a keylen, it's an HMAC.
1856 * A first or last (ie. not middle) descriptor must request HMAC.
1857 */
1858 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1859 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1860
1861 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1862 ahash_done);
1863}
1864
1865static int ahash_update(struct ahash_request *areq)
1866{
1867 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1868
1869 req_ctx->last = 0;
1870
1871 return ahash_process_req(areq, areq->nbytes);
1872}
1873
1874static int ahash_final(struct ahash_request *areq)
1875{
1876 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1877
1878 req_ctx->last = 1;
1879
1880 return ahash_process_req(areq, 0);
1881}
1882
1883static int ahash_finup(struct ahash_request *areq)
1884{
1885 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1886
1887 req_ctx->last = 1;
1888
1889 return ahash_process_req(areq, areq->nbytes);
1890}
1891
1892static int ahash_digest(struct ahash_request *areq)
1893{
1894 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
60f208d7 1895 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
497f2e6b 1896
60f208d7 1897 ahash->init(areq);
497f2e6b
LN		 1898 req_ctx->last = 1;
1899
1900 return ahash_process_req(areq, areq->nbytes);
1901}
1902
79b3a418
LN		 1903struct keyhash_result {
1904 struct completion completion;
1905 int err;
1906};
1907
1908static void keyhash_complete(struct crypto_async_request *req, int err)
1909{
1910 struct keyhash_result *res = req->data;
1911
1912 if (err == -EINPROGRESS)
1913 return;
1914
1915 res->err = err;
1916 complete(&res->completion);
1917}
1918
1919static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
1920 u8 *hash)
1921{
1922 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1923
1924 struct scatterlist sg[1];
1925 struct ahash_request *req;
1926 struct keyhash_result hresult;
1927 int ret;
1928
1929 init_completion(&hresult.completion);
1930
1931 req = ahash_request_alloc(tfm, GFP_KERNEL);
1932 if (!req)
1933 return -ENOMEM;
1934
1935 /* Keep tfm keylen == 0 during hash of the long key */
1936 ctx->keylen = 0;
1937 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1938 keyhash_complete, &hresult);
1939
1940 sg_init_one(&sg[0], key, keylen);
1941
1942 ahash_request_set_crypt(req, sg, hash, keylen);
1943 ret = crypto_ahash_digest(req);
1944 switch (ret) {
1945 case 0:
1946 break;
1947 case -EINPROGRESS:
1948 case -EBUSY:
1949 ret = wait_for_completion_interruptible(
1950 &hresult.completion);
1951 if (!ret)
1952 ret = hresult.err;
1953 break;
1954 default:
1955 break;
1956 }
1957 ahash_request_free(req);
1958
1959 return ret;
1960}
1961
1962static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1963 unsigned int keylen)
1964{
1965 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1966 unsigned int blocksize =
1967 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1968 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1969 unsigned int keysize = keylen;
1970 u8 hash[SHA512_DIGEST_SIZE];
1971 int ret;
1972
1973 if (keylen <= blocksize)
1974 memcpy(ctx->key, key, keysize);
1975 else {
1976 /* Must get the hash of the long key */
1977 ret = keyhash(tfm, key, keylen, hash);
1978
1979 if (ret) {
1980 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1981 return -EINVAL;
1982 }
1983
1984 keysize = digestsize;
1985 memcpy(ctx->key, hash, digestsize);
1986 }
1987
1988 ctx->keylen = keysize;
1989
1990 return 0;
1991}
1992
1993
9c4a7965 1994struct talitos_alg_template {
d5e4aaef
LN		 1995 u32 type;
1996 union {
1997 struct crypto_alg crypto;
acbf7c62 1998 struct ahash_alg hash;
d5e4aaef 1999 } alg;
9c4a7965
KP		 2000 __be32 desc_hdr_template;
2001};
2002
2003static struct talitos_alg_template driver_algs[] = {
56af8cd4 2004 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
d5e4aaef
LN		 2005 { .type = CRYPTO_ALG_TYPE_AEAD,
2006 .alg.crypto = {
56af8cd4
LN		 2007 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2008 .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
2009 .cra_blocksize = AES_BLOCK_SIZE,
2010 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2011 .cra_type = &crypto_aead_type,
2012 .cra_aead = {
2013 .setkey = aead_setkey,
2014 .setauthsize = aead_setauthsize,
2015 .encrypt = aead_encrypt,
2016 .decrypt = aead_decrypt,
2017 .givencrypt = aead_givencrypt,
2018 .geniv = "<built-in>",
2019 .ivsize = AES_BLOCK_SIZE,
2020 .maxauthsize = SHA1_DIGEST_SIZE,
2021 }
2022 },
9c4a7965
KP		 2023 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2024 DESC_HDR_SEL0_AESU |
2025 DESC_HDR_MODE0_AESU_CBC |
2026 DESC_HDR_SEL1_MDEUA |
2027 DESC_HDR_MODE1_MDEU_INIT |
2028 DESC_HDR_MODE1_MDEU_PAD |
2029 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
70bcaca7 2030 },
d5e4aaef
LN		 2031 { .type = CRYPTO_ALG_TYPE_AEAD,
2032 .alg.crypto = {
56af8cd4
LN		 2033 .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
2034 .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
2035 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2036 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2037 .cra_type = &crypto_aead_type,
2038 .cra_aead = {
2039 .setkey = aead_setkey,
2040 .setauthsize = aead_setauthsize,
2041 .encrypt = aead_encrypt,
2042 .decrypt = aead_decrypt,
2043 .givencrypt = aead_givencrypt,
2044 .geniv = "<built-in>",
2045 .ivsize = DES3_EDE_BLOCK_SIZE,
2046 .maxauthsize = SHA1_DIGEST_SIZE,
2047 }
2048 },
70bcaca7
LN		 2049 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2050 DESC_HDR_SEL0_DEU |
2051 DESC_HDR_MODE0_DEU_CBC |
2052 DESC_HDR_MODE0_DEU_3DES |
2053 DESC_HDR_SEL1_MDEUA |
2054 DESC_HDR_MODE1_MDEU_INIT |
2055 DESC_HDR_MODE1_MDEU_PAD |
2056 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
3952f17e 2057 },
d5e4aaef
LN		 2058 { .type = CRYPTO_ALG_TYPE_AEAD,
2059 .alg.crypto = {
56af8cd4
LN		 2060 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2061 .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
2062 .cra_blocksize = AES_BLOCK_SIZE,
2063 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2064 .cra_type = &crypto_aead_type,
2065 .cra_aead = {
2066 .setkey = aead_setkey,
2067 .setauthsize = aead_setauthsize,
2068 .encrypt = aead_encrypt,
2069 .decrypt = aead_decrypt,
2070 .givencrypt = aead_givencrypt,
2071 .geniv = "<built-in>",
2072 .ivsize = AES_BLOCK_SIZE,
2073 .maxauthsize = SHA256_DIGEST_SIZE,
2074 }
2075 },
3952f17e
LN		 2076 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2077 DESC_HDR_SEL0_AESU |
2078 DESC_HDR_MODE0_AESU_CBC |
2079 DESC_HDR_SEL1_MDEUA |
2080 DESC_HDR_MODE1_MDEU_INIT |
2081 DESC_HDR_MODE1_MDEU_PAD |
2082 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2083 },
d5e4aaef
LN		 2084 { .type = CRYPTO_ALG_TYPE_AEAD,
2085 .alg.crypto = {
56af8cd4
LN		 2086 .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
2087 .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
2088 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2089 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2090 .cra_type = &crypto_aead_type,
2091 .cra_aead = {
2092 .setkey = aead_setkey,
2093 .setauthsize = aead_setauthsize,
2094 .encrypt = aead_encrypt,
2095 .decrypt = aead_decrypt,
2096 .givencrypt = aead_givencrypt,
2097 .geniv = "<built-in>",
2098 .ivsize = DES3_EDE_BLOCK_SIZE,
2099 .maxauthsize = SHA256_DIGEST_SIZE,
2100 }
2101 },
3952f17e
LN		 2102 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2103 DESC_HDR_SEL0_DEU |
2104 DESC_HDR_MODE0_DEU_CBC |
2105 DESC_HDR_MODE0_DEU_3DES |
2106 DESC_HDR_SEL1_MDEUA |
2107 DESC_HDR_MODE1_MDEU_INIT |
2108 DESC_HDR_MODE1_MDEU_PAD |
2109 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2110 },
d5e4aaef
LN		 2111 { .type = CRYPTO_ALG_TYPE_AEAD,
2112 .alg.crypto = {
56af8cd4
LN		 2113 .cra_name = "authenc(hmac(md5),cbc(aes))",
2114 .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
2115 .cra_blocksize = AES_BLOCK_SIZE,
2116 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2117 .cra_type = &crypto_aead_type,
2118 .cra_aead = {
2119 .setkey = aead_setkey,
2120 .setauthsize = aead_setauthsize,
2121 .encrypt = aead_encrypt,
2122 .decrypt = aead_decrypt,
2123 .givencrypt = aead_givencrypt,
2124 .geniv = "<built-in>",
2125 .ivsize = AES_BLOCK_SIZE,
2126 .maxauthsize = MD5_DIGEST_SIZE,
2127 }
2128 },
3952f17e
LN		 2129 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2130 DESC_HDR_SEL0_AESU |
2131 DESC_HDR_MODE0_AESU_CBC |
2132 DESC_HDR_SEL1_MDEUA |
2133 DESC_HDR_MODE1_MDEU_INIT |
2134 DESC_HDR_MODE1_MDEU_PAD |
2135 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2136 },
d5e4aaef
LN		 2137 { .type = CRYPTO_ALG_TYPE_AEAD,
2138 .alg.crypto = {
56af8cd4
LN		 2139 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2140 .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
2141 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2142 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2143 .cra_type = &crypto_aead_type,
2144 .cra_aead = {
2145 .setkey = aead_setkey,
2146 .setauthsize = aead_setauthsize,
2147 .encrypt = aead_encrypt,
2148 .decrypt = aead_decrypt,
2149 .givencrypt = aead_givencrypt,
2150 .geniv = "<built-in>",
2151 .ivsize = DES3_EDE_BLOCK_SIZE,
2152 .maxauthsize = MD5_DIGEST_SIZE,
2153 }
2154 },
3952f17e
LN		 2155 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2156 DESC_HDR_SEL0_DEU |
2157 DESC_HDR_MODE0_DEU_CBC |
2158 DESC_HDR_MODE0_DEU_3DES |
2159 DESC_HDR_SEL1_MDEUA |
2160 DESC_HDR_MODE1_MDEU_INIT |
2161 DESC_HDR_MODE1_MDEU_PAD |
2162 DESC_HDR_MODE1_MDEU_MD5_HMAC,
4de9d0b5
LN		 2163 },
2164 /* ABLKCIPHER algorithms. */
d5e4aaef
LN		 2165 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2166 .alg.crypto = {
4de9d0b5
LN		 2167 .cra_name = "cbc(aes)",
2168 .cra_driver_name = "cbc-aes-talitos",
2169 .cra_blocksize = AES_BLOCK_SIZE,
2170 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2171 CRYPTO_ALG_ASYNC,
2172 .cra_type = &crypto_ablkcipher_type,
2173 .cra_ablkcipher = {
2174 .setkey = ablkcipher_setkey,
2175 .encrypt = ablkcipher_encrypt,
2176 .decrypt = ablkcipher_decrypt,
2177 .geniv = "eseqiv",
2178 .min_keysize = AES_MIN_KEY_SIZE,
2179 .max_keysize = AES_MAX_KEY_SIZE,
2180 .ivsize = AES_BLOCK_SIZE,
2181 }
2182 },
2183 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2184 DESC_HDR_SEL0_AESU |
2185 DESC_HDR_MODE0_AESU_CBC,
2186 },
d5e4aaef
LN		 2187 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2188 .alg.crypto = {
4de9d0b5
LN		 2189 .cra_name = "cbc(des3_ede)",
2190 .cra_driver_name = "cbc-3des-talitos",
2191 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2192 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2193 CRYPTO_ALG_ASYNC,
2194 .cra_type = &crypto_ablkcipher_type,
2195 .cra_ablkcipher = {
2196 .setkey = ablkcipher_setkey,
2197 .encrypt = ablkcipher_encrypt,
2198 .decrypt = ablkcipher_decrypt,
2199 .geniv = "eseqiv",
2200 .min_keysize = DES3_EDE_KEY_SIZE,
2201 .max_keysize = DES3_EDE_KEY_SIZE,
2202 .ivsize = DES3_EDE_BLOCK_SIZE,
2203 }
2204 },
2205 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2206 DESC_HDR_SEL0_DEU |
2207 DESC_HDR_MODE0_DEU_CBC |
2208 DESC_HDR_MODE0_DEU_3DES,
497f2e6b
LN		 2209 },
2210 /* AHASH algorithms. */
2211 { .type = CRYPTO_ALG_TYPE_AHASH,
2212 .alg.hash = {
2213 .init = ahash_init,
2214 .update = ahash_update,
2215 .final = ahash_final,
2216 .finup = ahash_finup,
2217 .digest = ahash_digest,
2218 .halg.digestsize = MD5_DIGEST_SIZE,
2219 .halg.base = {
2220 .cra_name = "md5",
2221 .cra_driver_name = "md5-talitos",
2222 .cra_blocksize = MD5_BLOCK_SIZE,
2223 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2224 CRYPTO_ALG_ASYNC,
2225 .cra_type = &crypto_ahash_type
2226 }
2227 },
2228 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2229 DESC_HDR_SEL0_MDEUA |
2230 DESC_HDR_MODE0_MDEU_MD5,
2231 },
2232 { .type = CRYPTO_ALG_TYPE_AHASH,
2233 .alg.hash = {
2234 .init = ahash_init,
2235 .update = ahash_update,
2236 .final = ahash_final,
2237 .finup = ahash_finup,
2238 .digest = ahash_digest,
2239 .halg.digestsize = SHA1_DIGEST_SIZE,
2240 .halg.base = {
2241 .cra_name = "sha1",
2242 .cra_driver_name = "sha1-talitos",
2243 .cra_blocksize = SHA1_BLOCK_SIZE,
2244 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2245 CRYPTO_ALG_ASYNC,
2246 .cra_type = &crypto_ahash_type
2247 }
2248 },
2249 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2250 DESC_HDR_SEL0_MDEUA |
2251 DESC_HDR_MODE0_MDEU_SHA1,
2252 },
60f208d7
KP		 2253 { .type = CRYPTO_ALG_TYPE_AHASH,
2254 .alg.hash = {
2255 .init = ahash_init,
2256 .update = ahash_update,
2257 .final = ahash_final,
2258 .finup = ahash_finup,
2259 .digest = ahash_digest,
2260 .halg.digestsize = SHA224_DIGEST_SIZE,
2261 .halg.base = {
2262 .cra_name = "sha224",
2263 .cra_driver_name = "sha224-talitos",
2264 .cra_blocksize = SHA224_BLOCK_SIZE,
2265 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2266 CRYPTO_ALG_ASYNC,
2267 .cra_type = &crypto_ahash_type
2268 }
2269 },
2270 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2271 DESC_HDR_SEL0_MDEUA |
2272 DESC_HDR_MODE0_MDEU_SHA224,
2273 },
497f2e6b
LN		 2274 { .type = CRYPTO_ALG_TYPE_AHASH,
2275 .alg.hash = {
2276 .init = ahash_init,
2277 .update = ahash_update,
2278 .final = ahash_final,
2279 .finup = ahash_finup,
2280 .digest = ahash_digest,
2281 .halg.digestsize = SHA256_DIGEST_SIZE,
2282 .halg.base = {
2283 .cra_name = "sha256",
2284 .cra_driver_name = "sha256-talitos",
2285 .cra_blocksize = SHA256_BLOCK_SIZE,
2286 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2287 CRYPTO_ALG_ASYNC,
2288 .cra_type = &crypto_ahash_type
2289 }
2290 },
2291 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2292 DESC_HDR_SEL0_MDEUA |
2293 DESC_HDR_MODE0_MDEU_SHA256,
2294 },
2295 { .type = CRYPTO_ALG_TYPE_AHASH,
2296 .alg.hash = {
2297 .init = ahash_init,
2298 .update = ahash_update,
2299 .final = ahash_final,
2300 .finup = ahash_finup,
2301 .digest = ahash_digest,
2302 .halg.digestsize = SHA384_DIGEST_SIZE,
2303 .halg.base = {
2304 .cra_name = "sha384",
2305 .cra_driver_name = "sha384-talitos",
2306 .cra_blocksize = SHA384_BLOCK_SIZE,
2307 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2308 CRYPTO_ALG_ASYNC,
2309 .cra_type = &crypto_ahash_type
2310 }
2311 },
2312 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2313 DESC_HDR_SEL0_MDEUB |
2314 DESC_HDR_MODE0_MDEUB_SHA384,
2315 },
2316 { .type = CRYPTO_ALG_TYPE_AHASH,
2317 .alg.hash = {
2318 .init = ahash_init,
2319 .update = ahash_update,
2320 .final = ahash_final,
2321 .finup = ahash_finup,
2322 .digest = ahash_digest,
2323 .halg.digestsize = SHA512_DIGEST_SIZE,
2324 .halg.base = {
2325 .cra_name = "sha512",
2326 .cra_driver_name = "sha512-talitos",
2327 .cra_blocksize = SHA512_BLOCK_SIZE,
2328 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2329 CRYPTO_ALG_ASYNC,
2330 .cra_type = &crypto_ahash_type
2331 }
2332 },
2333 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2334 DESC_HDR_SEL0_MDEUB |
2335 DESC_HDR_MODE0_MDEUB_SHA512,
2336 },
79b3a418
LN		 2337 { .type = CRYPTO_ALG_TYPE_AHASH,
2338 .alg.hash = {
2339 .init = ahash_init,
2340 .update = ahash_update,
2341 .final = ahash_final,
2342 .finup = ahash_finup,
2343 .digest = ahash_digest,
2344 .setkey = ahash_setkey,
2345 .halg.digestsize = MD5_DIGEST_SIZE,
2346 .halg.base = {
2347 .cra_name = "hmac(md5)",
2348 .cra_driver_name = "hmac-md5-talitos",
2349 .cra_blocksize = MD5_BLOCK_SIZE,
2350 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2351 CRYPTO_ALG_ASYNC,
2352 .cra_type = &crypto_ahash_type
2353 }
2354 },
2355 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2356 DESC_HDR_SEL0_MDEUA |
2357 DESC_HDR_MODE0_MDEU_MD5,
2358 },
2359 { .type = CRYPTO_ALG_TYPE_AHASH,
2360 .alg.hash = {
2361 .init = ahash_init,
2362 .update = ahash_update,
2363 .final = ahash_final,
2364 .finup = ahash_finup,
2365 .digest = ahash_digest,
2366 .setkey = ahash_setkey,
2367 .halg.digestsize = SHA1_DIGEST_SIZE,
2368 .halg.base = {
2369 .cra_name = "hmac(sha1)",
2370 .cra_driver_name = "hmac-sha1-talitos",
2371 .cra_blocksize = SHA1_BLOCK_SIZE,
2372 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2373 CRYPTO_ALG_ASYNC,
2374 .cra_type = &crypto_ahash_type
2375 }
2376 },
2377 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2378 DESC_HDR_SEL0_MDEUA |
2379 DESC_HDR_MODE0_MDEU_SHA1,
2380 },
2381 { .type = CRYPTO_ALG_TYPE_AHASH,
2382 .alg.hash = {
2383 .init = ahash_init,
2384 .update = ahash_update,
2385 .final = ahash_final,
2386 .finup = ahash_finup,
2387 .digest = ahash_digest,
2388 .setkey = ahash_setkey,
2389 .halg.digestsize = SHA224_DIGEST_SIZE,
2390 .halg.base = {
2391 .cra_name = "hmac(sha224)",
2392 .cra_driver_name = "hmac-sha224-talitos",
2393 .cra_blocksize = SHA224_BLOCK_SIZE,
2394 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2395 CRYPTO_ALG_ASYNC,
2396 .cra_type = &crypto_ahash_type
2397 }
2398 },
2399 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2400 DESC_HDR_SEL0_MDEUA |
2401 DESC_HDR_MODE0_MDEU_SHA224,
2402 },
2403 { .type = CRYPTO_ALG_TYPE_AHASH,
2404 .alg.hash = {
2405 .init = ahash_init,
2406 .update = ahash_update,
2407 .final = ahash_final,
2408 .finup = ahash_finup,
2409 .digest = ahash_digest,
2410 .setkey = ahash_setkey,
2411 .halg.digestsize = SHA256_DIGEST_SIZE,
2412 .halg.base = {
2413 .cra_name = "hmac(sha256)",
2414 .cra_driver_name = "hmac-sha256-talitos",
2415 .cra_blocksize = SHA256_BLOCK_SIZE,
2416 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2417 CRYPTO_ALG_ASYNC,
2418 .cra_type = &crypto_ahash_type
2419 }
2420 },
2421 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2422 DESC_HDR_SEL0_MDEUA |
2423 DESC_HDR_MODE0_MDEU_SHA256,
2424 },
2425 { .type = CRYPTO_ALG_TYPE_AHASH,
2426 .alg.hash = {
2427 .init = ahash_init,
2428 .update = ahash_update,
2429 .final = ahash_final,
2430 .finup = ahash_finup,
2431 .digest = ahash_digest,
2432 .setkey = ahash_setkey,
2433 .halg.digestsize = SHA384_DIGEST_SIZE,
2434 .halg.base = {
2435 .cra_name = "hmac(sha384)",
2436 .cra_driver_name = "hmac-sha384-talitos",
2437 .cra_blocksize = SHA384_BLOCK_SIZE,
2438 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2439 CRYPTO_ALG_ASYNC,
2440 .cra_type = &crypto_ahash_type
2441 }
2442 },
2443 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2444 DESC_HDR_SEL0_MDEUB |
2445 DESC_HDR_MODE0_MDEUB_SHA384,
2446 },
2447 { .type = CRYPTO_ALG_TYPE_AHASH,
2448 .alg.hash = {
2449 .init = ahash_init,
2450 .update = ahash_update,
2451 .final = ahash_final,
2452 .finup = ahash_finup,
2453 .digest = ahash_digest,
2454 .setkey = ahash_setkey,
2455 .halg.digestsize = SHA512_DIGEST_SIZE,
2456 .halg.base = {
2457 .cra_name = "hmac(sha512)",
2458 .cra_driver_name = "hmac-sha512-talitos",
2459 .cra_blocksize = SHA512_BLOCK_SIZE,
2460 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2461 CRYPTO_ALG_ASYNC,
2462 .cra_type = &crypto_ahash_type
2463 }
2464 },
2465 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2466 DESC_HDR_SEL0_MDEUB |
2467 DESC_HDR_MODE0_MDEUB_SHA512,
2468 }
9c4a7965
KP		 2469};
2470
2471struct talitos_crypto_alg {
2472 struct list_head entry;
2473 struct device *dev;
acbf7c62 2474 struct talitos_alg_template algt;
9c4a7965
KP		 2475};
2476
2477static int talitos_cra_init(struct crypto_tfm *tfm)
2478{
2479 struct crypto_alg *alg = tfm->__crt_alg;
19bbbc63 2480 struct talitos_crypto_alg *talitos_alg;
9c4a7965 2481 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
5228f0f7 2482 struct talitos_private *priv;
9c4a7965 2483
497f2e6b
LN		 2484 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2485 talitos_alg = container_of(__crypto_ahash_alg(alg),
2486 struct talitos_crypto_alg,
2487 algt.alg.hash);
2488 else
2489 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2490 algt.alg.crypto);
19bbbc63 2491
9c4a7965
KP		 2492 /* update context with ptr to dev */
2493 ctx->dev = talitos_alg->dev;
19bbbc63 2494
5228f0f7
KP		 2495 /* assign SEC channel to tfm in round-robin fashion */
2496 priv = dev_get_drvdata(ctx->dev);
2497 ctx->ch = atomic_inc_return(&priv->last_chan) &
2498 (priv->num_channels - 1);
2499
9c4a7965 2500 /* copy descriptor header template value */
acbf7c62 2501 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
9c4a7965 2502
602dba5a
KP		 2503 /* select done notification */
2504 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2505
497f2e6b
LN		 2506 return 0;
2507}
2508
2509static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2510{
2511 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2512
2513 talitos_cra_init(tfm);
9c4a7965
KP		 2514
2515 /* random first IV */
70bcaca7 2516 get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
9c4a7965
KP		 2517
2518 return 0;
2519}
2520
497f2e6b
LN		 2521static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2522{
2523 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2524
2525 talitos_cra_init(tfm);
2526
2527 ctx->keylen = 0;
2528 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2529 sizeof(struct talitos_ahash_req_ctx));
2530
2531 return 0;
2532}
2533
9c4a7965
KP		 2534/*
2535 * given the alg's descriptor header template, determine whether descriptor
2536 * type and primary/secondary execution units required match the hw
2537 * capabilities description provided in the device tree node.
2538 */
2539static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2540{
2541 struct talitos_private *priv = dev_get_drvdata(dev);
2542 int ret;
2543
2544 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2545 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2546
2547 if (SECONDARY_EU(desc_hdr_template))
2548 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2549 & priv->exec_units);
2550
2551 return ret;
2552}
2553
2dc11581 2554static int talitos_remove(struct platform_device *ofdev)
9c4a7965
KP		 2555{
2556 struct device *dev = &ofdev->dev;
2557 struct talitos_private *priv = dev_get_drvdata(dev);
2558 struct talitos_crypto_alg *t_alg, *n;
2559 int i;
2560
2561 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
acbf7c62
LN		 2562 switch (t_alg->algt.type) {
2563 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2564 case CRYPTO_ALG_TYPE_AEAD:
2565 crypto_unregister_alg(&t_alg->algt.alg.crypto);
2566 break;
2567 case CRYPTO_ALG_TYPE_AHASH:
2568 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2569 break;
2570 }
9c4a7965
KP		 2571 list_del(&t_alg->entry);
2572 kfree(t_alg);
2573 }
2574
2575 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2576 talitos_unregister_rng(dev);
2577
4b992628 2578 for (i = 0; i < priv->num_channels; i++)
0b798247 2579 kfree(priv->chan[i].fifo);
9c4a7965 2580
4b992628 2581 kfree(priv->chan);
9c4a7965 2582
c3e337f8
KP		 2583 for (i = 0; i < 2; i++)
2584 if (priv->irq[i] != NO_IRQ) {
2585 free_irq(priv->irq[i], dev);
2586 irq_dispose_mapping(priv->irq[i]);
2587 }
9c4a7965 2588
c3e337f8
KP		 2589 tasklet_kill(&priv->done_task[0]);
2590 if (priv->irq[1] != NO_IRQ)
2591 tasklet_kill(&priv->done_task[1]);
9c4a7965
KP		 2592
2593 iounmap(priv->reg);
2594
2595 dev_set_drvdata(dev, NULL);
2596
2597 kfree(priv);
2598
2599 return 0;
2600}
2601
2602static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2603 struct talitos_alg_template
2604 *template)
2605{
60f208d7 2606 struct talitos_private *priv = dev_get_drvdata(dev);
9c4a7965
KP		 2607 struct talitos_crypto_alg *t_alg;
2608 struct crypto_alg *alg;
2609
2610 t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2611 if (!t_alg)
2612 return ERR_PTR(-ENOMEM);
2613
acbf7c62
LN		 2614 t_alg->algt = *template;
2615
2616 switch (t_alg->algt.type) {
2617 case CRYPTO_ALG_TYPE_ABLKCIPHER:
497f2e6b
LN		 2618 alg = &t_alg->algt.alg.crypto;
2619 alg->cra_init = talitos_cra_init;
2620 break;
acbf7c62
LN		 2621 case CRYPTO_ALG_TYPE_AEAD:
2622 alg = &t_alg->algt.alg.crypto;
497f2e6b 2623 alg->cra_init = talitos_cra_init_aead;
acbf7c62
LN		 2624 break;
2625 case CRYPTO_ALG_TYPE_AHASH:
2626 alg = &t_alg->algt.alg.hash.halg.base;
497f2e6b 2627 alg->cra_init = talitos_cra_init_ahash;
79b3a418 2628 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
0b2730d8
KP		 2629 !strncmp(alg->cra_name, "hmac", 4)) {
2630 kfree(t_alg);
79b3a418 2631 return ERR_PTR(-ENOTSUPP);
0b2730d8 2632 }
60f208d7 2633 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
79b3a418
LN		 2634 (!strcmp(alg->cra_name, "sha224") ||
2635 !strcmp(alg->cra_name, "hmac(sha224)"))) {
60f208d7
KP		 2636 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2637 t_alg->algt.desc_hdr_template =
2638 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2639 DESC_HDR_SEL0_MDEUA |
2640 DESC_HDR_MODE0_MDEU_SHA256;
2641 }
497f2e6b 2642 break;
1d11911a
KP		 2643 default:
2644 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2645 return ERR_PTR(-EINVAL);
acbf7c62 2646 }
9c4a7965 2647
9c4a7965 2648 alg->cra_module = THIS_MODULE;
9c4a7965 2649 alg->cra_priority = TALITOS_CRA_PRIORITY;
9c4a7965 2650 alg->cra_alignmask = 0;
9c4a7965 2651 alg->cra_ctxsize = sizeof(struct talitos_ctx);
9c4a7965 2652
9c4a7965
KP		 2653 t_alg->dev = dev;
2654
2655 return t_alg;
2656}
2657
c3e337f8
KP		 2658static int talitos_probe_irq(struct platform_device *ofdev)
2659{
2660 struct device *dev = &ofdev->dev;
2661 struct device_node *np = ofdev->dev.of_node;
2662 struct talitos_private *priv = dev_get_drvdata(dev);
2663 int err;
2664
2665 priv->irq[0] = irq_of_parse_and_map(np, 0);
2666 if (priv->irq[0] == NO_IRQ) {
2667 dev_err(dev, "failed to map irq\n");
2668 return -EINVAL;
2669 }
2670
2671 priv->irq[1] = irq_of_parse_and_map(np, 1);
2672
2673 /* get the primary irq line */
2674 if (priv->irq[1] == NO_IRQ) {
2675 err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0,
2676 dev_driver_string(dev), dev);
2677 goto primary_out;
2678 }
2679
2680 err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0,
2681 dev_driver_string(dev), dev);
2682 if (err)
2683 goto primary_out;
2684
2685 /* get the secondary irq line */
2686 err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0,
2687 dev_driver_string(dev), dev);
2688 if (err) {
2689 dev_err(dev, "failed to request secondary irq\n");
2690 irq_dispose_mapping(priv->irq[1]);
2691 priv->irq[1] = NO_IRQ;
2692 }
2693
2694 return err;
2695
2696primary_out:
2697 if (err) {
2698 dev_err(dev, "failed to request primary irq\n");
2699 irq_dispose_mapping(priv->irq[0]);
2700 priv->irq[0] = NO_IRQ;
2701 }
2702
2703 return err;
2704}
2705
1c48a5c9 2706static int talitos_probe(struct platform_device *ofdev)
9c4a7965
KP		 2707{
2708 struct device *dev = &ofdev->dev;
61c7a080 2709 struct device_node *np = ofdev->dev.of_node;
9c4a7965
KP		 2710 struct talitos_private *priv;
2711 const unsigned int *prop;
2712 int i, err;
2713
2714 priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2715 if (!priv)
2716 return -ENOMEM;
2717
2718 dev_set_drvdata(dev, priv);
2719
2720 priv->ofdev = ofdev;
2721
c3e337f8
KP		 2722 err = talitos_probe_irq(ofdev);
2723 if (err)
9c4a7965 2724 goto err_out;
9c4a7965 2725
c3e337f8
KP		 2726 if (priv->irq[1] == NO_IRQ) {
2727 tasklet_init(&priv->done_task[0], talitos_done_4ch,
2728 (unsigned long)dev);
2729 } else {
2730 tasklet_init(&priv->done_task[0], talitos_done_ch0_2,
2731 (unsigned long)dev);
2732 tasklet_init(&priv->done_task[1], talitos_done_ch1_3,
2733 (unsigned long)dev);
9c4a7965
KP		 2734 }
2735
c3e337f8
KP		 2736 INIT_LIST_HEAD(&priv->alg_list);
2737
9c4a7965
KP		 2738 priv->reg = of_iomap(np, 0);
2739 if (!priv->reg) {
2740 dev_err(dev, "failed to of_iomap\n");
2741 err = -ENOMEM;
2742 goto err_out;
2743 }
2744
2745 /* get SEC version capabilities from device tree */
2746 prop = of_get_property(np, "fsl,num-channels", NULL);
2747 if (prop)
2748 priv->num_channels = *prop;
2749
2750 prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2751 if (prop)
2752 priv->chfifo_len = *prop;
2753
2754 prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2755 if (prop)
2756 priv->exec_units = *prop;
2757
2758 prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2759 if (prop)
2760 priv->desc_types = *prop;
2761
2762 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2763 !priv->exec_units || !priv->desc_types) {
2764 dev_err(dev, "invalid property data in device tree node\n");
2765 err = -EINVAL;
2766 goto err_out;
2767 }
2768
f3c85bc1
LN		 2769 if (of_device_is_compatible(np, "fsl,sec3.0"))
2770 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2771
fe5720e2 2772 if (of_device_is_compatible(np, "fsl,sec2.1"))
60f208d7 2773 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
79b3a418
LN		 2774 TALITOS_FTR_SHA224_HWINIT |
2775 TALITOS_FTR_HMAC_OK;
fe5720e2 2776
4b992628
KP		 2777 priv->chan = kzalloc(sizeof(struct talitos_channel) *
2778 priv->num_channels, GFP_KERNEL);
2779 if (!priv->chan) {
2780 dev_err(dev, "failed to allocate channel management space\n");
9c4a7965
KP		 2781 err = -ENOMEM;
2782 goto err_out;
2783 }
2784
c3e337f8
KP		 2785 for (i = 0; i < priv->num_channels; i++) {
2786 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
2787 if ((priv->irq[1] == NO_IRQ) || !(i & 1))
2788 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2789 }
ad42d5fc 2790
9c4a7965 2791 for (i = 0; i < priv->num_channels; i++) {
4b992628
KP		 2792 spin_lock_init(&priv->chan[i].head_lock);
2793 spin_lock_init(&priv->chan[i].tail_lock);
9c4a7965
KP		 2794 }
2795
2796 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2797
2798 for (i = 0; i < priv->num_channels; i++) {
4b992628
KP		 2799 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2800 priv->fifo_len, GFP_KERNEL);
2801 if (!priv->chan[i].fifo) {
9c4a7965
KP		 2802 dev_err(dev, "failed to allocate request fifo %d\n", i);
2803 err = -ENOMEM;
2804 goto err_out;
2805 }
2806 }
2807
ec6644d6 2808 for (i = 0; i < priv->num_channels; i++)
4b992628
KP		 2809 atomic_set(&priv->chan[i].submit_count,
2810 -(priv->chfifo_len - 1));
9c4a7965 2811
81eb024c
KP		 2812 dma_set_mask(dev, DMA_BIT_MASK(36));
2813
9c4a7965
KP		 2814 /* reset and initialize the h/w */
2815 err = init_device(dev);
2816 if (err) {
2817 dev_err(dev, "failed to initialize device\n");
2818 goto err_out;
2819 }
2820
2821 /* register the RNG, if available */
2822 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
2823 err = talitos_register_rng(dev);
2824 if (err) {
2825 dev_err(dev, "failed to register hwrng: %d\n", err);
2826 goto err_out;
2827 } else
2828 dev_info(dev, "hwrng\n");
2829 }
2830
2831 /* register crypto algorithms the device supports */
9c4a7965
KP		 2832 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2833 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
2834 struct talitos_crypto_alg *t_alg;
acbf7c62 2835 char *name = NULL;
9c4a7965
KP		 2836
2837 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
2838 if (IS_ERR(t_alg)) {
2839 err = PTR_ERR(t_alg);
0b2730d8 2840 if (err == -ENOTSUPP)
79b3a418 2841 continue;
9c4a7965
KP		 2842 goto err_out;
2843 }
2844
acbf7c62
LN		 2845 switch (t_alg->algt.type) {
2846 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2847 case CRYPTO_ALG_TYPE_AEAD:
2848 err = crypto_register_alg(
2849 &t_alg->algt.alg.crypto);
2850 name = t_alg->algt.alg.crypto.cra_driver_name;
2851 break;
2852 case CRYPTO_ALG_TYPE_AHASH:
2853 err = crypto_register_ahash(
2854 &t_alg->algt.alg.hash);
2855 name =
2856 t_alg->algt.alg.hash.halg.base.cra_driver_name;
2857 break;
2858 }
9c4a7965
KP		 2859 if (err) {
2860 dev_err(dev, "%s alg registration failed\n",
acbf7c62 2861 name);
9c4a7965 2862 kfree(t_alg);
5b859b6e 2863 } else
9c4a7965 2864 list_add_tail(&t_alg->entry, &priv->alg_list);
9c4a7965
KP		 2865 }
2866 }
5b859b6e
KP		 2867 if (!list_empty(&priv->alg_list))
2868 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
2869 (char *)of_get_property(np, "compatible", NULL));
9c4a7965
KP		 2870
2871 return 0;
2872
2873err_out:
2874 talitos_remove(ofdev);
9c4a7965
KP		 2875
2876 return err;
2877}
2878
6c3f975a 2879static const struct of_device_id talitos_match[] = {
9c4a7965
KP		 2880 {
2881 .compatible = "fsl,sec2.0",
2882 },
2883 {},
2884};
2885MODULE_DEVICE_TABLE(of, talitos_match);
2886
1c48a5c9 2887static struct platform_driver talitos_driver = {
4018294b
GL		 2888 .driver = {
2889 .name = "talitos",
2890 .owner = THIS_MODULE,
2891 .of_match_table = talitos_match,
2892 },
9c4a7965 2893 .probe = talitos_probe,
596f1034 2894 .remove = talitos_remove,
9c4a7965
KP		 2895};
2896
741e8c2d 2897module_platform_driver(talitos_driver);
9c4a7965
KP		 2898
2899MODULE_LICENSE("GPL");
2900MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2901MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
kernel source for telekom puls (alcatel/mediatek)