projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge 4.14.24 into android-4.14
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / crypto / inside-secure / safexcel.c
CommitLineData
1b44c5a6
AT		 1/*
2 * Copyright (C) 2017 Marvell
3 *
4 * Antoine Tenart <antoine.tenart@free-electrons.com>
5 *
6 * This file is licensed under the terms of the GNU General Public
7 * License version 2. This program is licensed "as is" without any
8 * warranty of any kind, whether express or implied.
9 */
10
11#include <linux/clk.h>
12#include <linux/device.h>
13#include <linux/dma-mapping.h>
14#include <linux/dmapool.h>
15#include <linux/firmware.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18#include <linux/of_platform.h>
19#include <linux/of_irq.h>
20#include <linux/platform_device.h>
21#include <linux/workqueue.h>
22
23#include <crypto/internal/hash.h>
24#include <crypto/internal/skcipher.h>
25
26#include "safexcel.h"
27
28static u32 max_rings = EIP197_MAX_RINGS;
29module_param(max_rings, uint, 0644);
30MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
31
32static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
33{
34 u32 val, htable_offset;
35 int i;
36
37 /* Enable the record cache memory access */
38 val = readl(priv->base + EIP197_CS_RAM_CTRL);
39 val &= ~EIP197_TRC_ENABLE_MASK;
40 val |= EIP197_TRC_ENABLE_0;
41 writel(val, priv->base + EIP197_CS_RAM_CTRL);
42
43 /* Clear all ECC errors */
44 writel(0, priv->base + EIP197_TRC_ECCCTRL);
45
46 /*
47 * Make sure the cache memory is accessible by taking record cache into
48 * reset.
49 */
50 val = readl(priv->base + EIP197_TRC_PARAMS);
51 val |= EIP197_TRC_PARAMS_SW_RESET;
52 val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
53 writel(val, priv->base + EIP197_TRC_PARAMS);
54
55 /* Clear all records */
56 for (i = 0; i < EIP197_CS_RC_MAX; i++) {
57 u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
58
59 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
60 EIP197_CS_RC_PREV(EIP197_RC_NULL),
61 priv->base + offset);
62
63 val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
64 if (i == 0)
65 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
66 else if (i == EIP197_CS_RC_MAX - 1)
67 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
68 writel(val, priv->base + offset + sizeof(u32));
69 }
70
71 /* Clear the hash table entries */
72 htable_offset = EIP197_CS_RC_MAX * EIP197_CS_RC_SIZE;
73 for (i = 0; i < 64; i++)
74 writel(GENMASK(29, 0),
75 priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
76
77 /* Disable the record cache memory access */
78 val = readl(priv->base + EIP197_CS_RAM_CTRL);
79 val &= ~EIP197_TRC_ENABLE_MASK;
80 writel(val, priv->base + EIP197_CS_RAM_CTRL);
81
82 /* Write head and tail pointers of the record free chain */
83 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
84 EIP197_TRC_FREECHAIN_TAIL_PTR(EIP197_CS_RC_MAX - 1);
85 writel(val, priv->base + EIP197_TRC_FREECHAIN);
86
87 /* Configure the record cache #1 */
88 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(EIP197_CS_TRC_REC_WC) |
89 EIP197_TRC_PARAMS2_HTABLE_PTR(EIP197_CS_RC_MAX);
90 writel(val, priv->base + EIP197_TRC_PARAMS2);
91
92 /* Configure the record cache #2 */
93 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(EIP197_CS_TRC_LG_REC_WC) |
94 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
95 EIP197_TRC_PARAMS_HTABLE_SZ(2);
96 writel(val, priv->base + EIP197_TRC_PARAMS);
97}
98
99static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
100 const struct firmware *fw, u32 ctrl,
101 u32 prog_en)
102{
103 const u32 *data = (const u32 *)fw->data;
104 u32 val;
105 int i;
106
107 /* Reset the engine to make its program memory accessible */
108 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
109 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
110 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
111 priv->base + ctrl);
112
113 /* Enable access to the program memory */
114 writel(prog_en, priv->base + EIP197_PE_ICE_RAM_CTRL);
115
116 /* Write the firmware */
117 for (i = 0; i < fw->size / sizeof(u32); i++)
118 writel(be32_to_cpu(data[i]),
119 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
120
121 /* Disable access to the program memory */
122 writel(0, priv->base + EIP197_PE_ICE_RAM_CTRL);
123
124 /* Release engine from reset */
125 val = readl(priv->base + ctrl);
126 val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
127 writel(val, priv->base + ctrl);
128}
129
130static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
131{
132 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
133 const struct firmware *fw[FW_NB];
134 int i, j, ret = 0;
135 u32 val;
136
137 for (i = 0; i < FW_NB; i++) {
138 ret = request_firmware(&fw[i], fw_name[i], priv->dev);
139 if (ret) {
140 dev_err(priv->dev,
141 "Failed to request firmware %s (%d)\n",
142 fw_name[i], ret);
143 goto release_fw;
144 }
145 }
146
147 /* Clear the scratchpad memory */
148 val = readl(priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
149 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
150 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
151 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
152 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
153 writel(val, priv->base + EIP197_PE_ICE_SCRATCH_CTRL);
154
155 memset(priv->base + EIP197_PE_ICE_SCRATCH_RAM, 0,
156 EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
157
158 eip197_write_firmware(priv, fw[FW_IFPP], EIP197_PE_ICE_FPP_CTRL,
159 EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
160
161 eip197_write_firmware(priv, fw[FW_IPUE], EIP197_PE_ICE_PUE_CTRL,
162 EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
163
164release_fw:
165 for (j = 0; j < i; j++)
166 release_firmware(fw[j]);
167
168 return ret;
169}
170
171static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
172{
173 u32 hdw, cd_size_rnd, val;
174 int i;
175
176 hdw = readl(priv->base + EIP197_HIA_OPTIONS);
177 hdw &= GENMASK(27, 25);
178 hdw >>= 25;
179
180 cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
181
182 for (i = 0; i < priv->config.rings; i++) {
183 /* ring base address */
184 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
185 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
186 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
187 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
188
189 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
190 priv->config.cd_size,
191 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DESC_SIZE);
192 writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
193 (EIP197_FETCH_COUNT * priv->config.cd_offset),
194 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
195
196 /* Configure DMA tx control */
197 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
198 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
199 writel(val,
200 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_DMA_CFG);
201
202 /* clear any pending interrupt */
203 writel(GENMASK(5, 0),
204 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_STAT);
205 }
206
207 return 0;
208}
209
210static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
211{
212 u32 hdw, rd_size_rnd, val;
213 int i;
214
215 hdw = readl(priv->base + EIP197_HIA_OPTIONS);
216 hdw &= GENMASK(27, 25);
217 hdw >>= 25;
218
219 rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
220
221 for (i = 0; i < priv->config.rings; i++) {
222 /* ring base address */
223 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
224 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
225 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
226 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
227
228 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
229 priv->config.rd_size,
230 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DESC_SIZE);
231
232 writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
233 (EIP197_FETCH_COUNT * priv->config.rd_offset),
234 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
235
236 /* Configure DMA tx control */
237 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
238 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
239 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUG;
240 writel(val,
241 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_DMA_CFG);
242
243 /* clear any pending interrupt */
244 writel(GENMASK(7, 0),
245 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_STAT);
246
247 /* enable ring interrupt */
248 val = readl(priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
249 val |= EIP197_RDR_IRQ(i);
250 writel(val, priv->base + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
251 }
252
253 return 0;
254}
255
256static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
257{
258 u32 version, val;
259 int i, ret;
260
261 /* Determine endianess and configure byte swap */
262 version = readl(priv->base + EIP197_HIA_VERSION);
263 val = readl(priv->base + EIP197_HIA_MST_CTRL);
264
265 if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
266 val |= EIP197_MST_CTRL_BYTE_SWAP;
267 else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
268 val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
269
270 writel(val, priv->base + EIP197_HIA_MST_CTRL);
271
272
273 /* Configure wr/rd cache values */
274 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
275 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
276 priv->base + EIP197_MST_CTRL);
277
278 /* Interrupts reset */
279
280 /* Disable all global interrupts */
281 writel(0, priv->base + EIP197_HIA_AIC_G_ENABLE_CTRL);
282
283 /* Clear any pending interrupt */
284 writel(GENMASK(31, 0), priv->base + EIP197_HIA_AIC_G_ACK);
285
286 /* Data Fetch Engine configuration */
287
288 /* Reset all DFE threads */
289 writel(EIP197_DxE_THR_CTRL_RESET_PE,
290 priv->base + EIP197_HIA_DFE_THR_CTRL);
291
292 /* Reset HIA input interface arbiter */
293 writel(EIP197_HIA_RA_PE_CTRL_RESET,
294 priv->base + EIP197_HIA_RA_PE_CTRL);
295
296 /* DMA transfer size to use */
297 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
298 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
299 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(5) | EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
300 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
301 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
302 writel(val, priv->base + EIP197_HIA_DFE_CFG);
303
304 /* Leave the DFE threads reset state */
305 writel(0, priv->base + EIP197_HIA_DFE_THR_CTRL);
306
307 /* Configure the procesing engine thresholds */
308 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(9),
309 priv->base + EIP197_PE_IN_DBUF_THRES);
310 writel(EIP197_PE_IN_xBUF_THRES_MIN(5) | EIP197_PE_IN_xBUF_THRES_MAX(7),
311 priv->base + EIP197_PE_IN_TBUF_THRES);
312
313 /* enable HIA input interface arbiter and rings */
314 writel(EIP197_HIA_RA_PE_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
315 priv->base + EIP197_HIA_RA_PE_CTRL);
316
317 /* Data Store Engine configuration */
318
319 /* Reset all DSE threads */
320 writel(EIP197_DxE_THR_CTRL_RESET_PE,
321 priv->base + EIP197_HIA_DSE_THR_CTRL);
322
323 /* Wait for all DSE threads to complete */
324 while ((readl(priv->base + EIP197_HIA_DSE_THR_STAT) &
325 GENMASK(15, 12)) != GENMASK(15, 12))
326 ;
327
328 /* DMA transfer size to use */
329 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
330 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) | EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
39ba1bb4 331 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
ee1fd870 332 val |= EIP197_HIA_DSE_CFG_ALLWAYS_BUFFERABLE;
c87925bf 333 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
1b44c5a6
AT		 334 writel(val, priv->base + EIP197_HIA_DSE_CFG);
335
336 /* Leave the DSE threads reset state */
337 writel(0, priv->base + EIP197_HIA_DSE_THR_CTRL);
338
339 /* Configure the procesing engine thresholds */
340 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) | EIP197_PE_OUT_DBUF_THRES_MAX(8),
341 priv->base + EIP197_PE_OUT_DBUF_THRES);
342
343 /* Processing Engine configuration */
344
345 /* H/W capabilities selection */
346 val = EIP197_FUNCTION_RSVD;
347 val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
348 val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
349 val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
350 val |= EIP197_ALG_SHA2;
351 writel(val, priv->base + EIP197_PE_EIP96_FUNCTION_EN);
352
353 /* Command Descriptor Rings prepare */
354 for (i = 0; i < priv->config.rings; i++) {
355 /* Clear interrupts for this ring */
356 writel(GENMASK(31, 0),
357 priv->base + EIP197_HIA_AIC_R_ENABLE_CLR(i));
358
359 /* Disable external triggering */
360 writel(0, priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_CFG);
361
362 /* Clear the pending prepared counter */
363 writel(EIP197_xDR_PREP_CLR_COUNT,
364 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT);
365
366 /* Clear the pending processed counter */
367 writel(EIP197_xDR_PROC_CLR_COUNT,
368 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_COUNT);
369
370 writel(0,
371 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_PNTR);
372 writel(0,
373 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PROC_PNTR);
374
375 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
376 priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_RING_SIZE);
377 }
378
379 /* Result Descriptor Ring prepare */
380 for (i = 0; i < priv->config.rings; i++) {
381 /* Disable external triggering*/
382 writel(0, priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_CFG);
383
384 /* Clear the pending prepared counter */
385 writel(EIP197_xDR_PREP_CLR_COUNT,
386 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT);
387
388 /* Clear the pending processed counter */
389 writel(EIP197_xDR_PROC_CLR_COUNT,
390 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_COUNT);
391
392 writel(0,
393 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_PNTR);
394 writel(0,
395 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PROC_PNTR);
396
397 /* Ring size */
398 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
399 priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_RING_SIZE);
400 }
401
402 /* Enable command descriptor rings */
403 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
404 priv->base + EIP197_HIA_DFE_THR_CTRL);
405
406 /* Enable result descriptor rings */
407 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
408 priv->base + EIP197_HIA_DSE_THR_CTRL);
409
410 /* Clear any HIA interrupt */
411 writel(GENMASK(30, 20), priv->base + EIP197_HIA_AIC_G_ACK);
412
413 eip197_trc_cache_init(priv);
414
415 ret = eip197_load_firmwares(priv);
416 if (ret)
417 return ret;
418
419 safexcel_hw_setup_cdesc_rings(priv);
420 safexcel_hw_setup_rdesc_rings(priv);
421
422 return 0;
423}
424
86671abb 425void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
1b44c5a6
AT		 426{
427 struct crypto_async_request *req, *backlog;
428 struct safexcel_context *ctx;
429 struct safexcel_request *request;
86671abb 430 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
1b44c5a6 431
5eb0cc66
AT		 432 priv->ring[ring].need_dequeue = false;
433
1b44c5a6 434 do {
86671abb 435 spin_lock_bh(&priv->ring[ring].queue_lock);
86671abb 436 backlog = crypto_get_backlog(&priv->ring[ring].queue);
b1deb47a 437 req = crypto_dequeue_request(&priv->ring[ring].queue);
86671abb 438 spin_unlock_bh(&priv->ring[ring].queue_lock);
1b44c5a6
AT		 439
440 if (!req)
441 goto finalize;
442
443 request = kzalloc(sizeof(*request), EIP197_GFP_FLAGS(*req));
c5acabd3
AT		 444 if (!request) {
445 spin_lock_bh(&priv->ring[ring].queue_lock);
446 crypto_enqueue_request(&priv->ring[ring].queue, req);
447 spin_unlock_bh(&priv->ring[ring].queue_lock);
448
449 priv->ring[ring].need_dequeue = true;
450 goto finalize;
451 }
1b44c5a6
AT		 452
453 ctx = crypto_tfm_ctx(req->tfm);
86671abb 454 ret = ctx->send(req, ring, request, &commands, &results);
1b44c5a6
AT		 455 if (ret) {
456 kfree(request);
c5acabd3 457 req->complete(req, ret);
86671abb 458 priv->ring[ring].need_dequeue = true;
c5acabd3 459 goto finalize;
1b44c5a6
AT		 460 }
461
462 if (backlog)
463 backlog->complete(backlog, -EINPROGRESS);
464
86671abb
AT		 465 spin_lock_bh(&priv->ring[ring].egress_lock);
466 list_add_tail(&request->list, &priv->ring[ring].list);
467 spin_unlock_bh(&priv->ring[ring].egress_lock);
1b44c5a6 468
86671abb
AT		 469 cdesc += commands;
470 rdesc += results;
471 } while (nreq++ < EIP197_MAX_BATCH_SZ);
1b44c5a6
AT		 472
473finalize:
86671abb
AT		 474 if (nreq == EIP197_MAX_BATCH_SZ)
475 priv->ring[ring].need_dequeue = true;
476 else if (!nreq)
1b44c5a6
AT		 477 return;
478
86671abb 479 spin_lock_bh(&priv->ring[ring].lock);
1b44c5a6 480
86671abb
AT		 481 /* Configure when we want an interrupt */
482 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
483 EIP197_HIA_RDR_THRESH_PROC_PKT(nreq),
484 priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH);
1b44c5a6 485
86671abb
AT		 486 /* let the RDR know we have pending descriptors */
487 writel((rdesc * priv->config.rd_offset) << 2,
488 priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
1b44c5a6 489
86671abb
AT		 490 /* let the CDR know we have pending descriptors */
491 writel((cdesc * priv->config.cd_offset) << 2,
492 priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT);
1b44c5a6 493
86671abb 494 spin_unlock_bh(&priv->ring[ring].lock);
1b44c5a6
AT		 495}
496
497void safexcel_free_context(struct safexcel_crypto_priv *priv,
498 struct crypto_async_request *req,
499 int result_sz)
500{
501 struct safexcel_context *ctx = crypto_tfm_ctx(req->tfm);
502
503 if (ctx->result_dma)
504 dma_unmap_single(priv->dev, ctx->result_dma, result_sz,
505 DMA_FROM_DEVICE);
506
507 if (ctx->cache) {
508 dma_unmap_single(priv->dev, ctx->cache_dma, ctx->cache_sz,
509 DMA_TO_DEVICE);
510 kfree(ctx->cache);
511 ctx->cache = NULL;
512 ctx->cache_sz = 0;
513 }
514}
515
516void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
517{
518 struct safexcel_command_desc *cdesc;
519
520 /* Acknowledge the command descriptors */
521 do {
522 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
523 if (IS_ERR(cdesc)) {
524 dev_err(priv->dev,
525 "Could not retrieve the command descriptor\n");
526 return;
527 }
528 } while (!cdesc->last_seg);
529}
530
531void safexcel_inv_complete(struct crypto_async_request *req, int error)
532{
533 struct safexcel_inv_result *result = req->data;
534
535 if (error == -EINPROGRESS)
536 return;
537
538 result->error = error;
539 complete(&result->completion);
540}
541
542int safexcel_invalidate_cache(struct crypto_async_request *async,
543 struct safexcel_context *ctx,
544 struct safexcel_crypto_priv *priv,
545 dma_addr_t ctxr_dma, int ring,
546 struct safexcel_request *request)
547{
548 struct safexcel_command_desc *cdesc;
549 struct safexcel_result_desc *rdesc;
550 int ret = 0;
551
552 spin_lock_bh(&priv->ring[ring].egress_lock);
553
554 /* Prepare command descriptor */
555 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
556 if (IS_ERR(cdesc)) {
557 ret = PTR_ERR(cdesc);
558 goto unlock;
559 }
560
561 cdesc->control_data.type = EIP197_TYPE_EXTENDED;
562 cdesc->control_data.options = 0;
563 cdesc->control_data.refresh = 0;
564 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
565
566 /* Prepare result descriptor */
567 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
568
569 if (IS_ERR(rdesc)) {
570 ret = PTR_ERR(rdesc);
571 goto cdesc_rollback;
572 }
573
574 request->req = async;
575 goto unlock;
576
577cdesc_rollback:
578 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
579
580unlock:
581 spin_unlock_bh(&priv->ring[ring].egress_lock);
582 return ret;
583}
584
585static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
586 int ring)
587{
588 struct safexcel_request *sreq;
589 struct safexcel_context *ctx;
590 int ret, i, nreq, ndesc = 0;
591 bool should_complete;
592
593 nreq = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
594 nreq >>= 24;
595 nreq &= GENMASK(6, 0);
596 if (!nreq)
597 return;
598
599 for (i = 0; i < nreq; i++) {
600 spin_lock_bh(&priv->ring[ring].egress_lock);
601 sreq = list_first_entry(&priv->ring[ring].list,
602 struct safexcel_request, list);
603 list_del(&sreq->list);
604 spin_unlock_bh(&priv->ring[ring].egress_lock);
605
606 ctx = crypto_tfm_ctx(sreq->req->tfm);
607 ndesc = ctx->handle_result(priv, ring, sreq->req,
608 &should_complete, &ret);
609 if (ndesc < 0) {
7cc6e841 610 kfree(sreq);
1b44c5a6
AT		 611 dev_err(priv->dev, "failed to handle result (%d)", ndesc);
612 return;
613 }
614
615 writel(EIP197_xDR_PROC_xD_PKT(1) |
616 EIP197_xDR_PROC_xD_COUNT(ndesc * priv->config.rd_offset),
617 priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PROC_COUNT);
618
619 if (should_complete) {
620 local_bh_disable();
621 sreq->req->complete(sreq->req, ret);
622 local_bh_enable();
623 }
624
625 kfree(sreq);
626 }
627}
628
629static void safexcel_handle_result_work(struct work_struct *work)
630{
631 struct safexcel_work_data *data =
632 container_of(work, struct safexcel_work_data, work);
633 struct safexcel_crypto_priv *priv = data->priv;
634
635 safexcel_handle_result_descriptor(priv, data->ring);
636
5eb0cc66 637 if (priv->ring[data->ring].need_dequeue)
86671abb 638 safexcel_dequeue(data->priv, data->ring);
1b44c5a6
AT		 639}
640
641struct safexcel_ring_irq_data {
642 struct safexcel_crypto_priv *priv;
643 int ring;
644};
645
646static irqreturn_t safexcel_irq_ring(int irq, void *data)
647{
648 struct safexcel_ring_irq_data *irq_data = data;
649 struct safexcel_crypto_priv *priv = irq_data->priv;
650 int ring = irq_data->ring;
651 u32 status, stat;
652
653 status = readl(priv->base + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
654 if (!status)
655 return IRQ_NONE;
656
657 /* RDR interrupts */
658 if (status & EIP197_RDR_IRQ(ring)) {
659 stat = readl(priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
660
661 if (unlikely(stat & EIP197_xDR_ERR)) {
662 /*
663 * Fatal error, the RDR is unusable and must be
664 * reinitialized. This should not happen under
665 * normal circumstances.
666 */
667 dev_err(priv->dev, "RDR: fatal error.");
668 } else if (likely(stat & EIP197_xDR_THRESH)) {
669 queue_work(priv->ring[ring].workqueue, &priv->ring[ring].work_data.work);
670 }
671
672 /* ACK the interrupts */
673 writel(stat & 0xff,
674 priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_STAT);
675 }
676
677 /* ACK the interrupts */
678 writel(status, priv->base + EIP197_HIA_AIC_R_ACK(ring));
679
680 return IRQ_HANDLED;
681}
682
683static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
684 irq_handler_t handler,
685 struct safexcel_ring_irq_data *ring_irq_priv)
686{
687 int ret, irq = platform_get_irq_byname(pdev, name);
688
689 if (irq < 0) {
690 dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
691 return irq;
692 }
693
694 ret = devm_request_irq(&pdev->dev, irq, handler, 0,
695 dev_name(&pdev->dev), ring_irq_priv);
696 if (ret) {
697 dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
698 return ret;
699 }
700
701 return irq;
702}
703
704static struct safexcel_alg_template *safexcel_algs[] = {
705 &safexcel_alg_ecb_aes,
706 &safexcel_alg_cbc_aes,
707 &safexcel_alg_sha1,
708 &safexcel_alg_sha224,
709 &safexcel_alg_sha256,
710 &safexcel_alg_hmac_sha1,
711};
712
713static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
714{
715 int i, j, ret = 0;
716
717 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
718 safexcel_algs[i]->priv = priv;
719
720 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
721 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
722 else
723 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
724
725 if (ret)
726 goto fail;
727 }
728
729 return 0;
730
731fail:
732 for (j = 0; j < i; j++) {
733 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
734 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
735 else
736 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
737 }
738
739 return ret;
740}
741
742static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
743{
744 int i;
745
746 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
747 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
748 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
749 else
750 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
751 }
752}
753
754static void safexcel_configure(struct safexcel_crypto_priv *priv)
755{
756 u32 val, mask;
757
758 val = readl(priv->base + EIP197_HIA_OPTIONS);
759 val = (val & GENMASK(27, 25)) >> 25;
760 mask = BIT(val) - 1;
761
762 val = readl(priv->base + EIP197_HIA_OPTIONS);
763 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
764
765 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
766 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
767
768 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
769 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
770}
771
772static int safexcel_probe(struct platform_device *pdev)
773{
774 struct device *dev = &pdev->dev;
775 struct resource *res;
776 struct safexcel_crypto_priv *priv;
1b44c5a6
AT		 777 int i, ret;
778
779 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
780 if (!priv)
781 return -ENOMEM;
782
783 priv->dev = dev;
784
785 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
786 priv->base = devm_ioremap_resource(dev, res);
787 if (IS_ERR(priv->base)) {
788 dev_err(dev, "failed to get resource\n");
789 return PTR_ERR(priv->base);
790 }
791
792 priv->clk = of_clk_get(dev->of_node, 0);
793 if (!IS_ERR(priv->clk)) {
794 ret = clk_prepare_enable(priv->clk);
795 if (ret) {
796 dev_err(dev, "unable to enable clk (%d)\n", ret);
797 return ret;
798 }
799 } else {
800 /* The clock isn't mandatory */
801 if (PTR_ERR(priv->clk) == -EPROBE_DEFER)
802 return -EPROBE_DEFER;
803 }
804
a2069aac 805 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1b44c5a6
AT		 806 if (ret)
807 goto err_clk;
808
809 priv->context_pool = dmam_pool_create("safexcel-context", dev,
810 sizeof(struct safexcel_context_record),
811 1, 0);
812 if (!priv->context_pool) {
813 ret = -ENOMEM;
814 goto err_clk;
815 }
816
817 safexcel_configure(priv);
818
819 for (i = 0; i < priv->config.rings; i++) {
820 char irq_name[6] = {0}; /* "ringX\0" */
821 char wq_name[9] = {0}; /* "wq_ringX\0" */
822 int irq;
823 struct safexcel_ring_irq_data *ring_irq;
824
825 ret = safexcel_init_ring_descriptors(priv,
826 &priv->ring[i].cdr,
827 &priv->ring[i].rdr);
828 if (ret)
829 goto err_clk;
830
831 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
832 if (!ring_irq) {
833 ret = -ENOMEM;
834 goto err_clk;
835 }
836
837 ring_irq->priv = priv;
838 ring_irq->ring = i;
839
840 snprintf(irq_name, 6, "ring%d", i);
841 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
842 ring_irq);
b7d65fe1
CJ		 843 if (irq < 0) {
844 ret = irq;
1b44c5a6 845 goto err_clk;
b7d65fe1 846 }
1b44c5a6
AT		 847
848 priv->ring[i].work_data.priv = priv;
849 priv->ring[i].work_data.ring = i;
850 INIT_WORK(&priv->ring[i].work_data.work, safexcel_handle_result_work);
851
852 snprintf(wq_name, 9, "wq_ring%d", i);
853 priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
854 if (!priv->ring[i].workqueue) {
855 ret = -ENOMEM;
856 goto err_clk;
857 }
858
86671abb
AT		 859 crypto_init_queue(&priv->ring[i].queue,
860 EIP197_DEFAULT_RING_SIZE);
861
1b44c5a6
AT		 862 INIT_LIST_HEAD(&priv->ring[i].list);
863 spin_lock_init(&priv->ring[i].lock);
864 spin_lock_init(&priv->ring[i].egress_lock);
86671abb 865 spin_lock_init(&priv->ring[i].queue_lock);
1b44c5a6
AT		 866 }
867
868 platform_set_drvdata(pdev, priv);
869 atomic_set(&priv->ring_used, 0);
870
1b44c5a6
AT		 871 ret = safexcel_hw_init(priv);
872 if (ret) {
873 dev_err(dev, "EIP h/w init failed (%d)\n", ret);
874 goto err_clk;
875 }
876
877 ret = safexcel_register_algorithms(priv);
878 if (ret) {
879 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
880 goto err_clk;
881 }
882
883 return 0;
884
885err_clk:
886 clk_disable_unprepare(priv->clk);
887 return ret;
888}
889
890
891static int safexcel_remove(struct platform_device *pdev)
892{
893 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
894 int i;
895
896 safexcel_unregister_algorithms(priv);
897 clk_disable_unprepare(priv->clk);
898
899 for (i = 0; i < priv->config.rings; i++)
900 destroy_workqueue(priv->ring[i].workqueue);
901
902 return 0;
903}
904
905static const struct of_device_id safexcel_of_match_table[] = {
906 { .compatible = "inside-secure,safexcel-eip197" },
907 {},
908};
909
910
911static struct platform_driver crypto_safexcel = {
912 .probe = safexcel_probe,
913 .remove = safexcel_remove,
914 .driver = {
915 .name = "crypto-safexcel",
916 .of_match_table = safexcel_of_match_table,
917 },
918};
919module_platform_driver(crypto_safexcel);
920
921MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
922MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
923MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
924MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
925MODULE_LICENSE("GPL v2");