projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/sam/kbuild
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / infiniband / hw / ehca / ehca_mrmw.c
1 /*
2 * IBM eServer eHCA Infiniband device driver for Linux on POWER
3 *
4 * MR/MW functions
5 *
6 * Authors: Dietmar Decker <ddecker@de.ibm.com>
7 * Christoph Raisch <raisch@de.ibm.com>
8 *
9 * Copyright (c) 2005 IBM Corporation
10 *
11 * All rights reserved.
12 *
13 * This source code is distributed under a dual license of GPL v2.0 and OpenIB
14 * BSD.
15 *
16 * OpenIB BSD License
17 *
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted provided that the following conditions are met:
20 *
21 * Redistributions of source code must retain the above copyright notice, this
22 * list of conditions and the following disclaimer.
23 *
24 * Redistributions in binary form must reproduce the above copyright notice,
25 * this list of conditions and the following disclaimer in the documentation
26 * and/or other materials
27 * provided with the distribution.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
30 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
33 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
34 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
35 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
36 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
37 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
38 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
39 * POSSIBILITY OF SUCH DAMAGE.
40 */
41
42 #include <rdma/ib_umem.h>
43
44 #include <asm/current.h>
45
46 #include "ehca_iverbs.h"
47 #include "ehca_mrmw.h"
48 #include "hcp_if.h"
49 #include "hipz_hw.h"
50
51 #define NUM_CHUNKS(length, chunk_size) \
52 (((length) + (chunk_size - 1)) / (chunk_size))
53 /* max number of rpages (per hcall register_rpages) */
54 #define MAX_RPAGES 512
55
56 static struct kmem_cache *mr_cache;
57 static struct kmem_cache *mw_cache;
58
59 static struct ehca_mr *ehca_mr_new(void)
60 {
61 struct ehca_mr *me;
62
63 me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
64 if (me)
65 spin_lock_init(&me->mrlock);
66 else
67 ehca_gen_err("alloc failed");
68
69 return me;
70 }
71
72 static void ehca_mr_delete(struct ehca_mr *me)
73 {
74 kmem_cache_free(mr_cache, me);
75 }
76
77 static struct ehca_mw *ehca_mw_new(void)
78 {
79 struct ehca_mw *me;
80
81 me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
82 if (me)
83 spin_lock_init(&me->mwlock);
84 else
85 ehca_gen_err("alloc failed");
86
87 return me;
88 }
89
90 static void ehca_mw_delete(struct ehca_mw *me)
91 {
92 kmem_cache_free(mw_cache, me);
93 }
94
95 /*----------------------------------------------------------------------*/
96
97 struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
98 {
99 struct ib_mr *ib_mr;
100 int ret;
101 struct ehca_mr *e_maxmr;
102 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
103 struct ehca_shca *shca =
104 container_of(pd->device, struct ehca_shca, ib_device);
105
106 if (shca->maxmr) {
107 e_maxmr = ehca_mr_new();
108 if (!e_maxmr) {
109 ehca_err(&shca->ib_device, "out of memory");
110 ib_mr = ERR_PTR(-ENOMEM);
111 goto get_dma_mr_exit0;
112 }
113
114 ret = ehca_reg_maxmr(shca, e_maxmr, (u64 *)KERNELBASE,
115 mr_access_flags, e_pd,
116 &e_maxmr->ib.ib_mr.lkey,
117 &e_maxmr->ib.ib_mr.rkey);
118 if (ret) {
119 ehca_mr_delete(e_maxmr);
120 ib_mr = ERR_PTR(ret);
121 goto get_dma_mr_exit0;
122 }
123 ib_mr = &e_maxmr->ib.ib_mr;
124 } else {
125 ehca_err(&shca->ib_device, "no internal max-MR exist!");
126 ib_mr = ERR_PTR(-EINVAL);
127 goto get_dma_mr_exit0;
128 }
129
130 get_dma_mr_exit0:
131 if (IS_ERR(ib_mr))
132 ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ",
133 PTR_ERR(ib_mr), pd, mr_access_flags);
134 return ib_mr;
135 } /* end ehca_get_dma_mr() */
136
137 /*----------------------------------------------------------------------*/
138
139 struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
140 struct ib_phys_buf *phys_buf_array,
141 int num_phys_buf,
142 int mr_access_flags,
143 u64 *iova_start)
144 {
145 struct ib_mr *ib_mr;
146 int ret;
147 struct ehca_mr *e_mr;
148 struct ehca_shca *shca =
149 container_of(pd->device, struct ehca_shca, ib_device);
150 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
151
152 u64 size;
153
154 if ((num_phys_buf <= 0) || !phys_buf_array) {
155 ehca_err(pd->device, "bad input values: num_phys_buf=%x "
156 "phys_buf_array=%p", num_phys_buf, phys_buf_array);
157 ib_mr = ERR_PTR(-EINVAL);
158 goto reg_phys_mr_exit0;
159 }
160 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
161 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
162 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
163 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
164 /*
165 * Remote Write Access requires Local Write Access
166 * Remote Atomic Access requires Local Write Access
167 */
168 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
169 mr_access_flags);
170 ib_mr = ERR_PTR(-EINVAL);
171 goto reg_phys_mr_exit0;
172 }
173
174 /* check physical buffer list and calculate size */
175 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
176 iova_start, &size);
177 if (ret) {
178 ib_mr = ERR_PTR(ret);
179 goto reg_phys_mr_exit0;
180 }
181 if ((size == 0) ||
182 (((u64)iova_start + size) < (u64)iova_start)) {
183 ehca_err(pd->device, "bad input values: size=%lx iova_start=%p",
184 size, iova_start);
185 ib_mr = ERR_PTR(-EINVAL);
186 goto reg_phys_mr_exit0;
187 }
188
189 e_mr = ehca_mr_new();
190 if (!e_mr) {
191 ehca_err(pd->device, "out of memory");
192 ib_mr = ERR_PTR(-ENOMEM);
193 goto reg_phys_mr_exit0;
194 }
195
196 /* register MR on HCA */
197 if (ehca_mr_is_maxmr(size, iova_start)) {
198 e_mr->flags |= EHCA_MR_FLAG_MAXMR;
199 ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
200 e_pd, &e_mr->ib.ib_mr.lkey,
201 &e_mr->ib.ib_mr.rkey);
202 if (ret) {
203 ib_mr = ERR_PTR(ret);
204 goto reg_phys_mr_exit1;
205 }
206 } else {
207 struct ehca_mr_pginfo pginfo;
208 u32 num_kpages;
209 u32 num_hwpages;
210
211 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
212 PAGE_SIZE);
213 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) +
214 size, EHCA_PAGESIZE);
215 memset(&pginfo, 0, sizeof(pginfo));
216 pginfo.type = EHCA_MR_PGI_PHYS;
217 pginfo.num_kpages = num_kpages;
218 pginfo.num_hwpages = num_hwpages;
219 pginfo.u.phy.num_phys_buf = num_phys_buf;
220 pginfo.u.phy.phys_buf_array = phys_buf_array;
221 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
222 EHCA_PAGESIZE);
223
224 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
225 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
226 &e_mr->ib.ib_mr.rkey);
227 if (ret) {
228 ib_mr = ERR_PTR(ret);
229 goto reg_phys_mr_exit1;
230 }
231 }
232
233 /* successful registration of all pages */
234 return &e_mr->ib.ib_mr;
235
236 reg_phys_mr_exit1:
237 ehca_mr_delete(e_mr);
238 reg_phys_mr_exit0:
239 if (IS_ERR(ib_mr))
240 ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p "
241 "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
242 PTR_ERR(ib_mr), pd, phys_buf_array,
243 num_phys_buf, mr_access_flags, iova_start);
244 return ib_mr;
245 } /* end ehca_reg_phys_mr() */
246
247 /*----------------------------------------------------------------------*/
248
249 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
250 u64 virt, int mr_access_flags,
251 struct ib_udata *udata)
252 {
253 struct ib_mr *ib_mr;
254 struct ehca_mr *e_mr;
255 struct ehca_shca *shca =
256 container_of(pd->device, struct ehca_shca, ib_device);
257 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
258 struct ehca_mr_pginfo pginfo;
259 int ret;
260 u32 num_kpages;
261 u32 num_hwpages;
262
263 if (!pd) {
264 ehca_gen_err("bad pd=%p", pd);
265 return ERR_PTR(-EFAULT);
266 }
267
268 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
269 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
270 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
271 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
272 /*
273 * Remote Write Access requires Local Write Access
274 * Remote Atomic Access requires Local Write Access
275 */
276 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
277 mr_access_flags);
278 ib_mr = ERR_PTR(-EINVAL);
279 goto reg_user_mr_exit0;
280 }
281
282 if (length == 0 || virt + length < virt) {
283 ehca_err(pd->device, "bad input values: length=%lx "
284 "virt_base=%lx", length, virt);
285 ib_mr = ERR_PTR(-EINVAL);
286 goto reg_user_mr_exit0;
287 }
288
289 e_mr = ehca_mr_new();
290 if (!e_mr) {
291 ehca_err(pd->device, "out of memory");
292 ib_mr = ERR_PTR(-ENOMEM);
293 goto reg_user_mr_exit0;
294 }
295
296 e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
297 mr_access_flags);
298 if (IS_ERR(e_mr->umem)) {
299 ib_mr = (void *)e_mr->umem;
300 goto reg_user_mr_exit1;
301 }
302
303 if (e_mr->umem->page_size != PAGE_SIZE) {
304 ehca_err(pd->device, "page size not supported, "
305 "e_mr->umem->page_size=%x", e_mr->umem->page_size);
306 ib_mr = ERR_PTR(-EINVAL);
307 goto reg_user_mr_exit2;
308 }
309
310 /* determine number of MR pages */
311 num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
312 num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length,
313 EHCA_PAGESIZE);
314
315 /* register MR on HCA */
316 memset(&pginfo, 0, sizeof(pginfo));
317 pginfo.type = EHCA_MR_PGI_USER;
318 pginfo.num_kpages = num_kpages;
319 pginfo.num_hwpages = num_hwpages;
320 pginfo.u.usr.region = e_mr->umem;
321 pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE;
322 pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk,
323 (&e_mr->umem->chunk_list),
324 list);
325
326 ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
327 e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
328 &e_mr->ib.ib_mr.rkey);
329 if (ret) {
330 ib_mr = ERR_PTR(ret);
331 goto reg_user_mr_exit2;
332 }
333
334 /* successful registration of all pages */
335 return &e_mr->ib.ib_mr;
336
337 reg_user_mr_exit2:
338 ib_umem_release(e_mr->umem);
339 reg_user_mr_exit1:
340 ehca_mr_delete(e_mr);
341 reg_user_mr_exit0:
342 if (IS_ERR(ib_mr))
343 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x"
344 " udata=%p",
345 PTR_ERR(ib_mr), pd, mr_access_flags, udata);
346 return ib_mr;
347 } /* end ehca_reg_user_mr() */
348
349 /*----------------------------------------------------------------------*/
350
351 int ehca_rereg_phys_mr(struct ib_mr *mr,
352 int mr_rereg_mask,
353 struct ib_pd *pd,
354 struct ib_phys_buf *phys_buf_array,
355 int num_phys_buf,
356 int mr_access_flags,
357 u64 *iova_start)
358 {
359 int ret;
360
361 struct ehca_shca *shca =
362 container_of(mr->device, struct ehca_shca, ib_device);
363 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
364 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
365 u64 new_size;
366 u64 *new_start;
367 u32 new_acl;
368 struct ehca_pd *new_pd;
369 u32 tmp_lkey, tmp_rkey;
370 unsigned long sl_flags;
371 u32 num_kpages = 0;
372 u32 num_hwpages = 0;
373 struct ehca_mr_pginfo pginfo;
374 u32 cur_pid = current->tgid;
375
376 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
377 (my_pd->ownpid != cur_pid)) {
378 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
379 cur_pid, my_pd->ownpid);
380 ret = -EINVAL;
381 goto rereg_phys_mr_exit0;
382 }
383
384 if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
385 /* TODO not supported, because PHYP rereg hCall needs pages */
386 ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
387 "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
388 ret = -EINVAL;
389 goto rereg_phys_mr_exit0;
390 }
391
392 if (mr_rereg_mask & IB_MR_REREG_PD) {
393 if (!pd) {
394 ehca_err(mr->device, "rereg with bad pd, pd=%p "
395 "mr_rereg_mask=%x", pd, mr_rereg_mask);
396 ret = -EINVAL;
397 goto rereg_phys_mr_exit0;
398 }
399 }
400
401 if ((mr_rereg_mask &
402 ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
403 (mr_rereg_mask == 0)) {
404 ret = -EINVAL;
405 goto rereg_phys_mr_exit0;
406 }
407
408 /* check other parameters */
409 if (e_mr == shca->maxmr) {
410 /* should be impossible, however reject to be sure */
411 ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
412 "shca->maxmr=%p mr->lkey=%x",
413 mr, shca->maxmr, mr->lkey);
414 ret = -EINVAL;
415 goto rereg_phys_mr_exit0;
416 }
417 if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
418 if (e_mr->flags & EHCA_MR_FLAG_FMR) {
419 ehca_err(mr->device, "not supported for FMR, mr=%p "
420 "flags=%x", mr, e_mr->flags);
421 ret = -EINVAL;
422 goto rereg_phys_mr_exit0;
423 }
424 if (!phys_buf_array || num_phys_buf <= 0) {
425 ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
426 " phys_buf_array=%p num_phys_buf=%x",
427 mr_rereg_mask, phys_buf_array, num_phys_buf);
428 ret = -EINVAL;
429 goto rereg_phys_mr_exit0;
430 }
431 }
432 if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */
433 (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
434 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
435 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
436 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
437 /*
438 * Remote Write Access requires Local Write Access
439 * Remote Atomic Access requires Local Write Access
440 */
441 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
442 "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
443 ret = -EINVAL;
444 goto rereg_phys_mr_exit0;
445 }
446
447 /* set requested values dependent on rereg request */
448 spin_lock_irqsave(&e_mr->mrlock, sl_flags);
449 new_start = e_mr->start;
450 new_size = e_mr->size;
451 new_acl = e_mr->acl;
452 new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
453
454 if (mr_rereg_mask & IB_MR_REREG_TRANS) {
455 new_start = iova_start; /* change address */
456 /* check physical buffer list and calculate size */
457 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
458 num_phys_buf, iova_start,
459 &new_size);
460 if (ret)
461 goto rereg_phys_mr_exit1;
462 if ((new_size == 0) ||
463 (((u64)iova_start + new_size) < (u64)iova_start)) {
464 ehca_err(mr->device, "bad input values: new_size=%lx "
465 "iova_start=%p", new_size, iova_start);
466 ret = -EINVAL;
467 goto rereg_phys_mr_exit1;
468 }
469 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
470 new_size, PAGE_SIZE);
471 num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) +
472 new_size, EHCA_PAGESIZE);
473 memset(&pginfo, 0, sizeof(pginfo));
474 pginfo.type = EHCA_MR_PGI_PHYS;
475 pginfo.num_kpages = num_kpages;
476 pginfo.num_hwpages = num_hwpages;
477 pginfo.u.phy.num_phys_buf = num_phys_buf;
478 pginfo.u.phy.phys_buf_array = phys_buf_array;
479 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
480 EHCA_PAGESIZE);
481 }
482 if (mr_rereg_mask & IB_MR_REREG_ACCESS)
483 new_acl = mr_access_flags;
484 if (mr_rereg_mask & IB_MR_REREG_PD)
485 new_pd = container_of(pd, struct ehca_pd, ib_pd);
486
487 ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
488 new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
489 if (ret)
490 goto rereg_phys_mr_exit1;
491
492 /* successful reregistration */
493 if (mr_rereg_mask & IB_MR_REREG_PD)
494 mr->pd = pd;
495 mr->lkey = tmp_lkey;
496 mr->rkey = tmp_rkey;
497
498 rereg_phys_mr_exit1:
499 spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
500 rereg_phys_mr_exit0:
501 if (ret)
502 ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
503 "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
504 "iova_start=%p",
505 ret, mr, mr_rereg_mask, pd, phys_buf_array,
506 num_phys_buf, mr_access_flags, iova_start);
507 return ret;
508 } /* end ehca_rereg_phys_mr() */
509
510 /*----------------------------------------------------------------------*/
511
512 int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
513 {
514 int ret = 0;
515 u64 h_ret;
516 struct ehca_shca *shca =
517 container_of(mr->device, struct ehca_shca, ib_device);
518 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
519 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
520 u32 cur_pid = current->tgid;
521 unsigned long sl_flags;
522 struct ehca_mr_hipzout_parms hipzout;
523
524 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
525 (my_pd->ownpid != cur_pid)) {
526 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
527 cur_pid, my_pd->ownpid);
528 ret = -EINVAL;
529 goto query_mr_exit0;
530 }
531
532 if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
533 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
534 "e_mr->flags=%x", mr, e_mr, e_mr->flags);
535 ret = -EINVAL;
536 goto query_mr_exit0;
537 }
538
539 memset(mr_attr, 0, sizeof(struct ib_mr_attr));
540 spin_lock_irqsave(&e_mr->mrlock, sl_flags);
541
542 h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
543 if (h_ret != H_SUCCESS) {
544 ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p "
545 "hca_hndl=%lx mr_hndl=%lx lkey=%x",
546 h_ret, mr, shca->ipz_hca_handle.handle,
547 e_mr->ipz_mr_handle.handle, mr->lkey);
548 ret = ehca2ib_return_code(h_ret);
549 goto query_mr_exit1;
550 }
551 mr_attr->pd = mr->pd;
552 mr_attr->device_virt_addr = hipzout.vaddr;
553 mr_attr->size = hipzout.len;
554 mr_attr->lkey = hipzout.lkey;
555 mr_attr->rkey = hipzout.rkey;
556 ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
557
558 query_mr_exit1:
559 spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
560 query_mr_exit0:
561 if (ret)
562 ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p",
563 ret, mr, mr_attr);
564 return ret;
565 } /* end ehca_query_mr() */
566
567 /*----------------------------------------------------------------------*/
568
569 int ehca_dereg_mr(struct ib_mr *mr)
570 {
571 int ret = 0;
572 u64 h_ret;
573 struct ehca_shca *shca =
574 container_of(mr->device, struct ehca_shca, ib_device);
575 struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
576 struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
577 u32 cur_pid = current->tgid;
578
579 if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
580 (my_pd->ownpid != cur_pid)) {
581 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
582 cur_pid, my_pd->ownpid);
583 ret = -EINVAL;
584 goto dereg_mr_exit0;
585 }
586
587 if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
588 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
589 "e_mr->flags=%x", mr, e_mr, e_mr->flags);
590 ret = -EINVAL;
591 goto dereg_mr_exit0;
592 } else if (e_mr == shca->maxmr) {
593 /* should be impossible, however reject to be sure */
594 ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
595 "shca->maxmr=%p mr->lkey=%x",
596 mr, shca->maxmr, mr->lkey);
597 ret = -EINVAL;
598 goto dereg_mr_exit0;
599 }
600
601 /* TODO: BUSY: MR still has bound window(s) */
602 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
603 if (h_ret != H_SUCCESS) {
604 ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p "
605 "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
606 h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
607 e_mr->ipz_mr_handle.handle, mr->lkey);
608 ret = ehca2ib_return_code(h_ret);
609 goto dereg_mr_exit0;
610 }
611
612 if (e_mr->umem)
613 ib_umem_release(e_mr->umem);
614
615 /* successful deregistration */
616 ehca_mr_delete(e_mr);
617
618 dereg_mr_exit0:
619 if (ret)
620 ehca_err(mr->device, "ret=%x mr=%p", ret, mr);
621 return ret;
622 } /* end ehca_dereg_mr() */
623
624 /*----------------------------------------------------------------------*/
625
626 struct ib_mw *ehca_alloc_mw(struct ib_pd *pd)
627 {
628 struct ib_mw *ib_mw;
629 u64 h_ret;
630 struct ehca_mw *e_mw;
631 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
632 struct ehca_shca *shca =
633 container_of(pd->device, struct ehca_shca, ib_device);
634 struct ehca_mw_hipzout_parms hipzout;
635
636 e_mw = ehca_mw_new();
637 if (!e_mw) {
638 ib_mw = ERR_PTR(-ENOMEM);
639 goto alloc_mw_exit0;
640 }
641
642 h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
643 e_pd->fw_pd, &hipzout);
644 if (h_ret != H_SUCCESS) {
645 ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx "
646 "shca=%p hca_hndl=%lx mw=%p",
647 h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
648 ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
649 goto alloc_mw_exit1;
650 }
651 /* successful MW allocation */
652 e_mw->ipz_mw_handle = hipzout.handle;
653 e_mw->ib_mw.rkey = hipzout.rkey;
654 return &e_mw->ib_mw;
655
656 alloc_mw_exit1:
657 ehca_mw_delete(e_mw);
658 alloc_mw_exit0:
659 if (IS_ERR(ib_mw))
660 ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd);
661 return ib_mw;
662 } /* end ehca_alloc_mw() */
663
664 /*----------------------------------------------------------------------*/
665
666 int ehca_bind_mw(struct ib_qp *qp,
667 struct ib_mw *mw,
668 struct ib_mw_bind *mw_bind)
669 {
670 /* TODO: not supported up to now */
671 ehca_gen_err("bind MW currently not supported by HCAD");
672
673 return -EPERM;
674 } /* end ehca_bind_mw() */
675
676 /*----------------------------------------------------------------------*/
677
678 int ehca_dealloc_mw(struct ib_mw *mw)
679 {
680 u64 h_ret;
681 struct ehca_shca *shca =
682 container_of(mw->device, struct ehca_shca, ib_device);
683 struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
684
685 h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
686 if (h_ret != H_SUCCESS) {
687 ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p "
688 "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
689 h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
690 e_mw->ipz_mw_handle.handle);
691 return ehca2ib_return_code(h_ret);
692 }
693 /* successful deallocation */
694 ehca_mw_delete(e_mw);
695 return 0;
696 } /* end ehca_dealloc_mw() */
697
698 /*----------------------------------------------------------------------*/
699
700 struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
701 int mr_access_flags,
702 struct ib_fmr_attr *fmr_attr)
703 {
704 struct ib_fmr *ib_fmr;
705 struct ehca_shca *shca =
706 container_of(pd->device, struct ehca_shca, ib_device);
707 struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
708 struct ehca_mr *e_fmr;
709 int ret;
710 u32 tmp_lkey, tmp_rkey;
711 struct ehca_mr_pginfo pginfo;
712
713 /* check other parameters */
714 if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
715 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
716 ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
717 !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
718 /*
719 * Remote Write Access requires Local Write Access
720 * Remote Atomic Access requires Local Write Access
721 */
722 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
723 mr_access_flags);
724 ib_fmr = ERR_PTR(-EINVAL);
725 goto alloc_fmr_exit0;
726 }
727 if (mr_access_flags & IB_ACCESS_MW_BIND) {
728 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
729 mr_access_flags);
730 ib_fmr = ERR_PTR(-EINVAL);
731 goto alloc_fmr_exit0;
732 }
733 if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
734 ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
735 "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
736 fmr_attr->max_pages, fmr_attr->max_maps,
737 fmr_attr->page_shift);
738 ib_fmr = ERR_PTR(-EINVAL);
739 goto alloc_fmr_exit0;
740 }
741 if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
742 ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
743 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
744 fmr_attr->page_shift);
745 ib_fmr = ERR_PTR(-EINVAL);
746 goto alloc_fmr_exit0;
747 }
748
749 e_fmr = ehca_mr_new();
750 if (!e_fmr) {
751 ib_fmr = ERR_PTR(-ENOMEM);
752 goto alloc_fmr_exit0;
753 }
754 e_fmr->flags |= EHCA_MR_FLAG_FMR;
755
756 /* register MR on HCA */
757 memset(&pginfo, 0, sizeof(pginfo));
758 ret = ehca_reg_mr(shca, e_fmr, NULL,
759 fmr_attr->max_pages * (1 << fmr_attr->page_shift),
760 mr_access_flags, e_pd, &pginfo,
761 &tmp_lkey, &tmp_rkey);
762 if (ret) {
763 ib_fmr = ERR_PTR(ret);
764 goto alloc_fmr_exit1;
765 }
766
767 /* successful */
768 e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
769 e_fmr->fmr_max_pages = fmr_attr->max_pages;
770 e_fmr->fmr_max_maps = fmr_attr->max_maps;
771 e_fmr->fmr_map_cnt = 0;
772 return &e_fmr->ib.ib_fmr;
773
774 alloc_fmr_exit1:
775 ehca_mr_delete(e_fmr);
776 alloc_fmr_exit0:
777 if (IS_ERR(ib_fmr))
778 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x "
779 "fmr_attr=%p", PTR_ERR(ib_fmr), pd,
780 mr_access_flags, fmr_attr);
781 return ib_fmr;
782 } /* end ehca_alloc_fmr() */
783
784 /*----------------------------------------------------------------------*/
785
786 int ehca_map_phys_fmr(struct ib_fmr *fmr,
787 u64 *page_list,
788 int list_len,
789 u64 iova)
790 {
791 int ret;
792 struct ehca_shca *shca =
793 container_of(fmr->device, struct ehca_shca, ib_device);
794 struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
795 struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
796 struct ehca_mr_pginfo pginfo;
797 u32 tmp_lkey, tmp_rkey;
798
799 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
800 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
801 e_fmr, e_fmr->flags);
802 ret = -EINVAL;
803 goto map_phys_fmr_exit0;
804 }
805 ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
806 if (ret)
807 goto map_phys_fmr_exit0;
808 if (iova % e_fmr->fmr_page_size) {
809 /* only whole-numbered pages */
810 ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x",
811 iova, e_fmr->fmr_page_size);
812 ret = -EINVAL;
813 goto map_phys_fmr_exit0;
814 }
815 if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
816 /* HCAD does not limit the maps, however trace this anyway */
817 ehca_info(fmr->device, "map limit exceeded, fmr=%p "
818 "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
819 fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
820 }
821
822 memset(&pginfo, 0, sizeof(pginfo));
823 pginfo.type = EHCA_MR_PGI_FMR;
824 pginfo.num_kpages = list_len;
825 pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
826 pginfo.u.fmr.page_list = page_list;
827 pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) /
828 EHCA_PAGESIZE);
829 pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
830
831 ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
832 list_len * e_fmr->fmr_page_size,
833 e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
834 if (ret)
835 goto map_phys_fmr_exit0;
836
837 /* successful reregistration */
838 e_fmr->fmr_map_cnt++;
839 e_fmr->ib.ib_fmr.lkey = tmp_lkey;
840 e_fmr->ib.ib_fmr.rkey = tmp_rkey;
841 return 0;
842
843 map_phys_fmr_exit0:
844 if (ret)
845 ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x "
846 "iova=%lx", ret, fmr, page_list, list_len, iova);
847 return ret;
848 } /* end ehca_map_phys_fmr() */
849
850 /*----------------------------------------------------------------------*/
851
852 int ehca_unmap_fmr(struct list_head *fmr_list)
853 {
854 int ret = 0;
855 struct ib_fmr *ib_fmr;
856 struct ehca_shca *shca = NULL;
857 struct ehca_shca *prev_shca;
858 struct ehca_mr *e_fmr;
859 u32 num_fmr = 0;
860 u32 unmap_fmr_cnt = 0;
861
862 /* check all FMR belong to same SHCA, and check internal flag */
863 list_for_each_entry(ib_fmr, fmr_list, list) {
864 prev_shca = shca;
865 if (!ib_fmr) {
866 ehca_gen_err("bad fmr=%p in list", ib_fmr);
867 ret = -EINVAL;
868 goto unmap_fmr_exit0;
869 }
870 shca = container_of(ib_fmr->device, struct ehca_shca,
871 ib_device);
872 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
873 if ((shca != prev_shca) && prev_shca) {
874 ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
875 "prev_shca=%p e_fmr=%p",
876 shca, prev_shca, e_fmr);
877 ret = -EINVAL;
878 goto unmap_fmr_exit0;
879 }
880 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
881 ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
882 "e_fmr->flags=%x", e_fmr, e_fmr->flags);
883 ret = -EINVAL;
884 goto unmap_fmr_exit0;
885 }
886 num_fmr++;
887 }
888
889 /* loop over all FMRs to unmap */
890 list_for_each_entry(ib_fmr, fmr_list, list) {
891 unmap_fmr_cnt++;
892 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
893 shca = container_of(ib_fmr->device, struct ehca_shca,
894 ib_device);
895 ret = ehca_unmap_one_fmr(shca, e_fmr);
896 if (ret) {
897 /* unmap failed, stop unmapping of rest of FMRs */
898 ehca_err(&shca->ib_device, "unmap of one FMR failed, "
899 "stop rest, e_fmr=%p num_fmr=%x "
900 "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
901 unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
902 goto unmap_fmr_exit0;
903 }
904 }
905
906 unmap_fmr_exit0:
907 if (ret)
908 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
909 ret, fmr_list, num_fmr, unmap_fmr_cnt);
910 return ret;
911 } /* end ehca_unmap_fmr() */
912
913 /*----------------------------------------------------------------------*/
914
915 int ehca_dealloc_fmr(struct ib_fmr *fmr)
916 {
917 int ret;
918 u64 h_ret;
919 struct ehca_shca *shca =
920 container_of(fmr->device, struct ehca_shca, ib_device);
921 struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
922
923 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
924 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
925 e_fmr, e_fmr->flags);
926 ret = -EINVAL;
927 goto free_fmr_exit0;
928 }
929
930 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
931 if (h_ret != H_SUCCESS) {
932 ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
933 "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
934 h_ret, e_fmr, shca->ipz_hca_handle.handle,
935 e_fmr->ipz_mr_handle.handle, fmr->lkey);
936 ret = ehca2ib_return_code(h_ret);
937 goto free_fmr_exit0;
938 }
939 /* successful deregistration */
940 ehca_mr_delete(e_fmr);
941 return 0;
942
943 free_fmr_exit0:
944 if (ret)
945 ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr);
946 return ret;
947 } /* end ehca_dealloc_fmr() */
948
949 /*----------------------------------------------------------------------*/
950
951 int ehca_reg_mr(struct ehca_shca *shca,
952 struct ehca_mr *e_mr,
953 u64 *iova_start,
954 u64 size,
955 int acl,
956 struct ehca_pd *e_pd,
957 struct ehca_mr_pginfo *pginfo,
958 u32 *lkey, /*OUT*/
959 u32 *rkey) /*OUT*/
960 {
961 int ret;
962 u64 h_ret;
963 u32 hipz_acl;
964 struct ehca_mr_hipzout_parms hipzout;
965
966 ehca_mrmw_map_acl(acl, &hipz_acl);
967 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
968 if (ehca_use_hp_mr == 1)
969 hipz_acl |= 0x00000001;
970
971 h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
972 (u64)iova_start, size, hipz_acl,
973 e_pd->fw_pd, &hipzout);
974 if (h_ret != H_SUCCESS) {
975 ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx "
976 "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle);
977 ret = ehca2ib_return_code(h_ret);
978 goto ehca_reg_mr_exit0;
979 }
980
981 e_mr->ipz_mr_handle = hipzout.handle;
982
983 ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
984 if (ret)
985 goto ehca_reg_mr_exit1;
986
987 /* successful registration */
988 e_mr->num_kpages = pginfo->num_kpages;
989 e_mr->num_hwpages = pginfo->num_hwpages;
990 e_mr->start = iova_start;
991 e_mr->size = size;
992 e_mr->acl = acl;
993 *lkey = hipzout.lkey;
994 *rkey = hipzout.rkey;
995 return 0;
996
997 ehca_reg_mr_exit1:
998 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
999 if (h_ret != H_SUCCESS) {
1000 ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p "
1001 "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
1002 "pginfo=%p num_kpages=%lx num_hwpages=%lx ret=%x",
1003 h_ret, shca, e_mr, iova_start, size, acl, e_pd,
1004 hipzout.lkey, pginfo, pginfo->num_kpages,
1005 pginfo->num_hwpages, ret);
1006 ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
1007 "not recoverable");
1008 }
1009 ehca_reg_mr_exit0:
1010 if (ret)
1011 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1012 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1013 "num_kpages=%lx num_hwpages=%lx",
1014 ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
1015 pginfo->num_kpages, pginfo->num_hwpages);
1016 return ret;
1017 } /* end ehca_reg_mr() */
1018
1019 /*----------------------------------------------------------------------*/
1020
1021 int ehca_reg_mr_rpages(struct ehca_shca *shca,
1022 struct ehca_mr *e_mr,
1023 struct ehca_mr_pginfo *pginfo)
1024 {
1025 int ret = 0;
1026 u64 h_ret;
1027 u32 rnum;
1028 u64 rpage;
1029 u32 i;
1030 u64 *kpage;
1031
1032 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1033 if (!kpage) {
1034 ehca_err(&shca->ib_device, "kpage alloc failed");
1035 ret = -ENOMEM;
1036 goto ehca_reg_mr_rpages_exit0;
1037 }
1038
1039 /* max 512 pages per shot */
1040 for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
1041
1042 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1043 rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
1044 if (rnum == 0)
1045 rnum = MAX_RPAGES; /* last shot is full */
1046 } else
1047 rnum = MAX_RPAGES;
1048
1049 ret = ehca_set_pagebuf(pginfo, rnum, kpage);
1050 if (ret) {
1051 ehca_err(&shca->ib_device, "ehca_set_pagebuf "
1052 "bad rc, ret=%x rnum=%x kpage=%p",
1053 ret, rnum, kpage);
1054 goto ehca_reg_mr_rpages_exit1;
1055 }
1056
1057 if (rnum > 1) {
1058 rpage = virt_to_abs(kpage);
1059 if (!rpage) {
1060 ehca_err(&shca->ib_device, "kpage=%p i=%x",
1061 kpage, i);
1062 ret = -EFAULT;
1063 goto ehca_reg_mr_rpages_exit1;
1064 }
1065 } else
1066 rpage = *kpage;
1067
1068 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
1069 0, /* pagesize 4k */
1070 0, rpage, rnum);
1071
1072 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1073 /*
1074 * check for 'registration complete'==H_SUCCESS
1075 * and for 'page registered'==H_PAGE_REGISTERED
1076 */
1077 if (h_ret != H_SUCCESS) {
1078 ehca_err(&shca->ib_device, "last "
1079 "hipz_reg_rpage_mr failed, h_ret=%lx "
1080 "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
1081 " lkey=%x", h_ret, e_mr, i,
1082 shca->ipz_hca_handle.handle,
1083 e_mr->ipz_mr_handle.handle,
1084 e_mr->ib.ib_mr.lkey);
1085 ret = ehca2ib_return_code(h_ret);
1086 break;
1087 } else
1088 ret = 0;
1089 } else if (h_ret != H_PAGE_REGISTERED) {
1090 ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
1091 "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
1092 "mr_hndl=%lx", h_ret, e_mr, i,
1093 e_mr->ib.ib_mr.lkey,
1094 shca->ipz_hca_handle.handle,
1095 e_mr->ipz_mr_handle.handle);
1096 ret = ehca2ib_return_code(h_ret);
1097 break;
1098 } else
1099 ret = 0;
1100 } /* end for(i) */
1101
1102
1103 ehca_reg_mr_rpages_exit1:
1104 ehca_free_fw_ctrlblock(kpage);
1105 ehca_reg_mr_rpages_exit0:
1106 if (ret)
1107 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p "
1108 "num_kpages=%lx num_hwpages=%lx", ret, shca, e_mr,
1109 pginfo, pginfo->num_kpages, pginfo->num_hwpages);
1110 return ret;
1111 } /* end ehca_reg_mr_rpages() */
1112
1113 /*----------------------------------------------------------------------*/
1114
1115 inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1116 struct ehca_mr *e_mr,
1117 u64 *iova_start,
1118 u64 size,
1119 u32 acl,
1120 struct ehca_pd *e_pd,
1121 struct ehca_mr_pginfo *pginfo,
1122 u32 *lkey, /*OUT*/
1123 u32 *rkey) /*OUT*/
1124 {
1125 int ret;
1126 u64 h_ret;
1127 u32 hipz_acl;
1128 u64 *kpage;
1129 u64 rpage;
1130 struct ehca_mr_pginfo pginfo_save;
1131 struct ehca_mr_hipzout_parms hipzout;
1132
1133 ehca_mrmw_map_acl(acl, &hipz_acl);
1134 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1135
1136 kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1137 if (!kpage) {
1138 ehca_err(&shca->ib_device, "kpage alloc failed");
1139 ret = -ENOMEM;
1140 goto ehca_rereg_mr_rereg1_exit0;
1141 }
1142
1143 pginfo_save = *pginfo;
1144 ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
1145 if (ret) {
1146 ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
1147 "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx "
1148 "kpage=%p", e_mr, pginfo, pginfo->type,
1149 pginfo->num_kpages, pginfo->num_hwpages, kpage);
1150 goto ehca_rereg_mr_rereg1_exit1;
1151 }
1152 rpage = virt_to_abs(kpage);
1153 if (!rpage) {
1154 ehca_err(&shca->ib_device, "kpage=%p", kpage);
1155 ret = -EFAULT;
1156 goto ehca_rereg_mr_rereg1_exit1;
1157 }
1158 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
1159 (u64)iova_start, size, hipz_acl,
1160 e_pd->fw_pd, rpage, &hipzout);
1161 if (h_ret != H_SUCCESS) {
1162 /*
1163 * reregistration unsuccessful, try it again with the 3 hCalls,
1164 * e.g. this is required in case H_MR_CONDITION
1165 * (MW bound or MR is shared)
1166 */
1167 ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
1168 "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr);
1169 *pginfo = pginfo_save;
1170 ret = -EAGAIN;
1171 } else if ((u64 *)hipzout.vaddr != iova_start) {
1172 ehca_err(&shca->ib_device, "PHYP changed iova_start in "
1173 "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
1174 "mr_handle=%lx lkey=%x lkey_out=%x", iova_start,
1175 hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
1176 e_mr->ib.ib_mr.lkey, hipzout.lkey);
1177 ret = -EFAULT;
1178 } else {
1179 /*
1180 * successful reregistration
1181 * note: start and start_out are identical for eServer HCAs
1182 */
1183 e_mr->num_kpages = pginfo->num_kpages;
1184 e_mr->num_hwpages = pginfo->num_hwpages;
1185 e_mr->start = iova_start;
1186 e_mr->size = size;
1187 e_mr->acl = acl;
1188 *lkey = hipzout.lkey;
1189 *rkey = hipzout.rkey;
1190 }
1191
1192 ehca_rereg_mr_rereg1_exit1:
1193 ehca_free_fw_ctrlblock(kpage);
1194 ehca_rereg_mr_rereg1_exit0:
1195 if ( ret && (ret != -EAGAIN) )
1196 ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x "
1197 "pginfo=%p num_kpages=%lx num_hwpages=%lx",
1198 ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
1199 pginfo->num_hwpages);
1200 return ret;
1201 } /* end ehca_rereg_mr_rereg1() */
1202
1203 /*----------------------------------------------------------------------*/
1204
1205 int ehca_rereg_mr(struct ehca_shca *shca,
1206 struct ehca_mr *e_mr,
1207 u64 *iova_start,
1208 u64 size,
1209 int acl,
1210 struct ehca_pd *e_pd,
1211 struct ehca_mr_pginfo *pginfo,
1212 u32 *lkey,
1213 u32 *rkey)
1214 {
1215 int ret = 0;
1216 u64 h_ret;
1217 int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
1218 int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
1219
1220 /* first determine reregistration hCall(s) */
1221 if ((pginfo->num_hwpages > MAX_RPAGES) ||
1222 (e_mr->num_hwpages > MAX_RPAGES) ||
1223 (pginfo->num_hwpages > e_mr->num_hwpages)) {
1224 ehca_dbg(&shca->ib_device, "Rereg3 case, "
1225 "pginfo->num_hwpages=%lx e_mr->num_hwpages=%x",
1226 pginfo->num_hwpages, e_mr->num_hwpages);
1227 rereg_1_hcall = 0;
1228 rereg_3_hcall = 1;
1229 }
1230
1231 if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
1232 rereg_1_hcall = 0;
1233 rereg_3_hcall = 1;
1234 e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
1235 ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
1236 e_mr);
1237 }
1238
1239 if (rereg_1_hcall) {
1240 ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
1241 acl, e_pd, pginfo, lkey, rkey);
1242 if (ret) {
1243 if (ret == -EAGAIN)
1244 rereg_3_hcall = 1;
1245 else
1246 goto ehca_rereg_mr_exit0;
1247 }
1248 }
1249
1250 if (rereg_3_hcall) {
1251 struct ehca_mr save_mr;
1252
1253 /* first deregister old MR */
1254 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
1255 if (h_ret != H_SUCCESS) {
1256 ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1257 "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
1258 "mr->lkey=%x",
1259 h_ret, e_mr, shca->ipz_hca_handle.handle,
1260 e_mr->ipz_mr_handle.handle,
1261 e_mr->ib.ib_mr.lkey);
1262 ret = ehca2ib_return_code(h_ret);
1263 goto ehca_rereg_mr_exit0;
1264 }
1265 /* clean ehca_mr_t, without changing struct ib_mr and lock */
1266 save_mr = *e_mr;
1267 ehca_mr_deletenew(e_mr);
1268
1269 /* set some MR values */
1270 e_mr->flags = save_mr.flags;
1271 e_mr->fmr_page_size = save_mr.fmr_page_size;
1272 e_mr->fmr_max_pages = save_mr.fmr_max_pages;
1273 e_mr->fmr_max_maps = save_mr.fmr_max_maps;
1274 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
1275
1276 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
1277 e_pd, pginfo, lkey, rkey);
1278 if (ret) {
1279 u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
1280 memcpy(&e_mr->flags, &(save_mr.flags),
1281 sizeof(struct ehca_mr) - offset);
1282 goto ehca_rereg_mr_exit0;
1283 }
1284 }
1285
1286 ehca_rereg_mr_exit0:
1287 if (ret)
1288 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1289 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1290 "num_kpages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
1291 "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
1292 acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
1293 rereg_1_hcall, rereg_3_hcall);
1294 return ret;
1295 } /* end ehca_rereg_mr() */
1296
1297 /*----------------------------------------------------------------------*/
1298
1299 int ehca_unmap_one_fmr(struct ehca_shca *shca,
1300 struct ehca_mr *e_fmr)
1301 {
1302 int ret = 0;
1303 u64 h_ret;
1304 struct ehca_pd *e_pd =
1305 container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
1306 struct ehca_mr save_fmr;
1307 u32 tmp_lkey, tmp_rkey;
1308 struct ehca_mr_pginfo pginfo;
1309 struct ehca_mr_hipzout_parms hipzout;
1310 struct ehca_mr save_mr;
1311
1312 if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
1313 /*
1314 * note: after using rereg hcall with len=0,
1315 * rereg hcall must be used again for registering pages
1316 */
1317 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
1318 0, 0, e_pd->fw_pd, 0, &hipzout);
1319 if (h_ret == H_SUCCESS) {
1320 /* successful reregistration */
1321 e_fmr->start = NULL;
1322 e_fmr->size = 0;
1323 tmp_lkey = hipzout.lkey;
1324 tmp_rkey = hipzout.rkey;
1325 return 0;
1326 }
1327 /*
1328 * should not happen, because length checked above,
1329 * FMRs are not shared and no MW bound to FMRs
1330 */
1331 ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
1332 "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
1333 "mr_hndl=%lx lkey=%x lkey_out=%x",
1334 h_ret, e_fmr, shca->ipz_hca_handle.handle,
1335 e_fmr->ipz_mr_handle.handle,
1336 e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
1337 /* try free and rereg */
1338 }
1339
1340 /* first free old FMR */
1341 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
1342 if (h_ret != H_SUCCESS) {
1343 ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1344 "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
1345 "lkey=%x",
1346 h_ret, e_fmr, shca->ipz_hca_handle.handle,
1347 e_fmr->ipz_mr_handle.handle,
1348 e_fmr->ib.ib_fmr.lkey);
1349 ret = ehca2ib_return_code(h_ret);
1350 goto ehca_unmap_one_fmr_exit0;
1351 }
1352 /* clean ehca_mr_t, without changing lock */
1353 save_fmr = *e_fmr;
1354 ehca_mr_deletenew(e_fmr);
1355
1356 /* set some MR values */
1357 e_fmr->flags = save_fmr.flags;
1358 e_fmr->fmr_page_size = save_fmr.fmr_page_size;
1359 e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
1360 e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
1361 e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
1362 e_fmr->acl = save_fmr.acl;
1363
1364 memset(&pginfo, 0, sizeof(pginfo));
1365 pginfo.type = EHCA_MR_PGI_FMR;
1366 pginfo.num_kpages = 0;
1367 pginfo.num_hwpages = 0;
1368 ret = ehca_reg_mr(shca, e_fmr, NULL,
1369 (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
1370 e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
1371 &tmp_rkey);
1372 if (ret) {
1373 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
1374 memcpy(&e_fmr->flags, &(save_mr.flags),
1375 sizeof(struct ehca_mr) - offset);
1376 goto ehca_unmap_one_fmr_exit0;
1377 }
1378
1379 ehca_unmap_one_fmr_exit0:
1380 if (ret)
1381 ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x "
1382 "fmr_max_pages=%x",
1383 ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
1384 return ret;
1385 } /* end ehca_unmap_one_fmr() */
1386
1387 /*----------------------------------------------------------------------*/
1388
1389 int ehca_reg_smr(struct ehca_shca *shca,
1390 struct ehca_mr *e_origmr,
1391 struct ehca_mr *e_newmr,
1392 u64 *iova_start,
1393 int acl,
1394 struct ehca_pd *e_pd,
1395 u32 *lkey, /*OUT*/
1396 u32 *rkey) /*OUT*/
1397 {
1398 int ret = 0;
1399 u64 h_ret;
1400 u32 hipz_acl;
1401 struct ehca_mr_hipzout_parms hipzout;
1402
1403 ehca_mrmw_map_acl(acl, &hipz_acl);
1404 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1405
1406 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1407 (u64)iova_start, hipz_acl, e_pd->fw_pd,
1408 &hipzout);
1409 if (h_ret != H_SUCCESS) {
1410 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1411 "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
1412 "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1413 h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
1414 shca->ipz_hca_handle.handle,
1415 e_origmr->ipz_mr_handle.handle,
1416 e_origmr->ib.ib_mr.lkey);
1417 ret = ehca2ib_return_code(h_ret);
1418 goto ehca_reg_smr_exit0;
1419 }
1420 /* successful registration */
1421 e_newmr->num_kpages = e_origmr->num_kpages;
1422 e_newmr->num_hwpages = e_origmr->num_hwpages;
1423 e_newmr->start = iova_start;
1424 e_newmr->size = e_origmr->size;
1425 e_newmr->acl = acl;
1426 e_newmr->ipz_mr_handle = hipzout.handle;
1427 *lkey = hipzout.lkey;
1428 *rkey = hipzout.rkey;
1429 return 0;
1430
1431 ehca_reg_smr_exit0:
1432 if (ret)
1433 ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p "
1434 "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
1435 ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
1436 return ret;
1437 } /* end ehca_reg_smr() */
1438
1439 /*----------------------------------------------------------------------*/
1440
1441 /* register internal max-MR to internal SHCA */
1442 int ehca_reg_internal_maxmr(
1443 struct ehca_shca *shca,
1444 struct ehca_pd *e_pd,
1445 struct ehca_mr **e_maxmr) /*OUT*/
1446 {
1447 int ret;
1448 struct ehca_mr *e_mr;
1449 u64 *iova_start;
1450 u64 size_maxmr;
1451 struct ehca_mr_pginfo pginfo;
1452 struct ib_phys_buf ib_pbuf;
1453 u32 num_kpages;
1454 u32 num_hwpages;
1455
1456 e_mr = ehca_mr_new();
1457 if (!e_mr) {
1458 ehca_err(&shca->ib_device, "out of memory");
1459 ret = -ENOMEM;
1460 goto ehca_reg_internal_maxmr_exit0;
1461 }
1462 e_mr->flags |= EHCA_MR_FLAG_MAXMR;
1463
1464 /* register internal max-MR on HCA */
1465 size_maxmr = (u64)high_memory - PAGE_OFFSET;
1466 iova_start = (u64 *)KERNELBASE;
1467 ib_pbuf.addr = 0;
1468 ib_pbuf.size = size_maxmr;
1469 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
1470 PAGE_SIZE);
1471 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr,
1472 EHCA_PAGESIZE);
1473
1474 memset(&pginfo, 0, sizeof(pginfo));
1475 pginfo.type = EHCA_MR_PGI_PHYS;
1476 pginfo.num_kpages = num_kpages;
1477 pginfo.num_hwpages = num_hwpages;
1478 pginfo.u.phy.num_phys_buf = 1;
1479 pginfo.u.phy.phys_buf_array = &ib_pbuf;
1480
1481 ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
1482 &pginfo, &e_mr->ib.ib_mr.lkey,
1483 &e_mr->ib.ib_mr.rkey);
1484 if (ret) {
1485 ehca_err(&shca->ib_device, "reg of internal max MR failed, "
1486 "e_mr=%p iova_start=%p size_maxmr=%lx num_kpages=%x "
1487 "num_hwpages=%x", e_mr, iova_start, size_maxmr,
1488 num_kpages, num_hwpages);
1489 goto ehca_reg_internal_maxmr_exit1;
1490 }
1491
1492 /* successful registration of all pages */
1493 e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
1494 e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
1495 e_mr->ib.ib_mr.uobject = NULL;
1496 atomic_inc(&(e_pd->ib_pd.usecnt));
1497 atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
1498 *e_maxmr = e_mr;
1499 return 0;
1500
1501 ehca_reg_internal_maxmr_exit1:
1502 ehca_mr_delete(e_mr);
1503 ehca_reg_internal_maxmr_exit0:
1504 if (ret)
1505 ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
1506 ret, shca, e_pd, e_maxmr);
1507 return ret;
1508 } /* end ehca_reg_internal_maxmr() */
1509
1510 /*----------------------------------------------------------------------*/
1511
1512 int ehca_reg_maxmr(struct ehca_shca *shca,
1513 struct ehca_mr *e_newmr,
1514 u64 *iova_start,
1515 int acl,
1516 struct ehca_pd *e_pd,
1517 u32 *lkey,
1518 u32 *rkey)
1519 {
1520 u64 h_ret;
1521 struct ehca_mr *e_origmr = shca->maxmr;
1522 u32 hipz_acl;
1523 struct ehca_mr_hipzout_parms hipzout;
1524
1525 ehca_mrmw_map_acl(acl, &hipz_acl);
1526 ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1527
1528 h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1529 (u64)iova_start, hipz_acl, e_pd->fw_pd,
1530 &hipzout);
1531 if (h_ret != H_SUCCESS) {
1532 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1533 "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1534 h_ret, e_origmr, shca->ipz_hca_handle.handle,
1535 e_origmr->ipz_mr_handle.handle,
1536 e_origmr->ib.ib_mr.lkey);
1537 return ehca2ib_return_code(h_ret);
1538 }
1539 /* successful registration */
1540 e_newmr->num_kpages = e_origmr->num_kpages;
1541 e_newmr->num_hwpages = e_origmr->num_hwpages;
1542 e_newmr->start = iova_start;
1543 e_newmr->size = e_origmr->size;
1544 e_newmr->acl = acl;
1545 e_newmr->ipz_mr_handle = hipzout.handle;
1546 *lkey = hipzout.lkey;
1547 *rkey = hipzout.rkey;
1548 return 0;
1549 } /* end ehca_reg_maxmr() */
1550
1551 /*----------------------------------------------------------------------*/
1552
1553 int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
1554 {
1555 int ret;
1556 struct ehca_mr *e_maxmr;
1557 struct ib_pd *ib_pd;
1558
1559 if (!shca->maxmr) {
1560 ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
1561 ret = -EINVAL;
1562 goto ehca_dereg_internal_maxmr_exit0;
1563 }
1564
1565 e_maxmr = shca->maxmr;
1566 ib_pd = e_maxmr->ib.ib_mr.pd;
1567 shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
1568
1569 ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
1570 if (ret) {
1571 ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
1572 "ret=%x e_maxmr=%p shca=%p lkey=%x",
1573 ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
1574 shca->maxmr = e_maxmr;
1575 goto ehca_dereg_internal_maxmr_exit0;
1576 }
1577
1578 atomic_dec(&ib_pd->usecnt);
1579
1580 ehca_dereg_internal_maxmr_exit0:
1581 if (ret)
1582 ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p",
1583 ret, shca, shca->maxmr);
1584 return ret;
1585 } /* end ehca_dereg_internal_maxmr() */
1586
1587 /*----------------------------------------------------------------------*/
1588
1589 /*
1590 * check physical buffer array of MR verbs for validness and
1591 * calculates MR size
1592 */
1593 int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
1594 int num_phys_buf,
1595 u64 *iova_start,
1596 u64 *size)
1597 {
1598 struct ib_phys_buf *pbuf = phys_buf_array;
1599 u64 size_count = 0;
1600 u32 i;
1601
1602 if (num_phys_buf == 0) {
1603 ehca_gen_err("bad phys buf array len, num_phys_buf=0");
1604 return -EINVAL;
1605 }
1606 /* check first buffer */
1607 if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
1608 ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
1609 "pbuf->addr=%lx pbuf->size=%lx",
1610 iova_start, pbuf->addr, pbuf->size);
1611 return -EINVAL;
1612 }
1613 if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
1614 (num_phys_buf > 1)) {
1615 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
1616 "pbuf->size=%lx", pbuf->addr, pbuf->size);
1617 return -EINVAL;
1618 }
1619
1620 for (i = 0; i < num_phys_buf; i++) {
1621 if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
1622 ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
1623 "pbuf->size=%lx",
1624 i, pbuf->addr, pbuf->size);
1625 return -EINVAL;
1626 }
1627 if (((i > 0) && /* not 1st */
1628 (i < (num_phys_buf - 1)) && /* not last */
1629 (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
1630 ehca_gen_err("bad size, i=%x pbuf->size=%lx",
1631 i, pbuf->size);
1632 return -EINVAL;
1633 }
1634 size_count += pbuf->size;
1635 pbuf++;
1636 }
1637
1638 *size = size_count;
1639 return 0;
1640 } /* end ehca_mr_chk_buf_and_calc_size() */
1641
1642 /*----------------------------------------------------------------------*/
1643
1644 /* check page list of map FMR verb for validness */
1645 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
1646 u64 *page_list,
1647 int list_len)
1648 {
1649 u32 i;
1650 u64 *page;
1651
1652 if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
1653 ehca_gen_err("bad list_len, list_len=%x "
1654 "e_fmr->fmr_max_pages=%x fmr=%p",
1655 list_len, e_fmr->fmr_max_pages, e_fmr);
1656 return -EINVAL;
1657 }
1658
1659 /* each page must be aligned */
1660 page = page_list;
1661 for (i = 0; i < list_len; i++) {
1662 if (*page % e_fmr->fmr_page_size) {
1663 ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
1664 "fmr_page_size=%x", i, *page, page, e_fmr,
1665 e_fmr->fmr_page_size);
1666 return -EINVAL;
1667 }
1668 page++;
1669 }
1670
1671 return 0;
1672 } /* end ehca_fmr_check_page_list() */
1673
1674 /*----------------------------------------------------------------------*/
1675
1676 /* PAGE_SIZE >= pginfo->hwpage_size */
1677 static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1678 u32 number,
1679 u64 *kpage)
1680 {
1681 int ret = 0;
1682 struct ib_umem_chunk *prev_chunk;
1683 struct ib_umem_chunk *chunk;
1684 u64 pgaddr;
1685 u32 i = 0;
1686 u32 j = 0;
1687
1688 /* loop over desired chunk entries */
1689 chunk = pginfo->u.usr.next_chunk;
1690 prev_chunk = pginfo->u.usr.next_chunk;
1691 list_for_each_entry_continue(
1692 chunk, (&(pginfo->u.usr.region->chunk_list)), list) {
1693 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) {
1694 pgaddr = page_to_pfn(chunk->page_list[i].page)
1695 << PAGE_SHIFT ;
1696 *kpage = phys_to_abs(pgaddr +
1697 (pginfo->next_hwpage *
1698 EHCA_PAGESIZE));
1699 if ( !(*kpage) ) {
1700 ehca_gen_err("pgaddr=%lx "
1701 "chunk->page_list[i]=%lx "
1702 "i=%x next_hwpage=%lx",
1703 pgaddr, (u64)sg_dma_address(
1704 &chunk->page_list[i]),
1705 i, pginfo->next_hwpage);
1706 return -EFAULT;
1707 }
1708 (pginfo->hwpage_cnt)++;
1709 (pginfo->next_hwpage)++;
1710 kpage++;
1711 if (pginfo->next_hwpage %
1712 (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1713 (pginfo->kpage_cnt)++;
1714 (pginfo->u.usr.next_nmap)++;
1715 pginfo->next_hwpage = 0;
1716 i++;
1717 }
1718 j++;
1719 if (j >= number) break;
1720 }
1721 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
1722 (j >= number)) {
1723 pginfo->u.usr.next_nmap = 0;
1724 prev_chunk = chunk;
1725 break;
1726 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
1727 pginfo->u.usr.next_nmap = 0;
1728 prev_chunk = chunk;
1729 } else if (j >= number)
1730 break;
1731 else
1732 prev_chunk = chunk;
1733 }
1734 pginfo->u.usr.next_chunk =
1735 list_prepare_entry(prev_chunk,
1736 (&(pginfo->u.usr.region->chunk_list)),
1737 list);
1738 return ret;
1739 }
1740
1741 int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1742 u32 number,
1743 u64 *kpage)
1744 {
1745 int ret = 0;
1746 struct ib_phys_buf *pbuf;
1747 u64 num_hw, offs_hw;
1748 u32 i = 0;
1749
1750 /* loop over desired phys_buf_array entries */
1751 while (i < number) {
1752 pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
1753 num_hw = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) +
1754 pbuf->size, EHCA_PAGESIZE);
1755 offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
1756 while (pginfo->next_hwpage < offs_hw + num_hw) {
1757 /* sanity check */
1758 if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
1759 (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
1760 ehca_gen_err("kpage_cnt >= num_kpages, "
1761 "kpage_cnt=%lx num_kpages=%lx "
1762 "hwpage_cnt=%lx "
1763 "num_hwpages=%lx i=%x",
1764 pginfo->kpage_cnt,
1765 pginfo->num_kpages,
1766 pginfo->hwpage_cnt,
1767 pginfo->num_hwpages, i);
1768 return -EFAULT;
1769 }
1770 *kpage = phys_to_abs(
1771 (pbuf->addr & EHCA_PAGEMASK)
1772 + (pginfo->next_hwpage * EHCA_PAGESIZE));
1773 if ( !(*kpage) && pbuf->addr ) {
1774 ehca_gen_err("pbuf->addr=%lx "
1775 "pbuf->size=%lx "
1776 "next_hwpage=%lx", pbuf->addr,
1777 pbuf->size,
1778 pginfo->next_hwpage);
1779 return -EFAULT;
1780 }
1781 (pginfo->hwpage_cnt)++;
1782 (pginfo->next_hwpage)++;
1783 if (pginfo->next_hwpage %
1784 (PAGE_SIZE / EHCA_PAGESIZE) == 0)
1785 (pginfo->kpage_cnt)++;
1786 kpage++;
1787 i++;
1788 if (i >= number) break;
1789 }
1790 if (pginfo->next_hwpage >= offs_hw + num_hw) {
1791 (pginfo->u.phy.next_buf)++;
1792 pginfo->next_hwpage = 0;
1793 }
1794 }
1795 return ret;
1796 }
1797
1798 int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
1799 u32 number,
1800 u64 *kpage)
1801 {
1802 int ret = 0;
1803 u64 *fmrlist;
1804 u32 i;
1805
1806 /* loop over desired page_list entries */
1807 fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
1808 for (i = 0; i < number; i++) {
1809 *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
1810 pginfo->next_hwpage * EHCA_PAGESIZE);
1811 if ( !(*kpage) ) {
1812 ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1813 "next_listelem=%lx next_hwpage=%lx",
1814 *fmrlist, fmrlist,
1815 pginfo->u.fmr.next_listelem,
1816 pginfo->next_hwpage);
1817 return -EFAULT;
1818 }
1819 (pginfo->hwpage_cnt)++;
1820 (pginfo->next_hwpage)++;
1821 kpage++;
1822 if (pginfo->next_hwpage %
1823 (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) {
1824 (pginfo->kpage_cnt)++;
1825 (pginfo->u.fmr.next_listelem)++;
1826 fmrlist++;
1827 pginfo->next_hwpage = 0;
1828 }
1829 }
1830 return ret;
1831 }
1832
1833 /* setup page buffer from page info */
1834 int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
1835 u32 number,
1836 u64 *kpage)
1837 {
1838 int ret;
1839
1840 switch (pginfo->type) {
1841 case EHCA_MR_PGI_PHYS:
1842 ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
1843 break;
1844 case EHCA_MR_PGI_USER:
1845 ret = ehca_set_pagebuf_user1(pginfo, number, kpage);
1846 break;
1847 case EHCA_MR_PGI_FMR:
1848 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
1849 break;
1850 default:
1851 ehca_gen_err("bad pginfo->type=%x", pginfo->type);
1852 ret = -EFAULT;
1853 break;
1854 }
1855 return ret;
1856 } /* end ehca_set_pagebuf() */
1857
1858 /*----------------------------------------------------------------------*/
1859
1860 /*
1861 * check MR if it is a max-MR, i.e. uses whole memory
1862 * in case it's a max-MR 1 is returned, else 0
1863 */
1864 int ehca_mr_is_maxmr(u64 size,
1865 u64 *iova_start)
1866 {
1867 /* a MR is treated as max-MR only if it fits following: */
1868 if ((size == ((u64)high_memory - PAGE_OFFSET)) &&
1869 (iova_start == (void *)KERNELBASE)) {
1870 ehca_gen_dbg("this is a max-MR");
1871 return 1;
1872 } else
1873 return 0;
1874 } /* end ehca_mr_is_maxmr() */
1875
1876 /*----------------------------------------------------------------------*/
1877
1878 /* map access control for MR/MW. This routine is used for MR and MW. */
1879 void ehca_mrmw_map_acl(int ib_acl,
1880 u32 *hipz_acl)
1881 {
1882 *hipz_acl = 0;
1883 if (ib_acl & IB_ACCESS_REMOTE_READ)
1884 *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
1885 if (ib_acl & IB_ACCESS_REMOTE_WRITE)
1886 *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
1887 if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
1888 *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
1889 if (ib_acl & IB_ACCESS_LOCAL_WRITE)
1890 *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
1891 if (ib_acl & IB_ACCESS_MW_BIND)
1892 *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
1893 } /* end ehca_mrmw_map_acl() */
1894
1895 /*----------------------------------------------------------------------*/
1896
1897 /* sets page size in hipz access control for MR/MW. */
1898 void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
1899 {
1900 return; /* HCA supports only 4k */
1901 } /* end ehca_mrmw_set_pgsize_hipz_acl() */
1902
1903 /*----------------------------------------------------------------------*/
1904
1905 /*
1906 * reverse map access control for MR/MW.
1907 * This routine is used for MR and MW.
1908 */
1909 void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
1910 int *ib_acl) /*OUT*/
1911 {
1912 *ib_acl = 0;
1913 if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
1914 *ib_acl |= IB_ACCESS_REMOTE_READ;
1915 if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
1916 *ib_acl |= IB_ACCESS_REMOTE_WRITE;
1917 if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
1918 *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
1919 if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
1920 *ib_acl |= IB_ACCESS_LOCAL_WRITE;
1921 if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
1922 *ib_acl |= IB_ACCESS_MW_BIND;
1923 } /* end ehca_mrmw_reverse_map_acl() */
1924
1925
1926 /*----------------------------------------------------------------------*/
1927
1928 /*
1929 * MR destructor and constructor
1930 * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
1931 * except struct ib_mr and spinlock
1932 */
1933 void ehca_mr_deletenew(struct ehca_mr *mr)
1934 {
1935 mr->flags = 0;
1936 mr->num_kpages = 0;
1937 mr->num_hwpages = 0;
1938 mr->acl = 0;
1939 mr->start = NULL;
1940 mr->fmr_page_size = 0;
1941 mr->fmr_max_pages = 0;
1942 mr->fmr_max_maps = 0;
1943 mr->fmr_map_cnt = 0;
1944 memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
1945 memset(&mr->galpas, 0, sizeof(mr->galpas));
1946 } /* end ehca_mr_deletenew() */
1947
1948 int ehca_init_mrmw_cache(void)
1949 {
1950 mr_cache = kmem_cache_create("ehca_cache_mr",
1951 sizeof(struct ehca_mr), 0,
1952 SLAB_HWCACHE_ALIGN,
1953 NULL, NULL);
1954 if (!mr_cache)
1955 return -ENOMEM;
1956 mw_cache = kmem_cache_create("ehca_cache_mw",
1957 sizeof(struct ehca_mw), 0,
1958 SLAB_HWCACHE_ALIGN,
1959 NULL, NULL);
1960 if (!mw_cache) {
1961 kmem_cache_destroy(mr_cache);
1962 mr_cache = NULL;
1963 return -ENOMEM;
1964 }
1965 return 0;
1966 }
1967
1968 void ehca_cleanup_mrmw_cache(void)
1969 {
1970 if (mr_cache)
1971 kmem_cache_destroy(mr_cache);
1972 if (mw_cache)
1973 kmem_cache_destroy(mw_cache);
1974 }
kernel source for telekom puls (alcatel/mediatek)