projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/addr: Keep pointer to netdevice in struct rdma_dev_addr
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / infiniband / core / cma.c
CommitLineData
e51060f0
SH		 1/*
2 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
6 *
a9474917
SH		 7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
e51060f0 12 *
a9474917
SH		 13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
e51060f0 16 *
a9474917
SH		 17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
e51060f0 20 *
a9474917
SH		 21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
e51060f0 25 *
a9474917
SH		 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
e51060f0
SH		 34 */
35
36#include <linux/completion.h>
37#include <linux/in.h>
38#include <linux/in6.h>
39#include <linux/mutex.h>
40#include <linux/random.h>
41#include <linux/idr.h>
07ebafba 42#include <linux/inetdevice.h>
e51060f0
SH		 43
44#include <net/tcp.h>
45
46#include <rdma/rdma_cm.h>
47#include <rdma/rdma_cm_ib.h>
48#include <rdma/ib_cache.h>
49#include <rdma/ib_cm.h>
50#include <rdma/ib_sa.h>
07ebafba 51#include <rdma/iw_cm.h>
e51060f0
SH		 52
53MODULE_AUTHOR("Sean Hefty");
54MODULE_DESCRIPTION("Generic RDMA CM Agent");
55MODULE_LICENSE("Dual BSD/GPL");
56
57#define CMA_CM_RESPONSE_TIMEOUT 20
d5bb7599 58#define CMA_MAX_CM_RETRIES 15
dcb3f974 59#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
e51060f0
SH		 60
61static void cma_add_one(struct ib_device *device);
62static void cma_remove_one(struct ib_device *device);
63
64static struct ib_client cma_client = {
65 .name = "cma",
66 .add = cma_add_one,
67 .remove = cma_remove_one
68};
69
c1a0b23b 70static struct ib_sa_client sa_client;
7a118df3 71static struct rdma_addr_client addr_client;
e51060f0
SH		 72static LIST_HEAD(dev_list);
73static LIST_HEAD(listen_any_list);
74static DEFINE_MUTEX(lock);
75static struct workqueue_struct *cma_wq;
76static DEFINE_IDR(sdp_ps);
77static DEFINE_IDR(tcp_ps);
628e5f6d 78static DEFINE_IDR(udp_ps);
c8f6a362 79static DEFINE_IDR(ipoib_ps);
aedec080 80static int next_port;
e51060f0
SH		 81
82struct cma_device {
83 struct list_head list;
84 struct ib_device *device;
e51060f0
SH		 85 struct completion comp;
86 atomic_t refcount;
87 struct list_head id_list;
88};
89
90enum cma_state {
91 CMA_IDLE,
92 CMA_ADDR_QUERY,
93 CMA_ADDR_RESOLVED,
94 CMA_ROUTE_QUERY,
95 CMA_ROUTE_RESOLVED,
96 CMA_CONNECT,
97 CMA_DISCONNECT,
98 CMA_ADDR_BOUND,
99 CMA_LISTEN,
100 CMA_DEVICE_REMOVAL,
101 CMA_DESTROYING
102};
103
104struct rdma_bind_list {
105 struct idr *ps;
106 struct hlist_head owners;
107 unsigned short port;
108};
109
110/*
111 * Device removal can occur at anytime, so we need extra handling to
112 * serialize notifying the user of device removal with other callbacks.
113 * We do this by disabling removal notification while a callback is in process,
114 * and reporting it after the callback completes.
115 */
116struct rdma_id_private {
117 struct rdma_cm_id id;
118
119 struct rdma_bind_list *bind_list;
120 struct hlist_node node;
d02d1f53
SH		 121 struct list_head list; /* listen_any_list or cma_device.list */
122 struct list_head listen_list; /* per device listens */
e51060f0 123 struct cma_device *cma_dev;
c8f6a362 124 struct list_head mc_list;
e51060f0 125
d02d1f53 126 int internal_id;
e51060f0
SH		 127 enum cma_state state;
128 spinlock_t lock;
c5483388
SH		 129 struct mutex qp_mutex;
130
e51060f0
SH		 131 struct completion comp;
132 atomic_t refcount;
133 wait_queue_head_t wait_remove;
134 atomic_t dev_remove;
135
136 int backlog;
137 int timeout_ms;
138 struct ib_sa_query *query;
139 int query_id;
140 union {
141 struct ib_cm_id *ib;
07ebafba 142 struct iw_cm_id *iw;
e51060f0
SH		 143 } cm_id;
144
145 u32 seq_num;
c8f6a362 146 u32 qkey;
e51060f0 147 u32 qp_num;
e51060f0 148 u8 srq;
a81c994d 149 u8 tos;
e51060f0
SH		 150};
151
c8f6a362
SH		 152struct cma_multicast {
153 struct rdma_id_private *id_priv;
154 union {
155 struct ib_sa_multicast *ib;
156 } multicast;
157 struct list_head list;
158 void *context;
159 struct sockaddr addr;
160 u8 pad[sizeof(struct sockaddr_in6) -
161 sizeof(struct sockaddr)];
162};
163
e51060f0
SH		 164struct cma_work {
165 struct work_struct work;
166 struct rdma_id_private *id;
167 enum cma_state old_state;
168 enum cma_state new_state;
169 struct rdma_cm_event event;
170};
171
172union cma_ip_addr {
173 struct in6_addr ip6;
174 struct {
1b90c137
AV		 175 __be32 pad[3];
176 __be32 addr;
e51060f0
SH		 177 } ip4;
178};
179
180struct cma_hdr {
181 u8 cma_version;
182 u8 ip_version; /* IP version: 7:4 */
1b90c137 183 __be16 port;
e51060f0
SH		 184 union cma_ip_addr src_addr;
185 union cma_ip_addr dst_addr;
186};
187
188struct sdp_hh {
189 u8 bsdh[16];
190 u8 sdp_version; /* Major version: 7:4 */
191 u8 ip_version; /* IP version: 7:4 */
192 u8 sdp_specific1[10];
1b90c137
AV		 193 __be16 port;
194 __be16 sdp_specific2;
e51060f0
SH		 195 union cma_ip_addr src_addr;
196 union cma_ip_addr dst_addr;
197};
198
199struct sdp_hah {
200 u8 bsdh[16];
201 u8 sdp_version;
202};
203
204#define CMA_VERSION 0x00
205#define SDP_MAJ_VERSION 0x2
206
207static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
208{
209 unsigned long flags;
210 int ret;
211
212 spin_lock_irqsave(&id_priv->lock, flags);
213 ret = (id_priv->state == comp);
214 spin_unlock_irqrestore(&id_priv->lock, flags);
215 return ret;
216}
217
218static int cma_comp_exch(struct rdma_id_private *id_priv,
219 enum cma_state comp, enum cma_state exch)
220{
221 unsigned long flags;
222 int ret;
223
224 spin_lock_irqsave(&id_priv->lock, flags);
225 if ((ret = (id_priv->state == comp)))
226 id_priv->state = exch;
227 spin_unlock_irqrestore(&id_priv->lock, flags);
228 return ret;
229}
230
231static enum cma_state cma_exch(struct rdma_id_private *id_priv,
232 enum cma_state exch)
233{
234 unsigned long flags;
235 enum cma_state old;
236
237 spin_lock_irqsave(&id_priv->lock, flags);
238 old = id_priv->state;
239 id_priv->state = exch;
240 spin_unlock_irqrestore(&id_priv->lock, flags);
241 return old;
242}
243
244static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
245{
246 return hdr->ip_version >> 4;
247}
248
249static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
250{
251 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
252}
253
254static inline u8 sdp_get_majv(u8 sdp_version)
255{
256 return sdp_version >> 4;
257}
258
259static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
260{
261 return hh->ip_version >> 4;
262}
263
264static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
265{
266 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
267}
268
c8f6a362
SH		 269static inline int cma_is_ud_ps(enum rdma_port_space ps)
270{
271 return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
272}
273
e51060f0
SH		 274static void cma_attach_to_dev(struct rdma_id_private *id_priv,
275 struct cma_device *cma_dev)
276{
277 atomic_inc(&cma_dev->refcount);
278 id_priv->cma_dev = cma_dev;
279 id_priv->id.device = cma_dev->device;
280 list_add_tail(&id_priv->list, &cma_dev->id_list);
281}
282
283static inline void cma_deref_dev(struct cma_device *cma_dev)
284{
285 if (atomic_dec_and_test(&cma_dev->refcount))
286 complete(&cma_dev->comp);
287}
288
289static void cma_detach_from_dev(struct rdma_id_private *id_priv)
290{
291 list_del(&id_priv->list);
292 cma_deref_dev(id_priv->cma_dev);
293 id_priv->cma_dev = NULL;
294}
295
c8f6a362
SH		 296static int cma_set_qkey(struct ib_device *device, u8 port_num,
297 enum rdma_port_space ps,
298 struct rdma_dev_addr *dev_addr, u32 *qkey)
299{
300 struct ib_sa_mcmember_rec rec;
301 int ret = 0;
302
303 switch (ps) {
304 case RDMA_PS_UDP:
305 *qkey = RDMA_UDP_QKEY;
306 break;
307 case RDMA_PS_IPOIB:
308 ib_addr_get_mgid(dev_addr, &rec.mgid);
309 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
310 *qkey = be32_to_cpu(rec.qkey);
311 break;
312 default:
313 break;
314 }
315 return ret;
316}
317
07ebafba 318static int cma_acquire_dev(struct rdma_id_private *id_priv)
e51060f0 319{
c8f6a362 320 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
e51060f0 321 struct cma_device *cma_dev;
f0ee3404 322 union ib_gid gid;
e51060f0
SH		 323 int ret = -ENODEV;
324
c8f6a362 325 switch (rdma_node_get_transport(dev_addr->dev_type)) {
07ebafba 326 case RDMA_TRANSPORT_IB:
c8f6a362 327 ib_addr_get_sgid(dev_addr, &gid);
07ebafba
TT		 328 break;
329 case RDMA_TRANSPORT_IWARP:
c8f6a362 330 iw_addr_get_sgid(dev_addr, &gid);
07ebafba
TT		 331 break;
332 default:
333 return -ENODEV;
334 }
61a73c70 335
e51060f0 336 list_for_each_entry(cma_dev, &dev_list, list) {
f0ee3404 337 ret = ib_find_cached_gid(cma_dev->device, &gid,
e51060f0
SH		 338 &id_priv->id.port_num, NULL);
339 if (!ret) {
c8f6a362
SH		 340 ret = cma_set_qkey(cma_dev->device,
341 id_priv->id.port_num,
342 id_priv->id.ps, dev_addr,
343 &id_priv->qkey);
344 if (!ret)
345 cma_attach_to_dev(id_priv, cma_dev);
e51060f0
SH		 346 break;
347 }
348 }
e51060f0
SH		 349 return ret;
350}
351
e51060f0
SH		 352static void cma_deref_id(struct rdma_id_private *id_priv)
353{
354 if (atomic_dec_and_test(&id_priv->refcount))
355 complete(&id_priv->comp);
356}
357
8aa08602
SH		 358static int cma_disable_remove(struct rdma_id_private *id_priv,
359 enum cma_state state)
360{
361 unsigned long flags;
362 int ret;
363
364 spin_lock_irqsave(&id_priv->lock, flags);
365 if (id_priv->state == state) {
366 atomic_inc(&id_priv->dev_remove);
367 ret = 0;
368 } else
369 ret = -EINVAL;
370 spin_unlock_irqrestore(&id_priv->lock, flags);
371 return ret;
372}
373
374static void cma_enable_remove(struct rdma_id_private *id_priv)
e51060f0
SH		 375{
376 if (atomic_dec_and_test(&id_priv->dev_remove))
377 wake_up(&id_priv->wait_remove);
378}
379
6c719f5c
SH		 380static int cma_has_cm_dev(struct rdma_id_private *id_priv)
381{
382 return (id_priv->id.device && id_priv->cm_id.ib);
383}
384
e51060f0
SH		 385struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
386 void *context, enum rdma_port_space ps)
387{
388 struct rdma_id_private *id_priv;
389
390 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
391 if (!id_priv)
392 return ERR_PTR(-ENOMEM);
393
394 id_priv->state = CMA_IDLE;
395 id_priv->id.context = context;
396 id_priv->id.event_handler = event_handler;
397 id_priv->id.ps = ps;
398 spin_lock_init(&id_priv->lock);
c5483388 399 mutex_init(&id_priv->qp_mutex);
e51060f0
SH		 400 init_completion(&id_priv->comp);
401 atomic_set(&id_priv->refcount, 1);
402 init_waitqueue_head(&id_priv->wait_remove);
403 atomic_set(&id_priv->dev_remove, 0);
404 INIT_LIST_HEAD(&id_priv->listen_list);
c8f6a362 405 INIT_LIST_HEAD(&id_priv->mc_list);
e51060f0
SH		 406 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
407
408 return &id_priv->id;
409}
410EXPORT_SYMBOL(rdma_create_id);
411
c8f6a362 412static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
e51060f0
SH		 413{
414 struct ib_qp_attr qp_attr;
c8f6a362 415 int qp_attr_mask, ret;
e51060f0 416
c8f6a362
SH		 417 qp_attr.qp_state = IB_QPS_INIT;
418 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0
SH		 419 if (ret)
420 return ret;
421
c8f6a362
SH		 422 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
423 if (ret)
424 return ret;
425
426 qp_attr.qp_state = IB_QPS_RTR;
427 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
428 if (ret)
429 return ret;
430
431 qp_attr.qp_state = IB_QPS_RTS;
432 qp_attr.sq_psn = 0;
433 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
434
435 return ret;
e51060f0
SH		 436}
437
c8f6a362 438static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
07ebafba
TT		 439{
440 struct ib_qp_attr qp_attr;
c8f6a362 441 int qp_attr_mask, ret;
07ebafba
TT		 442
443 qp_attr.qp_state = IB_QPS_INIT;
c8f6a362
SH		 444 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
445 if (ret)
446 return ret;
07ebafba 447
c8f6a362 448 return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
07ebafba
TT		 449}
450
e51060f0
SH		 451int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
452 struct ib_qp_init_attr *qp_init_attr)
453{
454 struct rdma_id_private *id_priv;
455 struct ib_qp *qp;
456 int ret;
457
458 id_priv = container_of(id, struct rdma_id_private, id);
459 if (id->device != pd->device)
460 return -EINVAL;
461
462 qp = ib_create_qp(pd, qp_init_attr);
463 if (IS_ERR(qp))
464 return PTR_ERR(qp);
465
c8f6a362
SH		 466 if (cma_is_ud_ps(id_priv->id.ps))
467 ret = cma_init_ud_qp(id_priv, qp);
468 else
469 ret = cma_init_conn_qp(id_priv, qp);
e51060f0
SH		 470 if (ret)
471 goto err;
472
473 id->qp = qp;
474 id_priv->qp_num = qp->qp_num;
e51060f0
SH		 475 id_priv->srq = (qp->srq != NULL);
476 return 0;
477err:
478 ib_destroy_qp(qp);
479 return ret;
480}
481EXPORT_SYMBOL(rdma_create_qp);
482
483void rdma_destroy_qp(struct rdma_cm_id *id)
484{
c5483388
SH		 485 struct rdma_id_private *id_priv;
486
487 id_priv = container_of(id, struct rdma_id_private, id);
488 mutex_lock(&id_priv->qp_mutex);
489 ib_destroy_qp(id_priv->id.qp);
490 id_priv->id.qp = NULL;
491 mutex_unlock(&id_priv->qp_mutex);
e51060f0
SH		 492}
493EXPORT_SYMBOL(rdma_destroy_qp);
494
5851bb89
SH		 495static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
496 struct rdma_conn_param *conn_param)
e51060f0
SH		 497{
498 struct ib_qp_attr qp_attr;
499 int qp_attr_mask, ret;
500
c5483388
SH		 501 mutex_lock(&id_priv->qp_mutex);
502 if (!id_priv->id.qp) {
503 ret = 0;
504 goto out;
505 }
e51060f0
SH		 506
507 /* Need to update QP attributes from default values. */
508 qp_attr.qp_state = IB_QPS_INIT;
c5483388 509 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 510 if (ret)
c5483388 511 goto out;
e51060f0 512
c5483388 513 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
e51060f0 514 if (ret)
c5483388 515 goto out;
e51060f0
SH		 516
517 qp_attr.qp_state = IB_QPS_RTR;
c5483388 518 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 519 if (ret)
c5483388 520 goto out;
e51060f0 521
5851bb89
SH		 522 if (conn_param)
523 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
c5483388
SH		 524 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
525out:
526 mutex_unlock(&id_priv->qp_mutex);
527 return ret;
e51060f0
SH		 528}
529
5851bb89
SH		 530static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
531 struct rdma_conn_param *conn_param)
e51060f0
SH		 532{
533 struct ib_qp_attr qp_attr;
534 int qp_attr_mask, ret;
535
c5483388
SH		 536 mutex_lock(&id_priv->qp_mutex);
537 if (!id_priv->id.qp) {
538 ret = 0;
539 goto out;
540 }
e51060f0
SH		 541
542 qp_attr.qp_state = IB_QPS_RTS;
c5483388 543 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 544 if (ret)
c5483388 545 goto out;
e51060f0 546
5851bb89
SH		 547 if (conn_param)
548 qp_attr.max_rd_atomic = conn_param->initiator_depth;
c5483388
SH		 549 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
550out:
551 mutex_unlock(&id_priv->qp_mutex);
552 return ret;
e51060f0
SH		 553}
554
c5483388 555static int cma_modify_qp_err(struct rdma_id_private *id_priv)
e51060f0
SH		 556{
557 struct ib_qp_attr qp_attr;
c5483388 558 int ret;
e51060f0 559
c5483388
SH		 560 mutex_lock(&id_priv->qp_mutex);
561 if (!id_priv->id.qp) {
562 ret = 0;
563 goto out;
564 }
e51060f0
SH		 565
566 qp_attr.qp_state = IB_QPS_ERR;
c5483388
SH		 567 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
568out:
569 mutex_unlock(&id_priv->qp_mutex);
570 return ret;
e51060f0
SH		 571}
572
c8f6a362
SH		 573static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
574 struct ib_qp_attr *qp_attr, int *qp_attr_mask)
575{
576 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
577 int ret;
578
579 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
580 ib_addr_get_pkey(dev_addr),
581 &qp_attr->pkey_index);
582 if (ret)
583 return ret;
584
585 qp_attr->port_num = id_priv->id.port_num;
586 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
587
588 if (cma_is_ud_ps(id_priv->id.ps)) {
589 qp_attr->qkey = id_priv->qkey;
590 *qp_attr_mask |= IB_QP_QKEY;
591 } else {
592 qp_attr->qp_access_flags = 0;
593 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
594 }
595 return 0;
596}
597
e51060f0
SH		 598int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
599 int *qp_attr_mask)
600{
601 struct rdma_id_private *id_priv;
c8f6a362 602 int ret = 0;
e51060f0
SH		 603
604 id_priv = container_of(id, struct rdma_id_private, id);
07ebafba
TT		 605 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
606 case RDMA_TRANSPORT_IB:
c8f6a362
SH		 607 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
608 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
609 else
610 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
611 qp_attr_mask);
e51060f0
SH		 612 if (qp_attr->qp_state == IB_QPS_RTR)
613 qp_attr->rq_psn = id_priv->seq_num;
614 break;
07ebafba 615 case RDMA_TRANSPORT_IWARP:
c8f6a362 616 if (!id_priv->cm_id.iw) {
8f076531 617 qp_attr->qp_access_flags = 0;
c8f6a362
SH		 618 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
619 } else
620 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
621 qp_attr_mask);
07ebafba 622 break;
e51060f0
SH		 623 default:
624 ret = -ENOSYS;
625 break;
626 }
627
628 return ret;
629}
630EXPORT_SYMBOL(rdma_init_qp_attr);
631
632static inline int cma_zero_addr(struct sockaddr *addr)
633{
634 struct in6_addr *ip6;
635
636 if (addr->sa_family == AF_INET)
6360a02a
JP		 637 return ipv4_is_zeronet(
638 ((struct sockaddr_in *)addr)->sin_addr.s_addr);
e51060f0
SH		 639 else {
640 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
641 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
5fd571cb 642 ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
e51060f0
SH		 643 }
644}
645
646static inline int cma_loopback_addr(struct sockaddr *addr)
647{
6360a02a 648 return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
e51060f0
SH		 649}
650
651static inline int cma_any_addr(struct sockaddr *addr)
652{
653 return cma_zero_addr(addr) || cma_loopback_addr(addr);
654}
655
628e5f6d
SH		 656static inline __be16 cma_port(struct sockaddr *addr)
657{
658 if (addr->sa_family == AF_INET)
659 return ((struct sockaddr_in *) addr)->sin_port;
660 else
661 return ((struct sockaddr_in6 *) addr)->sin6_port;
662}
663
e51060f0
SH		 664static inline int cma_any_port(struct sockaddr *addr)
665{
628e5f6d 666 return !cma_port(addr);
e51060f0
SH		 667}
668
669static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
1b90c137 670 u8 *ip_ver, __be16 *port,
e51060f0
SH		 671 union cma_ip_addr **src, union cma_ip_addr **dst)
672{
673 switch (ps) {
674 case RDMA_PS_SDP:
675 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
676 SDP_MAJ_VERSION)
677 return -EINVAL;
678
679 *ip_ver = sdp_get_ip_ver(hdr);
680 *port = ((struct sdp_hh *) hdr)->port;
681 *src = &((struct sdp_hh *) hdr)->src_addr;
682 *dst = &((struct sdp_hh *) hdr)->dst_addr;
683 break;
684 default:
685 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
686 return -EINVAL;
687
688 *ip_ver = cma_get_ip_ver(hdr);
689 *port = ((struct cma_hdr *) hdr)->port;
690 *src = &((struct cma_hdr *) hdr)->src_addr;
691 *dst = &((struct cma_hdr *) hdr)->dst_addr;
692 break;
693 }
694
695 if (*ip_ver != 4 && *ip_ver != 6)
696 return -EINVAL;
697 return 0;
698}
699
700static void cma_save_net_info(struct rdma_addr *addr,
701 struct rdma_addr *listen_addr,
1b90c137 702 u8 ip_ver, __be16 port,
e51060f0
SH		 703 union cma_ip_addr *src, union cma_ip_addr *dst)
704{
705 struct sockaddr_in *listen4, *ip4;
706 struct sockaddr_in6 *listen6, *ip6;
707
708 switch (ip_ver) {
709 case 4:
710 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
711 ip4 = (struct sockaddr_in *) &addr->src_addr;
712 ip4->sin_family = listen4->sin_family;
713 ip4->sin_addr.s_addr = dst->ip4.addr;
714 ip4->sin_port = listen4->sin_port;
715
716 ip4 = (struct sockaddr_in *) &addr->dst_addr;
717 ip4->sin_family = listen4->sin_family;
718 ip4->sin_addr.s_addr = src->ip4.addr;
719 ip4->sin_port = port;
720 break;
721 case 6:
722 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
723 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
724 ip6->sin6_family = listen6->sin6_family;
725 ip6->sin6_addr = dst->ip6;
726 ip6->sin6_port = listen6->sin6_port;
727
728 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
729 ip6->sin6_family = listen6->sin6_family;
730 ip6->sin6_addr = src->ip6;
731 ip6->sin6_port = port;
732 break;
733 default:
734 break;
735 }
736}
737
738static inline int cma_user_data_offset(enum rdma_port_space ps)
739{
740 switch (ps) {
741 case RDMA_PS_SDP:
742 return 0;
743 default:
744 return sizeof(struct cma_hdr);
745 }
746}
747
e51060f0
SH		 748static void cma_cancel_route(struct rdma_id_private *id_priv)
749{
07ebafba
TT		 750 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
751 case RDMA_TRANSPORT_IB:
e51060f0
SH		 752 if (id_priv->query)
753 ib_sa_cancel_query(id_priv->query_id, id_priv->query);
754 break;
755 default:
756 break;
757 }
758}
759
e51060f0
SH		 760static void cma_cancel_listens(struct rdma_id_private *id_priv)
761{
762 struct rdma_id_private *dev_id_priv;
763
d02d1f53
SH		 764 /*
765 * Remove from listen_any_list to prevent added devices from spawning
766 * additional listen requests.
767 */
e51060f0
SH		 768 mutex_lock(&lock);
769 list_del(&id_priv->list);
770
771 while (!list_empty(&id_priv->listen_list)) {
772 dev_id_priv = list_entry(id_priv->listen_list.next,
773 struct rdma_id_private, listen_list);
d02d1f53
SH		 774 /* sync with device removal to avoid duplicate destruction */
775 list_del_init(&dev_id_priv->list);
776 list_del(&dev_id_priv->listen_list);
777 mutex_unlock(&lock);
778
779 rdma_destroy_id(&dev_id_priv->id);
780 mutex_lock(&lock);
e51060f0
SH		 781 }
782 mutex_unlock(&lock);
783}
784
785static void cma_cancel_operation(struct rdma_id_private *id_priv,
786 enum cma_state state)
787{
788 switch (state) {
789 case CMA_ADDR_QUERY:
790 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
791 break;
792 case CMA_ROUTE_QUERY:
793 cma_cancel_route(id_priv);
794 break;
795 case CMA_LISTEN:
796 if (cma_any_addr(&id_priv->id.route.addr.src_addr) &&
797 !id_priv->cma_dev)
798 cma_cancel_listens(id_priv);
799 break;
800 default:
801 break;
802 }
803}
804
805static void cma_release_port(struct rdma_id_private *id_priv)
806{
807 struct rdma_bind_list *bind_list = id_priv->bind_list;
808
809 if (!bind_list)
810 return;
811
812 mutex_lock(&lock);
813 hlist_del(&id_priv->node);
814 if (hlist_empty(&bind_list->owners)) {
815 idr_remove(bind_list->ps, bind_list->port);
816 kfree(bind_list);
817 }
818 mutex_unlock(&lock);
819}
820
c8f6a362
SH		 821static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
822{
823 struct cma_multicast *mc;
824
825 while (!list_empty(&id_priv->mc_list)) {
826 mc = container_of(id_priv->mc_list.next,
827 struct cma_multicast, list);
828 list_del(&mc->list);
829 ib_sa_free_multicast(mc->multicast.ib);
830 kfree(mc);
831 }
832}
833
e51060f0
SH		 834void rdma_destroy_id(struct rdma_cm_id *id)
835{
836 struct rdma_id_private *id_priv;
837 enum cma_state state;
838
839 id_priv = container_of(id, struct rdma_id_private, id);
840 state = cma_exch(id_priv, CMA_DESTROYING);
841 cma_cancel_operation(id_priv, state);
842
61a73c70 843 mutex_lock(&lock);
e51060f0 844 if (id_priv->cma_dev) {
61a73c70 845 mutex_unlock(&lock);
07ebafba
TT		 846 switch (rdma_node_get_transport(id->device->node_type)) {
847 case RDMA_TRANSPORT_IB:
3cd96564 848 if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
e51060f0
SH		 849 ib_destroy_cm_id(id_priv->cm_id.ib);
850 break;
07ebafba
TT		 851 case RDMA_TRANSPORT_IWARP:
852 if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
853 iw_destroy_cm_id(id_priv->cm_id.iw);
854 break;
e51060f0
SH		 855 default:
856 break;
857 }
c8f6a362 858 cma_leave_mc_groups(id_priv);
3cd96564 859 mutex_lock(&lock);
e51060f0 860 cma_detach_from_dev(id_priv);
e51060f0 861 }
61a73c70 862 mutex_unlock(&lock);
e51060f0
SH		 863
864 cma_release_port(id_priv);
865 cma_deref_id(id_priv);
866 wait_for_completion(&id_priv->comp);
867
d02d1f53
SH		 868 if (id_priv->internal_id)
869 cma_deref_id(id_priv->id.context);
870
e51060f0
SH		 871 kfree(id_priv->id.route.path_rec);
872 kfree(id_priv);
873}
874EXPORT_SYMBOL(rdma_destroy_id);
875
876static int cma_rep_recv(struct rdma_id_private *id_priv)
877{
878 int ret;
879
5851bb89 880 ret = cma_modify_qp_rtr(id_priv, NULL);
e51060f0
SH		 881 if (ret)
882 goto reject;
883
5851bb89 884 ret = cma_modify_qp_rts(id_priv, NULL);
e51060f0
SH		 885 if (ret)
886 goto reject;
887
888 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
889 if (ret)
890 goto reject;
891
892 return 0;
893reject:
c5483388 894 cma_modify_qp_err(id_priv);
e51060f0
SH		 895 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
896 NULL, 0, NULL, 0);
897 return ret;
898}
899
900static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
901{
902 if (id_priv->id.ps == RDMA_PS_SDP &&
903 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
904 SDP_MAJ_VERSION)
905 return -EINVAL;
906
907 return 0;
908}
909
a1b1b61f
SH		 910static void cma_set_rep_event_data(struct rdma_cm_event *event,
911 struct ib_cm_rep_event_param *rep_data,
912 void *private_data)
913{
914 event->param.conn.private_data = private_data;
915 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
916 event->param.conn.responder_resources = rep_data->responder_resources;
917 event->param.conn.initiator_depth = rep_data->initiator_depth;
918 event->param.conn.flow_control = rep_data->flow_control;
919 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
920 event->param.conn.srq = rep_data->srq;
921 event->param.conn.qp_num = rep_data->remote_qpn;
922}
923
e51060f0
SH		 924static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
925{
926 struct rdma_id_private *id_priv = cm_id->context;
a1b1b61f
SH		 927 struct rdma_cm_event event;
928 int ret = 0;
e51060f0 929
8aa08602
SH		 930 if (cma_disable_remove(id_priv, CMA_CONNECT))
931 return 0;
e51060f0 932
a1b1b61f 933 memset(&event, 0, sizeof event);
e51060f0
SH		 934 switch (ib_event->event) {
935 case IB_CM_REQ_ERROR:
936 case IB_CM_REP_ERROR:
a1b1b61f
SH		 937 event.event = RDMA_CM_EVENT_UNREACHABLE;
938 event.status = -ETIMEDOUT;
e51060f0
SH		 939 break;
940 case IB_CM_REP_RECEIVED:
a1b1b61f
SH		 941 event.status = cma_verify_rep(id_priv, ib_event->private_data);
942 if (event.status)
943 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
e51060f0 944 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
a1b1b61f
SH		 945 event.status = cma_rep_recv(id_priv);
946 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
947 RDMA_CM_EVENT_ESTABLISHED;
e51060f0 948 } else
a1b1b61f
SH		 949 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
950 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
951 ib_event->private_data);
e51060f0
SH		 952 break;
953 case IB_CM_RTU_RECEIVED:
0fe313b0
SH		 954 case IB_CM_USER_ESTABLISHED:
955 event.event = RDMA_CM_EVENT_ESTABLISHED;
e51060f0
SH		 956 break;
957 case IB_CM_DREQ_ERROR:
a1b1b61f 958 event.status = -ETIMEDOUT; /* fall through */
e51060f0
SH		 959 case IB_CM_DREQ_RECEIVED:
960 case IB_CM_DREP_RECEIVED:
961 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
962 goto out;
a1b1b61f 963 event.event = RDMA_CM_EVENT_DISCONNECTED;
e51060f0
SH		 964 break;
965 case IB_CM_TIMEWAIT_EXIT:
966 case IB_CM_MRA_RECEIVED:
967 /* ignore event */
968 goto out;
969 case IB_CM_REJ_RECEIVED:
c5483388 970 cma_modify_qp_err(id_priv);
a1b1b61f
SH		 971 event.status = ib_event->param.rej_rcvd.reason;
972 event.event = RDMA_CM_EVENT_REJECTED;
973 event.param.conn.private_data = ib_event->private_data;
974 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
e51060f0
SH		 975 break;
976 default:
468f2239 977 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
e51060f0
SH		 978 ib_event->event);
979 goto out;
980 }
981
a1b1b61f 982 ret = id_priv->id.event_handler(&id_priv->id, &event);
e51060f0
SH		 983 if (ret) {
984 /* Destroy the CM ID by returning a non-zero value. */
985 id_priv->cm_id.ib = NULL;
986 cma_exch(id_priv, CMA_DESTROYING);
8aa08602 987 cma_enable_remove(id_priv);
e51060f0
SH		 988 rdma_destroy_id(&id_priv->id);
989 return ret;
990 }
991out:
8aa08602 992 cma_enable_remove(id_priv);
e51060f0
SH		 993 return ret;
994}
995
628e5f6d
SH		 996static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
997 struct ib_cm_event *ib_event)
e51060f0
SH		 998{
999 struct rdma_id_private *id_priv;
1000 struct rdma_cm_id *id;
1001 struct rdma_route *rt;
1002 union cma_ip_addr *src, *dst;
1b90c137 1003 __be16 port;
e51060f0 1004 u8 ip_ver;
64c5e613 1005 int ret;
e51060f0 1006
3f168d2b
KK		 1007 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1008 &ip_ver, &port, &src, &dst))
1009 goto err;
1010
e51060f0
SH		 1011 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1012 listen_id->ps);
1013 if (IS_ERR(id))
3f168d2b
KK		 1014 goto err;
1015
1016 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1017 ip_ver, port, src, dst);
e51060f0
SH		 1018
1019 rt = &id->route;
1020 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
3f168d2b
KK		 1021 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1022 GFP_KERNEL);
e51060f0 1023 if (!rt->path_rec)
3f168d2b 1024 goto destroy_id;
e51060f0 1025
e51060f0
SH		 1026 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1027 if (rt->num_paths == 2)
1028 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1029
e51060f0 1030 ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
64c5e613
OG		 1031 ret = rdma_translate_ip(&id->route.addr.src_addr,
1032 &id->route.addr.dev_addr);
1033 if (ret)
1034 goto destroy_id;
e51060f0
SH		 1035
1036 id_priv = container_of(id, struct rdma_id_private, id);
1037 id_priv->state = CMA_CONNECT;
1038 return id_priv;
3f168d2b
KK		 1039
1040destroy_id:
e51060f0 1041 rdma_destroy_id(id);
3f168d2b 1042err:
e51060f0
SH		 1043 return NULL;
1044}
1045
628e5f6d
SH		 1046static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1047 struct ib_cm_event *ib_event)
1048{
1049 struct rdma_id_private *id_priv;
1050 struct rdma_cm_id *id;
1051 union cma_ip_addr *src, *dst;
1b90c137 1052 __be16 port;
628e5f6d
SH		 1053 u8 ip_ver;
1054 int ret;
1055
1056 id = rdma_create_id(listen_id->event_handler, listen_id->context,
1057 listen_id->ps);
1058 if (IS_ERR(id))
1059 return NULL;
1060
1061
1062 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1063 &ip_ver, &port, &src, &dst))
1064 goto err;
1065
1066 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1067 ip_ver, port, src, dst);
1068
1069 ret = rdma_translate_ip(&id->route.addr.src_addr,
1070 &id->route.addr.dev_addr);
1071 if (ret)
1072 goto err;
1073
1074 id_priv = container_of(id, struct rdma_id_private, id);
1075 id_priv->state = CMA_CONNECT;
1076 return id_priv;
1077err:
1078 rdma_destroy_id(id);
1079 return NULL;
1080}
1081
a1b1b61f
SH		 1082static void cma_set_req_event_data(struct rdma_cm_event *event,
1083 struct ib_cm_req_event_param *req_data,
1084 void *private_data, int offset)
1085{
1086 event->param.conn.private_data = private_data + offset;
1087 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1088 event->param.conn.responder_resources = req_data->responder_resources;
1089 event->param.conn.initiator_depth = req_data->initiator_depth;
1090 event->param.conn.flow_control = req_data->flow_control;
1091 event->param.conn.retry_count = req_data->retry_count;
1092 event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1093 event->param.conn.srq = req_data->srq;
1094 event->param.conn.qp_num = req_data->remote_qpn;
1095}
1096
e51060f0
SH		 1097static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1098{
1099 struct rdma_id_private *listen_id, *conn_id;
a1b1b61f 1100 struct rdma_cm_event event;
e51060f0
SH		 1101 int offset, ret;
1102
1103 listen_id = cm_id->context;
8aa08602
SH		 1104 if (cma_disable_remove(listen_id, CMA_LISTEN))
1105 return -ECONNABORTED;
e51060f0 1106
628e5f6d
SH		 1107 memset(&event, 0, sizeof event);
1108 offset = cma_user_data_offset(listen_id->id.ps);
1109 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
c8f6a362 1110 if (cma_is_ud_ps(listen_id->id.ps)) {
628e5f6d
SH		 1111 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1112 event.param.ud.private_data = ib_event->private_data + offset;
1113 event.param.ud.private_data_len =
1114 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1115 } else {
1116 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1117 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1118 ib_event->private_data, offset);
1119 }
e51060f0
SH		 1120 if (!conn_id) {
1121 ret = -ENOMEM;
1122 goto out;
1123 }
1124
1125 atomic_inc(&conn_id->dev_remove);
61a73c70 1126 mutex_lock(&lock);
07ebafba 1127 ret = cma_acquire_dev(conn_id);
61a73c70 1128 mutex_unlock(&lock);
a1a733f6
KK		 1129 if (ret)
1130 goto release_conn_id;
e51060f0
SH		 1131
1132 conn_id->cm_id.ib = cm_id;
1133 cm_id->context = conn_id;
1134 cm_id->cm_handler = cma_ib_handler;
1135
a1b1b61f 1136 ret = conn_id->id.event_handler(&conn_id->id, &event);
45d9478d 1137 if (!ret) {
ead595ae
SH		 1138 /*
1139 * Acquire mutex to prevent user executing rdma_destroy_id()
1140 * while we're accessing the cm_id.
1141 */
1142 mutex_lock(&lock);
1143 if (cma_comp(conn_id, CMA_CONNECT) &&
1144 !cma_is_ud_ps(conn_id->id.ps))
1145 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1146 mutex_unlock(&lock);
45d9478d 1147 cma_enable_remove(conn_id);
a1a733f6 1148 goto out;
45d9478d 1149 }
a1a733f6
KK		 1150
1151 /* Destroy the CM ID by returning a non-zero value. */
1152 conn_id->cm_id.ib = NULL;
1153
1154release_conn_id:
1155 cma_exch(conn_id, CMA_DESTROYING);
8aa08602 1156 cma_enable_remove(conn_id);
a1a733f6
KK		 1157 rdma_destroy_id(&conn_id->id);
1158
e51060f0 1159out:
8aa08602 1160 cma_enable_remove(listen_id);
e51060f0
SH		 1161 return ret;
1162}
1163
1164static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1165{
628e5f6d 1166 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
e51060f0
SH		 1167}
1168
1169static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1170 struct ib_cm_compare_data *compare)
1171{
1172 struct cma_hdr *cma_data, *cma_mask;
1173 struct sdp_hh *sdp_data, *sdp_mask;
1b90c137 1174 __be32 ip4_addr;
e51060f0
SH		 1175 struct in6_addr ip6_addr;
1176
1177 memset(compare, 0, sizeof *compare);
1178 cma_data = (void *) compare->data;
1179 cma_mask = (void *) compare->mask;
1180 sdp_data = (void *) compare->data;
1181 sdp_mask = (void *) compare->mask;
1182
1183 switch (addr->sa_family) {
1184 case AF_INET:
1185 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1186 if (ps == RDMA_PS_SDP) {
1187 sdp_set_ip_ver(sdp_data, 4);
1188 sdp_set_ip_ver(sdp_mask, 0xF);
1189 sdp_data->dst_addr.ip4.addr = ip4_addr;
1b90c137 1190 sdp_mask->dst_addr.ip4.addr = htonl(~0);
e51060f0
SH		 1191 } else {
1192 cma_set_ip_ver(cma_data, 4);
1193 cma_set_ip_ver(cma_mask, 0xF);
1194 cma_data->dst_addr.ip4.addr = ip4_addr;
1b90c137 1195 cma_mask->dst_addr.ip4.addr = htonl(~0);
e51060f0
SH		 1196 }
1197 break;
1198 case AF_INET6:
1199 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1200 if (ps == RDMA_PS_SDP) {
1201 sdp_set_ip_ver(sdp_data, 6);
1202 sdp_set_ip_ver(sdp_mask, 0xF);
1203 sdp_data->dst_addr.ip6 = ip6_addr;
1204 memset(&sdp_mask->dst_addr.ip6, 0xFF,
1205 sizeof sdp_mask->dst_addr.ip6);
1206 } else {
1207 cma_set_ip_ver(cma_data, 6);
1208 cma_set_ip_ver(cma_mask, 0xF);
1209 cma_data->dst_addr.ip6 = ip6_addr;
1210 memset(&cma_mask->dst_addr.ip6, 0xFF,
1211 sizeof cma_mask->dst_addr.ip6);
1212 }
1213 break;
1214 default:
1215 break;
1216 }
1217}
1218
07ebafba
TT		 1219static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1220{
1221 struct rdma_id_private *id_priv = iw_id->context;
a1b1b61f 1222 struct rdma_cm_event event;
07ebafba
TT		 1223 struct sockaddr_in *sin;
1224 int ret = 0;
1225
be65f086
SH		 1226 if (cma_disable_remove(id_priv, CMA_CONNECT))
1227 return 0;
07ebafba 1228
be65f086 1229 memset(&event, 0, sizeof event);
07ebafba
TT		 1230 switch (iw_event->event) {
1231 case IW_CM_EVENT_CLOSE:
a1b1b61f 1232 event.event = RDMA_CM_EVENT_DISCONNECTED;
07ebafba
TT		 1233 break;
1234 case IW_CM_EVENT_CONNECT_REPLY:
1235 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1236 *sin = iw_event->local_addr;
1237 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1238 *sin = iw_event->remote_addr;
881a045f
SW		 1239 switch (iw_event->status) {
1240 case 0:
a1b1b61f 1241 event.event = RDMA_CM_EVENT_ESTABLISHED;
881a045f
SW		 1242 break;
1243 case -ECONNRESET:
1244 case -ECONNREFUSED:
1245 event.event = RDMA_CM_EVENT_REJECTED;
1246 break;
1247 case -ETIMEDOUT:
1248 event.event = RDMA_CM_EVENT_UNREACHABLE;
1249 break;
1250 default:
1251 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1252 break;
1253 }
07ebafba
TT		 1254 break;
1255 case IW_CM_EVENT_ESTABLISHED:
a1b1b61f 1256 event.event = RDMA_CM_EVENT_ESTABLISHED;
07ebafba
TT		 1257 break;
1258 default:
1259 BUG_ON(1);
1260 }
1261
a1b1b61f
SH		 1262 event.status = iw_event->status;
1263 event.param.conn.private_data = iw_event->private_data;
1264 event.param.conn.private_data_len = iw_event->private_data_len;
1265 ret = id_priv->id.event_handler(&id_priv->id, &event);
07ebafba
TT		 1266 if (ret) {
1267 /* Destroy the CM ID by returning a non-zero value. */
1268 id_priv->cm_id.iw = NULL;
1269 cma_exch(id_priv, CMA_DESTROYING);
8aa08602 1270 cma_enable_remove(id_priv);
07ebafba
TT		 1271 rdma_destroy_id(&id_priv->id);
1272 return ret;
1273 }
1274
8aa08602 1275 cma_enable_remove(id_priv);
07ebafba
TT		 1276 return ret;
1277}
1278
1279static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1280 struct iw_cm_event *iw_event)
1281{
1282 struct rdma_cm_id *new_cm_id;
1283 struct rdma_id_private *listen_id, *conn_id;
1284 struct sockaddr_in *sin;
1285 struct net_device *dev = NULL;
a1b1b61f 1286 struct rdma_cm_event event;
07ebafba 1287 int ret;
8d8293cf 1288 struct ib_device_attr attr;
07ebafba
TT		 1289
1290 listen_id = cm_id->context;
8aa08602
SH		 1291 if (cma_disable_remove(listen_id, CMA_LISTEN))
1292 return -ECONNABORTED;
07ebafba
TT		 1293
1294 /* Create a new RDMA id for the new IW CM ID */
1295 new_cm_id = rdma_create_id(listen_id->id.event_handler,
1296 listen_id->id.context,
1297 RDMA_PS_TCP);
10f32065 1298 if (IS_ERR(new_cm_id)) {
07ebafba
TT		 1299 ret = -ENOMEM;
1300 goto out;
1301 }
1302 conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1303 atomic_inc(&conn_id->dev_remove);
1304 conn_id->state = CMA_CONNECT;
1305
1ab35276 1306 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
07ebafba
TT		 1307 if (!dev) {
1308 ret = -EADDRNOTAVAIL;
8aa08602 1309 cma_enable_remove(conn_id);
07ebafba
TT		 1310 rdma_destroy_id(new_cm_id);
1311 goto out;
1312 }
1313 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1314 if (ret) {
8aa08602 1315 cma_enable_remove(conn_id);
07ebafba
TT		 1316 rdma_destroy_id(new_cm_id);
1317 goto out;
1318 }
1319
61a73c70 1320 mutex_lock(&lock);
07ebafba 1321 ret = cma_acquire_dev(conn_id);
61a73c70 1322 mutex_unlock(&lock);
07ebafba 1323 if (ret) {
8aa08602 1324 cma_enable_remove(conn_id);
07ebafba
TT		 1325 rdma_destroy_id(new_cm_id);
1326 goto out;
1327 }
1328
1329 conn_id->cm_id.iw = cm_id;
1330 cm_id->context = conn_id;
1331 cm_id->cm_handler = cma_iw_handler;
1332
1333 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1334 *sin = iw_event->local_addr;
1335 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1336 *sin = iw_event->remote_addr;
1337
8d8293cf
SW		 1338 ret = ib_query_device(conn_id->id.device, &attr);
1339 if (ret) {
1340 cma_enable_remove(conn_id);
1341 rdma_destroy_id(new_cm_id);
1342 goto out;
1343 }
1344
a1b1b61f
SH		 1345 memset(&event, 0, sizeof event);
1346 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1347 event.param.conn.private_data = iw_event->private_data;
1348 event.param.conn.private_data_len = iw_event->private_data_len;
8d8293cf
SW		 1349 event.param.conn.initiator_depth = attr.max_qp_init_rd_atom;
1350 event.param.conn.responder_resources = attr.max_qp_rd_atom;
a1b1b61f 1351 ret = conn_id->id.event_handler(&conn_id->id, &event);
07ebafba
TT		 1352 if (ret) {
1353 /* User wants to destroy the CM ID */
1354 conn_id->cm_id.iw = NULL;
1355 cma_exch(conn_id, CMA_DESTROYING);
8aa08602 1356 cma_enable_remove(conn_id);
07ebafba
TT		 1357 rdma_destroy_id(&conn_id->id);
1358 }
1359
1360out:
1361 if (dev)
1362 dev_put(dev);
8aa08602 1363 cma_enable_remove(listen_id);
07ebafba
TT		 1364 return ret;
1365}
1366
e51060f0
SH		 1367static int cma_ib_listen(struct rdma_id_private *id_priv)
1368{
1369 struct ib_cm_compare_data compare_data;
1370 struct sockaddr *addr;
1371 __be64 svc_id;
1372 int ret;
1373
1374 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1375 id_priv);
1376 if (IS_ERR(id_priv->cm_id.ib))
1377 return PTR_ERR(id_priv->cm_id.ib);
1378
1379 addr = &id_priv->id.route.addr.src_addr;
1380 svc_id = cma_get_service_id(id_priv->id.ps, addr);
1381 if (cma_any_addr(addr))
1382 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1383 else {
1384 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1385 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1386 }
1387
1388 if (ret) {
1389 ib_destroy_cm_id(id_priv->cm_id.ib);
1390 id_priv->cm_id.ib = NULL;
1391 }
1392
1393 return ret;
1394}
1395
07ebafba
TT		 1396static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1397{
1398 int ret;
1399 struct sockaddr_in *sin;
1400
1401 id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1402 iw_conn_req_handler,
1403 id_priv);
1404 if (IS_ERR(id_priv->cm_id.iw))
1405 return PTR_ERR(id_priv->cm_id.iw);
1406
1407 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1408 id_priv->cm_id.iw->local_addr = *sin;
1409
1410 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1411
1412 if (ret) {
1413 iw_destroy_cm_id(id_priv->cm_id.iw);
1414 id_priv->cm_id.iw = NULL;
1415 }
1416
1417 return ret;
1418}
1419
e51060f0
SH		 1420static int cma_listen_handler(struct rdma_cm_id *id,
1421 struct rdma_cm_event *event)
1422{
1423 struct rdma_id_private *id_priv = id->context;
1424
1425 id->context = id_priv->id.context;
1426 id->event_handler = id_priv->id.event_handler;
1427 return id_priv->id.event_handler(id, event);
1428}
1429
1430static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1431 struct cma_device *cma_dev)
1432{
1433 struct rdma_id_private *dev_id_priv;
1434 struct rdma_cm_id *id;
1435 int ret;
1436
1437 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1438 if (IS_ERR(id))
1439 return;
1440
1441 dev_id_priv = container_of(id, struct rdma_id_private, id);
1442
1443 dev_id_priv->state = CMA_ADDR_BOUND;
1444 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1445 ip_addr_size(&id_priv->id.route.addr.src_addr));
1446
1447 cma_attach_to_dev(dev_id_priv, cma_dev);
1448 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
d02d1f53
SH		 1449 atomic_inc(&id_priv->refcount);
1450 dev_id_priv->internal_id = 1;
e51060f0
SH		 1451
1452 ret = rdma_listen(id, id_priv->backlog);
1453 if (ret)
d02d1f53 1454 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
468f2239 1455 "listening on device %s\n", ret, cma_dev->device->name);
e51060f0
SH		 1456}
1457
1458static void cma_listen_on_all(struct rdma_id_private *id_priv)
1459{
1460 struct cma_device *cma_dev;
1461
1462 mutex_lock(&lock);
1463 list_add_tail(&id_priv->list, &listen_any_list);
1464 list_for_each_entry(cma_dev, &dev_list, list)
1465 cma_listen_on_dev(id_priv, cma_dev);
1466 mutex_unlock(&lock);
1467}
1468
1469static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1470{
1471 struct sockaddr_in addr_in;
1472
1473 memset(&addr_in, 0, sizeof addr_in);
1474 addr_in.sin_family = af;
1475 return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1476}
1477
1478int rdma_listen(struct rdma_cm_id *id, int backlog)
1479{
1480 struct rdma_id_private *id_priv;
1481 int ret;
1482
1483 id_priv = container_of(id, struct rdma_id_private, id);
1484 if (id_priv->state == CMA_IDLE) {
1485 ret = cma_bind_any(id, AF_INET);
1486 if (ret)
1487 return ret;
1488 }
1489
1490 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1491 return -EINVAL;
1492
1493 id_priv->backlog = backlog;
1494 if (id->device) {
07ebafba
TT		 1495 switch (rdma_node_get_transport(id->device->node_type)) {
1496 case RDMA_TRANSPORT_IB:
e51060f0
SH		 1497 ret = cma_ib_listen(id_priv);
1498 if (ret)
1499 goto err;
1500 break;
07ebafba
TT		 1501 case RDMA_TRANSPORT_IWARP:
1502 ret = cma_iw_listen(id_priv, backlog);
1503 if (ret)
1504 goto err;
1505 break;
e51060f0
SH		 1506 default:
1507 ret = -ENOSYS;
1508 goto err;
1509 }
1510 } else
1511 cma_listen_on_all(id_priv);
1512
1513 return 0;
1514err:
1515 id_priv->backlog = 0;
1516 cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1517 return ret;
1518}
1519EXPORT_SYMBOL(rdma_listen);
1520
a81c994d
SH		 1521void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1522{
1523 struct rdma_id_private *id_priv;
1524
1525 id_priv = container_of(id, struct rdma_id_private, id);
1526 id_priv->tos = (u8) tos;
1527}
1528EXPORT_SYMBOL(rdma_set_service_type);
1529
e51060f0
SH		 1530static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1531 void *context)
1532{
1533 struct cma_work *work = context;
1534 struct rdma_route *route;
1535
1536 route = &work->id->id.route;
1537
1538 if (!status) {
1539 route->num_paths = 1;
1540 *route->path_rec = *path_rec;
1541 } else {
1542 work->old_state = CMA_ROUTE_QUERY;
1543 work->new_state = CMA_ADDR_RESOLVED;
1544 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
8f0472d3 1545 work->event.status = status;
e51060f0
SH		 1546 }
1547
1548 queue_work(cma_wq, &work->work);
1549}
1550
1551static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1552 struct cma_work *work)
1553{
a81c994d 1554 struct rdma_addr *addr = &id_priv->id.route.addr;
e51060f0 1555 struct ib_sa_path_rec path_rec;
a81c994d
SH		 1556 ib_sa_comp_mask comp_mask;
1557 struct sockaddr_in6 *sin6;
e51060f0
SH		 1558
1559 memset(&path_rec, 0, sizeof path_rec);
a81c994d
SH		 1560 ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1561 ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1562 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
e51060f0 1563 path_rec.numb_path = 1;
962063e6 1564 path_rec.reversible = 1;
a81c994d
SH		 1565 path_rec.service_id = cma_get_service_id(id_priv->id.ps, &addr->dst_addr);
1566
1567 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1568 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1569 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1570
1571 if (addr->src_addr.sa_family == AF_INET) {
1572 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1573 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1574 } else {
1575 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1576 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1577 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1578 }
e51060f0 1579
c1a0b23b 1580 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
a81c994d
SH		 1581 id_priv->id.port_num, &path_rec,
1582 comp_mask, timeout_ms,
1583 GFP_KERNEL, cma_query_handler,
1584 work, &id_priv->query);
e51060f0
SH		 1585
1586 return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1587}
1588
c4028958 1589static void cma_work_handler(struct work_struct *_work)
e51060f0 1590{
c4028958 1591 struct cma_work *work = container_of(_work, struct cma_work, work);
e51060f0
SH		 1592 struct rdma_id_private *id_priv = work->id;
1593 int destroy = 0;
1594
1595 atomic_inc(&id_priv->dev_remove);
1596 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1597 goto out;
1598
1599 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1600 cma_exch(id_priv, CMA_DESTROYING);
1601 destroy = 1;
1602 }
1603out:
8aa08602 1604 cma_enable_remove(id_priv);
e51060f0
SH		 1605 cma_deref_id(id_priv);
1606 if (destroy)
1607 rdma_destroy_id(&id_priv->id);
1608 kfree(work);
1609}
1610
1611static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1612{
1613 struct rdma_route *route = &id_priv->id.route;
1614 struct cma_work *work;
1615 int ret;
1616
1617 work = kzalloc(sizeof *work, GFP_KERNEL);
1618 if (!work)
1619 return -ENOMEM;
1620
1621 work->id = id_priv;
c4028958 1622 INIT_WORK(&work->work, cma_work_handler);
e51060f0
SH		 1623 work->old_state = CMA_ROUTE_QUERY;
1624 work->new_state = CMA_ROUTE_RESOLVED;
1625 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1626
1627 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1628 if (!route->path_rec) {
1629 ret = -ENOMEM;
1630 goto err1;
1631 }
1632
1633 ret = cma_query_ib_route(id_priv, timeout_ms, work);
1634 if (ret)
1635 goto err2;
1636
1637 return 0;
1638err2:
1639 kfree(route->path_rec);
1640 route->path_rec = NULL;
1641err1:
1642 kfree(work);
1643 return ret;
1644}
1645
1646int rdma_set_ib_paths(struct rdma_cm_id *id,
1647 struct ib_sa_path_rec *path_rec, int num_paths)
1648{
1649 struct rdma_id_private *id_priv;
1650 int ret;
1651
1652 id_priv = container_of(id, struct rdma_id_private, id);
1653 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1654 return -EINVAL;
1655
1656 id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1657 if (!id->route.path_rec) {
1658 ret = -ENOMEM;
1659 goto err;
1660 }
1661
1662 memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1663 return 0;
1664err:
1665 cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1666 return ret;
1667}
1668EXPORT_SYMBOL(rdma_set_ib_paths);
1669
07ebafba
TT		 1670static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1671{
1672 struct cma_work *work;
1673
1674 work = kzalloc(sizeof *work, GFP_KERNEL);
1675 if (!work)
1676 return -ENOMEM;
1677
1678 work->id = id_priv;
c4028958 1679 INIT_WORK(&work->work, cma_work_handler);
07ebafba
TT		 1680 work->old_state = CMA_ROUTE_QUERY;
1681 work->new_state = CMA_ROUTE_RESOLVED;
1682 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1683 queue_work(cma_wq, &work->work);
1684 return 0;
1685}
1686
e51060f0
SH		 1687int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1688{
1689 struct rdma_id_private *id_priv;
1690 int ret;
1691
1692 id_priv = container_of(id, struct rdma_id_private, id);
1693 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1694 return -EINVAL;
1695
1696 atomic_inc(&id_priv->refcount);
07ebafba
TT		 1697 switch (rdma_node_get_transport(id->device->node_type)) {
1698 case RDMA_TRANSPORT_IB:
e51060f0
SH		 1699 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1700 break;
07ebafba
TT		 1701 case RDMA_TRANSPORT_IWARP:
1702 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1703 break;
e51060f0
SH		 1704 default:
1705 ret = -ENOSYS;
1706 break;
1707 }
1708 if (ret)
1709 goto err;
1710
1711 return 0;
1712err:
1713 cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1714 cma_deref_id(id_priv);
1715 return ret;
1716}
1717EXPORT_SYMBOL(rdma_resolve_route);
1718
1719static int cma_bind_loopback(struct rdma_id_private *id_priv)
1720{
1721 struct cma_device *cma_dev;
1722 struct ib_port_attr port_attr;
f0ee3404 1723 union ib_gid gid;
e51060f0
SH		 1724 u16 pkey;
1725 int ret;
1726 u8 p;
1727
1728 mutex_lock(&lock);
e82153b5
KK		 1729 if (list_empty(&dev_list)) {
1730 ret = -ENODEV;
1731 goto out;
1732 }
e51060f0
SH		 1733 list_for_each_entry(cma_dev, &dev_list, list)
1734 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
e82153b5 1735 if (!ib_query_port(cma_dev->device, p, &port_attr) &&
e51060f0
SH		 1736 port_attr.state == IB_PORT_ACTIVE)
1737 goto port_found;
1738
e82153b5
KK		 1739 p = 1;
1740 cma_dev = list_entry(dev_list.next, struct cma_device, list);
e51060f0
SH		 1741
1742port_found:
f0ee3404 1743 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
e51060f0
SH		 1744 if (ret)
1745 goto out;
1746
1747 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1748 if (ret)
1749 goto out;
1750
f0ee3404 1751 ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
e51060f0
SH		 1752 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1753 id_priv->id.port_num = p;
1754 cma_attach_to_dev(id_priv, cma_dev);
1755out:
1756 mutex_unlock(&lock);
1757 return ret;
1758}
1759
1760static void addr_handler(int status, struct sockaddr *src_addr,
1761 struct rdma_dev_addr *dev_addr, void *context)
1762{
1763 struct rdma_id_private *id_priv = context;
a1b1b61f 1764 struct rdma_cm_event event;
e51060f0 1765
a1b1b61f 1766 memset(&event, 0, sizeof event);
e51060f0 1767 atomic_inc(&id_priv->dev_remove);
61a73c70
SH		 1768
1769 /*
1770 * Grab mutex to block rdma_destroy_id() from removing the device while
1771 * we're trying to acquire it.
1772 */
1773 mutex_lock(&lock);
1774 if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1775 mutex_unlock(&lock);
1776 goto out;
1777 }
1778
1779 if (!status && !id_priv->cma_dev)
e51060f0 1780 status = cma_acquire_dev(id_priv);
61a73c70 1781 mutex_unlock(&lock);
e51060f0
SH		 1782
1783 if (status) {
61a73c70 1784 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
e51060f0 1785 goto out;
a1b1b61f
SH		 1786 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1787 event.status = status;
e51060f0 1788 } else {
e51060f0
SH		 1789 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1790 ip_addr_size(src_addr));
a1b1b61f 1791 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
e51060f0
SH		 1792 }
1793
a1b1b61f 1794 if (id_priv->id.event_handler(&id_priv->id, &event)) {
e51060f0 1795 cma_exch(id_priv, CMA_DESTROYING);
8aa08602 1796 cma_enable_remove(id_priv);
e51060f0
SH		 1797 cma_deref_id(id_priv);
1798 rdma_destroy_id(&id_priv->id);
1799 return;
1800 }
1801out:
8aa08602 1802 cma_enable_remove(id_priv);
e51060f0
SH		 1803 cma_deref_id(id_priv);
1804}
1805
1806static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1807{
1808 struct cma_work *work;
1809 struct sockaddr_in *src_in, *dst_in;
f0ee3404 1810 union ib_gid gid;
e51060f0
SH		 1811 int ret;
1812
1813 work = kzalloc(sizeof *work, GFP_KERNEL);
1814 if (!work)
1815 return -ENOMEM;
1816
1817 if (!id_priv->cma_dev) {
1818 ret = cma_bind_loopback(id_priv);
1819 if (ret)
1820 goto err;
1821 }
1822
f0ee3404
MT		 1823 ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1824 ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
e51060f0
SH		 1825
1826 if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) {
1827 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1828 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1829 src_in->sin_family = dst_in->sin_family;
1830 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1831 }
1832
1833 work->id = id_priv;
c4028958 1834 INIT_WORK(&work->work, cma_work_handler);
e51060f0
SH		 1835 work->old_state = CMA_ADDR_QUERY;
1836 work->new_state = CMA_ADDR_RESOLVED;
1837 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1838 queue_work(cma_wq, &work->work);
1839 return 0;
1840err:
1841 kfree(work);
1842 return ret;
1843}
1844
1845static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1846 struct sockaddr *dst_addr)
1847{
1848 if (src_addr && src_addr->sa_family)
1849 return rdma_bind_addr(id, src_addr);
1850 else
1851 return cma_bind_any(id, dst_addr->sa_family);
1852}
1853
1854int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1855 struct sockaddr *dst_addr, int timeout_ms)
1856{
1857 struct rdma_id_private *id_priv;
1858 int ret;
1859
1860 id_priv = container_of(id, struct rdma_id_private, id);
1861 if (id_priv->state == CMA_IDLE) {
1862 ret = cma_bind_addr(id, src_addr, dst_addr);
1863 if (ret)
1864 return ret;
1865 }
1866
1867 if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1868 return -EINVAL;
1869
1870 atomic_inc(&id_priv->refcount);
1871 memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1872 if (cma_any_addr(dst_addr))
1873 ret = cma_resolve_loopback(id_priv);
1874 else
7a118df3
SH		 1875 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr,
1876 dst_addr, &id->route.addr.dev_addr,
e51060f0
SH		 1877 timeout_ms, addr_handler, id_priv);
1878 if (ret)
1879 goto err;
1880
1881 return 0;
1882err:
1883 cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1884 cma_deref_id(id_priv);
1885 return ret;
1886}
1887EXPORT_SYMBOL(rdma_resolve_addr);
1888
1889static void cma_bind_port(struct rdma_bind_list *bind_list,
1890 struct rdma_id_private *id_priv)
1891{
1892 struct sockaddr_in *sin;
1893
1894 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1895 sin->sin_port = htons(bind_list->port);
1896 id_priv->bind_list = bind_list;
1897 hlist_add_head(&id_priv->node, &bind_list->owners);
1898}
1899
1900static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1901 unsigned short snum)
1902{
1903 struct rdma_bind_list *bind_list;
aedec080 1904 int port, ret;
e51060f0 1905
cb164b8c 1906 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
e51060f0
SH		 1907 if (!bind_list)
1908 return -ENOMEM;
1909
aedec080
SH		 1910 do {
1911 ret = idr_get_new_above(ps, bind_list, snum, &port);
1912 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1913
1914 if (ret)
1915 goto err1;
1916
1917 if (port != snum) {
1918 ret = -EADDRNOTAVAIL;
1919 goto err2;
1920 }
1921
1922 bind_list->ps = ps;
1923 bind_list->port = (unsigned short) port;
1924 cma_bind_port(bind_list, id_priv);
1925 return 0;
1926err2:
1927 idr_remove(ps, port);
1928err1:
1929 kfree(bind_list);
1930 return ret;
1931}
e51060f0 1932
aedec080
SH		 1933static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1934{
1935 struct rdma_bind_list *bind_list;
227b60f5 1936 int port, ret, low, high;
aedec080
SH		 1937
1938 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1939 if (!bind_list)
1940 return -ENOMEM;
1941
1942retry:
227b60f5 1943 /* FIXME: add proper port randomization per like inet_csk_get_port */
e51060f0 1944 do {
aedec080 1945 ret = idr_get_new_above(ps, bind_list, next_port, &port);
e51060f0
SH		 1946 } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1947
1948 if (ret)
aedec080 1949 goto err1;
e51060f0 1950
227b60f5
SH		 1951 inet_get_local_port_range(&low, &high);
1952 if (port > high) {
1953 if (next_port != low) {
aedec080 1954 idr_remove(ps, port);
227b60f5 1955 next_port = low;
aedec080
SH		 1956 goto retry;
1957 }
e51060f0 1958 ret = -EADDRNOTAVAIL;
aedec080 1959 goto err2;
e51060f0
SH		 1960 }
1961
227b60f5
SH		 1962 if (port == high)
1963 next_port = low;
aedec080
SH		 1964 else
1965 next_port = port + 1;
1966
e51060f0
SH		 1967 bind_list->ps = ps;
1968 bind_list->port = (unsigned short) port;
1969 cma_bind_port(bind_list, id_priv);
1970 return 0;
aedec080
SH		 1971err2:
1972 idr_remove(ps, port);
1973err1:
e51060f0
SH		 1974 kfree(bind_list);
1975 return ret;
1976}
1977
1978static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
1979{
1980 struct rdma_id_private *cur_id;
1981 struct sockaddr_in *sin, *cur_sin;
1982 struct rdma_bind_list *bind_list;
1983 struct hlist_node *node;
1984 unsigned short snum;
1985
1986 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1987 snum = ntohs(sin->sin_port);
1988 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
1989 return -EACCES;
1990
1991 bind_list = idr_find(ps, snum);
1992 if (!bind_list)
1993 return cma_alloc_port(ps, id_priv, snum);
1994
1995 /*
1996 * We don't support binding to any address if anyone is bound to
1997 * a specific address on the same port.
1998 */
1999 if (cma_any_addr(&id_priv->id.route.addr.src_addr))
2000 return -EADDRNOTAVAIL;
2001
2002 hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2003 if (cma_any_addr(&cur_id->id.route.addr.src_addr))
2004 return -EADDRNOTAVAIL;
3cd96564 2005
e51060f0
SH		 2006 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
2007 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
2008 return -EADDRINUSE;
2009 }
2010
2011 cma_bind_port(bind_list, id_priv);
2012 return 0;
2013}
2014
2015static int cma_get_port(struct rdma_id_private *id_priv)
2016{
2017 struct idr *ps;
2018 int ret;
2019
2020 switch (id_priv->id.ps) {
2021 case RDMA_PS_SDP:
2022 ps = &sdp_ps;
2023 break;
2024 case RDMA_PS_TCP:
2025 ps = &tcp_ps;
2026 break;
628e5f6d
SH		 2027 case RDMA_PS_UDP:
2028 ps = &udp_ps;
2029 break;
c8f6a362
SH		 2030 case RDMA_PS_IPOIB:
2031 ps = &ipoib_ps;
2032 break;
e51060f0
SH		 2033 default:
2034 return -EPROTONOSUPPORT;
2035 }
2036
2037 mutex_lock(&lock);
2038 if (cma_any_port(&id_priv->id.route.addr.src_addr))
aedec080 2039 ret = cma_alloc_any_port(ps, id_priv);
e51060f0
SH		 2040 else
2041 ret = cma_use_port(ps, id_priv);
2042 mutex_unlock(&lock);
2043
2044 return ret;
2045}
2046
2047int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2048{
2049 struct rdma_id_private *id_priv;
2050 int ret;
2051
2052 if (addr->sa_family != AF_INET)
2053 return -EAFNOSUPPORT;
2054
2055 id_priv = container_of(id, struct rdma_id_private, id);
2056 if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
2057 return -EINVAL;
2058
2059 if (!cma_any_addr(addr)) {
2060 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
e51060f0 2061 if (ret)
255d0c14
KK		 2062 goto err1;
2063
2064 mutex_lock(&lock);
2065 ret = cma_acquire_dev(id_priv);
2066 mutex_unlock(&lock);
2067 if (ret)
2068 goto err1;
e51060f0
SH		 2069 }
2070
2071 memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2072 ret = cma_get_port(id_priv);
2073 if (ret)
255d0c14 2074 goto err2;
e51060f0
SH		 2075
2076 return 0;
255d0c14
KK		 2077err2:
2078 if (!cma_any_addr(addr)) {
2079 mutex_lock(&lock);
2080 cma_detach_from_dev(id_priv);
2081 mutex_unlock(&lock);
2082 }
2083err1:
e51060f0
SH		 2084 cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2085 return ret;
2086}
2087EXPORT_SYMBOL(rdma_bind_addr);
2088
2089static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2090 struct rdma_route *route)
2091{
2092 struct sockaddr_in *src4, *dst4;
2093 struct cma_hdr *cma_hdr;
2094 struct sdp_hh *sdp_hdr;
2095
2096 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2097 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2098
2099 switch (ps) {
2100 case RDMA_PS_SDP:
2101 sdp_hdr = hdr;
2102 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2103 return -EINVAL;
2104 sdp_set_ip_ver(sdp_hdr, 4);
2105 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2106 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2107 sdp_hdr->port = src4->sin_port;
2108 break;
2109 default:
2110 cma_hdr = hdr;
2111 cma_hdr->cma_version = CMA_VERSION;
2112 cma_set_ip_ver(cma_hdr, 4);
2113 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2114 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2115 cma_hdr->port = src4->sin_port;
2116 break;
2117 }
2118 return 0;
2119}
2120
628e5f6d
SH		 2121static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2122 struct ib_cm_event *ib_event)
2123{
2124 struct rdma_id_private *id_priv = cm_id->context;
2125 struct rdma_cm_event event;
2126 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2127 int ret = 0;
2128
8aa08602
SH		 2129 if (cma_disable_remove(id_priv, CMA_CONNECT))
2130 return 0;
628e5f6d 2131
8aa08602 2132 memset(&event, 0, sizeof event);
628e5f6d
SH		 2133 switch (ib_event->event) {
2134 case IB_CM_SIDR_REQ_ERROR:
2135 event.event = RDMA_CM_EVENT_UNREACHABLE;
2136 event.status = -ETIMEDOUT;
2137 break;
2138 case IB_CM_SIDR_REP_RECEIVED:
2139 event.param.ud.private_data = ib_event->private_data;
2140 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2141 if (rep->status != IB_SIDR_SUCCESS) {
2142 event.event = RDMA_CM_EVENT_UNREACHABLE;
2143 event.status = ib_event->param.sidr_rep_rcvd.status;
2144 break;
2145 }
c8f6a362 2146 if (id_priv->qkey != rep->qkey) {
628e5f6d
SH		 2147 event.event = RDMA_CM_EVENT_UNREACHABLE;
2148 event.status = -EINVAL;
2149 break;
2150 }
2151 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2152 id_priv->id.route.path_rec,
2153 &event.param.ud.ah_attr);
2154 event.param.ud.qp_num = rep->qpn;
2155 event.param.ud.qkey = rep->qkey;
2156 event.event = RDMA_CM_EVENT_ESTABLISHED;
2157 event.status = 0;
2158 break;
2159 default:
468f2239 2160 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
628e5f6d
SH		 2161 ib_event->event);
2162 goto out;
2163 }
2164
2165 ret = id_priv->id.event_handler(&id_priv->id, &event);
2166 if (ret) {
2167 /* Destroy the CM ID by returning a non-zero value. */
2168 id_priv->cm_id.ib = NULL;
2169 cma_exch(id_priv, CMA_DESTROYING);
8aa08602 2170 cma_enable_remove(id_priv);
628e5f6d
SH		 2171 rdma_destroy_id(&id_priv->id);
2172 return ret;
2173 }
2174out:
8aa08602 2175 cma_enable_remove(id_priv);
628e5f6d
SH		 2176 return ret;
2177}
2178
2179static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2180 struct rdma_conn_param *conn_param)
2181{
2182 struct ib_cm_sidr_req_param req;
2183 struct rdma_route *route;
2184 int ret;
2185
2186 req.private_data_len = sizeof(struct cma_hdr) +
2187 conn_param->private_data_len;
2188 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2189 if (!req.private_data)
2190 return -ENOMEM;
2191
2192 if (conn_param->private_data && conn_param->private_data_len)
2193 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2194 conn_param->private_data, conn_param->private_data_len);
2195
2196 route = &id_priv->id.route;
2197 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2198 if (ret)
2199 goto out;
2200
2201 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2202 cma_sidr_rep_handler, id_priv);
2203 if (IS_ERR(id_priv->cm_id.ib)) {
2204 ret = PTR_ERR(id_priv->cm_id.ib);
2205 goto out;
2206 }
2207
2208 req.path = route->path_rec;
2209 req.service_id = cma_get_service_id(id_priv->id.ps,
2210 &route->addr.dst_addr);
2211 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2212 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2213
2214 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2215 if (ret) {
2216 ib_destroy_cm_id(id_priv->cm_id.ib);
2217 id_priv->cm_id.ib = NULL;
2218 }
2219out:
2220 kfree(req.private_data);
2221 return ret;
2222}
2223
e51060f0
SH		 2224static int cma_connect_ib(struct rdma_id_private *id_priv,
2225 struct rdma_conn_param *conn_param)
2226{
2227 struct ib_cm_req_param req;
2228 struct rdma_route *route;
2229 void *private_data;
2230 int offset, ret;
2231
2232 memset(&req, 0, sizeof req);
2233 offset = cma_user_data_offset(id_priv->id.ps);
2234 req.private_data_len = offset + conn_param->private_data_len;
2235 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2236 if (!private_data)
2237 return -ENOMEM;
2238
2239 if (conn_param->private_data && conn_param->private_data_len)
2240 memcpy(private_data + offset, conn_param->private_data,
2241 conn_param->private_data_len);
2242
2243 id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2244 id_priv);
2245 if (IS_ERR(id_priv->cm_id.ib)) {
2246 ret = PTR_ERR(id_priv->cm_id.ib);
2247 goto out;
2248 }
2249
2250 route = &id_priv->id.route;
2251 ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2252 if (ret)
2253 goto out;
2254 req.private_data = private_data;
2255
2256 req.primary_path = &route->path_rec[0];
2257 if (route->num_paths == 2)
2258 req.alternate_path = &route->path_rec[1];
2259
2260 req.service_id = cma_get_service_id(id_priv->id.ps,
2261 &route->addr.dst_addr);
2262 req.qp_num = id_priv->qp_num;
9b2e9c0c 2263 req.qp_type = IB_QPT_RC;
e51060f0
SH		 2264 req.starting_psn = id_priv->seq_num;
2265 req.responder_resources = conn_param->responder_resources;
2266 req.initiator_depth = conn_param->initiator_depth;
2267 req.flow_control = conn_param->flow_control;
2268 req.retry_count = conn_param->retry_count;
2269 req.rnr_retry_count = conn_param->rnr_retry_count;
2270 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2271 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2272 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2273 req.srq = id_priv->srq ? 1 : 0;
2274
2275 ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2276out:
675a027c
KK		 2277 if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2278 ib_destroy_cm_id(id_priv->cm_id.ib);
2279 id_priv->cm_id.ib = NULL;
2280 }
2281
e51060f0
SH		 2282 kfree(private_data);
2283 return ret;
2284}
2285
07ebafba
TT		 2286static int cma_connect_iw(struct rdma_id_private *id_priv,
2287 struct rdma_conn_param *conn_param)
2288{
2289 struct iw_cm_id *cm_id;
2290 struct sockaddr_in* sin;
2291 int ret;
2292 struct iw_cm_conn_param iw_param;
2293
2294 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2295 if (IS_ERR(cm_id)) {
2296 ret = PTR_ERR(cm_id);
2297 goto out;
2298 }
2299
2300 id_priv->cm_id.iw = cm_id;
2301
2302 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2303 cm_id->local_addr = *sin;
2304
2305 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2306 cm_id->remote_addr = *sin;
2307
5851bb89 2308 ret = cma_modify_qp_rtr(id_priv, conn_param);
675a027c
KK		 2309 if (ret)
2310 goto out;
07ebafba
TT		 2311
2312 iw_param.ord = conn_param->initiator_depth;
2313 iw_param.ird = conn_param->responder_resources;
2314 iw_param.private_data = conn_param->private_data;
2315 iw_param.private_data_len = conn_param->private_data_len;
2316 if (id_priv->id.qp)
2317 iw_param.qpn = id_priv->qp_num;
2318 else
2319 iw_param.qpn = conn_param->qp_num;
2320 ret = iw_cm_connect(cm_id, &iw_param);
2321out:
675a027c
KK		 2322 if (ret && !IS_ERR(cm_id)) {
2323 iw_destroy_cm_id(cm_id);
2324 id_priv->cm_id.iw = NULL;
2325 }
07ebafba
TT		 2326 return ret;
2327}
2328
e51060f0
SH		 2329int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2330{
2331 struct rdma_id_private *id_priv;
2332 int ret;
2333
2334 id_priv = container_of(id, struct rdma_id_private, id);
2335 if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2336 return -EINVAL;
2337
2338 if (!id->qp) {
2339 id_priv->qp_num = conn_param->qp_num;
e51060f0
SH		 2340 id_priv->srq = conn_param->srq;
2341 }
2342
07ebafba
TT		 2343 switch (rdma_node_get_transport(id->device->node_type)) {
2344 case RDMA_TRANSPORT_IB:
c8f6a362 2345 if (cma_is_ud_ps(id->ps))
628e5f6d
SH		 2346 ret = cma_resolve_ib_udp(id_priv, conn_param);
2347 else
2348 ret = cma_connect_ib(id_priv, conn_param);
e51060f0 2349 break;
07ebafba
TT		 2350 case RDMA_TRANSPORT_IWARP:
2351 ret = cma_connect_iw(id_priv, conn_param);
2352 break;
e51060f0
SH		 2353 default:
2354 ret = -ENOSYS;
2355 break;
2356 }
2357 if (ret)
2358 goto err;
2359
2360 return 0;
2361err:
2362 cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2363 return ret;
2364}
2365EXPORT_SYMBOL(rdma_connect);
2366
2367static int cma_accept_ib(struct rdma_id_private *id_priv,
2368 struct rdma_conn_param *conn_param)
2369{
2370 struct ib_cm_rep_param rep;
5851bb89 2371 int ret;
0fe313b0 2372
5851bb89
SH		 2373 ret = cma_modify_qp_rtr(id_priv, conn_param);
2374 if (ret)
2375 goto out;
0fe313b0 2376
5851bb89
SH		 2377 ret = cma_modify_qp_rts(id_priv, conn_param);
2378 if (ret)
2379 goto out;
e51060f0
SH		 2380
2381 memset(&rep, 0, sizeof rep);
2382 rep.qp_num = id_priv->qp_num;
2383 rep.starting_psn = id_priv->seq_num;
2384 rep.private_data = conn_param->private_data;
2385 rep.private_data_len = conn_param->private_data_len;
2386 rep.responder_resources = conn_param->responder_resources;
2387 rep.initiator_depth = conn_param->initiator_depth;
e51060f0
SH		 2388 rep.failover_accepted = 0;
2389 rep.flow_control = conn_param->flow_control;
2390 rep.rnr_retry_count = conn_param->rnr_retry_count;
2391 rep.srq = id_priv->srq ? 1 : 0;
2392
0fe313b0
SH		 2393 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2394out:
2395 return ret;
e51060f0
SH		 2396}
2397
07ebafba
TT		 2398static int cma_accept_iw(struct rdma_id_private *id_priv,
2399 struct rdma_conn_param *conn_param)
2400{
2401 struct iw_cm_conn_param iw_param;
2402 int ret;
2403
5851bb89 2404 ret = cma_modify_qp_rtr(id_priv, conn_param);
07ebafba
TT		 2405 if (ret)
2406 return ret;
2407
2408 iw_param.ord = conn_param->initiator_depth;
2409 iw_param.ird = conn_param->responder_resources;
2410 iw_param.private_data = conn_param->private_data;
2411 iw_param.private_data_len = conn_param->private_data_len;
2412 if (id_priv->id.qp) {
2413 iw_param.qpn = id_priv->qp_num;
2414 } else
2415 iw_param.qpn = conn_param->qp_num;
2416
2417 return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2418}
2419
628e5f6d
SH		 2420static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2421 enum ib_cm_sidr_status status,
2422 const void *private_data, int private_data_len)
2423{
2424 struct ib_cm_sidr_rep_param rep;
2425
2426 memset(&rep, 0, sizeof rep);
2427 rep.status = status;
2428 if (status == IB_SIDR_SUCCESS) {
2429 rep.qp_num = id_priv->qp_num;
c8f6a362 2430 rep.qkey = id_priv->qkey;
628e5f6d
SH		 2431 }
2432 rep.private_data = private_data;
2433 rep.private_data_len = private_data_len;
2434
2435 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2436}
2437
e51060f0
SH		 2438int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2439{
2440 struct rdma_id_private *id_priv;
2441 int ret;
2442
2443 id_priv = container_of(id, struct rdma_id_private, id);
2444 if (!cma_comp(id_priv, CMA_CONNECT))
2445 return -EINVAL;
2446
2447 if (!id->qp && conn_param) {
2448 id_priv->qp_num = conn_param->qp_num;
e51060f0
SH		 2449 id_priv->srq = conn_param->srq;
2450 }
2451
07ebafba
TT		 2452 switch (rdma_node_get_transport(id->device->node_type)) {
2453 case RDMA_TRANSPORT_IB:
c8f6a362 2454 if (cma_is_ud_ps(id->ps))
628e5f6d
SH		 2455 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2456 conn_param->private_data,
2457 conn_param->private_data_len);
2458 else if (conn_param)
e51060f0
SH		 2459 ret = cma_accept_ib(id_priv, conn_param);
2460 else
2461 ret = cma_rep_recv(id_priv);
2462 break;
07ebafba
TT		 2463 case RDMA_TRANSPORT_IWARP:
2464 ret = cma_accept_iw(id_priv, conn_param);
2465 break;
e51060f0
SH		 2466 default:
2467 ret = -ENOSYS;
2468 break;
2469 }
2470
2471 if (ret)
2472 goto reject;
2473
2474 return 0;
2475reject:
c5483388 2476 cma_modify_qp_err(id_priv);
e51060f0
SH		 2477 rdma_reject(id, NULL, 0);
2478 return ret;
2479}
2480EXPORT_SYMBOL(rdma_accept);
2481
0fe313b0
SH		 2482int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2483{
2484 struct rdma_id_private *id_priv;
2485 int ret;
2486
2487 id_priv = container_of(id, struct rdma_id_private, id);
6c719f5c 2488 if (!cma_has_cm_dev(id_priv))
0fe313b0
SH		 2489 return -EINVAL;
2490
2491 switch (id->device->node_type) {
2492 case RDMA_NODE_IB_CA:
2493 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2494 break;
2495 default:
2496 ret = 0;
2497 break;
2498 }
2499 return ret;
2500}
2501EXPORT_SYMBOL(rdma_notify);
2502
e51060f0
SH		 2503int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2504 u8 private_data_len)
2505{
2506 struct rdma_id_private *id_priv;
2507 int ret;
2508
2509 id_priv = container_of(id, struct rdma_id_private, id);
6c719f5c 2510 if (!cma_has_cm_dev(id_priv))
e51060f0
SH		 2511 return -EINVAL;
2512
07ebafba
TT		 2513 switch (rdma_node_get_transport(id->device->node_type)) {
2514 case RDMA_TRANSPORT_IB:
c8f6a362 2515 if (cma_is_ud_ps(id->ps))
628e5f6d
SH		 2516 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2517 private_data, private_data_len);
2518 else
2519 ret = ib_send_cm_rej(id_priv->cm_id.ib,
2520 IB_CM_REJ_CONSUMER_DEFINED, NULL,
2521 0, private_data, private_data_len);
e51060f0 2522 break;
07ebafba
TT		 2523 case RDMA_TRANSPORT_IWARP:
2524 ret = iw_cm_reject(id_priv->cm_id.iw,
2525 private_data, private_data_len);
2526 break;
e51060f0
SH		 2527 default:
2528 ret = -ENOSYS;
2529 break;
2530 }
2531 return ret;
2532}
2533EXPORT_SYMBOL(rdma_reject);
2534
2535int rdma_disconnect(struct rdma_cm_id *id)
2536{
2537 struct rdma_id_private *id_priv;
2538 int ret;
2539
2540 id_priv = container_of(id, struct rdma_id_private, id);
6c719f5c 2541 if (!cma_has_cm_dev(id_priv))
e51060f0
SH		 2542 return -EINVAL;
2543
07ebafba
TT		 2544 switch (rdma_node_get_transport(id->device->node_type)) {
2545 case RDMA_TRANSPORT_IB:
c5483388 2546 ret = cma_modify_qp_err(id_priv);
07ebafba
TT		 2547 if (ret)
2548 goto out;
e51060f0
SH		 2549 /* Initiate or respond to a disconnect. */
2550 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2551 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2552 break;
07ebafba
TT		 2553 case RDMA_TRANSPORT_IWARP:
2554 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2555 break;
e51060f0 2556 default:
07ebafba 2557 ret = -EINVAL;
e51060f0
SH		 2558 break;
2559 }
2560out:
2561 return ret;
2562}
2563EXPORT_SYMBOL(rdma_disconnect);
2564
c8f6a362
SH		 2565static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2566{
2567 struct rdma_id_private *id_priv;
2568 struct cma_multicast *mc = multicast->context;
2569 struct rdma_cm_event event;
2570 int ret;
2571
2572 id_priv = mc->id_priv;
8aa08602
SH		 2573 if (cma_disable_remove(id_priv, CMA_ADDR_BOUND) &&
2574 cma_disable_remove(id_priv, CMA_ADDR_RESOLVED))
2575 return 0;
c8f6a362 2576
c5483388 2577 mutex_lock(&id_priv->qp_mutex);
c8f6a362
SH		 2578 if (!status && id_priv->id.qp)
2579 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2580 multicast->rec.mlid);
c5483388 2581 mutex_unlock(&id_priv->qp_mutex);
c8f6a362
SH		 2582
2583 memset(&event, 0, sizeof event);
2584 event.status = status;
2585 event.param.ud.private_data = mc->context;
2586 if (!status) {
2587 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2588 ib_init_ah_from_mcmember(id_priv->id.device,
2589 id_priv->id.port_num, &multicast->rec,
2590 &event.param.ud.ah_attr);
2591 event.param.ud.qp_num = 0xFFFFFF;
2592 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2593 } else
2594 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2595
2596 ret = id_priv->id.event_handler(&id_priv->id, &event);
2597 if (ret) {
2598 cma_exch(id_priv, CMA_DESTROYING);
8aa08602 2599 cma_enable_remove(id_priv);
c8f6a362
SH		 2600 rdma_destroy_id(&id_priv->id);
2601 return 0;
2602 }
8aa08602
SH		 2603
2604 cma_enable_remove(id_priv);
c8f6a362
SH		 2605 return 0;
2606}
2607
2608static void cma_set_mgid(struct rdma_id_private *id_priv,
2609 struct sockaddr *addr, union ib_gid *mgid)
2610{
2611 unsigned char mc_map[MAX_ADDR_LEN];
2612 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2613 struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2614 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2615
2616 if (cma_any_addr(addr)) {
2617 memset(mgid, 0, sizeof *mgid);
2618 } else if ((addr->sa_family == AF_INET6) &&
2619 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2620 0xFF10A01B)) {
2621 /* IPv6 address is an SA assigned MGID. */
2622 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2623 } else {
a9e527e3 2624 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
c8f6a362
SH		 2625 if (id_priv->id.ps == RDMA_PS_UDP)
2626 mc_map[7] = 0x01; /* Use RDMA CM signature */
c8f6a362
SH		 2627 *mgid = *(union ib_gid *) (mc_map + 4);
2628 }
2629}
2630
2631static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2632 struct cma_multicast *mc)
2633{
2634 struct ib_sa_mcmember_rec rec;
2635 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2636 ib_sa_comp_mask comp_mask;
2637 int ret;
2638
2639 ib_addr_get_mgid(dev_addr, &rec.mgid);
2640 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2641 &rec.mgid, &rec);
2642 if (ret)
2643 return ret;
2644
2645 cma_set_mgid(id_priv, &mc->addr, &rec.mgid);
2646 if (id_priv->id.ps == RDMA_PS_UDP)
2647 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2648 ib_addr_get_sgid(dev_addr, &rec.port_gid);
2649 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2650 rec.join_state = 1;
2651
2652 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2653 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2654 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2655 IB_SA_MCMEMBER_REC_FLOW_LABEL |
2656 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2657
2658 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2659 id_priv->id.port_num, &rec,
2660 comp_mask, GFP_KERNEL,
2661 cma_ib_mc_handler, mc);
2662 if (IS_ERR(mc->multicast.ib))
2663 return PTR_ERR(mc->multicast.ib);
2664
2665 return 0;
2666}
2667
2668int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2669 void *context)
2670{
2671 struct rdma_id_private *id_priv;
2672 struct cma_multicast *mc;
2673 int ret;
2674
2675 id_priv = container_of(id, struct rdma_id_private, id);
2676 if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2677 !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2678 return -EINVAL;
2679
2680 mc = kmalloc(sizeof *mc, GFP_KERNEL);
2681 if (!mc)
2682 return -ENOMEM;
2683
2684 memcpy(&mc->addr, addr, ip_addr_size(addr));
2685 mc->context = context;
2686 mc->id_priv = id_priv;
2687
2688 spin_lock(&id_priv->lock);
2689 list_add(&mc->list, &id_priv->mc_list);
2690 spin_unlock(&id_priv->lock);
2691
2692 switch (rdma_node_get_transport(id->device->node_type)) {
2693 case RDMA_TRANSPORT_IB:
2694 ret = cma_join_ib_multicast(id_priv, mc);
2695 break;
2696 default:
2697 ret = -ENOSYS;
2698 break;
2699 }
2700
2701 if (ret) {
2702 spin_lock_irq(&id_priv->lock);
2703 list_del(&mc->list);
2704 spin_unlock_irq(&id_priv->lock);
2705 kfree(mc);
2706 }
2707 return ret;
2708}
2709EXPORT_SYMBOL(rdma_join_multicast);
2710
2711void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2712{
2713 struct rdma_id_private *id_priv;
2714 struct cma_multicast *mc;
2715
2716 id_priv = container_of(id, struct rdma_id_private, id);
2717 spin_lock_irq(&id_priv->lock);
2718 list_for_each_entry(mc, &id_priv->mc_list, list) {
2719 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2720 list_del(&mc->list);
2721 spin_unlock_irq(&id_priv->lock);
2722
2723 if (id->qp)
2724 ib_detach_mcast(id->qp,
2725 &mc->multicast.ib->rec.mgid,
2726 mc->multicast.ib->rec.mlid);
2727 ib_sa_free_multicast(mc->multicast.ib);
2728 kfree(mc);
2729 return;
2730 }
2731 }
2732 spin_unlock_irq(&id_priv->lock);
2733}
2734EXPORT_SYMBOL(rdma_leave_multicast);
2735
e51060f0
SH		 2736static void cma_add_one(struct ib_device *device)
2737{
2738 struct cma_device *cma_dev;
2739 struct rdma_id_private *id_priv;
2740
2741 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2742 if (!cma_dev)
2743 return;
2744
2745 cma_dev->device = device;
e51060f0
SH		 2746
2747 init_completion(&cma_dev->comp);
2748 atomic_set(&cma_dev->refcount, 1);
2749 INIT_LIST_HEAD(&cma_dev->id_list);
2750 ib_set_client_data(device, &cma_client, cma_dev);
2751
2752 mutex_lock(&lock);
2753 list_add_tail(&cma_dev->list, &dev_list);
2754 list_for_each_entry(id_priv, &listen_any_list, list)
2755 cma_listen_on_dev(id_priv, cma_dev);
2756 mutex_unlock(&lock);
e51060f0
SH		 2757}
2758
2759static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2760{
a1b1b61f 2761 struct rdma_cm_event event;
e51060f0
SH		 2762 enum cma_state state;
2763
2764 /* Record that we want to remove the device */
2765 state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2766 if (state == CMA_DESTROYING)
2767 return 0;
2768
2769 cma_cancel_operation(id_priv, state);
2770 wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
2771
2772 /* Check for destruction from another callback. */
2773 if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2774 return 0;
2775
a1b1b61f
SH		 2776 memset(&event, 0, sizeof event);
2777 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2778 return id_priv->id.event_handler(&id_priv->id, &event);
e51060f0
SH		 2779}
2780
2781static void cma_process_remove(struct cma_device *cma_dev)
2782{
e51060f0
SH		 2783 struct rdma_id_private *id_priv;
2784 int ret;
2785
e51060f0
SH		 2786 mutex_lock(&lock);
2787 while (!list_empty(&cma_dev->id_list)) {
2788 id_priv = list_entry(cma_dev->id_list.next,
2789 struct rdma_id_private, list);
2790
d02d1f53 2791 list_del(&id_priv->listen_list);
94de178a 2792 list_del_init(&id_priv->list);
e51060f0
SH		 2793 atomic_inc(&id_priv->refcount);
2794 mutex_unlock(&lock);
2795
d02d1f53 2796 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
e51060f0
SH		 2797 cma_deref_id(id_priv);
2798 if (ret)
2799 rdma_destroy_id(&id_priv->id);
2800
2801 mutex_lock(&lock);
2802 }
2803 mutex_unlock(&lock);
2804
2805 cma_deref_dev(cma_dev);
2806 wait_for_completion(&cma_dev->comp);
2807}
2808
2809static void cma_remove_one(struct ib_device *device)
2810{
2811 struct cma_device *cma_dev;
2812
2813 cma_dev = ib_get_client_data(device, &cma_client);
2814 if (!cma_dev)
2815 return;
2816
2817 mutex_lock(&lock);
2818 list_del(&cma_dev->list);
2819 mutex_unlock(&lock);
2820
2821 cma_process_remove(cma_dev);
2822 kfree(cma_dev);
2823}
2824
2825static int cma_init(void)
2826{
a25de534 2827 int ret, low, high, remaining;
e51060f0 2828
aedec080 2829 get_random_bytes(&next_port, sizeof next_port);
227b60f5 2830 inet_get_local_port_range(&low, &high);
a25de534
AA		 2831 remaining = (high - low) + 1;
2832 next_port = ((unsigned int) next_port % remaining) + low;
227b60f5 2833
c7f743a6 2834 cma_wq = create_singlethread_workqueue("rdma_cm");
e51060f0
SH		 2835 if (!cma_wq)
2836 return -ENOMEM;
2837
c1a0b23b 2838 ib_sa_register_client(&sa_client);
7a118df3 2839 rdma_addr_register_client(&addr_client);
c1a0b23b 2840
e51060f0
SH		 2841 ret = ib_register_client(&cma_client);
2842 if (ret)
2843 goto err;
2844 return 0;
2845
2846err:
7a118df3 2847 rdma_addr_unregister_client(&addr_client);
c1a0b23b 2848 ib_sa_unregister_client(&sa_client);
e51060f0
SH		 2849 destroy_workqueue(cma_wq);
2850 return ret;
2851}
2852
2853static void cma_cleanup(void)
2854{
2855 ib_unregister_client(&cma_client);
7a118df3 2856 rdma_addr_unregister_client(&addr_client);
c1a0b23b 2857 ib_sa_unregister_client(&sa_client);
e51060f0
SH		 2858 destroy_workqueue(cma_wq);
2859 idr_destroy(&sdp_ps);
2860 idr_destroy(&tcp_ps);
628e5f6d 2861 idr_destroy(&udp_ps);
c8f6a362 2862 idr_destroy(&ipoib_ps);
e51060f0
SH		 2863}
2864
2865module_init(cma_init);
2866module_exit(cma_cleanup);
kernel source for telekom puls (alcatel/mediatek)