From 15723f06fb9d80cbfd895c32c6023881c7d0e0b4 Mon Sep 17 00:00:00 2001 From: Dennis Dalessandro Date: Tue, 19 Jan 2016 14:42:17 -0800 Subject: [PATCH] staging/rdma/hfi1: Use address handle in rdmavt and remove from hfi1 Original patch from Kamal Heib , split apart from original and modified to accomodate recent changes in rdmavt. Remove AH from hfi1 and use rdmavt version. Signed-off-by: Kamal Heib Signed-off-by: Dennis Dalessandro Signed-off-by: Doug Ledford --- drivers/staging/rdma/hfi1/common.h | 2 - drivers/staging/rdma/hfi1/mad.c | 2 +- drivers/staging/rdma/hfi1/qp.c | 6 +- drivers/staging/rdma/hfi1/ruc.c | 2 +- drivers/staging/rdma/hfi1/ud.c | 4 +- drivers/staging/rdma/hfi1/verbs.c | 131 ++--------------------------- drivers/staging/rdma/hfi1/verbs.h | 20 +---- 7 files changed, 18 insertions(+), 149 deletions(-) diff --git a/drivers/staging/rdma/hfi1/common.h b/drivers/staging/rdma/hfi1/common.h index e4b1dc6d0328..cb5ca794ac08 100644 --- a/drivers/staging/rdma/hfi1/common.h +++ b/drivers/staging/rdma/hfi1/common.h @@ -341,7 +341,6 @@ struct hfi1_message_header { #define FULL_MGMT_P_KEY 0xFFFF #define DEFAULT_P_KEY LIM_MGMT_P_KEY -#define HFI1_PERMISSIVE_LID 0xFFFF #define HFI1_AETH_CREDIT_SHIFT 24 #define HFI1_AETH_CREDIT_MASK 0x1F #define HFI1_AETH_CREDIT_INVAL 0x1F @@ -353,7 +352,6 @@ struct hfi1_message_header { #define HFI1_BECN_SHIFT 30 #define HFI1_BECN_MASK 1 #define HFI1_BECN_SMASK BIT(HFI1_BECN_SHIFT) -#define HFI1_MULTICAST_LID_BASE 0xC000 static inline __u64 rhf_to_cpu(const __le32 *rbuf) { diff --git a/drivers/staging/rdma/hfi1/mad.c b/drivers/staging/rdma/hfi1/mad.c index 13994850396c..1190f8dd2629 100644 --- a/drivers/staging/rdma/hfi1/mad.c +++ b/drivers/staging/rdma/hfi1/mad.c @@ -137,7 +137,7 @@ static void send_trap(struct hfi1_ibport *ibp, void *data, unsigned len) ret = PTR_ERR(ah); else { send_buf->ah = ah; - ibp->sm_ah = to_iah(ah); + ibp->sm_ah = ibah_to_rvtah(ah); ret = 0; } } else diff --git a/drivers/staging/rdma/hfi1/qp.c b/drivers/staging/rdma/hfi1/qp.c index 735253b7d3e8..9fcf052b62bd 100644 --- a/drivers/staging/rdma/hfi1/qp.c +++ b/drivers/staging/rdma/hfi1/qp.c @@ -424,7 +424,7 @@ static void clear_mr_refs(struct hfi1_qp *qp, int clr_sends) if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_SMI || qp->ibqp.qp_type == IB_QPT_GSI) - atomic_dec(&to_iah(wqe->ud_wr.ah)->refcount); + atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount); if (++qp->s_last >= qp->s_size) qp->s_last = 0; } @@ -642,7 +642,7 @@ int hfi1_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (attr->ah_attr.dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) goto inval; - if (hfi1_check_ah(qp->ibqp.device, &attr->ah_attr)) + if (rvt_check_ah(qp->ibqp.device, &attr->ah_attr)) goto inval; sc = ah_to_sc(ibqp->device, &attr->ah_attr); if (!qp_to_sdma_engine(qp, sc) && @@ -656,7 +656,7 @@ int hfi1_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (attr->alt_ah_attr.dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) goto inval; - if (hfi1_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) + if (rvt_check_ah(qp->ibqp.device, &attr->alt_ah_attr)) goto inval; if (attr->alt_pkey_index >= hfi1_get_npkeys(dd)) goto inval; diff --git a/drivers/staging/rdma/hfi1/ruc.c b/drivers/staging/rdma/hfi1/ruc.c index 55ed00dd0218..33bcfe5bfd13 100644 --- a/drivers/staging/rdma/hfi1/ruc.c +++ b/drivers/staging/rdma/hfi1/ruc.c @@ -893,7 +893,7 @@ void hfi1_send_complete(struct hfi1_qp *qp, struct hfi1_swqe *wqe, if (qp->ibqp.qp_type == IB_QPT_UD || qp->ibqp.qp_type == IB_QPT_SMI || qp->ibqp.qp_type == IB_QPT_GSI) - atomic_dec(&to_iah(wqe->ud_wr.ah)->refcount); + atomic_dec(&ibah_to_rvtah(wqe->ud_wr.ah)->refcount); /* See ch. 11.2.4.1 and 10.7.3.1 */ if (!(qp->s_flags & HFI1_S_SIGNAL_REQ_WR) || diff --git a/drivers/staging/rdma/hfi1/ud.c b/drivers/staging/rdma/hfi1/ud.c index 55a4eec37331..820fef211edf 100644 --- a/drivers/staging/rdma/hfi1/ud.c +++ b/drivers/staging/rdma/hfi1/ud.c @@ -98,7 +98,7 @@ static void ud_loopback(struct hfi1_qp *sqp, struct hfi1_swqe *swqe) goto drop; } - ah_attr = &to_iah(swqe->ud_wr.ah)->attr; + ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -309,7 +309,7 @@ int hfi1_make_ud_req(struct hfi1_qp *qp) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &to_iah(wqe->ud_wr.ah)->attr; + ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; if (ah_attr->dlid < be16_to_cpu(IB_MULTICAST_LID_BASE) || ah_attr->dlid == be16_to_cpu(IB_LID_PERMISSIVE)) { lid = ah_attr->dlid & ~((1 << ppd->lmc) - 1); diff --git a/drivers/staging/rdma/hfi1/verbs.c b/drivers/staging/rdma/hfi1/verbs.c index 5c952d82a023..021e21153b90 100644 --- a/drivers/staging/rdma/hfi1/verbs.c +++ b/drivers/staging/rdma/hfi1/verbs.c @@ -455,9 +455,7 @@ static int post_one_send(struct hfi1_qp *qp, struct ib_send_wr *wr) if (wqe->length > 0x80000000U) goto bail_inval_free; } else { - struct hfi1_ah *ah = to_iah(ud_wr(wr)->ah); - - atomic_inc(&ah->refcount); + atomic_inc(&ibah_to_rvtah(ud_wr(wr)->ah)->refcount); } wqe->ssn = qp->s_ssn++; qp->s_head = next; @@ -1615,88 +1613,21 @@ u8 ah_to_sc(struct ib_device *ibdev, struct ib_ah_attr *ah) return ibp->sl_to_sc[ah->sl]; } -int hfi1_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr) +static int hfi1_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr) { struct hfi1_ibport *ibp; struct hfi1_pportdata *ppd; struct hfi1_devdata *dd; u8 sc5; - /* A multicast address requires a GRH (see ch. 8.4.1). */ - if (ah_attr->dlid >= HFI1_MULTICAST_LID_BASE && - ah_attr->dlid != HFI1_PERMISSIVE_LID && - !(ah_attr->ah_flags & IB_AH_GRH)) - goto bail; - if ((ah_attr->ah_flags & IB_AH_GRH) && - ah_attr->grh.sgid_index >= HFI1_GUIDS_PER_PORT) - goto bail; - if (ah_attr->dlid == 0) - goto bail; - if (ah_attr->port_num < 1 || - ah_attr->port_num > ibdev->phys_port_cnt) - goto bail; - if (ah_attr->static_rate != IB_RATE_PORT_CURRENT && - ib_rate_to_mbps(ah_attr->static_rate) < 0) - goto bail; - if (ah_attr->sl >= OPA_MAX_SLS) - goto bail; /* test the mapping for validity */ ibp = to_iport(ibdev, ah_attr->port_num); ppd = ppd_from_ibp(ibp); sc5 = ibp->sl_to_sc[ah_attr->sl]; dd = dd_from_ppd(ppd); if (sc_to_vlt(dd, sc5) > num_vls && sc_to_vlt(dd, sc5) != 0xf) - goto bail; + return -EINVAL; return 0; -bail: - return -EINVAL; -} - -/** - * create_ah - create an address handle - * @pd: the protection domain - * @ah_attr: the attributes of the AH - * - * This may be called from interrupt context. - */ -static struct ib_ah *create_ah(struct ib_pd *pd, - struct ib_ah_attr *ah_attr) -{ - struct hfi1_ah *ah; - struct ib_ah *ret; - struct hfi1_ibdev *dev = to_idev(pd->device); - unsigned long flags; - - if (hfi1_check_ah(pd->device, ah_attr)) { - ret = ERR_PTR(-EINVAL); - goto bail; - } - - ah = kmalloc(sizeof(*ah), GFP_ATOMIC); - if (!ah) { - ret = ERR_PTR(-ENOMEM); - goto bail; - } - - spin_lock_irqsave(&dev->n_ahs_lock, flags); - if (dev->n_ahs_allocated == hfi1_max_ahs) { - spin_unlock_irqrestore(&dev->n_ahs_lock, flags); - kfree(ah); - ret = ERR_PTR(-ENOMEM); - goto bail; - } - - dev->n_ahs_allocated++; - spin_unlock_irqrestore(&dev->n_ahs_lock, flags); - - /* ib_create_ah() will initialize ah->ibah. */ - ah->attr = *ah_attr; - atomic_set(&ah->refcount, 0); - - ret = &ah->ibah; - -bail: - return ret; } struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid) @@ -1716,51 +1647,6 @@ struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid) return ah; } -/** - * destroy_ah - destroy an address handle - * @ibah: the AH to destroy - * - * This may be called from interrupt context. - */ -static int destroy_ah(struct ib_ah *ibah) -{ - struct hfi1_ibdev *dev = to_idev(ibah->device); - struct hfi1_ah *ah = to_iah(ibah); - unsigned long flags; - - if (atomic_read(&ah->refcount) != 0) - return -EBUSY; - - spin_lock_irqsave(&dev->n_ahs_lock, flags); - dev->n_ahs_allocated--; - spin_unlock_irqrestore(&dev->n_ahs_lock, flags); - - kfree(ah); - - return 0; -} - -static int modify_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) -{ - struct hfi1_ah *ah = to_iah(ibah); - - if (hfi1_check_ah(ibah->device, ah_attr)) - return -EINVAL; - - ah->attr = *ah_attr; - - return 0; -} - -static int query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr) -{ - struct hfi1_ah *ah = to_iah(ibah); - - *ah_attr = ah->attr; - - return 0; -} - /** * hfi1_get_npkeys - return the size of the PKEY table for context 0 * @dd: the hfi1_ib device @@ -1879,7 +1765,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) /* Only need to initialize non-zero fields. */ - spin_lock_init(&dev->n_ahs_lock); spin_lock_init(&dev->n_cqs_lock); spin_lock_init(&dev->n_qps_lock); spin_lock_init(&dev->n_srqs_lock); @@ -1989,10 +1874,10 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) ibdev->dealloc_ucontext = dealloc_ucontext; ibdev->alloc_pd = NULL; ibdev->dealloc_pd = NULL; - ibdev->create_ah = create_ah; - ibdev->destroy_ah = destroy_ah; - ibdev->modify_ah = modify_ah; - ibdev->query_ah = query_ah; + ibdev->create_ah = NULL; + ibdev->destroy_ah = NULL; + ibdev->modify_ah = NULL; + ibdev->query_ah = NULL; ibdev->create_srq = hfi1_create_srq; ibdev->modify_srq = hfi1_modify_srq; ibdev->query_srq = hfi1_query_srq; @@ -2033,6 +1918,8 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) dd->verbs_dev.rdi.driver_f.port_callback = hfi1_create_port_files; dd->verbs_dev.rdi.driver_f.get_card_name = get_card_name; dd->verbs_dev.rdi.driver_f.get_pci_dev = get_pci_dev; + dd->verbs_dev.rdi.driver_f.check_ah = hfi1_check_ah; + dd->verbs_dev.rdi.dparms.props.max_ah = hfi1_max_ahs; dd->verbs_dev.rdi.dparms.props.max_pd = hfi1_max_pds; dd->verbs_dev.rdi.flags = (RVT_FLAG_MR_INIT_DRIVER | RVT_FLAG_QP_INIT_DRIVER | diff --git a/drivers/staging/rdma/hfi1/verbs.h b/drivers/staging/rdma/hfi1/verbs.h index dc5aa9aacd02..4db6136f0384 100644 --- a/drivers/staging/rdma/hfi1/verbs.h +++ b/drivers/staging/rdma/hfi1/verbs.h @@ -234,13 +234,6 @@ struct hfi1_mcast { int n_attached; }; -/* Address Handle */ -struct hfi1_ah { - struct ib_ah ibah; - struct ib_ah_attr attr; - atomic_t refcount; -}; - /* * This structure is used by hfi1_mmap() to validate an offset * when an mmap() request is made. The vm_area_struct then uses @@ -652,8 +645,8 @@ static inline void inc_opstats( struct hfi1_ibport { struct hfi1_qp __rcu *qp[2]; struct ib_mad_agent *send_agent; /* agent for SMI (traps) */ - struct hfi1_ah *sm_ah; - struct hfi1_ah *smi_ah; + struct rvt_ah *sm_ah; + struct rvt_ah *smi_ah; struct rb_root mcast_tree; spinlock_t lock; /* protect changes in this struct */ @@ -735,8 +728,6 @@ struct hfi1_ibdev { u64 n_kmem_wait; u64 n_send_schedule; - u32 n_ahs_allocated; /* number of AHs allocated for device */ - spinlock_t n_ahs_lock; u32 n_cqs_allocated; /* number of CQs allocated for device */ spinlock_t n_cqs_lock; u32 n_qps_allocated; /* number of QPs allocated for device */ @@ -774,11 +765,6 @@ static inline struct hfi1_mr *to_imr(struct ib_mr *ibmr) return container_of(ibmr, struct hfi1_mr, ibmr); } -static inline struct hfi1_ah *to_iah(struct ib_ah *ibah) -{ - return container_of(ibah, struct hfi1_ah, ibah); -} - static inline struct hfi1_cq *to_icq(struct ib_cq *ibcq) { return container_of(ibcq, struct hfi1_cq, ibcq); @@ -925,8 +911,6 @@ void hfi1_rc_hdrerr( u8 ah_to_sc(struct ib_device *ibdev, struct ib_ah_attr *ah_attr); -int hfi1_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr); - struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid); void hfi1_rc_rnr_retry(unsigned long arg); -- 2.20.1