The ehca driver is only supported on IBM machines with a custom EBus.
As they have opted to build their newer machines using more industry
standard technology and haven't really been pushing EBus capable
machines for a while, this driver can now safely be moved to the
staging area and scheduled for eventual removal. This plan was brought
to IBM's attention and received their sign-off.
Cc: alexs@linux.vnet.ibm.com
Cc: hnguyen@de.ibm.com
Cc: raisch@de.ibm.com
Cc: stefan.roscher@de.ibm.com
Signed-off-by: Doug Ledford <dledford@redhat.com>
source "drivers/infiniband/hw/mthca/Kconfig"
source "drivers/infiniband/hw/qib/Kconfig"
-source "drivers/infiniband/hw/ehca/Kconfig"
source "drivers/infiniband/hw/cxgb3/Kconfig"
source "drivers/infiniband/hw/cxgb4/Kconfig"
source "drivers/infiniband/hw/mlx4/Kconfig"
obj-$(CONFIG_INFINIBAND_MTHCA) += mthca/
obj-$(CONFIG_INFINIBAND_QIB) += qib/
-obj-$(CONFIG_INFINIBAND_EHCA) += ehca/
obj-$(CONFIG_INFINIBAND_CXGB3) += cxgb3/
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/
+++ /dev/null
-config INFINIBAND_EHCA
- tristate "eHCA support"
- depends on IBMEBUS
- ---help---
- This driver supports the IBM pSeries eHCA InfiniBand adapter.
-
- To compile the driver as a module, choose M here. The module
- will be called ib_ehca.
-
+++ /dev/null
-# Authors: Heiko J Schick <schickhj@de.ibm.com>
-# Christoph Raisch <raisch@de.ibm.com>
-# Joachim Fenkes <fenkes@de.ibm.com>
-#
-# Copyright (c) 2005 IBM Corporation
-#
-# All rights reserved.
-#
-# This source code is distributed under a dual license of GPL v2.0 and OpenIB BSD.
-
-obj-$(CONFIG_INFINIBAND_EHCA) += ib_ehca.o
-
-ib_ehca-objs = ehca_main.o ehca_hca.o ehca_mcast.o ehca_pd.o ehca_av.o ehca_eq.o \
- ehca_cq.o ehca_qp.o ehca_sqp.o ehca_mrmw.o ehca_reqs.o ehca_irq.o \
- ehca_uverbs.o ipz_pt_fn.o hcp_if.o hcp_phyp.o
-
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * address vector functions
- *
- * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Khadija Souissi <souissik@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-static struct kmem_cache *av_cache;
-
-int ehca_calc_ipd(struct ehca_shca *shca, int port,
- enum ib_rate path_rate, u32 *ipd)
-{
- int path = ib_rate_to_mult(path_rate);
- int link, ret;
- struct ib_port_attr pa;
-
- if (path_rate == IB_RATE_PORT_CURRENT) {
- *ipd = 0;
- return 0;
- }
-
- if (unlikely(path < 0)) {
- ehca_err(&shca->ib_device, "Invalid static rate! path_rate=%x",
- path_rate);
- return -EINVAL;
- }
-
- ret = ehca_query_port(&shca->ib_device, port, &pa);
- if (unlikely(ret < 0)) {
- ehca_err(&shca->ib_device, "Failed to query port ret=%i", ret);
- return ret;
- }
-
- link = ib_width_enum_to_int(pa.active_width) * pa.active_speed;
-
- if (path >= link)
- /* no need to throttle if path faster than link */
- *ipd = 0;
- else
- /* IPD = round((link / path) - 1) */
- *ipd = ((link + (path >> 1)) / path) - 1;
-
- return 0;
-}
-
-struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
-{
- int ret;
- struct ehca_av *av;
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
-
- av = kmem_cache_alloc(av_cache, GFP_KERNEL);
- if (!av) {
- ehca_err(pd->device, "Out of memory pd=%p ah_attr=%p",
- pd, ah_attr);
- return ERR_PTR(-ENOMEM);
- }
-
- av->av.sl = ah_attr->sl;
- av->av.dlid = ah_attr->dlid;
- av->av.slid_path_bits = ah_attr->src_path_bits;
-
- if (ehca_static_rate < 0) {
- u32 ipd;
- if (ehca_calc_ipd(shca, ah_attr->port_num,
- ah_attr->static_rate, &ipd)) {
- ret = -EINVAL;
- goto create_ah_exit1;
- }
- av->av.ipd = ipd;
- } else
- av->av.ipd = ehca_static_rate;
-
- av->av.lnh = ah_attr->ah_flags;
- av->av.grh.word_0 = EHCA_BMASK_SET(GRH_IPVERSION_MASK, 6);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_TCLASS_MASK,
- ah_attr->grh.traffic_class);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
- ah_attr->grh.flow_label);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
- ah_attr->grh.hop_limit);
- av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1B);
- /* set sgid in grh.word_1 */
- if (ah_attr->ah_flags & IB_AH_GRH) {
- int rc;
- struct ib_port_attr port_attr;
- union ib_gid gid;
- memset(&port_attr, 0, sizeof(port_attr));
- rc = ehca_query_port(pd->device, ah_attr->port_num,
- &port_attr);
- if (rc) { /* invalid port number */
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid port number "
- "ehca_query_port() returned %x "
- "pd=%p ah_attr=%p", rc, pd, ah_attr);
- goto create_ah_exit1;
- }
- memset(&gid, 0, sizeof(gid));
- rc = ehca_query_gid(pd->device,
- ah_attr->port_num,
- ah_attr->grh.sgid_index, &gid);
- if (rc) {
- ret = -EINVAL;
- ehca_err(pd->device, "Failed to retrieve sgid "
- "ehca_query_gid() returned %x "
- "pd=%p ah_attr=%p", rc, pd, ah_attr);
- goto create_ah_exit1;
- }
- memcpy(&av->av.grh.word_1, &gid, sizeof(gid));
- }
- av->av.pmtu = shca->max_mtu;
-
- /* dgid comes in grh.word_3 */
- memcpy(&av->av.grh.word_3, &ah_attr->grh.dgid,
- sizeof(ah_attr->grh.dgid));
-
- return &av->ib_ah;
-
-create_ah_exit1:
- kmem_cache_free(av_cache, av);
-
- return ERR_PTR(ret);
-}
-
-int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
-{
- struct ehca_av *av;
- struct ehca_ud_av new_ehca_av;
- struct ehca_shca *shca = container_of(ah->pd->device, struct ehca_shca,
- ib_device);
-
- memset(&new_ehca_av, 0, sizeof(new_ehca_av));
- new_ehca_av.sl = ah_attr->sl;
- new_ehca_av.dlid = ah_attr->dlid;
- new_ehca_av.slid_path_bits = ah_attr->src_path_bits;
- new_ehca_av.ipd = ah_attr->static_rate;
- new_ehca_av.lnh = EHCA_BMASK_SET(GRH_FLAG_MASK,
- (ah_attr->ah_flags & IB_AH_GRH) > 0);
- new_ehca_av.grh.word_0 = EHCA_BMASK_SET(GRH_TCLASS_MASK,
- ah_attr->grh.traffic_class);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
- ah_attr->grh.flow_label);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
- ah_attr->grh.hop_limit);
- new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1b);
-
- /* set sgid in grh.word_1 */
- if (ah_attr->ah_flags & IB_AH_GRH) {
- int rc;
- struct ib_port_attr port_attr;
- union ib_gid gid;
- memset(&port_attr, 0, sizeof(port_attr));
- rc = ehca_query_port(ah->device, ah_attr->port_num,
- &port_attr);
- if (rc) { /* invalid port number */
- ehca_err(ah->device, "Invalid port number "
- "ehca_query_port() returned %x "
- "ah=%p ah_attr=%p port_num=%x",
- rc, ah, ah_attr, ah_attr->port_num);
- return -EINVAL;
- }
- memset(&gid, 0, sizeof(gid));
- rc = ehca_query_gid(ah->device,
- ah_attr->port_num,
- ah_attr->grh.sgid_index, &gid);
- if (rc) {
- ehca_err(ah->device, "Failed to retrieve sgid "
- "ehca_query_gid() returned %x "
- "ah=%p ah_attr=%p port_num=%x "
- "sgid_index=%x",
- rc, ah, ah_attr, ah_attr->port_num,
- ah_attr->grh.sgid_index);
- return -EINVAL;
- }
- memcpy(&new_ehca_av.grh.word_1, &gid, sizeof(gid));
- }
-
- new_ehca_av.pmtu = shca->max_mtu;
-
- memcpy(&new_ehca_av.grh.word_3, &ah_attr->grh.dgid,
- sizeof(ah_attr->grh.dgid));
-
- av = container_of(ah, struct ehca_av, ib_ah);
- av->av = new_ehca_av;
-
- return 0;
-}
-
-int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
-{
- struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
-
- memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
- sizeof(ah_attr->grh.dgid));
- ah_attr->sl = av->av.sl;
-
- ah_attr->dlid = av->av.dlid;
-
- ah_attr->src_path_bits = av->av.slid_path_bits;
- ah_attr->static_rate = av->av.ipd;
- ah_attr->ah_flags = EHCA_BMASK_GET(GRH_FLAG_MASK, av->av.lnh);
- ah_attr->grh.traffic_class = EHCA_BMASK_GET(GRH_TCLASS_MASK,
- av->av.grh.word_0);
- ah_attr->grh.hop_limit = EHCA_BMASK_GET(GRH_HOPLIMIT_MASK,
- av->av.grh.word_0);
- ah_attr->grh.flow_label = EHCA_BMASK_GET(GRH_FLOWLABEL_MASK,
- av->av.grh.word_0);
-
- return 0;
-}
-
-int ehca_destroy_ah(struct ib_ah *ah)
-{
- kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
-
- return 0;
-}
-
-int ehca_init_av_cache(void)
-{
- av_cache = kmem_cache_create("ehca_cache_av",
- sizeof(struct ehca_av), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!av_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_av_cache(void)
-{
- if (av_cache)
- kmem_cache_destroy(av_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Struct definition for eHCA internal structures
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_CLASSES_H__
-#define __EHCA_CLASSES_H__
-
-struct ehca_module;
-struct ehca_qp;
-struct ehca_cq;
-struct ehca_eq;
-struct ehca_mr;
-struct ehca_mw;
-struct ehca_pd;
-struct ehca_av;
-
-#include <linux/wait.h>
-#include <linux/mutex.h>
-
-#include <rdma/ib_verbs.h>
-#include <rdma/ib_user_verbs.h>
-
-#ifdef CONFIG_PPC64
-#include "ehca_classes_pSeries.h"
-#endif
-#include "ipz_pt_fn.h"
-#include "ehca_qes.h"
-#include "ehca_irq.h"
-
-#define EHCA_EQE_CACHE_SIZE 20
-#define EHCA_MAX_NUM_QUEUES 0xffff
-
-struct ehca_eqe_cache_entry {
- struct ehca_eqe *eqe;
- struct ehca_cq *cq;
-};
-
-struct ehca_eq {
- u32 length;
- struct ipz_queue ipz_queue;
- struct ipz_eq_handle ipz_eq_handle;
- struct work_struct work;
- struct h_galpas galpas;
- int is_initialized;
- struct ehca_pfeq pf;
- spinlock_t spinlock;
- struct tasklet_struct interrupt_task;
- u32 ist;
- spinlock_t irq_spinlock;
- struct ehca_eqe_cache_entry eqe_cache[EHCA_EQE_CACHE_SIZE];
-};
-
-struct ehca_sma_attr {
- u16 lid, lmc, sm_sl, sm_lid;
- u16 pkey_tbl_len, pkeys[16];
-};
-
-struct ehca_sport {
- struct ib_cq *ibcq_aqp1;
- struct ib_qp *ibqp_sqp[2];
- /* lock to serialze modify_qp() calls for sqp in normal
- * and irq path (when event PORT_ACTIVE is received first time)
- */
- spinlock_t mod_sqp_lock;
- enum ib_port_state port_state;
- struct ehca_sma_attr saved_attr;
- u32 pma_qp_nr;
-};
-
-#define HCA_CAP_MR_PGSIZE_4K 0x80000000
-#define HCA_CAP_MR_PGSIZE_64K 0x40000000
-#define HCA_CAP_MR_PGSIZE_1M 0x20000000
-#define HCA_CAP_MR_PGSIZE_16M 0x10000000
-
-struct ehca_shca {
- struct ib_device ib_device;
- struct platform_device *ofdev;
- u8 num_ports;
- int hw_level;
- struct list_head shca_list;
- struct ipz_adapter_handle ipz_hca_handle;
- struct ehca_sport sport[2];
- struct ehca_eq eq;
- struct ehca_eq neq;
- struct ehca_mr *maxmr;
- struct ehca_pd *pd;
- struct h_galpas galpas;
- struct mutex modify_mutex;
- u64 hca_cap;
- /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
- u32 hca_cap_mr_pgsize;
- int max_mtu;
- int max_num_qps;
- int max_num_cqs;
- atomic_t num_cqs;
- atomic_t num_qps;
-};
-
-struct ehca_pd {
- struct ib_pd ib_pd;
- struct ipz_pd fw_pd;
- /* small queue mgmt */
- struct mutex lock;
- struct list_head free[2];
- struct list_head full[2];
-};
-
-enum ehca_ext_qp_type {
- EQPT_NORMAL = 0,
- EQPT_LLQP = 1,
- EQPT_SRQBASE = 2,
- EQPT_SRQ = 3,
-};
-
-/* struct to cache modify_qp()'s parms for GSI/SMI qp */
-struct ehca_mod_qp_parm {
- int mask;
- struct ib_qp_attr attr;
-};
-
-#define EHCA_MOD_QP_PARM_MAX 4
-
-#define QMAP_IDX_MASK 0xFFFFULL
-
-/* struct for tracking if cqes have been reported to the application */
-struct ehca_qmap_entry {
- u16 app_wr_id;
- u8 reported;
- u8 cqe_req;
-};
-
-struct ehca_queue_map {
- struct ehca_qmap_entry *map;
- unsigned int entries;
- unsigned int tail;
- unsigned int left_to_poll;
- unsigned int next_wqe_idx; /* Idx to first wqe to be flushed */
-};
-
-/* function to calculate the next index for the qmap */
-static inline unsigned int next_index(unsigned int cur_index, unsigned int limit)
-{
- unsigned int temp = cur_index + 1;
- return (temp == limit) ? 0 : temp;
-}
-
-struct ehca_qp {
- union {
- struct ib_qp ib_qp;
- struct ib_srq ib_srq;
- };
- u32 qp_type;
- enum ehca_ext_qp_type ext_type;
- enum ib_qp_state state;
- struct ipz_queue ipz_squeue;
- struct ehca_queue_map sq_map;
- struct ipz_queue ipz_rqueue;
- struct ehca_queue_map rq_map;
- struct h_galpas galpas;
- u32 qkey;
- u32 real_qp_num;
- u32 token;
- spinlock_t spinlock_s;
- spinlock_t spinlock_r;
- u32 sq_max_inline_data_size;
- struct ipz_qp_handle ipz_qp_handle;
- struct ehca_pfqp pf;
- struct ib_qp_init_attr init_attr;
- struct ehca_cq *send_cq;
- struct ehca_cq *recv_cq;
- unsigned int sqerr_purgeflag;
- struct hlist_node list_entries;
- /* array to cache modify_qp()'s parms for GSI/SMI qp */
- struct ehca_mod_qp_parm *mod_qp_parm;
- int mod_qp_parm_idx;
- /* mmap counter for resources mapped into user space */
- u32 mm_count_squeue;
- u32 mm_count_rqueue;
- u32 mm_count_galpa;
- /* unsolicited ack circumvention */
- int unsol_ack_circ;
- int mtu_shift;
- u32 message_count;
- u32 packet_count;
- atomic_t nr_events; /* events seen */
- wait_queue_head_t wait_completion;
- int mig_armed;
- struct list_head sq_err_node;
- struct list_head rq_err_node;
-};
-
-#define IS_SRQ(qp) (qp->ext_type == EQPT_SRQ)
-#define HAS_SQ(qp) (qp->ext_type != EQPT_SRQ)
-#define HAS_RQ(qp) (qp->ext_type != EQPT_SRQBASE)
-
-/* must be power of 2 */
-#define QP_HASHTAB_LEN 8
-
-struct ehca_cq {
- struct ib_cq ib_cq;
- struct ipz_queue ipz_queue;
- struct h_galpas galpas;
- spinlock_t spinlock;
- u32 cq_number;
- u32 token;
- u32 nr_of_entries;
- struct ipz_cq_handle ipz_cq_handle;
- struct ehca_pfcq pf;
- spinlock_t cb_lock;
- struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
- struct list_head entry;
- u32 nr_callbacks; /* #events assigned to cpu by scaling code */
- atomic_t nr_events; /* #events seen */
- wait_queue_head_t wait_completion;
- spinlock_t task_lock;
- /* mmap counter for resources mapped into user space */
- u32 mm_count_queue;
- u32 mm_count_galpa;
- struct list_head sqp_err_list;
- struct list_head rqp_err_list;
-};
-
-enum ehca_mr_flag {
- EHCA_MR_FLAG_FMR = 0x80000000, /* FMR, created with ehca_alloc_fmr */
- EHCA_MR_FLAG_MAXMR = 0x40000000, /* max-MR */
-};
-
-struct ehca_mr {
- union {
- struct ib_mr ib_mr; /* must always be first in ehca_mr */
- struct ib_fmr ib_fmr; /* must always be first in ehca_mr */
- } ib;
- struct ib_umem *umem;
- spinlock_t mrlock;
-
- enum ehca_mr_flag flags;
- u32 num_kpages; /* number of kernel pages */
- u32 num_hwpages; /* number of hw pages to form MR */
- u64 hwpage_size; /* hw page size used for this MR */
- int acl; /* ACL (stored here for usage in reregister) */
- u64 *start; /* virtual start address (stored here for */
- /* usage in reregister) */
- u64 size; /* size (stored here for usage in reregister) */
- u32 fmr_page_size; /* page size for FMR */
- u32 fmr_max_pages; /* max pages for FMR */
- u32 fmr_max_maps; /* max outstanding maps for FMR */
- u32 fmr_map_cnt; /* map counter for FMR */
- /* fw specific data */
- struct ipz_mrmw_handle ipz_mr_handle; /* MR handle for h-calls */
- struct h_galpas galpas;
-};
-
-struct ehca_mw {
- struct ib_mw ib_mw; /* gen2 mw, must always be first in ehca_mw */
- spinlock_t mwlock;
-
- u8 never_bound; /* indication MW was never bound */
- struct ipz_mrmw_handle ipz_mw_handle; /* MW handle for h-calls */
- struct h_galpas galpas;
-};
-
-enum ehca_mr_pgi_type {
- EHCA_MR_PGI_PHYS = 1, /* type of ehca_reg_phys_mr,
- * ehca_rereg_phys_mr,
- * ehca_reg_internal_maxmr */
- EHCA_MR_PGI_USER = 2, /* type of ehca_reg_user_mr */
- EHCA_MR_PGI_FMR = 3 /* type of ehca_map_phys_fmr */
-};
-
-struct ehca_mr_pginfo {
- enum ehca_mr_pgi_type type;
- u64 num_kpages;
- u64 kpage_cnt;
- u64 hwpage_size; /* hw page size used for this MR */
- u64 num_hwpages; /* number of hw pages */
- u64 hwpage_cnt; /* counter for hw pages */
- u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
-
- union {
- struct { /* type EHCA_MR_PGI_PHYS section */
- int num_phys_buf;
- struct ib_phys_buf *phys_buf_array;
- u64 next_buf;
- } phy;
- struct { /* type EHCA_MR_PGI_USER section */
- struct ib_umem *region;
- struct scatterlist *next_sg;
- u64 next_nmap;
- } usr;
- struct { /* type EHCA_MR_PGI_FMR section */
- u64 fmr_pgsize;
- u64 *page_list;
- u64 next_listelem;
- } fmr;
- } u;
-};
-
-/* output parameters for MR/FMR hipz calls */
-struct ehca_mr_hipzout_parms {
- struct ipz_mrmw_handle handle;
- u32 lkey;
- u32 rkey;
- u64 len;
- u64 vaddr;
- u32 acl;
-};
-
-/* output parameters for MW hipz calls */
-struct ehca_mw_hipzout_parms {
- struct ipz_mrmw_handle handle;
- u32 rkey;
-};
-
-struct ehca_av {
- struct ib_ah ib_ah;
- struct ehca_ud_av av;
-};
-
-struct ehca_ucontext {
- struct ib_ucontext ib_ucontext;
-};
-
-int ehca_init_pd_cache(void);
-void ehca_cleanup_pd_cache(void);
-int ehca_init_cq_cache(void);
-void ehca_cleanup_cq_cache(void);
-int ehca_init_qp_cache(void);
-void ehca_cleanup_qp_cache(void);
-int ehca_init_av_cache(void);
-void ehca_cleanup_av_cache(void);
-int ehca_init_mrmw_cache(void);
-void ehca_cleanup_mrmw_cache(void);
-int ehca_init_small_qp_cache(void);
-void ehca_cleanup_small_qp_cache(void);
-
-extern rwlock_t ehca_qp_idr_lock;
-extern rwlock_t ehca_cq_idr_lock;
-extern struct idr ehca_qp_idr;
-extern struct idr ehca_cq_idr;
-extern spinlock_t shca_list_lock;
-
-extern int ehca_static_rate;
-extern int ehca_port_act_time;
-extern bool ehca_use_hp_mr;
-extern bool ehca_scaling_code;
-extern int ehca_lock_hcalls;
-extern int ehca_nr_ports;
-extern int ehca_max_cq;
-extern int ehca_max_qp;
-
-struct ipzu_queue_resp {
- u32 qe_size; /* queue entry size */
- u32 act_nr_of_sg;
- u32 queue_length; /* queue length allocated in bytes */
- u32 pagesize;
- u32 toggle_state;
- u32 offset; /* save offset within a page for small_qp */
-};
-
-struct ehca_create_cq_resp {
- u32 cq_number;
- u32 token;
- struct ipzu_queue_resp ipz_queue;
- u32 fw_handle_ofs;
- u32 dummy;
-};
-
-struct ehca_create_qp_resp {
- u32 qp_num;
- u32 token;
- u32 qp_type;
- u32 ext_type;
- u32 qkey;
- /* qp_num assigned by ehca: sqp0/1 may have got different numbers */
- u32 real_qp_num;
- u32 fw_handle_ofs;
- u32 dummy;
- struct ipzu_queue_resp ipz_squeue;
- struct ipzu_queue_resp ipz_rqueue;
-};
-
-struct ehca_alloc_cq_parms {
- u32 nr_cqe;
- u32 act_nr_of_entries;
- u32 act_pages;
- struct ipz_eq_handle eq_handle;
-};
-
-enum ehca_service_type {
- ST_RC = 0,
- ST_UC = 1,
- ST_RD = 2,
- ST_UD = 3,
-};
-
-enum ehca_ll_comp_flags {
- LLQP_SEND_COMP = 0x20,
- LLQP_RECV_COMP = 0x40,
- LLQP_COMP_MASK = 0x60,
-};
-
-struct ehca_alloc_queue_parms {
- /* input parameters */
- int max_wr;
- int max_sge;
- int page_size;
- int is_small;
-
- /* output parameters */
- u16 act_nr_wqes;
- u8 act_nr_sges;
- u32 queue_size; /* bytes for small queues, pages otherwise */
-};
-
-struct ehca_alloc_qp_parms {
- struct ehca_alloc_queue_parms squeue;
- struct ehca_alloc_queue_parms rqueue;
-
- /* input parameters */
- enum ehca_service_type servicetype;
- int qp_storage;
- int sigtype;
- enum ehca_ext_qp_type ext_type;
- enum ehca_ll_comp_flags ll_comp_flags;
- int ud_av_l_key_ctl;
-
- u32 token;
- struct ipz_eq_handle eq_handle;
- struct ipz_pd pd;
- struct ipz_cq_handle send_cq_handle, recv_cq_handle;
-
- u32 srq_qpn, srq_token, srq_limit;
-
- /* output parameters */
- u32 real_qp_num;
- struct ipz_qp_handle qp_handle;
- struct h_galpas galpas;
-};
-
-int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
-int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int qp_num);
-struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int qp_num);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * pSeries interface definitions
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_CLASSES_PSERIES_H__
-#define __EHCA_CLASSES_PSERIES_H__
-
-#include "hcp_phyp.h"
-#include "ipz_pt_fn.h"
-
-
-struct ehca_pfqp {
- struct ipz_qpt sqpt;
- struct ipz_qpt rqpt;
-};
-
-struct ehca_pfcq {
- struct ipz_qpt qpt;
- u32 cqnr;
-};
-
-struct ehca_pfeq {
- struct ipz_qpt qpt;
- struct h_galpa galpa;
- u32 eqnr;
-};
-
-struct ipz_adapter_handle {
- u64 handle;
-};
-
-struct ipz_cq_handle {
- u64 handle;
-};
-
-struct ipz_eq_handle {
- u64 handle;
-};
-
-struct ipz_qp_handle {
- u64 handle;
-};
-struct ipz_mrmw_handle {
- u64 handle;
-};
-
-struct ipz_pd {
- u32 value;
-};
-
-struct hcp_modify_qp_control_block {
- u32 qkey; /* 00 */
- u32 rdd; /* reliable datagram domain */
- u32 send_psn; /* 02 */
- u32 receive_psn; /* 03 */
- u32 prim_phys_port; /* 04 */
- u32 alt_phys_port; /* 05 */
- u32 prim_p_key_idx; /* 06 */
- u32 alt_p_key_idx; /* 07 */
- u32 rdma_atomic_ctrl; /* 08 */
- u32 qp_state; /* 09 */
- u32 reserved_10; /* 10 */
- u32 rdma_nr_atomic_resp_res; /* 11 */
- u32 path_migration_state; /* 12 */
- u32 rdma_atomic_outst_dest_qp; /* 13 */
- u32 dest_qp_nr; /* 14 */
- u32 min_rnr_nak_timer_field; /* 15 */
- u32 service_level; /* 16 */
- u32 send_grh_flag; /* 17 */
- u32 retry_count; /* 18 */
- u32 timeout; /* 19 */
- u32 path_mtu; /* 20 */
- u32 max_static_rate; /* 21 */
- u32 dlid; /* 22 */
- u32 rnr_retry_count; /* 23 */
- u32 source_path_bits; /* 24 */
- u32 traffic_class; /* 25 */
- u32 hop_limit; /* 26 */
- u32 source_gid_idx; /* 27 */
- u32 flow_label; /* 28 */
- u32 reserved_29; /* 29 */
- union { /* 30 */
- u64 dw[2];
- u8 byte[16];
- } dest_gid;
- u32 service_level_al; /* 34 */
- u32 send_grh_flag_al; /* 35 */
- u32 retry_count_al; /* 36 */
- u32 timeout_al; /* 37 */
- u32 max_static_rate_al; /* 38 */
- u32 dlid_al; /* 39 */
- u32 rnr_retry_count_al; /* 40 */
- u32 source_path_bits_al; /* 41 */
- u32 traffic_class_al; /* 42 */
- u32 hop_limit_al; /* 43 */
- u32 source_gid_idx_al; /* 44 */
- u32 flow_label_al; /* 45 */
- u32 reserved_46; /* 46 */
- u32 reserved_47; /* 47 */
- union { /* 48 */
- u64 dw[2];
- u8 byte[16];
- } dest_gid_al;
- u32 max_nr_outst_send_wr; /* 52 */
- u32 max_nr_outst_recv_wr; /* 53 */
- u32 disable_ete_credit_check; /* 54 */
- u32 qp_number; /* 55 */
- u64 send_queue_handle; /* 56 */
- u64 recv_queue_handle; /* 58 */
- u32 actual_nr_sges_in_sq_wqe; /* 60 */
- u32 actual_nr_sges_in_rq_wqe; /* 61 */
- u32 qp_enable; /* 62 */
- u32 curr_srq_limit; /* 63 */
- u64 qp_aff_asyn_ev_log_reg; /* 64 */
- u64 shared_rq_hndl; /* 66 */
- u64 trigg_doorbell_qp_hndl; /* 68 */
- u32 reserved_70_127[58]; /* 70 */
-};
-
-#define MQPCB_MASK_QKEY EHCA_BMASK_IBM( 0, 0)
-#define MQPCB_MASK_SEND_PSN EHCA_BMASK_IBM( 2, 2)
-#define MQPCB_MASK_RECEIVE_PSN EHCA_BMASK_IBM( 3, 3)
-#define MQPCB_MASK_PRIM_PHYS_PORT EHCA_BMASK_IBM( 4, 4)
-#define MQPCB_PRIM_PHYS_PORT EHCA_BMASK_IBM(24, 31)
-#define MQPCB_MASK_ALT_PHYS_PORT EHCA_BMASK_IBM( 5, 5)
-#define MQPCB_MASK_PRIM_P_KEY_IDX EHCA_BMASK_IBM( 6, 6)
-#define MQPCB_PRIM_P_KEY_IDX EHCA_BMASK_IBM(24, 31)
-#define MQPCB_MASK_ALT_P_KEY_IDX EHCA_BMASK_IBM( 7, 7)
-#define MQPCB_MASK_RDMA_ATOMIC_CTRL EHCA_BMASK_IBM( 8, 8)
-#define MQPCB_MASK_QP_STATE EHCA_BMASK_IBM( 9, 9)
-#define MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES EHCA_BMASK_IBM(11, 11)
-#define MQPCB_MASK_PATH_MIGRATION_STATE EHCA_BMASK_IBM(12, 12)
-#define MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP EHCA_BMASK_IBM(13, 13)
-#define MQPCB_MASK_DEST_QP_NR EHCA_BMASK_IBM(14, 14)
-#define MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD EHCA_BMASK_IBM(15, 15)
-#define MQPCB_MASK_SERVICE_LEVEL EHCA_BMASK_IBM(16, 16)
-#define MQPCB_MASK_SEND_GRH_FLAG EHCA_BMASK_IBM(17, 17)
-#define MQPCB_MASK_RETRY_COUNT EHCA_BMASK_IBM(18, 18)
-#define MQPCB_MASK_TIMEOUT EHCA_BMASK_IBM(19, 19)
-#define MQPCB_MASK_PATH_MTU EHCA_BMASK_IBM(20, 20)
-#define MQPCB_MASK_MAX_STATIC_RATE EHCA_BMASK_IBM(21, 21)
-#define MQPCB_MASK_DLID EHCA_BMASK_IBM(22, 22)
-#define MQPCB_MASK_RNR_RETRY_COUNT EHCA_BMASK_IBM(23, 23)
-#define MQPCB_MASK_SOURCE_PATH_BITS EHCA_BMASK_IBM(24, 24)
-#define MQPCB_MASK_TRAFFIC_CLASS EHCA_BMASK_IBM(25, 25)
-#define MQPCB_MASK_HOP_LIMIT EHCA_BMASK_IBM(26, 26)
-#define MQPCB_MASK_SOURCE_GID_IDX EHCA_BMASK_IBM(27, 27)
-#define MQPCB_MASK_FLOW_LABEL EHCA_BMASK_IBM(28, 28)
-#define MQPCB_MASK_DEST_GID EHCA_BMASK_IBM(30, 30)
-#define MQPCB_MASK_SERVICE_LEVEL_AL EHCA_BMASK_IBM(31, 31)
-#define MQPCB_MASK_SEND_GRH_FLAG_AL EHCA_BMASK_IBM(32, 32)
-#define MQPCB_MASK_RETRY_COUNT_AL EHCA_BMASK_IBM(33, 33)
-#define MQPCB_MASK_TIMEOUT_AL EHCA_BMASK_IBM(34, 34)
-#define MQPCB_MASK_MAX_STATIC_RATE_AL EHCA_BMASK_IBM(35, 35)
-#define MQPCB_MASK_DLID_AL EHCA_BMASK_IBM(36, 36)
-#define MQPCB_MASK_RNR_RETRY_COUNT_AL EHCA_BMASK_IBM(37, 37)
-#define MQPCB_MASK_SOURCE_PATH_BITS_AL EHCA_BMASK_IBM(38, 38)
-#define MQPCB_MASK_TRAFFIC_CLASS_AL EHCA_BMASK_IBM(39, 39)
-#define MQPCB_MASK_HOP_LIMIT_AL EHCA_BMASK_IBM(40, 40)
-#define MQPCB_MASK_SOURCE_GID_IDX_AL EHCA_BMASK_IBM(41, 41)
-#define MQPCB_MASK_FLOW_LABEL_AL EHCA_BMASK_IBM(42, 42)
-#define MQPCB_MASK_DEST_GID_AL EHCA_BMASK_IBM(44, 44)
-#define MQPCB_MASK_MAX_NR_OUTST_SEND_WR EHCA_BMASK_IBM(45, 45)
-#define MQPCB_MASK_MAX_NR_OUTST_RECV_WR EHCA_BMASK_IBM(46, 46)
-#define MQPCB_MASK_DISABLE_ETE_CREDIT_CHECK EHCA_BMASK_IBM(47, 47)
-#define MQPCB_MASK_QP_ENABLE EHCA_BMASK_IBM(48, 48)
-#define MQPCB_MASK_CURR_SRQ_LIMIT EHCA_BMASK_IBM(49, 49)
-#define MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG EHCA_BMASK_IBM(50, 50)
-#define MQPCB_MASK_SHARED_RQ_HNDL EHCA_BMASK_IBM(51, 51)
-
-#endif /* __EHCA_CLASSES_PSERIES_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Completion queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_iverbs.h"
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "hcp_if.h"
-
-static struct kmem_cache *cq_cache;
-
-int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
-{
- unsigned int qp_num = qp->real_qp_num;
- unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
- hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
- spin_unlock_irqrestore(&cq->spinlock, flags);
-
- ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
- cq->cq_number, qp_num);
-
- return 0;
-}
-
-int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
-{
- int ret = -EINVAL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct hlist_node *iter;
- struct ehca_qp *qp;
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
- hlist_for_each(iter, &cq->qp_hashtab[key]) {
- qp = hlist_entry(iter, struct ehca_qp, list_entries);
- if (qp->real_qp_num == real_qp_num) {
- hlist_del(iter);
- ehca_dbg(cq->ib_cq.device,
- "removed qp from cq .cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
- ret = 0;
- break;
- }
- }
- spin_unlock_irqrestore(&cq->spinlock, flags);
- if (ret)
- ehca_err(cq->ib_cq.device,
- "qp not found cq_num=%x real_qp_num=%x",
- cq->cq_number, real_qp_num);
-
- return ret;
-}
-
-struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
-{
- struct ehca_qp *ret = NULL;
- unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
- struct hlist_node *iter;
- struct ehca_qp *qp;
- hlist_for_each(iter, &cq->qp_hashtab[key]) {
- qp = hlist_entry(iter, struct ehca_qp, list_entries);
- if (qp->real_qp_num == real_qp_num) {
- ret = qp;
- break;
- }
- }
- return ret;
-}
-
-struct ib_cq *ehca_create_cq(struct ib_device *device,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata)
-{
- int cqe = attr->cqe;
- static const u32 additional_cqe = 20;
- struct ib_cq *cq;
- struct ehca_cq *my_cq;
- struct ehca_shca *shca =
- container_of(device, struct ehca_shca, ib_device);
- struct ipz_adapter_handle adapter_handle;
- struct ehca_alloc_cq_parms param; /* h_call's out parameters */
- struct h_galpa gal;
- void *vpage;
- u32 counter;
- u64 rpage, cqx_fec, h_ret;
- int ipz_rc, i;
- unsigned long flags;
-
- if (attr->flags)
- return ERR_PTR(-EINVAL);
-
- if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
- return ERR_PTR(-EINVAL);
-
- if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
- ehca_err(device, "Unable to create CQ, max number of %i "
- "CQs reached.", shca->max_num_cqs);
- ehca_err(device, "To increase the maximum number of CQs "
- "use the number_of_cqs module parameter.\n");
- return ERR_PTR(-ENOSPC);
- }
-
- my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
- if (!my_cq) {
- ehca_err(device, "Out of memory for ehca_cq struct device=%p",
- device);
- atomic_dec(&shca->num_cqs);
- return ERR_PTR(-ENOMEM);
- }
-
- memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms));
-
- spin_lock_init(&my_cq->spinlock);
- spin_lock_init(&my_cq->cb_lock);
- spin_lock_init(&my_cq->task_lock);
- atomic_set(&my_cq->nr_events, 0);
- init_waitqueue_head(&my_cq->wait_completion);
-
- cq = &my_cq->ib_cq;
-
- adapter_handle = shca->ipz_hca_handle;
- param.eq_handle = shca->eq.ipz_eq_handle;
-
- idr_preload(GFP_KERNEL);
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
- idr_preload_end();
-
- if (my_cq->token < 0) {
- cq = ERR_PTR(-ENOMEM);
- ehca_err(device, "Can't allocate new idr entry. device=%p",
- device);
- goto create_cq_exit1;
- }
-
- /*
- * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
- * for receiving errors CQEs.
- */
- param.nr_cqe = cqe + additional_cqe;
- h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(device, "hipz_h_alloc_resource_cq() failed "
- "h_ret=%lli device=%p", h_ret, device);
- cq = ERR_PTR(ehca2ib_return_code(h_ret));
- goto create_cq_exit2;
- }
-
- ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
- if (!ipz_rc) {
- ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
- ipz_rc, device);
- cq = ERR_PTR(-EINVAL);
- goto create_cq_exit3;
- }
-
- for (counter = 0; counter < param.act_pages; counter++) {
- vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
- if (!vpage) {
- ehca_err(device, "ipz_qpageit_get_inc() "
- "returns NULL device=%p", device);
- cq = ERR_PTR(-EAGAIN);
- goto create_cq_exit4;
- }
- rpage = __pa(vpage);
-
- h_ret = hipz_h_register_rpage_cq(adapter_handle,
- my_cq->ipz_cq_handle,
- &my_cq->pf,
- 0,
- 0,
- rpage,
- 1,
- my_cq->galpas.
- kernel);
-
- if (h_ret < H_SUCCESS) {
- ehca_err(device, "hipz_h_register_rpage_cq() failed "
- "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
- "act_pages=%i", my_cq, my_cq->cq_number,
- h_ret, counter, param.act_pages);
- cq = ERR_PTR(-EINVAL);
- goto create_cq_exit4;
- }
-
- if (counter == (param.act_pages - 1)) {
- vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
- if ((h_ret != H_SUCCESS) || vpage) {
- ehca_err(device, "Registration of pages not "
- "complete ehca_cq=%p cq_num=%x "
- "h_ret=%lli", my_cq, my_cq->cq_number,
- h_ret);
- cq = ERR_PTR(-EAGAIN);
- goto create_cq_exit4;
- }
- } else {
- if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(device, "Registration of page failed "
- "ehca_cq=%p cq_num=%x h_ret=%lli "
- "counter=%i act_pages=%i",
- my_cq, my_cq->cq_number,
- h_ret, counter, param.act_pages);
- cq = ERR_PTR(-ENOMEM);
- goto create_cq_exit4;
- }
- }
- }
-
- ipz_qeit_reset(&my_cq->ipz_queue);
-
- gal = my_cq->galpas.kernel;
- cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
- ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
- my_cq, my_cq->cq_number, cqx_fec);
-
- my_cq->ib_cq.cqe = my_cq->nr_of_entries =
- param.act_nr_of_entries - additional_cqe;
- my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
-
- for (i = 0; i < QP_HASHTAB_LEN; i++)
- INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
-
- INIT_LIST_HEAD(&my_cq->sqp_err_list);
- INIT_LIST_HEAD(&my_cq->rqp_err_list);
-
- if (context) {
- struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
- struct ehca_create_cq_resp resp;
- memset(&resp, 0, sizeof(resp));
- resp.cq_number = my_cq->cq_number;
- resp.token = my_cq->token;
- resp.ipz_queue.qe_size = ipz_queue->qe_size;
- resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
- resp.ipz_queue.queue_length = ipz_queue->queue_length;
- resp.ipz_queue.pagesize = ipz_queue->pagesize;
- resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
- resp.fw_handle_ofs = (u32)
- (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
- if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
- ehca_err(device, "Copy to udata failed.");
- cq = ERR_PTR(-EFAULT);
- goto create_cq_exit4;
- }
- }
-
- return cq;
-
-create_cq_exit4:
- ipz_queue_dtor(NULL, &my_cq->ipz_queue);
-
-create_cq_exit3:
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
- if (h_ret != H_SUCCESS)
- ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
- "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
-
-create_cq_exit2:
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- idr_remove(&ehca_cq_idr, my_cq->token);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-
-create_cq_exit1:
- kmem_cache_free(cq_cache, my_cq);
-
- atomic_dec(&shca->num_cqs);
- return cq;
-}
-
-int ehca_destroy_cq(struct ib_cq *cq)
-{
- u64 h_ret;
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int cq_num = my_cq->cq_number;
- struct ib_device *device = cq->device;
- struct ehca_shca *shca = container_of(device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- unsigned long flags;
-
- if (cq->uobject) {
- if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
- ehca_err(device, "Resources still referenced in "
- "user space cq_num=%x", my_cq->cq_number);
- return -EINVAL;
- }
- }
-
- /*
- * remove the CQ from the idr first to make sure
- * no more interrupt tasklets will touch this CQ
- */
- write_lock_irqsave(&ehca_cq_idr_lock, flags);
- idr_remove(&ehca_cq_idr, my_cq->token);
- write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
-
- /* now wait until all pending events have completed */
- wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
-
- /* nobody's using our CQ any longer -- we can destroy it */
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
- if (h_ret == H_R_STATE) {
- /* cq in err: read err data and destroy it forcibly */
- ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
- "state. Try to delete it forcibly.",
- my_cq, cq_num, my_cq->ipz_cq_handle.handle);
- ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
- h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
- if (h_ret == H_SUCCESS)
- ehca_dbg(device, "cq_num=%x deleted successfully.",
- cq_num);
- }
- if (h_ret != H_SUCCESS) {
- ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
- "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
- return ehca2ib_return_code(h_ret);
- }
- ipz_queue_dtor(NULL, &my_cq->ipz_queue);
- kmem_cache_free(cq_cache, my_cq);
-
- atomic_dec(&shca->num_cqs);
- return 0;
-}
-
-int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
-{
- /* TODO: proper resize needs to be done */
- ehca_err(cq->device, "not implemented yet");
-
- return -EFAULT;
-}
-
-int ehca_init_cq_cache(void)
-{
- cq_cache = kmem_cache_create("ehca_cache_cq",
- sizeof(struct ehca_cq), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!cq_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_cq_cache(void)
-{
- if (cq_cache)
- kmem_cache_destroy(cq_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Event queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "ehca_iverbs.h"
-#include "ehca_qes.h"
-#include "hcp_if.h"
-#include "ipz_pt_fn.h"
-
-int ehca_create_eq(struct ehca_shca *shca,
- struct ehca_eq *eq,
- const enum ehca_eq_type type, const u32 length)
-{
- int ret;
- u64 h_ret;
- u32 nr_pages;
- u32 i;
- void *vpage;
- struct ib_device *ib_dev = &shca->ib_device;
-
- spin_lock_init(&eq->spinlock);
- spin_lock_init(&eq->irq_spinlock);
- eq->is_initialized = 0;
-
- if (type != EHCA_EQ && type != EHCA_NEQ) {
- ehca_err(ib_dev, "Invalid EQ type %x. eq=%p", type, eq);
- return -EINVAL;
- }
- if (!length) {
- ehca_err(ib_dev, "EQ length must not be zero. eq=%p", eq);
- return -EINVAL;
- }
-
- h_ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
- &eq->pf,
- type,
- length,
- &eq->ipz_eq_handle,
- &eq->length,
- &nr_pages, &eq->ist);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
- return -EINVAL;
- }
-
- ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
- EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
- if (!ret) {
- ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
- goto create_eq_exit1;
- }
-
- for (i = 0; i < nr_pages; i++) {
- u64 rpage;
-
- vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (!vpage)
- goto create_eq_exit2;
-
- rpage = __pa(vpage);
- h_ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
- eq->ipz_eq_handle,
- &eq->pf,
- 0, 0, rpage, 1);
-
- if (i == (nr_pages - 1)) {
- /* last page */
- vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
- if (h_ret != H_SUCCESS || vpage)
- goto create_eq_exit2;
- } else {
- if (h_ret != H_PAGE_REGISTERED)
- goto create_eq_exit2;
- }
- }
-
- ipz_qeit_reset(&eq->ipz_queue);
-
- /* register interrupt handlers and initialize work queues */
- if (type == EHCA_EQ) {
- tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca);
-
- ret = ibmebus_request_irq(eq->ist, ehca_interrupt_eq,
- 0, "ehca_eq",
- (void *)shca);
- if (ret < 0)
- ehca_err(ib_dev, "Can't map interrupt handler.");
- } else if (type == EHCA_NEQ) {
- tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca);
-
- ret = ibmebus_request_irq(eq->ist, ehca_interrupt_neq,
- 0, "ehca_neq",
- (void *)shca);
- if (ret < 0)
- ehca_err(ib_dev, "Can't map interrupt handler.");
- }
-
- eq->is_initialized = 1;
-
- return 0;
-
-create_eq_exit2:
- ipz_queue_dtor(NULL, &eq->ipz_queue);
-
-create_eq_exit1:
- hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
-
- return -EINVAL;
-}
-
-void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq)
-{
- unsigned long flags;
- void *eqe;
-
- spin_lock_irqsave(&eq->spinlock, flags);
- eqe = ipz_eqit_eq_get_inc_valid(&eq->ipz_queue);
- spin_unlock_irqrestore(&eq->spinlock, flags);
-
- return eqe;
-}
-
-int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
-{
- unsigned long flags;
- u64 h_ret;
-
- ibmebus_free_irq(eq->ist, (void *)shca);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- eq->is_initialized = 0;
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- tasklet_kill(&eq->interrupt_task);
-
- h_ret = hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
-
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't free EQ resources.");
- return -EINVAL;
- }
- ipz_queue_dtor(NULL, &eq->ipz_queue);
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HCA query functions
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/gfp.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-static unsigned int limit_uint(unsigned int value)
-{
- return min_t(unsigned int, value, INT_MAX);
-}
-
-int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
- struct ib_udata *uhw)
-{
- int i, ret = 0;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_hca *rblock;
-
- static const u32 cap_mapping[] = {
- IB_DEVICE_RESIZE_MAX_WR, HCA_CAP_WQE_RESIZE,
- IB_DEVICE_BAD_PKEY_CNTR, HCA_CAP_BAD_P_KEY_CTR,
- IB_DEVICE_BAD_QKEY_CNTR, HCA_CAP_Q_KEY_VIOL_CTR,
- IB_DEVICE_RAW_MULTI, HCA_CAP_RAW_PACKET_MCAST,
- IB_DEVICE_AUTO_PATH_MIG, HCA_CAP_AUTO_PATH_MIG,
- IB_DEVICE_CHANGE_PHY_PORT, HCA_CAP_SQD_RTS_PORT_CHANGE,
- IB_DEVICE_UD_AV_PORT_ENFORCE, HCA_CAP_AH_PORT_NR_CHECK,
- IB_DEVICE_CURR_QP_STATE_MOD, HCA_CAP_CUR_QP_STATE_MOD,
- IB_DEVICE_SHUTDOWN_PORT, HCA_CAP_SHUTDOWN_PORT,
- IB_DEVICE_INIT_TYPE, HCA_CAP_INIT_TYPE,
- IB_DEVICE_PORT_ACTIVE_EVENT, HCA_CAP_PORT_ACTIVE_EVENT,
- };
-
- if (uhw->inlen || uhw->outlen)
- return -EINVAL;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query device properties");
- ret = -EINVAL;
- goto query_device1;
- }
-
- memset(props, 0, sizeof(struct ib_device_attr));
- props->page_size_cap = shca->hca_cap_mr_pgsize;
- props->fw_ver = rblock->hw_ver;
- props->max_mr_size = rblock->max_mr_size;
- props->vendor_id = rblock->vendor_id >> 8;
- props->vendor_part_id = rblock->vendor_part_id >> 16;
- props->hw_ver = rblock->hw_ver;
- props->max_qp = limit_uint(rblock->max_qp);
- props->max_qp_wr = limit_uint(rblock->max_wqes_wq);
- props->max_sge = limit_uint(rblock->max_sge);
- props->max_sge_rd = limit_uint(rblock->max_sge_rd);
- props->max_cq = limit_uint(rblock->max_cq);
- props->max_cqe = limit_uint(rblock->max_cqe);
- props->max_mr = limit_uint(rblock->max_mr);
- props->max_mw = limit_uint(rblock->max_mw);
- props->max_pd = limit_uint(rblock->max_pd);
- props->max_ah = limit_uint(rblock->max_ah);
- props->max_ee = limit_uint(rblock->max_rd_ee_context);
- props->max_rdd = limit_uint(rblock->max_rd_domain);
- props->max_fmr = limit_uint(rblock->max_mr);
- props->max_qp_rd_atom = limit_uint(rblock->max_rr_qp);
- props->max_ee_rd_atom = limit_uint(rblock->max_rr_ee_context);
- props->max_res_rd_atom = limit_uint(rblock->max_rr_hca);
- props->max_qp_init_rd_atom = limit_uint(rblock->max_act_wqs_qp);
- props->max_ee_init_rd_atom = limit_uint(rblock->max_act_wqs_ee_context);
-
- if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
- props->max_srq = limit_uint(props->max_qp);
- props->max_srq_wr = limit_uint(props->max_qp_wr);
- props->max_srq_sge = 3;
- }
-
- props->max_pkeys = 16;
- /* Some FW versions say 0 here; insert sensible value in that case */
- props->local_ca_ack_delay = rblock->local_ca_ack_delay ?
- min_t(u8, rblock->local_ca_ack_delay, 255) : 12;
- props->max_raw_ipv6_qp = limit_uint(rblock->max_raw_ipv6_qp);
- props->max_raw_ethy_qp = limit_uint(rblock->max_raw_ethy_qp);
- props->max_mcast_grp = limit_uint(rblock->max_mcast_grp);
- props->max_mcast_qp_attach = limit_uint(rblock->max_mcast_qp_attach);
- props->max_total_mcast_qp_attach
- = limit_uint(rblock->max_total_mcast_qp_attach);
-
- /* translate device capabilities */
- props->device_cap_flags = IB_DEVICE_SYS_IMAGE_GUID |
- IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_N_NOTIFY_CQ;
- for (i = 0; i < ARRAY_SIZE(cap_mapping); i += 2)
- if (rblock->hca_cap_indicators & cap_mapping[i + 1])
- props->device_cap_flags |= cap_mapping[i];
-
-query_device1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-static enum ib_mtu map_mtu(struct ehca_shca *shca, u32 fw_mtu)
-{
- switch (fw_mtu) {
- case 0x1:
- return IB_MTU_256;
- case 0x2:
- return IB_MTU_512;
- case 0x3:
- return IB_MTU_1024;
- case 0x4:
- return IB_MTU_2048;
- case 0x5:
- return IB_MTU_4096;
- default:
- ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
- fw_mtu);
- return 0;
- }
-}
-
-static u8 map_number_of_vls(struct ehca_shca *shca, u32 vl_cap)
-{
- switch (vl_cap) {
- case 0x1:
- return 1;
- case 0x2:
- return 2;
- case 0x3:
- return 4;
- case 0x4:
- return 8;
- case 0x5:
- return 15;
- default:
- ehca_err(&shca->ib_device, "invalid Vl Capability: %x.",
- vl_cap);
- return 0;
- }
-}
-
-int ehca_query_port(struct ib_device *ibdev,
- u8 port, struct ib_port_attr *props)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_port *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_port1;
- }
-
- memset(props, 0, sizeof(struct ib_port_attr));
-
- props->active_mtu = props->max_mtu = map_mtu(shca, rblock->max_mtu);
- props->port_cap_flags = rblock->capability_mask;
- props->gid_tbl_len = rblock->gid_tbl_len;
- if (rblock->max_msg_sz)
- props->max_msg_sz = rblock->max_msg_sz;
- else
- props->max_msg_sz = 0x1 << 31;
- props->bad_pkey_cntr = rblock->bad_pkey_cntr;
- props->qkey_viol_cntr = rblock->qkey_viol_cntr;
- props->pkey_tbl_len = rblock->pkey_tbl_len;
- props->lid = rblock->lid;
- props->sm_lid = rblock->sm_lid;
- props->lmc = rblock->lmc;
- props->sm_sl = rblock->sm_sl;
- props->subnet_timeout = rblock->subnet_timeout;
- props->init_type_reply = rblock->init_type_reply;
- props->max_vl_num = map_number_of_vls(shca, rblock->vl_cap);
-
- if (rblock->state && rblock->phys_width) {
- props->phys_state = rblock->phys_pstate;
- props->state = rblock->phys_state;
- props->active_width = rblock->phys_width;
- props->active_speed = rblock->phys_speed;
- } else {
- /* old firmware releases don't report physical
- * port info, so use default values
- */
- props->phys_state = 5;
- props->state = rblock->state;
- props->active_width = IB_WIDTH_12X;
- props->active_speed = IB_SPEED_SDR;
- }
-
-query_port1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_sma_attr(struct ehca_shca *shca,
- u8 port, struct ehca_sma_attr *attr)
-{
- int ret = 0;
- u64 h_ret;
- struct hipz_query_port *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_sma_attr1;
- }
-
- memset(attr, 0, sizeof(struct ehca_sma_attr));
-
- attr->lid = rblock->lid;
- attr->lmc = rblock->lmc;
- attr->sm_sl = rblock->sm_sl;
- attr->sm_lid = rblock->sm_lid;
-
- attr->pkey_tbl_len = rblock->pkey_tbl_len;
- memcpy(attr->pkeys, rblock->pkey_entries, sizeof(attr->pkeys));
-
-query_sma_attr1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca;
- struct hipz_query_port *rblock;
-
- shca = container_of(ibdev, struct ehca_shca, ib_device);
- if (index > 16) {
- ehca_err(&shca->ib_device, "Invalid index: %x.", index);
- return -EINVAL;
- }
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_pkey1;
- }
-
- memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
-
-query_pkey1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-int ehca_query_gid(struct ib_device *ibdev, u8 port,
- int index, union ib_gid *gid)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
- ib_device);
- struct hipz_query_port *rblock;
-
- if (index < 0 || index > 255) {
- ehca_err(&shca->ib_device, "Invalid index: %x.", index);
- return -EINVAL;
- }
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto query_gid1;
- }
-
- memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
- memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
-
-query_gid1:
- ehca_free_fw_ctrlblock(rblock);
-
- return ret;
-}
-
-static const u32 allowed_port_caps = (
- IB_PORT_SM | IB_PORT_LED_INFO_SUP | IB_PORT_CM_SUP |
- IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_DEVICE_MGMT_SUP |
- IB_PORT_VENDOR_CLASS_SUP);
-
-int ehca_modify_port(struct ib_device *ibdev,
- u8 port, int port_modify_mask,
- struct ib_port_modify *props)
-{
- int ret = 0;
- struct ehca_shca *shca;
- struct hipz_query_port *rblock;
- u32 cap;
- u64 hret;
-
- shca = container_of(ibdev, struct ehca_shca, ib_device);
- if ((props->set_port_cap_mask | props->clr_port_cap_mask)
- & ~allowed_port_caps) {
- ehca_err(&shca->ib_device, "Non-changeable bits set in masks "
- "set=%x clr=%x allowed=%x", props->set_port_cap_mask,
- props->clr_port_cap_mask, allowed_port_caps);
- return -EINVAL;
- }
-
- if (mutex_lock_interruptible(&shca->modify_mutex))
- return -ERESTARTSYS;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- ret = -ENOMEM;
- goto modify_port1;
- }
-
- hret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
- if (hret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query port properties");
- ret = -EINVAL;
- goto modify_port2;
- }
-
- cap = (rblock->capability_mask | props->set_port_cap_mask)
- & ~props->clr_port_cap_mask;
-
- hret = hipz_h_modify_port(shca->ipz_hca_handle, port,
- cap, props->init_type, port_modify_mask);
- if (hret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Modify port failed h_ret=%lli",
- hret);
- ret = -EINVAL;
- }
-
-modify_port2:
- ehca_free_fw_ctrlblock(rblock);
-
-modify_port1:
- mutex_unlock(&shca->modify_mutex);
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Functions for EQs, NEQs and interrupts
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-#include <linux/smpboot.h>
-
-#include "ehca_classes.h"
-#include "ehca_irq.h"
-#include "ehca_iverbs.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-#include "ipz_pt_fn.h"
-
-#define EQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
-#define EQE_CQ_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_EE_IDENTIFIER EHCA_BMASK_IBM( 2, 7)
-#define EQE_CQ_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
-#define EQE_QP_TOKEN EHCA_BMASK_IBM(32, 63)
-#define EQE_CQ_TOKEN EHCA_BMASK_IBM(32, 63)
-
-#define NEQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
-#define NEQE_EVENT_CODE EHCA_BMASK_IBM( 2, 7)
-#define NEQE_PORT_NUMBER EHCA_BMASK_IBM( 8, 15)
-#define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16, 16)
-#define NEQE_DISRUPTIVE EHCA_BMASK_IBM(16, 16)
-#define NEQE_SPECIFIC_EVENT EHCA_BMASK_IBM(16, 23)
-
-#define ERROR_DATA_LENGTH EHCA_BMASK_IBM(52, 63)
-#define ERROR_DATA_TYPE EHCA_BMASK_IBM( 0, 7)
-
-static void queue_comp_task(struct ehca_cq *__cq);
-
-static struct ehca_comp_pool *pool;
-
-static inline void comp_event_callback(struct ehca_cq *cq)
-{
- if (!cq->ib_cq.comp_handler)
- return;
-
- spin_lock(&cq->cb_lock);
- cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context);
- spin_unlock(&cq->cb_lock);
-
- return;
-}
-
-static void print_error_data(struct ehca_shca *shca, void *data,
- u64 *rblock, int length)
-{
- u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
- u64 resource = rblock[1];
-
- switch (type) {
- case 0x1: /* Queue Pair */
- {
- struct ehca_qp *qp = (struct ehca_qp *)data;
-
- /* only print error data if AER is set */
- if (rblock[6] == 0)
- return;
-
- ehca_err(&shca->ib_device,
- "QP 0x%x (resource=%llx) has errors.",
- qp->ib_qp.qp_num, resource);
- break;
- }
- case 0x4: /* Completion Queue */
- {
- struct ehca_cq *cq = (struct ehca_cq *)data;
-
- ehca_err(&shca->ib_device,
- "CQ 0x%x (resource=%llx) has errors.",
- cq->cq_number, resource);
- break;
- }
- default:
- ehca_err(&shca->ib_device,
- "Unknown error type: %llx on %s.",
- type, shca->ib_device.name);
- break;
- }
-
- ehca_err(&shca->ib_device, "Error data is available: %llx.", resource);
- ehca_err(&shca->ib_device, "EHCA ----- error data begin "
- "---------------------------------------------------");
- ehca_dmp(rblock, length, "resource=%llx", resource);
- ehca_err(&shca->ib_device, "EHCA ----- error data end "
- "----------------------------------------------------");
-
- return;
-}
-
-int ehca_error_data(struct ehca_shca *shca, void *data,
- u64 resource)
-{
-
- unsigned long ret;
- u64 *rblock;
- unsigned long block_count;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Cannot allocate rblock memory.");
- ret = -ENOMEM;
- goto error_data1;
- }
-
- /* rblock must be 4K aligned and should be 4K large */
- ret = hipz_h_error_data(shca->ipz_hca_handle,
- resource,
- rblock,
- &block_count);
-
- if (ret == H_R_STATE)
- ehca_err(&shca->ib_device,
- "No error data is available: %llx.", resource);
- else if (ret == H_SUCCESS) {
- int length;
-
- length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]);
-
- if (length > EHCA_PAGESIZE)
- length = EHCA_PAGESIZE;
-
- print_error_data(shca, data, rblock, length);
- } else
- ehca_err(&shca->ib_device,
- "Error data could not be fetched: %llx", resource);
-
- ehca_free_fw_ctrlblock(rblock);
-
-error_data1:
- return ret;
-
-}
-
-static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp,
- enum ib_event_type event_type)
-{
- struct ib_event event;
-
- /* PATH_MIG without the QP ever having been armed is false alarm */
- if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed)
- return;
-
- event.device = &shca->ib_device;
- event.event = event_type;
-
- if (qp->ext_type == EQPT_SRQ) {
- if (!qp->ib_srq.event_handler)
- return;
-
- event.element.srq = &qp->ib_srq;
- qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
- } else {
- if (!qp->ib_qp.event_handler)
- return;
-
- event.element.qp = &qp->ib_qp;
- qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
- }
-}
-
-static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
- enum ib_event_type event_type, int fatal)
-{
- struct ehca_qp *qp;
- u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe);
-
- read_lock(&ehca_qp_idr_lock);
- qp = idr_find(&ehca_qp_idr, token);
- if (qp)
- atomic_inc(&qp->nr_events);
- read_unlock(&ehca_qp_idr_lock);
-
- if (!qp)
- return;
-
- if (fatal)
- ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
-
- dispatch_qp_event(shca, qp, fatal && qp->ext_type == EQPT_SRQ ?
- IB_EVENT_SRQ_ERR : event_type);
-
- /*
- * eHCA only processes one WQE at a time for SRQ base QPs,
- * so the last WQE has been processed as soon as the QP enters
- * error state.
- */
- if (fatal && qp->ext_type == EQPT_SRQBASE)
- dispatch_qp_event(shca, qp, IB_EVENT_QP_LAST_WQE_REACHED);
-
- if (atomic_dec_and_test(&qp->nr_events))
- wake_up(&qp->wait_completion);
- return;
-}
-
-static void cq_event_callback(struct ehca_shca *shca,
- u64 eqe)
-{
- struct ehca_cq *cq;
- u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe);
-
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, token);
- if (cq)
- atomic_inc(&cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
-
- if (!cq)
- return;
-
- ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
-
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
-
- return;
-}
-
-static void parse_identifier(struct ehca_shca *shca, u64 eqe)
-{
- u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe);
-
- switch (identifier) {
- case 0x02: /* path migrated */
- qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
- break;
- case 0x03: /* communication established */
- qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
- break;
- case 0x04: /* send queue drained */
- qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
- break;
- case 0x05: /* QP error */
- case 0x06: /* QP error */
- qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
- break;
- case 0x07: /* CQ error */
- case 0x08: /* CQ error */
- cq_event_callback(shca, eqe);
- break;
- case 0x09: /* MRMWPTE error */
- ehca_err(&shca->ib_device, "MRMWPTE error.");
- break;
- case 0x0A: /* port event */
- ehca_err(&shca->ib_device, "Port event.");
- break;
- case 0x0B: /* MR access error */
- ehca_err(&shca->ib_device, "MR access error.");
- break;
- case 0x0C: /* EQ error */
- ehca_err(&shca->ib_device, "EQ error.");
- break;
- case 0x0D: /* P/Q_Key mismatch */
- ehca_err(&shca->ib_device, "P/Q_Key mismatch.");
- break;
- case 0x10: /* sampling complete */
- ehca_err(&shca->ib_device, "Sampling complete.");
- break;
- case 0x11: /* unaffiliated access error */
- ehca_err(&shca->ib_device, "Unaffiliated access error.");
- break;
- case 0x12: /* path migrating */
- ehca_err(&shca->ib_device, "Path migrating.");
- break;
- case 0x13: /* interface trace stopped */
- ehca_err(&shca->ib_device, "Interface trace stopped.");
- break;
- case 0x14: /* first error capture info available */
- ehca_info(&shca->ib_device, "First error capture available");
- break;
- case 0x15: /* SRQ limit reached */
- qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
- break;
- default:
- ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
- identifier, shca->ib_device.name);
- break;
- }
-
- return;
-}
-
-static void dispatch_port_event(struct ehca_shca *shca, int port_num,
- enum ib_event_type type, const char *msg)
-{
- struct ib_event event;
-
- ehca_info(&shca->ib_device, "port %d %s.", port_num, msg);
- event.device = &shca->ib_device;
- event.event = type;
- event.element.port_num = port_num;
- ib_dispatch_event(&event);
-}
-
-static void notify_port_conf_change(struct ehca_shca *shca, int port_num)
-{
- struct ehca_sma_attr new_attr;
- struct ehca_sma_attr *old_attr = &shca->sport[port_num - 1].saved_attr;
-
- ehca_query_sma_attr(shca, port_num, &new_attr);
-
- if (new_attr.sm_sl != old_attr->sm_sl ||
- new_attr.sm_lid != old_attr->sm_lid)
- dispatch_port_event(shca, port_num, IB_EVENT_SM_CHANGE,
- "SM changed");
-
- if (new_attr.lid != old_attr->lid ||
- new_attr.lmc != old_attr->lmc)
- dispatch_port_event(shca, port_num, IB_EVENT_LID_CHANGE,
- "LID changed");
-
- if (new_attr.pkey_tbl_len != old_attr->pkey_tbl_len ||
- memcmp(new_attr.pkeys, old_attr->pkeys,
- sizeof(u16) * new_attr.pkey_tbl_len))
- dispatch_port_event(shca, port_num, IB_EVENT_PKEY_CHANGE,
- "P_Key changed");
-
- *old_attr = new_attr;
-}
-
-/* replay modify_qp for sqps -- return 0 if all is well, 1 if AQP1 destroyed */
-static int replay_modify_qp(struct ehca_sport *sport)
-{
- int aqp1_destroyed;
- unsigned long flags;
-
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
-
- aqp1_destroyed = !sport->ibqp_sqp[IB_QPT_GSI];
-
- if (sport->ibqp_sqp[IB_QPT_SMI])
- ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_SMI]);
- if (!aqp1_destroyed)
- ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_GSI]);
-
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
-
- return aqp1_destroyed;
-}
-
-static void parse_ec(struct ehca_shca *shca, u64 eqe)
-{
- u8 ec = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe);
- u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe);
- u8 spec_event;
- struct ehca_sport *sport = &shca->sport[port - 1];
-
- switch (ec) {
- case 0x30: /* port availability change */
- if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) {
- /* only replay modify_qp calls in autodetect mode;
- * if AQP1 was destroyed, the port is already down
- * again and we can drop the event.
- */
- if (ehca_nr_ports < 0)
- if (replay_modify_qp(sport))
- break;
-
- sport->port_state = IB_PORT_ACTIVE;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
- "is active");
- ehca_query_sma_attr(shca, port, &sport->saved_attr);
- } else {
- sport->port_state = IB_PORT_DOWN;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
- "is inactive");
- }
- break;
- case 0x31:
- /* port configuration change
- * disruptive change is caused by
- * LID, PKEY or SM change
- */
- if (EHCA_BMASK_GET(NEQE_DISRUPTIVE, eqe)) {
- ehca_warn(&shca->ib_device, "disruptive port "
- "%d configuration change", port);
-
- sport->port_state = IB_PORT_DOWN;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
- "is inactive");
-
- sport->port_state = IB_PORT_ACTIVE;
- dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
- "is active");
- ehca_query_sma_attr(shca, port,
- &sport->saved_attr);
- } else
- notify_port_conf_change(shca, port);
- break;
- case 0x32: /* adapter malfunction */
- ehca_err(&shca->ib_device, "Adapter malfunction.");
- break;
- case 0x33: /* trace stopped */
- ehca_err(&shca->ib_device, "Traced stopped.");
- break;
- case 0x34: /* util async event */
- spec_event = EHCA_BMASK_GET(NEQE_SPECIFIC_EVENT, eqe);
- if (spec_event == 0x80) /* client reregister required */
- dispatch_port_event(shca, port,
- IB_EVENT_CLIENT_REREGISTER,
- "client reregister req.");
- else
- ehca_warn(&shca->ib_device, "Unknown util async "
- "event %x on port %x", spec_event, port);
- break;
- default:
- ehca_err(&shca->ib_device, "Unknown event code: %x on %s.",
- ec, shca->ib_device.name);
- break;
- }
-
- return;
-}
-
-static inline void reset_eq_pending(struct ehca_cq *cq)
-{
- u64 CQx_EP;
- struct h_galpa gal = cq->galpas.kernel;
-
- hipz_galpa_store_cq(gal, cqx_ep, 0x0);
- CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep));
-
- return;
-}
-
-irqreturn_t ehca_interrupt_neq(int irq, void *dev_id)
-{
- struct ehca_shca *shca = (struct ehca_shca*)dev_id;
-
- tasklet_hi_schedule(&shca->neq.interrupt_task);
-
- return IRQ_HANDLED;
-}
-
-void ehca_tasklet_neq(unsigned long data)
-{
- struct ehca_shca *shca = (struct ehca_shca*)data;
- struct ehca_eqe *eqe;
- u64 ret;
-
- eqe = ehca_poll_eq(shca, &shca->neq);
-
- while (eqe) {
- if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry))
- parse_ec(shca, eqe->entry);
-
- eqe = ehca_poll_eq(shca, &shca->neq);
- }
-
- ret = hipz_h_reset_event(shca->ipz_hca_handle,
- shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL);
-
- if (ret != H_SUCCESS)
- ehca_err(&shca->ib_device, "Can't clear notification events.");
-
- return;
-}
-
-irqreturn_t ehca_interrupt_eq(int irq, void *dev_id)
-{
- struct ehca_shca *shca = (struct ehca_shca*)dev_id;
-
- tasklet_hi_schedule(&shca->eq.interrupt_task);
-
- return IRQ_HANDLED;
-}
-
-
-static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe)
-{
- u64 eqe_value;
- u32 token;
- struct ehca_cq *cq;
-
- eqe_value = eqe->entry;
- ehca_dbg(&shca->ib_device, "eqe_value=%llx", eqe_value);
- if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
- ehca_dbg(&shca->ib_device, "Got completion event");
- token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, token);
- if (cq)
- atomic_inc(&cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
- if (cq == NULL) {
- ehca_err(&shca->ib_device,
- "Invalid eqe for non-existing cq token=%x",
- token);
- return;
- }
- reset_eq_pending(cq);
- if (ehca_scaling_code)
- queue_comp_task(cq);
- else {
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
- }
- } else {
- ehca_dbg(&shca->ib_device, "Got non completion event");
- parse_identifier(shca, eqe_value);
- }
-}
-
-void ehca_process_eq(struct ehca_shca *shca, int is_irq)
-{
- struct ehca_eq *eq = &shca->eq;
- struct ehca_eqe_cache_entry *eqe_cache = eq->eqe_cache;
- u64 eqe_value, ret;
- int eqe_cnt, i;
- int eq_empty = 0;
-
- spin_lock(&eq->irq_spinlock);
- if (is_irq) {
- const int max_query_cnt = 100;
- int query_cnt = 0;
- int int_state = 1;
- do {
- int_state = hipz_h_query_int_state(
- shca->ipz_hca_handle, eq->ist);
- query_cnt++;
- iosync();
- } while (int_state && query_cnt < max_query_cnt);
- if (unlikely((query_cnt == max_query_cnt)))
- ehca_dbg(&shca->ib_device, "int_state=%x query_cnt=%x",
- int_state, query_cnt);
- }
-
- /* read out all eqes */
- eqe_cnt = 0;
- do {
- u32 token;
- eqe_cache[eqe_cnt].eqe = ehca_poll_eq(shca, eq);
- if (!eqe_cache[eqe_cnt].eqe)
- break;
- eqe_value = eqe_cache[eqe_cnt].eqe->entry;
- if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
- token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
- read_lock(&ehca_cq_idr_lock);
- eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token);
- if (eqe_cache[eqe_cnt].cq)
- atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events);
- read_unlock(&ehca_cq_idr_lock);
- if (!eqe_cache[eqe_cnt].cq) {
- ehca_err(&shca->ib_device,
- "Invalid eqe for non-existing cq "
- "token=%x", token);
- continue;
- }
- } else
- eqe_cache[eqe_cnt].cq = NULL;
- eqe_cnt++;
- } while (eqe_cnt < EHCA_EQE_CACHE_SIZE);
- if (!eqe_cnt) {
- if (is_irq)
- ehca_dbg(&shca->ib_device,
- "No eqe found for irq event");
- goto unlock_irq_spinlock;
- } else if (!is_irq) {
- ret = hipz_h_eoi(eq->ist);
- if (ret != H_SUCCESS)
- ehca_err(&shca->ib_device,
- "bad return code EOI -rc = %lld\n", ret);
- ehca_dbg(&shca->ib_device, "deadman found %x eqe", eqe_cnt);
- }
- if (unlikely(eqe_cnt == EHCA_EQE_CACHE_SIZE))
- ehca_dbg(&shca->ib_device, "too many eqes for one irq event");
- /* enable irq for new packets */
- for (i = 0; i < eqe_cnt; i++) {
- if (eq->eqe_cache[i].cq)
- reset_eq_pending(eq->eqe_cache[i].cq);
- }
- /* check eq */
- spin_lock(&eq->spinlock);
- eq_empty = (!ipz_eqit_eq_peek_valid(&shca->eq.ipz_queue));
- spin_unlock(&eq->spinlock);
- /* call completion handler for cached eqes */
- for (i = 0; i < eqe_cnt; i++)
- if (eq->eqe_cache[i].cq) {
- if (ehca_scaling_code)
- queue_comp_task(eq->eqe_cache[i].cq);
- else {
- struct ehca_cq *cq = eq->eqe_cache[i].cq;
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
- }
- } else {
- ehca_dbg(&shca->ib_device, "Got non completion event");
- parse_identifier(shca, eq->eqe_cache[i].eqe->entry);
- }
- /* poll eq if not empty */
- if (eq_empty)
- goto unlock_irq_spinlock;
- do {
- struct ehca_eqe *eqe;
- eqe = ehca_poll_eq(shca, &shca->eq);
- if (!eqe)
- break;
- process_eqe(shca, eqe);
- } while (1);
-
-unlock_irq_spinlock:
- spin_unlock(&eq->irq_spinlock);
-}
-
-void ehca_tasklet_eq(unsigned long data)
-{
- ehca_process_eq((struct ehca_shca*)data, 1);
-}
-
-static int find_next_online_cpu(struct ehca_comp_pool *pool)
-{
- int cpu;
- unsigned long flags;
-
- WARN_ON_ONCE(!in_interrupt());
- if (ehca_debug_level >= 3)
- ehca_dmp(cpu_online_mask, cpumask_size(), "");
-
- spin_lock_irqsave(&pool->last_cpu_lock, flags);
- do {
- cpu = cpumask_next(pool->last_cpu, cpu_online_mask);
- if (cpu >= nr_cpu_ids)
- cpu = cpumask_first(cpu_online_mask);
- pool->last_cpu = cpu;
- } while (!per_cpu_ptr(pool->cpu_comp_tasks, cpu)->active);
- spin_unlock_irqrestore(&pool->last_cpu_lock, flags);
-
- return cpu;
-}
-
-static void __queue_comp_task(struct ehca_cq *__cq,
- struct ehca_cpu_comp_task *cct,
- struct task_struct *thread)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cct->task_lock, flags);
- spin_lock(&__cq->task_lock);
-
- if (__cq->nr_callbacks == 0) {
- __cq->nr_callbacks++;
- list_add_tail(&__cq->entry, &cct->cq_list);
- cct->cq_jobs++;
- wake_up_process(thread);
- } else
- __cq->nr_callbacks++;
-
- spin_unlock(&__cq->task_lock);
- spin_unlock_irqrestore(&cct->task_lock, flags);
-}
-
-static void queue_comp_task(struct ehca_cq *__cq)
-{
- int cpu_id;
- struct ehca_cpu_comp_task *cct;
- struct task_struct *thread;
- int cq_jobs;
- unsigned long flags;
-
- cpu_id = find_next_online_cpu(pool);
- BUG_ON(!cpu_online(cpu_id));
-
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
- BUG_ON(!cct || !thread);
-
- spin_lock_irqsave(&cct->task_lock, flags);
- cq_jobs = cct->cq_jobs;
- spin_unlock_irqrestore(&cct->task_lock, flags);
- if (cq_jobs > 0) {
- cpu_id = find_next_online_cpu(pool);
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
- BUG_ON(!cct || !thread);
- }
- __queue_comp_task(__cq, cct, thread);
-}
-
-static void run_comp_task(struct ehca_cpu_comp_task *cct)
-{
- struct ehca_cq *cq;
-
- while (!list_empty(&cct->cq_list)) {
- cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
- spin_unlock_irq(&cct->task_lock);
-
- comp_event_callback(cq);
- if (atomic_dec_and_test(&cq->nr_events))
- wake_up(&cq->wait_completion);
-
- spin_lock_irq(&cct->task_lock);
- spin_lock(&cq->task_lock);
- cq->nr_callbacks--;
- if (!cq->nr_callbacks) {
- list_del_init(cct->cq_list.next);
- cct->cq_jobs--;
- }
- spin_unlock(&cq->task_lock);
- }
-}
-
-static void comp_task_park(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- struct ehca_cpu_comp_task *target;
- struct task_struct *thread;
- struct ehca_cq *cq, *tmp;
- LIST_HEAD(list);
-
- spin_lock_irq(&cct->task_lock);
- cct->cq_jobs = 0;
- cct->active = 0;
- list_splice_init(&cct->cq_list, &list);
- spin_unlock_irq(&cct->task_lock);
-
- cpu = find_next_online_cpu(pool);
- target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
- spin_lock_irq(&target->task_lock);
- list_for_each_entry_safe(cq, tmp, &list, entry) {
- list_del(&cq->entry);
- __queue_comp_task(cq, target, thread);
- }
- spin_unlock_irq(&target->task_lock);
-}
-
-static void comp_task_stop(unsigned int cpu, bool online)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
-
- spin_lock_irq(&cct->task_lock);
- cct->cq_jobs = 0;
- cct->active = 0;
- WARN_ON(!list_empty(&cct->cq_list));
- spin_unlock_irq(&cct->task_lock);
-}
-
-static int comp_task_should_run(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
-
- return cct->cq_jobs;
-}
-
-static void comp_task(unsigned int cpu)
-{
- struct ehca_cpu_comp_task *cct = this_cpu_ptr(pool->cpu_comp_tasks);
- int cql_empty;
-
- spin_lock_irq(&cct->task_lock);
- cql_empty = list_empty(&cct->cq_list);
- if (!cql_empty) {
- __set_current_state(TASK_RUNNING);
- run_comp_task(cct);
- }
- spin_unlock_irq(&cct->task_lock);
-}
-
-static struct smp_hotplug_thread comp_pool_threads = {
- .thread_should_run = comp_task_should_run,
- .thread_fn = comp_task,
- .thread_comm = "ehca_comp/%u",
- .cleanup = comp_task_stop,
- .park = comp_task_park,
-};
-
-int ehca_create_comp_pool(void)
-{
- int cpu, ret = -ENOMEM;
-
- if (!ehca_scaling_code)
- return 0;
-
- pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL);
- if (pool == NULL)
- return -ENOMEM;
-
- spin_lock_init(&pool->last_cpu_lock);
- pool->last_cpu = cpumask_any(cpu_online_mask);
-
- pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task);
- if (!pool->cpu_comp_tasks)
- goto out_pool;
-
- pool->cpu_comp_threads = alloc_percpu(struct task_struct *);
- if (!pool->cpu_comp_threads)
- goto out_tasks;
-
- for_each_present_cpu(cpu) {
- struct ehca_cpu_comp_task *cct;
-
- cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
- spin_lock_init(&cct->task_lock);
- INIT_LIST_HEAD(&cct->cq_list);
- }
-
- comp_pool_threads.store = pool->cpu_comp_threads;
- ret = smpboot_register_percpu_thread(&comp_pool_threads);
- if (ret)
- goto out_threads;
-
- pr_info("eHCA scaling code enabled\n");
- return ret;
-
-out_threads:
- free_percpu(pool->cpu_comp_threads);
-out_tasks:
- free_percpu(pool->cpu_comp_tasks);
-out_pool:
- kfree(pool);
- return ret;
-}
-
-void ehca_destroy_comp_pool(void)
-{
- if (!ehca_scaling_code)
- return;
-
- smpboot_unregister_percpu_thread(&comp_pool_threads);
-
- free_percpu(pool->cpu_comp_threads);
- free_percpu(pool->cpu_comp_tasks);
- kfree(pool);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Function definitions and structs for EQs, NEQs and interrupts
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Khadija Souissi <souissi@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_IRQ_H
-#define __EHCA_IRQ_H
-
-
-struct ehca_shca;
-
-#include <linux/interrupt.h>
-#include <linux/types.h>
-
-int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
-
-irqreturn_t ehca_interrupt_neq(int irq, void *dev_id);
-void ehca_tasklet_neq(unsigned long data);
-
-irqreturn_t ehca_interrupt_eq(int irq, void *dev_id);
-void ehca_tasklet_eq(unsigned long data);
-void ehca_process_eq(struct ehca_shca *shca, int is_irq);
-
-struct ehca_cpu_comp_task {
- struct list_head cq_list;
- spinlock_t task_lock;
- int cq_jobs;
- int active;
-};
-
-struct ehca_comp_pool {
- struct ehca_cpu_comp_task __percpu *cpu_comp_tasks;
- struct task_struct * __percpu *cpu_comp_threads;
- int last_cpu;
- spinlock_t last_cpu_lock;
-};
-
-int ehca_create_comp_pool(void);
-void ehca_destroy_comp_pool(void);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Function definitions for internal functions
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Dietmar Decker <ddecker@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __EHCA_IVERBS_H__
-#define __EHCA_IVERBS_H__
-
-#include "ehca_classes.h"
-
-int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
- struct ib_udata *uhw);
-
-int ehca_query_port(struct ib_device *ibdev, u8 port,
- struct ib_port_attr *props);
-
-enum rdma_protocol_type
-ehca_query_protocol(struct ib_device *device, u8 port_num);
-
-int ehca_query_sma_attr(struct ehca_shca *shca, u8 port,
- struct ehca_sma_attr *attr);
-
-int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey);
-
-int ehca_query_gid(struct ib_device *ibdev, u8 port, int index,
- union ib_gid *gid);
-
-int ehca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
- struct ib_port_modify *props);
-
-struct ib_pd *ehca_alloc_pd(struct ib_device *device,
- struct ib_ucontext *context,
- struct ib_udata *udata);
-
-int ehca_dealloc_pd(struct ib_pd *pd);
-
-struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
-
-int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
-
-int ehca_destroy_ah(struct ib_ah *ah);
-
-struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
-
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags, u64 *iova_start);
-
-struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
- u64 virt, int mr_access_flags,
- struct ib_udata *udata);
-
-int ehca_rereg_phys_mr(struct ib_mr *mr,
- int mr_rereg_mask,
- struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf, int mr_access_flags, u64 *iova_start);
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
-
-int ehca_dereg_mr(struct ib_mr *mr);
-
-struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
-
-int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
- struct ib_mw_bind *mw_bind);
-
-int ehca_dealloc_mw(struct ib_mw *mw);
-
-struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
- int mr_access_flags,
- struct ib_fmr_attr *fmr_attr);
-
-int ehca_map_phys_fmr(struct ib_fmr *fmr,
- u64 *page_list, int list_len, u64 iova);
-
-int ehca_unmap_fmr(struct list_head *fmr_list);
-
-int ehca_dealloc_fmr(struct ib_fmr *fmr);
-
-enum ehca_eq_type {
- EHCA_EQ = 0, /* Event Queue */
- EHCA_NEQ /* Notification Event Queue */
-};
-
-int ehca_create_eq(struct ehca_shca *shca, struct ehca_eq *eq,
- enum ehca_eq_type type, const u32 length);
-
-int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq);
-
-void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq);
-
-
-struct ib_cq *ehca_create_cq(struct ib_device *device,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata);
-
-int ehca_destroy_cq(struct ib_cq *cq);
-
-int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
-
-int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
-
-int ehca_peek_cq(struct ib_cq *cq, int wc_cnt);
-
-int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags);
-
-struct ib_qp *ehca_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_udata *udata);
-
-int ehca_destroy_qp(struct ib_qp *qp);
-
-int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
- struct ib_udata *udata);
-
-int ehca_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
- int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
-
-int ehca_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
- struct ib_send_wr **bad_send_wr);
-
-int ehca_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr);
-
-int ehca_post_srq_recv(struct ib_srq *srq,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr);
-
-struct ib_srq *ehca_create_srq(struct ib_pd *pd,
- struct ib_srq_init_attr *init_attr,
- struct ib_udata *udata);
-
-int ehca_modify_srq(struct ib_srq *srq, struct ib_srq_attr *attr,
- enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
-
-int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
-
-int ehca_destroy_srq(struct ib_srq *srq);
-
-u64 ehca_define_sqp(struct ehca_shca *shca, struct ehca_qp *ibqp,
- struct ib_qp_init_attr *qp_init_attr);
-
-int ehca_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
-
-int ehca_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
-
-struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
- struct ib_udata *udata);
-
-int ehca_dealloc_ucontext(struct ib_ucontext *context);
-
-int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
-
-int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad_hdr *in, size_t in_mad_size,
- struct ib_mad_hdr *out, size_t *out_mad_size,
- u16 *out_mad_pkey_index);
-
-void ehca_poll_eqs(unsigned long data);
-
-int ehca_calc_ipd(struct ehca_shca *shca, int port,
- enum ib_rate path_rate, u32 *ipd);
-
-void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq);
-
-#ifdef CONFIG_PPC_64K_PAGES
-void *ehca_alloc_fw_ctrlblock(gfp_t flags);
-void ehca_free_fw_ctrlblock(void *ptr);
-#else
-#define ehca_alloc_fw_ctrlblock(flags) ((void *)get_zeroed_page(flags))
-#define ehca_free_fw_ctrlblock(ptr) free_page((unsigned long)(ptr))
-#endif
-
-void ehca_recover_sqp(struct ib_qp *sqp);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * module start stop, hca detection
- *
- * Authors: Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifdef CONFIG_PPC_64K_PAGES
-#include <linux/slab.h>
-#endif
-
-#include <linux/notifier.h>
-#include <linux/memory.h>
-#include <rdma/ib_mad.h>
-#include "ehca_classes.h"
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-
-#define HCAD_VERSION "0029"
-
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
-MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
-MODULE_VERSION(HCAD_VERSION);
-
-static bool ehca_open_aqp1 = 0;
-static int ehca_hw_level = 0;
-static bool ehca_poll_all_eqs = 1;
-
-int ehca_debug_level = 0;
-int ehca_nr_ports = -1;
-bool ehca_use_hp_mr = 0;
-int ehca_port_act_time = 30;
-int ehca_static_rate = -1;
-bool ehca_scaling_code = 0;
-int ehca_lock_hcalls = -1;
-int ehca_max_cq = -1;
-int ehca_max_qp = -1;
-
-module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
-module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
-module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
-module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
-module_param_named(use_hp_mr, ehca_use_hp_mr, bool, S_IRUGO);
-module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
-module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
-module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
-module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
-module_param_named(lock_hcalls, ehca_lock_hcalls, bint, S_IRUGO);
-module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
-module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);
-
-MODULE_PARM_DESC(open_aqp1,
- "Open AQP1 on startup (default: no)");
-MODULE_PARM_DESC(debug_level,
- "Amount of debug output (0: none (default), 1: traces, "
- "2: some dumps, 3: lots)");
-MODULE_PARM_DESC(hw_level,
- "Hardware level (0: autosensing (default), "
- "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
-MODULE_PARM_DESC(nr_ports,
- "number of connected ports (-1: autodetect (default), "
- "1: port one only, 2: two ports)");
-MODULE_PARM_DESC(use_hp_mr,
- "Use high performance MRs (default: no)");
-MODULE_PARM_DESC(port_act_time,
- "Time to wait for port activation (default: 30 sec)");
-MODULE_PARM_DESC(poll_all_eqs,
- "Poll all event queues periodically (default: yes)");
-MODULE_PARM_DESC(static_rate,
- "Set permanent static rate (default: no static rate)");
-MODULE_PARM_DESC(scaling_code,
- "Enable scaling code (default: no)");
-MODULE_PARM_DESC(lock_hcalls,
- "Serialize all hCalls made by the driver "
- "(default: autodetect)");
-MODULE_PARM_DESC(number_of_cqs,
- "Max number of CQs which can be allocated "
- "(default: autodetect)");
-MODULE_PARM_DESC(number_of_qps,
- "Max number of QPs which can be allocated "
- "(default: autodetect)");
-
-DEFINE_RWLOCK(ehca_qp_idr_lock);
-DEFINE_RWLOCK(ehca_cq_idr_lock);
-DEFINE_IDR(ehca_qp_idr);
-DEFINE_IDR(ehca_cq_idr);
-
-static LIST_HEAD(shca_list); /* list of all registered ehcas */
-DEFINE_SPINLOCK(shca_list_lock);
-
-static struct timer_list poll_eqs_timer;
-
-#ifdef CONFIG_PPC_64K_PAGES
-static struct kmem_cache *ctblk_cache;
-
-void *ehca_alloc_fw_ctrlblock(gfp_t flags)
-{
- void *ret = kmem_cache_zalloc(ctblk_cache, flags);
- if (!ret)
- ehca_gen_err("Out of memory for ctblk");
- return ret;
-}
-
-void ehca_free_fw_ctrlblock(void *ptr)
-{
- if (ptr)
- kmem_cache_free(ctblk_cache, ptr);
-
-}
-#endif
-
-int ehca2ib_return_code(u64 ehca_rc)
-{
- switch (ehca_rc) {
- case H_SUCCESS:
- return 0;
- case H_RESOURCE: /* Resource in use */
- case H_BUSY:
- return -EBUSY;
- case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
- case H_CONSTRAINED: /* resource constraint */
- case H_NO_MEM:
- return -ENOMEM;
- default:
- return -EINVAL;
- }
-}
-
-static int ehca_create_slab_caches(void)
-{
- int ret;
-
- ret = ehca_init_pd_cache();
- if (ret) {
- ehca_gen_err("Cannot create PD SLAB cache.");
- return ret;
- }
-
- ret = ehca_init_cq_cache();
- if (ret) {
- ehca_gen_err("Cannot create CQ SLAB cache.");
- goto create_slab_caches2;
- }
-
- ret = ehca_init_qp_cache();
- if (ret) {
- ehca_gen_err("Cannot create QP SLAB cache.");
- goto create_slab_caches3;
- }
-
- ret = ehca_init_av_cache();
- if (ret) {
- ehca_gen_err("Cannot create AV SLAB cache.");
- goto create_slab_caches4;
- }
-
- ret = ehca_init_mrmw_cache();
- if (ret) {
- ehca_gen_err("Cannot create MR&MW SLAB cache.");
- goto create_slab_caches5;
- }
-
- ret = ehca_init_small_qp_cache();
- if (ret) {
- ehca_gen_err("Cannot create small queue SLAB cache.");
- goto create_slab_caches6;
- }
-
-#ifdef CONFIG_PPC_64K_PAGES
- ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
- EHCA_PAGESIZE, H_CB_ALIGNMENT,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!ctblk_cache) {
- ehca_gen_err("Cannot create ctblk SLAB cache.");
- ehca_cleanup_small_qp_cache();
- ret = -ENOMEM;
- goto create_slab_caches6;
- }
-#endif
- return 0;
-
-create_slab_caches6:
- ehca_cleanup_mrmw_cache();
-
-create_slab_caches5:
- ehca_cleanup_av_cache();
-
-create_slab_caches4:
- ehca_cleanup_qp_cache();
-
-create_slab_caches3:
- ehca_cleanup_cq_cache();
-
-create_slab_caches2:
- ehca_cleanup_pd_cache();
-
- return ret;
-}
-
-static void ehca_destroy_slab_caches(void)
-{
- ehca_cleanup_small_qp_cache();
- ehca_cleanup_mrmw_cache();
- ehca_cleanup_av_cache();
- ehca_cleanup_qp_cache();
- ehca_cleanup_cq_cache();
- ehca_cleanup_pd_cache();
-#ifdef CONFIG_PPC_64K_PAGES
- if (ctblk_cache)
- kmem_cache_destroy(ctblk_cache);
-#endif
-}
-
-#define EHCA_HCAAVER EHCA_BMASK_IBM(32, 39)
-#define EHCA_REVID EHCA_BMASK_IBM(40, 63)
-
-static struct cap_descr {
- u64 mask;
- char *descr;
-} hca_cap_descr[] = {
- { HCA_CAP_AH_PORT_NR_CHECK, "HCA_CAP_AH_PORT_NR_CHECK" },
- { HCA_CAP_ATOMIC, "HCA_CAP_ATOMIC" },
- { HCA_CAP_AUTO_PATH_MIG, "HCA_CAP_AUTO_PATH_MIG" },
- { HCA_CAP_BAD_P_KEY_CTR, "HCA_CAP_BAD_P_KEY_CTR" },
- { HCA_CAP_SQD_RTS_PORT_CHANGE, "HCA_CAP_SQD_RTS_PORT_CHANGE" },
- { HCA_CAP_CUR_QP_STATE_MOD, "HCA_CAP_CUR_QP_STATE_MOD" },
- { HCA_CAP_INIT_TYPE, "HCA_CAP_INIT_TYPE" },
- { HCA_CAP_PORT_ACTIVE_EVENT, "HCA_CAP_PORT_ACTIVE_EVENT" },
- { HCA_CAP_Q_KEY_VIOL_CTR, "HCA_CAP_Q_KEY_VIOL_CTR" },
- { HCA_CAP_WQE_RESIZE, "HCA_CAP_WQE_RESIZE" },
- { HCA_CAP_RAW_PACKET_MCAST, "HCA_CAP_RAW_PACKET_MCAST" },
- { HCA_CAP_SHUTDOWN_PORT, "HCA_CAP_SHUTDOWN_PORT" },
- { HCA_CAP_RC_LL_QP, "HCA_CAP_RC_LL_QP" },
- { HCA_CAP_SRQ, "HCA_CAP_SRQ" },
- { HCA_CAP_UD_LL_QP, "HCA_CAP_UD_LL_QP" },
- { HCA_CAP_RESIZE_MR, "HCA_CAP_RESIZE_MR" },
- { HCA_CAP_MINI_QP, "HCA_CAP_MINI_QP" },
- { HCA_CAP_H_ALLOC_RES_SYNC, "HCA_CAP_H_ALLOC_RES_SYNC" },
-};
-
-static int ehca_sense_attributes(struct ehca_shca *shca)
-{
- int i, ret = 0;
- u64 h_ret;
- struct hipz_query_hca *rblock;
- struct hipz_query_port *port;
- const char *loc_code;
-
- static const u32 pgsize_map[] = {
- HCA_CAP_MR_PGSIZE_4K, 0x1000,
- HCA_CAP_MR_PGSIZE_64K, 0x10000,
- HCA_CAP_MR_PGSIZE_1M, 0x100000,
- HCA_CAP_MR_PGSIZE_16M, 0x1000000,
- };
-
- ehca_gen_dbg("Probing adapter %s...",
- shca->ofdev->dev.of_node->full_name);
- loc_code = of_get_property(shca->ofdev->dev.of_node, "ibm,loc-code",
- NULL);
- if (loc_code)
- ehca_gen_dbg(" ... location lode=%s", loc_code);
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_gen_err("Cannot allocate rblock memory.");
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
- if (h_ret != H_SUCCESS) {
- ehca_gen_err("Cannot query device properties. h_ret=%lli",
- h_ret);
- ret = -EPERM;
- goto sense_attributes1;
- }
-
- if (ehca_nr_ports == 1)
- shca->num_ports = 1;
- else
- shca->num_ports = (u8)rblock->num_ports;
-
- ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
-
- if (ehca_hw_level == 0) {
- u32 hcaaver;
- u32 revid;
-
- hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
- revid = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
-
- ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
-
- if (hcaaver == 1) {
- if (revid <= 3)
- shca->hw_level = 0x10 | (revid + 1);
- else
- shca->hw_level = 0x14;
- } else if (hcaaver == 2) {
- if (revid == 0)
- shca->hw_level = 0x21;
- else if (revid == 0x10)
- shca->hw_level = 0x22;
- else if (revid == 0x20 || revid == 0x21)
- shca->hw_level = 0x23;
- }
-
- if (!shca->hw_level) {
- ehca_gen_warn("unknown hardware version"
- " - assuming default level");
- shca->hw_level = 0x22;
- }
- } else
- shca->hw_level = ehca_hw_level;
- ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
-
- shca->hca_cap = rblock->hca_cap_indicators;
- ehca_gen_dbg(" ... HCA capabilities:");
- for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++)
- if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
- ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
-
- /* Autodetect hCall locking -- the "H_ALLOC_RESOURCE synced" flag is
- * a firmware property, so it's valid across all adapters
- */
- if (ehca_lock_hcalls == -1)
- ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC,
- shca->hca_cap);
-
- /* translate supported MR page sizes; always support 4K */
- shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
- for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
- if (rblock->memory_page_size_supported & pgsize_map[i])
- shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
-
- /* Set maximum number of CQs and QPs to calculate EQ size */
- if (shca->max_num_qps == -1)
- shca->max_num_qps = min_t(int, rblock->max_qp,
- EHCA_MAX_NUM_QUEUES);
- else if (shca->max_num_qps < 1 || shca->max_num_qps > rblock->max_qp) {
- ehca_gen_warn("The requested number of QPs is out of range "
- "(1 - %i) specified by HW. Value is set to %i",
- rblock->max_qp, rblock->max_qp);
- shca->max_num_qps = rblock->max_qp;
- }
-
- if (shca->max_num_cqs == -1)
- shca->max_num_cqs = min_t(int, rblock->max_cq,
- EHCA_MAX_NUM_QUEUES);
- else if (shca->max_num_cqs < 1 || shca->max_num_cqs > rblock->max_cq) {
- ehca_gen_warn("The requested number of CQs is out of range "
- "(1 - %i) specified by HW. Value is set to %i",
- rblock->max_cq, rblock->max_cq);
- }
-
- /* query max MTU from first port -- it's the same for all ports */
- port = (struct hipz_query_port *)rblock;
- h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
- if (h_ret != H_SUCCESS) {
- ehca_gen_err("Cannot query port properties. h_ret=%lli",
- h_ret);
- ret = -EPERM;
- goto sense_attributes1;
- }
-
- shca->max_mtu = port->max_mtu;
-
-sense_attributes1:
- ehca_free_fw_ctrlblock(rblock);
- return ret;
-}
-
-static int init_node_guid(struct ehca_shca *shca)
-{
- int ret = 0;
- struct hipz_query_hca *rblock;
-
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!rblock) {
- ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
- return -ENOMEM;
- }
-
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
- ehca_err(&shca->ib_device, "Can't query device properties");
- ret = -EINVAL;
- goto init_node_guid1;
- }
-
- memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
-
-init_node_guid1:
- ehca_free_fw_ctrlblock(rblock);
- return ret;
-}
-
-static int ehca_port_immutable(struct ib_device *ibdev, u8 port_num,
- struct ib_port_immutable *immutable)
-{
- struct ib_port_attr attr;
- int err;
-
- err = ehca_query_port(ibdev, port_num, &attr);
- if (err)
- return err;
-
- immutable->pkey_tbl_len = attr.pkey_tbl_len;
- immutable->gid_tbl_len = attr.gid_tbl_len;
- immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
- immutable->max_mad_size = IB_MGMT_MAD_SIZE;
-
- return 0;
-}
-
-static int ehca_init_device(struct ehca_shca *shca)
-{
- int ret;
-
- ret = init_node_guid(shca);
- if (ret)
- return ret;
-
- strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
- shca->ib_device.owner = THIS_MODULE;
-
- shca->ib_device.uverbs_abi_ver = 8;
- shca->ib_device.uverbs_cmd_mask =
- (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
- (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
- (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
- (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
- (1ull << IB_USER_VERBS_CMD_REG_MR) |
- (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
- (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
- (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
- (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
- (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
- (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
- (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
-
- shca->ib_device.node_type = RDMA_NODE_IB_CA;
- shca->ib_device.phys_port_cnt = shca->num_ports;
- shca->ib_device.num_comp_vectors = 1;
- shca->ib_device.dma_device = &shca->ofdev->dev;
- shca->ib_device.query_device = ehca_query_device;
- shca->ib_device.query_port = ehca_query_port;
- shca->ib_device.query_gid = ehca_query_gid;
- shca->ib_device.query_pkey = ehca_query_pkey;
- /* shca->in_device.modify_device = ehca_modify_device */
- shca->ib_device.modify_port = ehca_modify_port;
- shca->ib_device.alloc_ucontext = ehca_alloc_ucontext;
- shca->ib_device.dealloc_ucontext = ehca_dealloc_ucontext;
- shca->ib_device.alloc_pd = ehca_alloc_pd;
- shca->ib_device.dealloc_pd = ehca_dealloc_pd;
- shca->ib_device.create_ah = ehca_create_ah;
- /* shca->ib_device.modify_ah = ehca_modify_ah; */
- shca->ib_device.query_ah = ehca_query_ah;
- shca->ib_device.destroy_ah = ehca_destroy_ah;
- shca->ib_device.create_qp = ehca_create_qp;
- shca->ib_device.modify_qp = ehca_modify_qp;
- shca->ib_device.query_qp = ehca_query_qp;
- shca->ib_device.destroy_qp = ehca_destroy_qp;
- shca->ib_device.post_send = ehca_post_send;
- shca->ib_device.post_recv = ehca_post_recv;
- shca->ib_device.create_cq = ehca_create_cq;
- shca->ib_device.destroy_cq = ehca_destroy_cq;
- shca->ib_device.resize_cq = ehca_resize_cq;
- shca->ib_device.poll_cq = ehca_poll_cq;
- /* shca->ib_device.peek_cq = ehca_peek_cq; */
- shca->ib_device.req_notify_cq = ehca_req_notify_cq;
- /* shca->ib_device.req_ncomp_notif = ehca_req_ncomp_notif; */
- shca->ib_device.get_dma_mr = ehca_get_dma_mr;
- shca->ib_device.reg_phys_mr = ehca_reg_phys_mr;
- shca->ib_device.reg_user_mr = ehca_reg_user_mr;
- shca->ib_device.query_mr = ehca_query_mr;
- shca->ib_device.dereg_mr = ehca_dereg_mr;
- shca->ib_device.rereg_phys_mr = ehca_rereg_phys_mr;
- shca->ib_device.alloc_mw = ehca_alloc_mw;
- shca->ib_device.bind_mw = ehca_bind_mw;
- shca->ib_device.dealloc_mw = ehca_dealloc_mw;
- shca->ib_device.alloc_fmr = ehca_alloc_fmr;
- shca->ib_device.map_phys_fmr = ehca_map_phys_fmr;
- shca->ib_device.unmap_fmr = ehca_unmap_fmr;
- shca->ib_device.dealloc_fmr = ehca_dealloc_fmr;
- shca->ib_device.attach_mcast = ehca_attach_mcast;
- shca->ib_device.detach_mcast = ehca_detach_mcast;
- shca->ib_device.process_mad = ehca_process_mad;
- shca->ib_device.mmap = ehca_mmap;
- shca->ib_device.dma_ops = &ehca_dma_mapping_ops;
- shca->ib_device.get_port_immutable = ehca_port_immutable;
-
- if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
- shca->ib_device.uverbs_cmd_mask |=
- (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
- (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
- (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
- (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
-
- shca->ib_device.create_srq = ehca_create_srq;
- shca->ib_device.modify_srq = ehca_modify_srq;
- shca->ib_device.query_srq = ehca_query_srq;
- shca->ib_device.destroy_srq = ehca_destroy_srq;
- shca->ib_device.post_srq_recv = ehca_post_srq_recv;
- }
-
- return ret;
-}
-
-static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
-{
- struct ehca_sport *sport = &shca->sport[port - 1];
- struct ib_cq *ibcq;
- struct ib_qp *ibqp;
- struct ib_qp_init_attr qp_init_attr;
- struct ib_cq_init_attr cq_attr = {};
- int ret;
-
- if (sport->ibcq_aqp1) {
- ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
- return -EPERM;
- }
-
- cq_attr.cqe = 10;
- ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void *)(-1),
- &cq_attr);
- if (IS_ERR(ibcq)) {
- ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
- return PTR_ERR(ibcq);
- }
- sport->ibcq_aqp1 = ibcq;
-
- if (sport->ibqp_sqp[IB_QPT_GSI]) {
- ehca_err(&shca->ib_device, "AQP1 QP is already created.");
- ret = -EPERM;
- goto create_aqp1;
- }
-
- memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
- qp_init_attr.send_cq = ibcq;
- qp_init_attr.recv_cq = ibcq;
- qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
- qp_init_attr.cap.max_send_wr = 100;
- qp_init_attr.cap.max_recv_wr = 100;
- qp_init_attr.cap.max_send_sge = 2;
- qp_init_attr.cap.max_recv_sge = 1;
- qp_init_attr.qp_type = IB_QPT_GSI;
- qp_init_attr.port_num = port;
- qp_init_attr.qp_context = NULL;
- qp_init_attr.event_handler = NULL;
- qp_init_attr.srq = NULL;
-
- ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
- if (IS_ERR(ibqp)) {
- ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
- ret = PTR_ERR(ibqp);
- goto create_aqp1;
- }
- sport->ibqp_sqp[IB_QPT_GSI] = ibqp;
-
- return 0;
-
-create_aqp1:
- ib_destroy_cq(sport->ibcq_aqp1);
- return ret;
-}
-
-static int ehca_destroy_aqp1(struct ehca_sport *sport)
-{
- int ret;
-
- ret = ib_destroy_qp(sport->ibqp_sqp[IB_QPT_GSI]);
- if (ret) {
- ehca_gen_err("Cannot destroy AQP1 QP. ret=%i", ret);
- return ret;
- }
-
- ret = ib_destroy_cq(sport->ibcq_aqp1);
- if (ret)
- ehca_gen_err("Cannot destroy AQP1 CQ. ret=%i", ret);
-
- return ret;
-}
-
-static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
-{
- return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
-}
-
-static ssize_t ehca_store_debug_level(struct device_driver *ddp,
- const char *buf, size_t count)
-{
- int value = (*buf) - '0';
- if (value >= 0 && value <= 9)
- ehca_debug_level = value;
- return 1;
-}
-
-static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
- ehca_show_debug_level, ehca_store_debug_level);
-
-static struct attribute *ehca_drv_attrs[] = {
- &driver_attr_debug_level.attr,
- NULL
-};
-
-static struct attribute_group ehca_drv_attr_grp = {
- .attrs = ehca_drv_attrs
-};
-
-static const struct attribute_group *ehca_drv_attr_groups[] = {
- &ehca_drv_attr_grp,
- NULL,
-};
-
-#define EHCA_RESOURCE_ATTR(name) \
-static ssize_t ehca_show_##name(struct device *dev, \
- struct device_attribute *attr, \
- char *buf) \
-{ \
- struct ehca_shca *shca; \
- struct hipz_query_hca *rblock; \
- int data; \
- \
- shca = dev_get_drvdata(dev); \
- \
- rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL); \
- if (!rblock) { \
- dev_err(dev, "Can't allocate rblock memory.\n"); \
- return 0; \
- } \
- \
- if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
- dev_err(dev, "Can't query device properties\n"); \
- ehca_free_fw_ctrlblock(rblock); \
- return 0; \
- } \
- \
- data = rblock->name; \
- ehca_free_fw_ctrlblock(rblock); \
- \
- if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1)) \
- return snprintf(buf, 256, "1\n"); \
- else \
- return snprintf(buf, 256, "%d\n", data); \
- \
-} \
-static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
-
-EHCA_RESOURCE_ATTR(num_ports);
-EHCA_RESOURCE_ATTR(hw_ver);
-EHCA_RESOURCE_ATTR(max_eq);
-EHCA_RESOURCE_ATTR(cur_eq);
-EHCA_RESOURCE_ATTR(max_cq);
-EHCA_RESOURCE_ATTR(cur_cq);
-EHCA_RESOURCE_ATTR(max_qp);
-EHCA_RESOURCE_ATTR(cur_qp);
-EHCA_RESOURCE_ATTR(max_mr);
-EHCA_RESOURCE_ATTR(cur_mr);
-EHCA_RESOURCE_ATTR(max_mw);
-EHCA_RESOURCE_ATTR(cur_mw);
-EHCA_RESOURCE_ATTR(max_pd);
-EHCA_RESOURCE_ATTR(max_ah);
-
-static ssize_t ehca_show_adapter_handle(struct device *dev,
- struct device_attribute *attr,
- char *buf)
-{
- struct ehca_shca *shca = dev_get_drvdata(dev);
-
- return sprintf(buf, "%llx\n", shca->ipz_hca_handle.handle);
-
-}
-static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
-
-static struct attribute *ehca_dev_attrs[] = {
- &dev_attr_adapter_handle.attr,
- &dev_attr_num_ports.attr,
- &dev_attr_hw_ver.attr,
- &dev_attr_max_eq.attr,
- &dev_attr_cur_eq.attr,
- &dev_attr_max_cq.attr,
- &dev_attr_cur_cq.attr,
- &dev_attr_max_qp.attr,
- &dev_attr_cur_qp.attr,
- &dev_attr_max_mr.attr,
- &dev_attr_cur_mr.attr,
- &dev_attr_max_mw.attr,
- &dev_attr_cur_mw.attr,
- &dev_attr_max_pd.attr,
- &dev_attr_max_ah.attr,
- NULL
-};
-
-static struct attribute_group ehca_dev_attr_grp = {
- .attrs = ehca_dev_attrs
-};
-
-static int ehca_probe(struct platform_device *dev)
-{
- struct ehca_shca *shca;
- const u64 *handle;
- struct ib_pd *ibpd;
- int ret, i, eq_size;
- unsigned long flags;
-
- handle = of_get_property(dev->dev.of_node, "ibm,hca-handle", NULL);
- if (!handle) {
- ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
- dev->dev.of_node->full_name);
- return -ENODEV;
- }
-
- if (!(*handle)) {
- ehca_gen_err("Wrong eHCA handle for adapter: %s.",
- dev->dev.of_node->full_name);
- return -ENODEV;
- }
-
- shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
- if (!shca) {
- ehca_gen_err("Cannot allocate shca memory.");
- return -ENOMEM;
- }
-
- mutex_init(&shca->modify_mutex);
- atomic_set(&shca->num_cqs, 0);
- atomic_set(&shca->num_qps, 0);
- shca->max_num_qps = ehca_max_qp;
- shca->max_num_cqs = ehca_max_cq;
-
- for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
- spin_lock_init(&shca->sport[i].mod_sqp_lock);
-
- shca->ofdev = dev;
- shca->ipz_hca_handle.handle = *handle;
- dev_set_drvdata(&dev->dev, shca);
-
- ret = ehca_sense_attributes(shca);
- if (ret < 0) {
- ehca_gen_err("Cannot sense eHCA attributes.");
- goto probe1;
- }
-
- ret = ehca_init_device(shca);
- if (ret) {
- ehca_gen_err("Cannot init ehca device struct");
- goto probe1;
- }
-
- eq_size = 2 * shca->max_num_cqs + 4 * shca->max_num_qps;
- /* create event queues */
- ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create EQ.");
- goto probe1;
- }
-
- ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create NEQ.");
- goto probe3;
- }
-
- /* create internal protection domain */
- ibpd = ehca_alloc_pd(&shca->ib_device, (void *)(-1), NULL);
- if (IS_ERR(ibpd)) {
- ehca_err(&shca->ib_device, "Cannot create internal PD.");
- ret = PTR_ERR(ibpd);
- goto probe4;
- }
-
- shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
- shca->pd->ib_pd.device = &shca->ib_device;
-
- /* create internal max MR */
- ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
-
- if (ret) {
- ehca_err(&shca->ib_device, "Cannot create internal MR ret=%i",
- ret);
- goto probe5;
- }
-
- ret = ib_register_device(&shca->ib_device, NULL);
- if (ret) {
- ehca_err(&shca->ib_device,
- "ib_register_device() failed ret=%i", ret);
- goto probe6;
- }
-
- /* create AQP1 for port 1 */
- if (ehca_open_aqp1 == 1) {
- shca->sport[0].port_state = IB_PORT_DOWN;
- ret = ehca_create_aqp1(shca, 1);
- if (ret) {
- ehca_err(&shca->ib_device,
- "Cannot create AQP1 for port 1.");
- goto probe7;
- }
- }
-
- /* create AQP1 for port 2 */
- if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
- shca->sport[1].port_state = IB_PORT_DOWN;
- ret = ehca_create_aqp1(shca, 2);
- if (ret) {
- ehca_err(&shca->ib_device,
- "Cannot create AQP1 for port 2.");
- goto probe8;
- }
- }
-
- ret = sysfs_create_group(&dev->dev.kobj, &ehca_dev_attr_grp);
- if (ret) /* only complain; we can live without attributes */
- ehca_err(&shca->ib_device,
- "Cannot create device attributes ret=%d", ret);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- list_add(&shca->shca_list, &shca_list);
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- return 0;
-
-probe8:
- ret = ehca_destroy_aqp1(&shca->sport[0]);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy AQP1 for port 1. ret=%i", ret);
-
-probe7:
- ib_unregister_device(&shca->ib_device);
-
-probe6:
- ret = ehca_dereg_internal_maxmr(shca);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal MR. ret=%x", ret);
-
-probe5:
- ret = ehca_dealloc_pd(&shca->pd->ib_pd);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal PD. ret=%x", ret);
-
-probe4:
- ret = ehca_destroy_eq(shca, &shca->neq);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy NEQ. ret=%x", ret);
-
-probe3:
- ret = ehca_destroy_eq(shca, &shca->eq);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy EQ. ret=%x", ret);
-
-probe1:
- ib_dealloc_device(&shca->ib_device);
-
- return -EINVAL;
-}
-
-static int ehca_remove(struct platform_device *dev)
-{
- struct ehca_shca *shca = dev_get_drvdata(&dev->dev);
- unsigned long flags;
- int ret;
-
- sysfs_remove_group(&dev->dev.kobj, &ehca_dev_attr_grp);
-
- if (ehca_open_aqp1 == 1) {
- int i;
- for (i = 0; i < shca->num_ports; i++) {
- ret = ehca_destroy_aqp1(&shca->sport[i]);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy AQP1 for port %x "
- "ret=%i", ret, i);
- }
- }
-
- ib_unregister_device(&shca->ib_device);
-
- ret = ehca_dereg_internal_maxmr(shca);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal MR. ret=%i", ret);
-
- ret = ehca_dealloc_pd(&shca->pd->ib_pd);
- if (ret)
- ehca_err(&shca->ib_device,
- "Cannot destroy internal PD. ret=%i", ret);
-
- ret = ehca_destroy_eq(shca, &shca->eq);
- if (ret)
- ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%i", ret);
-
- ret = ehca_destroy_eq(shca, &shca->neq);
- if (ret)
- ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%i", ret);
-
- ib_dealloc_device(&shca->ib_device);
-
- spin_lock_irqsave(&shca_list_lock, flags);
- list_del(&shca->shca_list);
- spin_unlock_irqrestore(&shca_list_lock, flags);
-
- return ret;
-}
-
-static struct of_device_id ehca_device_table[] =
-{
- {
- .name = "lhca",
- .compatible = "IBM,lhca",
- },
- {},
-};
-MODULE_DEVICE_TABLE(of, ehca_device_table);
-
-static struct platform_driver ehca_driver = {
- .probe = ehca_probe,
- .remove = ehca_remove,
- .driver = {
- .name = "ehca",
- .owner = THIS_MODULE,
- .groups = ehca_drv_attr_groups,
- .of_match_table = ehca_device_table,
- },
-};
-
-void ehca_poll_eqs(unsigned long data)
-{
- struct ehca_shca *shca;
-
- spin_lock(&shca_list_lock);
- list_for_each_entry(shca, &shca_list, shca_list) {
- if (shca->eq.is_initialized) {
- /* call deadman proc only if eq ptr does not change */
- struct ehca_eq *eq = &shca->eq;
- int max = 3;
- volatile u64 q_ofs, q_ofs2;
- unsigned long flags;
- spin_lock_irqsave(&eq->spinlock, flags);
- q_ofs = eq->ipz_queue.current_q_offset;
- spin_unlock_irqrestore(&eq->spinlock, flags);
- do {
- spin_lock_irqsave(&eq->spinlock, flags);
- q_ofs2 = eq->ipz_queue.current_q_offset;
- spin_unlock_irqrestore(&eq->spinlock, flags);
- max--;
- } while (q_ofs == q_ofs2 && max > 0);
- if (q_ofs == q_ofs2)
- ehca_process_eq(shca, 0);
- }
- }
- mod_timer(&poll_eqs_timer, round_jiffies(jiffies + HZ));
- spin_unlock(&shca_list_lock);
-}
-
-static int ehca_mem_notifier(struct notifier_block *nb,
- unsigned long action, void *data)
-{
- static unsigned long ehca_dmem_warn_time;
- unsigned long flags;
-
- switch (action) {
- case MEM_CANCEL_OFFLINE:
- case MEM_CANCEL_ONLINE:
- case MEM_ONLINE:
- case MEM_OFFLINE:
- return NOTIFY_OK;
- case MEM_GOING_ONLINE:
- case MEM_GOING_OFFLINE:
- /* only ok if no hca is attached to the lpar */
- spin_lock_irqsave(&shca_list_lock, flags);
- if (list_empty(&shca_list)) {
- spin_unlock_irqrestore(&shca_list_lock, flags);
- return NOTIFY_OK;
- } else {
- spin_unlock_irqrestore(&shca_list_lock, flags);
- if (printk_timed_ratelimit(&ehca_dmem_warn_time,
- 30 * 1000))
- ehca_gen_err("DMEM operations are not allowed"
- "in conjunction with eHCA");
- return NOTIFY_BAD;
- }
- }
- return NOTIFY_OK;
-}
-
-static struct notifier_block ehca_mem_nb = {
- .notifier_call = ehca_mem_notifier,
-};
-
-static int __init ehca_module_init(void)
-{
- int ret;
-
- printk(KERN_INFO "eHCA Infiniband Device Driver "
- "(Version " HCAD_VERSION ")\n");
-
- ret = ehca_create_comp_pool();
- if (ret) {
- ehca_gen_err("Cannot create comp pool.");
- return ret;
- }
-
- ret = ehca_create_slab_caches();
- if (ret) {
- ehca_gen_err("Cannot create SLAB caches");
- ret = -ENOMEM;
- goto module_init1;
- }
-
- ret = ehca_create_busmap();
- if (ret) {
- ehca_gen_err("Cannot create busmap.");
- goto module_init2;
- }
-
- ret = ibmebus_register_driver(&ehca_driver);
- if (ret) {
- ehca_gen_err("Cannot register eHCA device driver");
- ret = -EINVAL;
- goto module_init3;
- }
-
- ret = register_memory_notifier(&ehca_mem_nb);
- if (ret) {
- ehca_gen_err("Failed registering memory add/remove notifier");
- goto module_init4;
- }
-
- if (ehca_poll_all_eqs != 1) {
- ehca_gen_err("WARNING!!!");
- ehca_gen_err("It is possible to lose interrupts.");
- } else {
- init_timer(&poll_eqs_timer);
- poll_eqs_timer.function = ehca_poll_eqs;
- poll_eqs_timer.expires = jiffies + HZ;
- add_timer(&poll_eqs_timer);
- }
-
- return 0;
-
-module_init4:
- ibmebus_unregister_driver(&ehca_driver);
-
-module_init3:
- ehca_destroy_busmap();
-
-module_init2:
- ehca_destroy_slab_caches();
-
-module_init1:
- ehca_destroy_comp_pool();
- return ret;
-};
-
-static void __exit ehca_module_exit(void)
-{
- if (ehca_poll_all_eqs == 1)
- del_timer_sync(&poll_eqs_timer);
-
- ibmebus_unregister_driver(&ehca_driver);
-
- unregister_memory_notifier(&ehca_mem_nb);
-
- ehca_destroy_busmap();
-
- ehca_destroy_slab_caches();
-
- ehca_destroy_comp_pool();
-
- idr_destroy(&ehca_cq_idr);
- idr_destroy(&ehca_qp_idr);
-};
-
-module_init(ehca_module_init);
-module_exit(ehca_module_exit);
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * mcast functions
- *
- * Authors: Khadija Souissi <souissik@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/module.h>
-#include <linux/err.h>
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-#define MAX_MC_LID 0xFFFE
-#define MIN_MC_LID 0xC000 /* Multicast limits */
-#define EHCA_VALID_MULTICAST_GID(gid) ((gid)[0] == 0xFF)
-#define EHCA_VALID_MULTICAST_LID(lid) \
- (((lid) >= MIN_MC_LID) && ((lid) <= MAX_MC_LID))
-
-int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
-{
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
- ib_device);
- union ib_gid my_gid;
- u64 subnet_prefix, interface_id, h_ret;
-
- if (ibqp->qp_type != IB_QPT_UD) {
- ehca_err(ibqp->device, "invalid qp_type=%x", ibqp->qp_type);
- return -EINVAL;
- }
-
- if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
- ehca_err(ibqp->device, "invalid mulitcast gid");
- return -EINVAL;
- } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
- ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
- return -EINVAL;
- }
-
- memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
-
- subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
- interface_id = be64_to_cpu(my_gid.global.interface_id);
- h_ret = hipz_h_attach_mcqp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- my_qp->galpas.kernel,
- lid, subnet_prefix, interface_id);
- if (h_ret != H_SUCCESS)
- ehca_err(ibqp->device,
- "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
- "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
-
- return ehca2ib_return_code(h_ret);
-}
-
-int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
-{
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(ibqp->pd->device,
- struct ehca_shca, ib_device);
- union ib_gid my_gid;
- u64 subnet_prefix, interface_id, h_ret;
-
- if (ibqp->qp_type != IB_QPT_UD) {
- ehca_err(ibqp->device, "invalid qp_type %x", ibqp->qp_type);
- return -EINVAL;
- }
-
- if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
- ehca_err(ibqp->device, "invalid mulitcast gid");
- return -EINVAL;
- } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
- ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
- return -EINVAL;
- }
-
- memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
-
- subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
- interface_id = be64_to_cpu(my_gid.global.interface_id);
- h_ret = hipz_h_detach_mcqp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- my_qp->galpas.kernel,
- lid, subnet_prefix, interface_id);
- if (h_ret != H_SUCCESS)
- ehca_err(ibqp->device,
- "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
- "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
-
- return ehca2ib_return_code(h_ret);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * MR/MW functions
- *
- * Authors: Dietmar Decker <ddecker@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-#include <rdma/ib_umem.h>
-
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "hcp_if.h"
-#include "hipz_hw.h"
-
-#define NUM_CHUNKS(length, chunk_size) \
- (((length) + (chunk_size - 1)) / (chunk_size))
-
-/* max number of rpages (per hcall register_rpages) */
-#define MAX_RPAGES 512
-
-/* DMEM toleration management */
-#define EHCA_SECTSHIFT SECTION_SIZE_BITS
-#define EHCA_SECTSIZE (1UL << EHCA_SECTSHIFT)
-#define EHCA_HUGEPAGESHIFT 34
-#define EHCA_HUGEPAGE_SIZE (1UL << EHCA_HUGEPAGESHIFT)
-#define EHCA_HUGEPAGE_PFN_MASK ((EHCA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT)
-#define EHCA_INVAL_ADDR 0xFFFFFFFFFFFFFFFFULL
-#define EHCA_DIR_INDEX_SHIFT 13 /* 8k Entries in 64k block */
-#define EHCA_TOP_INDEX_SHIFT (EHCA_DIR_INDEX_SHIFT * 2)
-#define EHCA_MAP_ENTRIES (1 << EHCA_DIR_INDEX_SHIFT)
-#define EHCA_TOP_MAP_SIZE (0x10000) /* currently fixed map size */
-#define EHCA_DIR_MAP_SIZE (0x10000)
-#define EHCA_ENT_MAP_SIZE (0x10000)
-#define EHCA_INDEX_MASK (EHCA_MAP_ENTRIES - 1)
-
-static unsigned long ehca_mr_len;
-
-/*
- * Memory map data structures
- */
-struct ehca_dir_bmap {
- u64 ent[EHCA_MAP_ENTRIES];
-};
-struct ehca_top_bmap {
- struct ehca_dir_bmap *dir[EHCA_MAP_ENTRIES];
-};
-struct ehca_bmap {
- struct ehca_top_bmap *top[EHCA_MAP_ENTRIES];
-};
-
-static struct ehca_bmap *ehca_bmap;
-
-static struct kmem_cache *mr_cache;
-static struct kmem_cache *mw_cache;
-
-enum ehca_mr_pgsize {
- EHCA_MR_PGSIZE4K = 0x1000L,
- EHCA_MR_PGSIZE64K = 0x10000L,
- EHCA_MR_PGSIZE1M = 0x100000L,
- EHCA_MR_PGSIZE16M = 0x1000000L
-};
-
-#define EHCA_MR_PGSHIFT4K 12
-#define EHCA_MR_PGSHIFT64K 16
-#define EHCA_MR_PGSHIFT1M 20
-#define EHCA_MR_PGSHIFT16M 24
-
-static u64 ehca_map_vaddr(void *caddr);
-
-static u32 ehca_encode_hwpage_size(u32 pgsize)
-{
- int log = ilog2(pgsize);
- WARN_ON(log < 12 || log > 24 || log & 3);
- return (log - 12) / 4;
-}
-
-static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
-{
- return rounddown_pow_of_two(shca->hca_cap_mr_pgsize);
-}
-
-static struct ehca_mr *ehca_mr_new(void)
-{
- struct ehca_mr *me;
-
- me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
- if (me)
- spin_lock_init(&me->mrlock);
- else
- ehca_gen_err("alloc failed");
-
- return me;
-}
-
-static void ehca_mr_delete(struct ehca_mr *me)
-{
- kmem_cache_free(mr_cache, me);
-}
-
-static struct ehca_mw *ehca_mw_new(void)
-{
- struct ehca_mw *me;
-
- me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
- if (me)
- spin_lock_init(&me->mwlock);
- else
- ehca_gen_err("alloc failed");
-
- return me;
-}
-
-static void ehca_mw_delete(struct ehca_mw *me)
-{
- kmem_cache_free(mw_cache, me);
-}
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
-{
- struct ib_mr *ib_mr;
- int ret;
- struct ehca_mr *e_maxmr;
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
-
- if (shca->maxmr) {
- e_maxmr = ehca_mr_new();
- if (!e_maxmr) {
- ehca_err(&shca->ib_device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto get_dma_mr_exit0;
- }
-
- ret = ehca_reg_maxmr(shca, e_maxmr,
- (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)),
- mr_access_flags, e_pd,
- &e_maxmr->ib.ib_mr.lkey,
- &e_maxmr->ib.ib_mr.rkey);
- if (ret) {
- ehca_mr_delete(e_maxmr);
- ib_mr = ERR_PTR(ret);
- goto get_dma_mr_exit0;
- }
- ib_mr = &e_maxmr->ib.ib_mr;
- } else {
- ehca_err(&shca->ib_device, "no internal max-MR exist!");
- ib_mr = ERR_PTR(-EINVAL);
- goto get_dma_mr_exit0;
- }
-
-get_dma_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x",
- PTR_ERR(ib_mr), pd, mr_access_flags);
- return ib_mr;
-} /* end ehca_get_dma_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags,
- u64 *iova_start)
-{
- struct ib_mr *ib_mr;
- int ret;
- struct ehca_mr *e_mr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
-
- u64 size;
-
- if ((num_phys_buf <= 0) || !phys_buf_array) {
- ehca_err(pd->device, "bad input values: num_phys_buf=%x "
- "phys_buf_array=%p", num_phys_buf, phys_buf_array);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
-
- /* check physical buffer list and calculate size */
- ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
- iova_start, &size);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit0;
- }
- if ((size == 0) ||
- (((u64)iova_start + size) < (u64)iova_start)) {
- ehca_err(pd->device, "bad input values: size=%llx iova_start=%p",
- size, iova_start);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_phys_mr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(pd->device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto reg_phys_mr_exit0;
- }
-
- /* register MR on HCA */
- if (ehca_mr_is_maxmr(size, iova_start)) {
- e_mr->flags |= EHCA_MR_FLAG_MAXMR;
- ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
- e_pd, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit1;
- }
- } else {
- struct ehca_mr_pginfo pginfo;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hw_pgsize;
-
- num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
- PAGE_SIZE);
- /* for kernel space we try most possible pgsize */
- hw_pgsize = ehca_get_max_hwpage_size(shca);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
- hw_pgsize);
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.phy.num_phys_buf = num_phys_buf;
- pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage =
- ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
- e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_phys_mr_exit1;
- }
- }
-
- /* successful registration of all pages */
- return &e_mr->ib.ib_mr;
-
-reg_phys_mr_exit1:
- ehca_mr_delete(e_mr);
-reg_phys_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p "
- "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
- PTR_ERR(ib_mr), pd, phys_buf_array,
- num_phys_buf, mr_access_flags, iova_start);
- return ib_mr;
-} /* end ehca_reg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
- u64 virt, int mr_access_flags,
- struct ib_udata *udata)
-{
- struct ib_mr *ib_mr;
- struct ehca_mr *e_mr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_mr_pginfo pginfo;
- int ret, page_shift;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hwpage_size;
-
- if (!pd) {
- ehca_gen_err("bad pd=%p", pd);
- return ERR_PTR(-EFAULT);
- }
-
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit0;
- }
-
- if (length == 0 || virt + length < virt) {
- ehca_err(pd->device, "bad input values: length=%llx "
- "virt_base=%llx", length, virt);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(pd->device, "out of memory");
- ib_mr = ERR_PTR(-ENOMEM);
- goto reg_user_mr_exit0;
- }
-
- e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
- mr_access_flags, 0);
- if (IS_ERR(e_mr->umem)) {
- ib_mr = (void *)e_mr->umem;
- goto reg_user_mr_exit1;
- }
-
- if (e_mr->umem->page_size != PAGE_SIZE) {
- ehca_err(pd->device, "page size not supported, "
- "e_mr->umem->page_size=%x", e_mr->umem->page_size);
- ib_mr = ERR_PTR(-EINVAL);
- goto reg_user_mr_exit2;
- }
-
- /* determine number of MR pages */
- num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
- /* select proper hw_pgsize */
- page_shift = PAGE_SHIFT;
- if (e_mr->umem->hugetlb) {
- /* determine page_shift, clamp between 4K and 16M */
- page_shift = (fls64(length - 1) + 3) & ~3;
- page_shift = min(max(page_shift, EHCA_MR_PGSHIFT4K),
- EHCA_MR_PGSHIFT16M);
- }
- hwpage_size = 1UL << page_shift;
-
- /* now that we have the desired page size, shift until it's
- * supported, too. 4K is always supported, so this terminates.
- */
- while (!(hwpage_size & shca->hca_cap_mr_pgsize))
- hwpage_size >>= 4;
-
-reg_user_mr_fallback:
- num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
- /* register MR on HCA */
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_USER;
- pginfo.hwpage_size = hwpage_size;
- pginfo.num_kpages = num_kpages;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.usr.region = e_mr->umem;
- pginfo.next_hwpage = ib_umem_offset(e_mr->umem) / hwpage_size;
- pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
- ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
- e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
- if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
- ehca_warn(pd->device, "failed to register mr "
- "with hwpage_size=%llx", hwpage_size);
- ehca_info(pd->device, "try to register mr with "
- "kpage_size=%lx", PAGE_SIZE);
- /*
- * this means kpages are not contiguous for a hw page
- * try kernel page size as fallback solution
- */
- hwpage_size = PAGE_SIZE;
- goto reg_user_mr_fallback;
- }
- if (ret) {
- ib_mr = ERR_PTR(ret);
- goto reg_user_mr_exit2;
- }
-
- /* successful registration of all pages */
- return &e_mr->ib.ib_mr;
-
-reg_user_mr_exit2:
- ib_umem_release(e_mr->umem);
-reg_user_mr_exit1:
- ehca_mr_delete(e_mr);
-reg_user_mr_exit0:
- if (IS_ERR(ib_mr))
- ehca_err(pd->device, "rc=%li pd=%p mr_access_flags=%x udata=%p",
- PTR_ERR(ib_mr), pd, mr_access_flags, udata);
- return ib_mr;
-} /* end ehca_reg_user_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_rereg_phys_mr(struct ib_mr *mr,
- int mr_rereg_mask,
- struct ib_pd *pd,
- struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- int mr_access_flags,
- u64 *iova_start)
-{
- int ret;
-
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- u64 new_size;
- u64 *new_start;
- u32 new_acl;
- struct ehca_pd *new_pd;
- u32 tmp_lkey, tmp_rkey;
- unsigned long sl_flags;
- u32 num_kpages = 0;
- u32 num_hwpages = 0;
- struct ehca_mr_pginfo pginfo;
-
- if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
- /* TODO not supported, because PHYP rereg hCall needs pages */
- ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
- "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- if (mr_rereg_mask & IB_MR_REREG_PD) {
- if (!pd) {
- ehca_err(mr->device, "rereg with bad pd, pd=%p "
- "mr_rereg_mask=%x", pd, mr_rereg_mask);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- }
-
- if ((mr_rereg_mask &
- ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
- (mr_rereg_mask == 0)) {
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- /* check other parameters */
- if (e_mr == shca->maxmr) {
- /* should be impossible, however reject to be sure */
- ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
- "shca->maxmr=%p mr->lkey=%x",
- mr, shca->maxmr, mr->lkey);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
- if (e_mr->flags & EHCA_MR_FLAG_FMR) {
- ehca_err(mr->device, "not supported for FMR, mr=%p "
- "flags=%x", mr, e_mr->flags);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- if (!phys_buf_array || num_phys_buf <= 0) {
- ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
- " phys_buf_array=%p num_phys_buf=%x",
- mr_rereg_mask, phys_buf_array, num_phys_buf);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
- }
- if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */
- (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
- "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
- ret = -EINVAL;
- goto rereg_phys_mr_exit0;
- }
-
- /* set requested values dependent on rereg request */
- spin_lock_irqsave(&e_mr->mrlock, sl_flags);
- new_start = e_mr->start;
- new_size = e_mr->size;
- new_acl = e_mr->acl;
- new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
-
- if (mr_rereg_mask & IB_MR_REREG_TRANS) {
- u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
-
- new_start = iova_start; /* change address */
- /* check physical buffer list and calculate size */
- ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
- num_phys_buf, iova_start,
- &new_size);
- if (ret)
- goto rereg_phys_mr_exit1;
- if ((new_size == 0) ||
- (((u64)iova_start + new_size) < (u64)iova_start)) {
- ehca_err(mr->device, "bad input values: new_size=%llx "
- "iova_start=%p", new_size, iova_start);
- ret = -EINVAL;
- goto rereg_phys_mr_exit1;
- }
- num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
- new_size, PAGE_SIZE);
- num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
- new_size, hw_pgsize);
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.num_hwpages = num_hwpages;
- pginfo.u.phy.num_phys_buf = num_phys_buf;
- pginfo.u.phy.phys_buf_array = phys_buf_array;
- pginfo.next_hwpage =
- ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
- }
- if (mr_rereg_mask & IB_MR_REREG_ACCESS)
- new_acl = mr_access_flags;
- if (mr_rereg_mask & IB_MR_REREG_PD)
- new_pd = container_of(pd, struct ehca_pd, ib_pd);
-
- ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
- new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
- if (ret)
- goto rereg_phys_mr_exit1;
-
- /* successful reregistration */
- if (mr_rereg_mask & IB_MR_REREG_PD)
- mr->pd = pd;
- mr->lkey = tmp_lkey;
- mr->rkey = tmp_rkey;
-
-rereg_phys_mr_exit1:
- spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-rereg_phys_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p "
- "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
- "iova_start=%p",
- ret, mr, mr_rereg_mask, pd, phys_buf_array,
- num_phys_buf, mr_access_flags, iova_start);
- return ret;
-} /* end ehca_rereg_phys_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
- unsigned long sl_flags;
- struct ehca_mr_hipzout_parms hipzout;
-
- if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
- "e_mr->flags=%x", mr, e_mr, e_mr->flags);
- ret = -EINVAL;
- goto query_mr_exit0;
- }
-
- memset(mr_attr, 0, sizeof(struct ib_mr_attr));
- spin_lock_irqsave(&e_mr->mrlock, sl_flags);
-
- h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p "
- "hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle, mr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto query_mr_exit1;
- }
- mr_attr->pd = mr->pd;
- mr_attr->device_virt_addr = hipzout.vaddr;
- mr_attr->size = hipzout.len;
- mr_attr->lkey = hipzout.lkey;
- mr_attr->rkey = hipzout.rkey;
- ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
-
-query_mr_exit1:
- spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
-query_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p",
- ret, mr, mr_attr);
- return ret;
-} /* end ehca_query_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dereg_mr(struct ib_mr *mr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
-
- if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
- "e_mr->flags=%x", mr, e_mr, e_mr->flags);
- ret = -EINVAL;
- goto dereg_mr_exit0;
- } else if (e_mr == shca->maxmr) {
- /* should be impossible, however reject to be sure */
- ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
- "shca->maxmr=%p mr->lkey=%x",
- mr, shca->maxmr, mr->lkey);
- ret = -EINVAL;
- goto dereg_mr_exit0;
- }
-
- /* TODO: BUSY: MR still has bound window(s) */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lli shca=%p "
- "e_mr=%p hca_hndl=%llx mr_hndl=%llx mr->lkey=%x",
- h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle, mr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto dereg_mr_exit0;
- }
-
- if (e_mr->umem)
- ib_umem_release(e_mr->umem);
-
- /* successful deregistration */
- ehca_mr_delete(e_mr);
-
-dereg_mr_exit0:
- if (ret)
- ehca_err(mr->device, "ret=%i mr=%p", ret, mr);
- return ret;
-} /* end ehca_dereg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
-{
- struct ib_mw *ib_mw;
- u64 h_ret;
- struct ehca_mw *e_mw;
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_mw_hipzout_parms hipzout;
-
- if (type != IB_MW_TYPE_1)
- return ERR_PTR(-EINVAL);
-
- e_mw = ehca_mw_new();
- if (!e_mw) {
- ib_mw = ERR_PTR(-ENOMEM);
- goto alloc_mw_exit0;
- }
-
- h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
- e_pd->fw_pd, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lli "
- "shca=%p hca_hndl=%llx mw=%p",
- h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
- ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
- goto alloc_mw_exit1;
- }
- /* successful MW allocation */
- e_mw->ipz_mw_handle = hipzout.handle;
- e_mw->ib_mw.rkey = hipzout.rkey;
- return &e_mw->ib_mw;
-
-alloc_mw_exit1:
- ehca_mw_delete(e_mw);
-alloc_mw_exit0:
- if (IS_ERR(ib_mw))
- ehca_err(pd->device, "h_ret=%li pd=%p", PTR_ERR(ib_mw), pd);
- return ib_mw;
-} /* end ehca_alloc_mw() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_bind_mw(struct ib_qp *qp,
- struct ib_mw *mw,
- struct ib_mw_bind *mw_bind)
-{
- /* TODO: not supported up to now */
- ehca_gen_err("bind MW currently not supported by HCAD");
-
- return -EPERM;
-} /* end ehca_bind_mw() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dealloc_mw(struct ib_mw *mw)
-{
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(mw->device, struct ehca_shca, ib_device);
- struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
-
- h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
- if (h_ret != H_SUCCESS) {
- ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lli shca=%p "
- "mw=%p rkey=%x hca_hndl=%llx mw_hndl=%llx",
- h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
- e_mw->ipz_mw_handle.handle);
- return ehca2ib_return_code(h_ret);
- }
- /* successful deallocation */
- ehca_mw_delete(e_mw);
- return 0;
-} /* end ehca_dealloc_mw() */
-
-/*----------------------------------------------------------------------*/
-
-struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
- int mr_access_flags,
- struct ib_fmr_attr *fmr_attr)
-{
- struct ib_fmr *ib_fmr;
- struct ehca_shca *shca =
- container_of(pd->device, struct ehca_shca, ib_device);
- struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_mr *e_fmr;
- int ret;
- u32 tmp_lkey, tmp_rkey;
- struct ehca_mr_pginfo pginfo;
- u64 hw_pgsize;
-
- /* check other parameters */
- if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
- ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
- !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
- /*
- * Remote Write Access requires Local Write Access
- * Remote Atomic Access requires Local Write Access
- */
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
- if (mr_access_flags & IB_ACCESS_MW_BIND) {
- ehca_err(pd->device, "bad input values: mr_access_flags=%x",
- mr_access_flags);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
- if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
- ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
- "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
- fmr_attr->max_pages, fmr_attr->max_maps,
- fmr_attr->page_shift);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
-
- hw_pgsize = 1 << fmr_attr->page_shift;
- if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) {
- ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
- fmr_attr->page_shift);
- ib_fmr = ERR_PTR(-EINVAL);
- goto alloc_fmr_exit0;
- }
-
- e_fmr = ehca_mr_new();
- if (!e_fmr) {
- ib_fmr = ERR_PTR(-ENOMEM);
- goto alloc_fmr_exit0;
- }
- e_fmr->flags |= EHCA_MR_FLAG_FMR;
-
- /* register MR on HCA */
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.hwpage_size = hw_pgsize;
- /*
- * pginfo.num_hwpages==0, ie register_rpages() will not be called
- * but deferred to map_phys_fmr()
- */
- ret = ehca_reg_mr(shca, e_fmr, NULL,
- fmr_attr->max_pages * (1 << fmr_attr->page_shift),
- mr_access_flags, e_pd, &pginfo,
- &tmp_lkey, &tmp_rkey, EHCA_REG_MR);
- if (ret) {
- ib_fmr = ERR_PTR(ret);
- goto alloc_fmr_exit1;
- }
-
- /* successful */
- e_fmr->hwpage_size = hw_pgsize;
- e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
- e_fmr->fmr_max_pages = fmr_attr->max_pages;
- e_fmr->fmr_max_maps = fmr_attr->max_maps;
- e_fmr->fmr_map_cnt = 0;
- return &e_fmr->ib.ib_fmr;
-
-alloc_fmr_exit1:
- ehca_mr_delete(e_fmr);
-alloc_fmr_exit0:
- return ib_fmr;
-} /* end ehca_alloc_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_map_phys_fmr(struct ib_fmr *fmr,
- u64 *page_list,
- int list_len,
- u64 iova)
-{
- int ret;
- struct ehca_shca *shca =
- container_of(fmr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
- struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
- struct ehca_mr_pginfo pginfo;
- u32 tmp_lkey, tmp_rkey;
-
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
- e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto map_phys_fmr_exit0;
- }
- ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
- if (ret)
- goto map_phys_fmr_exit0;
- if (iova % e_fmr->fmr_page_size) {
- /* only whole-numbered pages */
- ehca_err(fmr->device, "bad iova, iova=%llx fmr_page_size=%x",
- iova, e_fmr->fmr_page_size);
- ret = -EINVAL;
- goto map_phys_fmr_exit0;
- }
- if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
- /* HCAD does not limit the maps, however trace this anyway */
- ehca_info(fmr->device, "map limit exceeded, fmr=%p "
- "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
- fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
- }
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_FMR;
- pginfo.num_kpages = list_len;
- pginfo.hwpage_size = e_fmr->hwpage_size;
- pginfo.num_hwpages =
- list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
- pginfo.u.fmr.page_list = page_list;
- pginfo.next_hwpage =
- (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
- pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
-
- ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
- list_len * e_fmr->fmr_page_size,
- e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
- if (ret)
- goto map_phys_fmr_exit0;
-
- /* successful reregistration */
- e_fmr->fmr_map_cnt++;
- e_fmr->ib.ib_fmr.lkey = tmp_lkey;
- e_fmr->ib.ib_fmr.rkey = tmp_rkey;
- return 0;
-
-map_phys_fmr_exit0:
- if (ret)
- ehca_err(fmr->device, "ret=%i fmr=%p page_list=%p list_len=%x "
- "iova=%llx", ret, fmr, page_list, list_len, iova);
- return ret;
-} /* end ehca_map_phys_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_unmap_fmr(struct list_head *fmr_list)
-{
- int ret = 0;
- struct ib_fmr *ib_fmr;
- struct ehca_shca *shca = NULL;
- struct ehca_shca *prev_shca;
- struct ehca_mr *e_fmr;
- u32 num_fmr = 0;
- u32 unmap_fmr_cnt = 0;
-
- /* check all FMR belong to same SHCA, and check internal flag */
- list_for_each_entry(ib_fmr, fmr_list, list) {
- prev_shca = shca;
- shca = container_of(ib_fmr->device, struct ehca_shca,
- ib_device);
- e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
- if ((shca != prev_shca) && prev_shca) {
- ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
- "prev_shca=%p e_fmr=%p",
- shca, prev_shca, e_fmr);
- ret = -EINVAL;
- goto unmap_fmr_exit0;
- }
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
- "e_fmr->flags=%x", e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto unmap_fmr_exit0;
- }
- num_fmr++;
- }
-
- /* loop over all FMRs to unmap */
- list_for_each_entry(ib_fmr, fmr_list, list) {
- unmap_fmr_cnt++;
- e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
- shca = container_of(ib_fmr->device, struct ehca_shca,
- ib_device);
- ret = ehca_unmap_one_fmr(shca, e_fmr);
- if (ret) {
- /* unmap failed, stop unmapping of rest of FMRs */
- ehca_err(&shca->ib_device, "unmap of one FMR failed, "
- "stop rest, e_fmr=%p num_fmr=%x "
- "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
- unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
- goto unmap_fmr_exit0;
- }
- }
-
-unmap_fmr_exit0:
- if (ret)
- ehca_gen_err("ret=%i fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
- ret, fmr_list, num_fmr, unmap_fmr_cnt);
- return ret;
-} /* end ehca_unmap_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dealloc_fmr(struct ib_fmr *fmr)
-{
- int ret;
- u64 h_ret;
- struct ehca_shca *shca =
- container_of(fmr->device, struct ehca_shca, ib_device);
- struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
-
- if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
- ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
- e_fmr, e_fmr->flags);
- ret = -EINVAL;
- goto free_fmr_exit0;
- }
-
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
- if (h_ret != H_SUCCESS) {
- ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lli e_fmr=%p "
- "hca_hndl=%llx fmr_hndl=%llx fmr->lkey=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle, fmr->lkey);
- ret = ehca2ib_return_code(h_ret);
- goto free_fmr_exit0;
- }
- /* successful deregistration */
- ehca_mr_delete(e_fmr);
- return 0;
-
-free_fmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr);
- return ret;
-} /* end ehca_dealloc_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo);
-
-int ehca_reg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey, /*OUT*/
- u32 *rkey, /*OUT*/
- enum ehca_reg_type reg_type)
-{
- int ret;
- u64 h_ret;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
- if (ehca_use_hp_mr == 1)
- hipz_acl |= 0x00000001;
-
- h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
- (u64)iova_start, size, hipz_acl,
- e_pd->fw_pd, &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli "
- "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_reg_mr_exit0;
- }
-
- e_mr->ipz_mr_handle = hipzout.handle;
-
- if (reg_type == EHCA_REG_BUSMAP_MR)
- ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);
- else if (reg_type == EHCA_REG_MR)
- ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
- else
- ret = -EINVAL;
-
- if (ret)
- goto ehca_reg_mr_exit1;
-
- /* successful registration */
- e_mr->num_kpages = pginfo->num_kpages;
- e_mr->num_hwpages = pginfo->num_hwpages;
- e_mr->hwpage_size = pginfo->hwpage_size;
- e_mr->start = iova_start;
- e_mr->size = size;
- e_mr->acl = acl;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-
-ehca_reg_mr_exit1:
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p lkey=%x "
- "pginfo=%p num_kpages=%llx num_hwpages=%llx ret=%i",
- h_ret, shca, e_mr, iova_start, size, acl, e_pd,
- hipzout.lkey, pginfo, pginfo->num_kpages,
- pginfo->num_hwpages, ret);
- ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
- "not recoverable");
- }
-ehca_reg_mr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
- "num_kpages=%llx num_hwpages=%llx",
- ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
- pginfo->num_kpages, pginfo->num_hwpages);
- return ret;
-} /* end ehca_reg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo)
-{
- int ret = 0;
- u64 h_ret;
- u32 rnum;
- u64 rpage;
- u32 i;
- u64 *kpage;
-
- if (!pginfo->num_hwpages) /* in case of fmr */
- return 0;
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- ret = -ENOMEM;
- goto ehca_reg_mr_rpages_exit0;
- }
-
- /* max MAX_RPAGES ehca mr pages per register call */
- for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
-
- if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
- rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
- if (rnum == 0)
- rnum = MAX_RPAGES; /* last shot is full */
- } else
- rnum = MAX_RPAGES;
-
- ret = ehca_set_pagebuf(pginfo, rnum, kpage);
- if (ret) {
- ehca_err(&shca->ib_device, "ehca_set_pagebuf "
- "bad rc, ret=%i rnum=%x kpage=%p",
- ret, rnum, kpage);
- goto ehca_reg_mr_rpages_exit1;
- }
-
- if (rnum > 1) {
- rpage = __pa(kpage);
- if (!rpage) {
- ehca_err(&shca->ib_device, "kpage=%p i=%x",
- kpage, i);
- ret = -EFAULT;
- goto ehca_reg_mr_rpages_exit1;
- }
- } else
- rpage = *kpage;
-
- h_ret = hipz_h_register_rpage_mr(
- shca->ipz_hca_handle, e_mr,
- ehca_encode_hwpage_size(pginfo->hwpage_size),
- 0, rpage, rnum);
-
- if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
- /*
- * check for 'registration complete'==H_SUCCESS
- * and for 'page registered'==H_PAGE_REGISTERED
- */
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "last "
- "hipz_reg_rpage_mr failed, h_ret=%lli "
- "e_mr=%p i=%x hca_hndl=%llx mr_hndl=%llx"
- " lkey=%x", h_ret, e_mr, i,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- break;
- } else
- ret = 0;
- } else if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
- "h_ret=%lli e_mr=%p i=%x lkey=%x hca_hndl=%llx "
- "mr_hndl=%llx", h_ret, e_mr, i,
- e_mr->ib.ib_mr.lkey,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle);
- ret = ehca2ib_return_code(h_ret);
- break;
- } else
- ret = 0;
- } /* end for(i) */
-
-
-ehca_reg_mr_rpages_exit1:
- ehca_free_fw_ctrlblock(kpage);
-ehca_reg_mr_rpages_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p "
- "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr,
- pginfo, pginfo->num_kpages, pginfo->num_hwpages);
- return ret;
-} /* end ehca_reg_mr_rpages() */
-
-/*----------------------------------------------------------------------*/
-
-inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- u32 acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey, /*OUT*/
- u32 *rkey) /*OUT*/
-{
- int ret;
- u64 h_ret;
- u32 hipz_acl;
- u64 *kpage;
- u64 rpage;
- struct ehca_mr_pginfo pginfo_save;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- ret = -ENOMEM;
- goto ehca_rereg_mr_rereg1_exit0;
- }
-
- pginfo_save = *pginfo;
- ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
- if (ret) {
- ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
- "pginfo=%p type=%x num_kpages=%llx num_hwpages=%llx "
- "kpage=%p", e_mr, pginfo, pginfo->type,
- pginfo->num_kpages, pginfo->num_hwpages, kpage);
- goto ehca_rereg_mr_rereg1_exit1;
- }
- rpage = __pa(kpage);
- if (!rpage) {
- ehca_err(&shca->ib_device, "kpage=%p", kpage);
- ret = -EFAULT;
- goto ehca_rereg_mr_rereg1_exit1;
- }
- h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
- (u64)iova_start, size, hipz_acl,
- e_pd->fw_pd, rpage, &hipzout);
- if (h_ret != H_SUCCESS) {
- /*
- * reregistration unsuccessful, try it again with the 3 hCalls,
- * e.g. this is required in case H_MR_CONDITION
- * (MW bound or MR is shared)
- */
- ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
- "(Rereg1), h_ret=%lli e_mr=%p", h_ret, e_mr);
- *pginfo = pginfo_save;
- ret = -EAGAIN;
- } else if ((u64 *)hipzout.vaddr != iova_start) {
- ehca_err(&shca->ib_device, "PHYP changed iova_start in "
- "rereg_pmr, iova_start=%p iova_start_out=%llx e_mr=%p "
- "mr_handle=%llx lkey=%x lkey_out=%x", iova_start,
- hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey, hipzout.lkey);
- ret = -EFAULT;
- } else {
- /*
- * successful reregistration
- * note: start and start_out are identical for eServer HCAs
- */
- e_mr->num_kpages = pginfo->num_kpages;
- e_mr->num_hwpages = pginfo->num_hwpages;
- e_mr->hwpage_size = pginfo->hwpage_size;
- e_mr->start = iova_start;
- e_mr->size = size;
- e_mr->acl = acl;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- }
-
-ehca_rereg_mr_rereg1_exit1:
- ehca_free_fw_ctrlblock(kpage);
-ehca_rereg_mr_rereg1_exit0:
- if ( ret && (ret != -EAGAIN) )
- ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x "
- "pginfo=%p num_kpages=%llx num_hwpages=%llx",
- ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
- pginfo->num_hwpages);
- return ret;
-} /* end ehca_rereg_mr_rereg1() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_rereg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey)
-{
- int ret = 0;
- u64 h_ret;
- int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
- int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
-
- /* first determine reregistration hCall(s) */
- if ((pginfo->num_hwpages > MAX_RPAGES) ||
- (e_mr->num_hwpages > MAX_RPAGES) ||
- (pginfo->num_hwpages > e_mr->num_hwpages)) {
- ehca_dbg(&shca->ib_device, "Rereg3 case, "
- "pginfo->num_hwpages=%llx e_mr->num_hwpages=%x",
- pginfo->num_hwpages, e_mr->num_hwpages);
- rereg_1_hcall = 0;
- rereg_3_hcall = 1;
- }
-
- if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
- rereg_1_hcall = 0;
- rereg_3_hcall = 1;
- e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
- ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
- e_mr);
- }
-
- if (rereg_1_hcall) {
- ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
- acl, e_pd, pginfo, lkey, rkey);
- if (ret) {
- if (ret == -EAGAIN)
- rereg_3_hcall = 1;
- else
- goto ehca_rereg_mr_exit0;
- }
- }
-
- if (rereg_3_hcall) {
- struct ehca_mr save_mr;
-
- /* first deregister old MR */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_free_mr failed, "
- "h_ret=%lli e_mr=%p hca_hndl=%llx mr_hndl=%llx "
- "mr->lkey=%x",
- h_ret, e_mr, shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle,
- e_mr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_rereg_mr_exit0;
- }
- /* clean ehca_mr_t, without changing struct ib_mr and lock */
- save_mr = *e_mr;
- ehca_mr_deletenew(e_mr);
-
- /* set some MR values */
- e_mr->flags = save_mr.flags;
- e_mr->hwpage_size = save_mr.hwpage_size;
- e_mr->fmr_page_size = save_mr.fmr_page_size;
- e_mr->fmr_max_pages = save_mr.fmr_max_pages;
- e_mr->fmr_max_maps = save_mr.fmr_max_maps;
- e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
- e_pd, pginfo, lkey, rkey, EHCA_REG_MR);
- if (ret) {
- u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
- memcpy(&e_mr->flags, &(save_mr.flags),
- sizeof(struct ehca_mr) - offset);
- goto ehca_rereg_mr_exit0;
- }
- }
-
-ehca_rereg_mr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
- "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
- "num_kpages=%llx lkey=%x rkey=%x rereg_1_hcall=%x "
- "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
- acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
- rereg_1_hcall, rereg_3_hcall);
- return ret;
-} /* end ehca_rereg_mr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_unmap_one_fmr(struct ehca_shca *shca,
- struct ehca_mr *e_fmr)
-{
- int ret = 0;
- u64 h_ret;
- struct ehca_pd *e_pd =
- container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
- struct ehca_mr save_fmr;
- u32 tmp_lkey, tmp_rkey;
- struct ehca_mr_pginfo pginfo;
- struct ehca_mr_hipzout_parms hipzout;
- struct ehca_mr save_mr;
-
- if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
- /*
- * note: after using rereg hcall with len=0,
- * rereg hcall must be used again for registering pages
- */
- h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
- 0, 0, e_pd->fw_pd, 0, &hipzout);
- if (h_ret == H_SUCCESS) {
- /* successful reregistration */
- e_fmr->start = NULL;
- e_fmr->size = 0;
- tmp_lkey = hipzout.lkey;
- tmp_rkey = hipzout.rkey;
- return 0;
- }
- /*
- * should not happen, because length checked above,
- * FMRs are not shared and no MW bound to FMRs
- */
- ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
- "(Rereg1), h_ret=%lli e_fmr=%p hca_hndl=%llx "
- "mr_hndl=%llx lkey=%x lkey_out=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle,
- e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
- /* try free and rereg */
- }
-
- /* first free old FMR */
- h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_free_mr failed, "
- "h_ret=%lli e_fmr=%p hca_hndl=%llx mr_hndl=%llx "
- "lkey=%x",
- h_ret, e_fmr, shca->ipz_hca_handle.handle,
- e_fmr->ipz_mr_handle.handle,
- e_fmr->ib.ib_fmr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_unmap_one_fmr_exit0;
- }
- /* clean ehca_mr_t, without changing lock */
- save_fmr = *e_fmr;
- ehca_mr_deletenew(e_fmr);
-
- /* set some MR values */
- e_fmr->flags = save_fmr.flags;
- e_fmr->hwpage_size = save_fmr.hwpage_size;
- e_fmr->fmr_page_size = save_fmr.fmr_page_size;
- e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
- e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
- e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
- e_fmr->acl = save_fmr.acl;
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_FMR;
- ret = ehca_reg_mr(shca, e_fmr, NULL,
- (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
- e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
- &tmp_rkey, EHCA_REG_MR);
- if (ret) {
- u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
- memcpy(&e_fmr->flags, &(save_mr.flags),
- sizeof(struct ehca_mr) - offset);
- }
-
-ehca_unmap_one_fmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x "
- "fmr_max_pages=%x",
- ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
- return ret;
-} /* end ehca_unmap_one_fmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_smr(struct ehca_shca *shca,
- struct ehca_mr *e_origmr,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey, /*OUT*/
- u32 *rkey) /*OUT*/
-{
- int ret = 0;
- u64 h_ret;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
-
- h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
- (u64)iova_start, hipz_acl, e_pd->fw_pd,
- &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
- "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
- "e_pd=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
- shca->ipz_hca_handle.handle,
- e_origmr->ipz_mr_handle.handle,
- e_origmr->ib.ib_mr.lkey);
- ret = ehca2ib_return_code(h_ret);
- goto ehca_reg_smr_exit0;
- }
- /* successful registration */
- e_newmr->num_kpages = e_origmr->num_kpages;
- e_newmr->num_hwpages = e_origmr->num_hwpages;
- e_newmr->hwpage_size = e_origmr->hwpage_size;
- e_newmr->start = iova_start;
- e_newmr->size = e_origmr->size;
- e_newmr->acl = acl;
- e_newmr->ipz_mr_handle = hipzout.handle;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-
-ehca_reg_smr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p "
- "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
- ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
- return ret;
-} /* end ehca_reg_smr() */
-
-/*----------------------------------------------------------------------*/
-static inline void *ehca_calc_sectbase(int top, int dir, int idx)
-{
- unsigned long ret = idx;
- ret |= dir << EHCA_DIR_INDEX_SHIFT;
- ret |= top << EHCA_TOP_INDEX_SHIFT;
- return __va(ret << SECTION_SIZE_BITS);
-}
-
-#define ehca_bmap_valid(entry) \
- ((u64)entry != (u64)EHCA_INVAL_ADDR)
-
-static u64 ehca_reg_mr_section(int top, int dir, int idx, u64 *kpage,
- struct ehca_shca *shca, struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 h_ret = 0;
- unsigned long page = 0;
- u64 rpage = __pa(kpage);
- int page_count;
-
- void *sectbase = ehca_calc_sectbase(top, dir, idx);
- if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) {
- ehca_err(&shca->ib_device, "reg_mr_section will probably fail:"
- "hwpage_size does not fit to "
- "section start address");
- }
- page_count = EHCA_SECTSIZE / pginfo->hwpage_size;
-
- while (page < page_count) {
- u64 rnum;
- for (rnum = 0; (rnum < MAX_RPAGES) && (page < page_count);
- rnum++) {
- void *pg = sectbase + ((page++) * pginfo->hwpage_size);
- kpage[rnum] = __pa(pg);
- }
-
- h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr,
- ehca_encode_hwpage_size(pginfo->hwpage_size),
- 0, rpage, rnum);
-
- if ((h_ret != H_SUCCESS) && (h_ret != H_PAGE_REGISTERED)) {
- ehca_err(&shca->ib_device, "register_rpage_mr failed");
- return h_ret;
- }
- }
- return h_ret;
-}
-
-static u64 ehca_reg_mr_sections(int top, int dir, u64 *kpage,
- struct ehca_shca *shca, struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 hret = H_SUCCESS;
- int idx;
-
- for (idx = 0; idx < EHCA_MAP_ENTRIES; idx++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]->ent[idx]))
- continue;
-
- hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr,
- pginfo);
- if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
- return hret;
- }
- return hret;
-}
-
-static u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca,
- struct ehca_mr *mr,
- struct ehca_mr_pginfo *pginfo)
-{
- u64 hret = H_SUCCESS;
- int dir;
-
- for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- continue;
-
- hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);
- if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
- return hret;
- }
- return hret;
-}
-
-/* register internal max-MR to internal SHCA */
-int ehca_reg_internal_maxmr(
- struct ehca_shca *shca,
- struct ehca_pd *e_pd,
- struct ehca_mr **e_maxmr) /*OUT*/
-{
- int ret;
- struct ehca_mr *e_mr;
- u64 *iova_start;
- u64 size_maxmr;
- struct ehca_mr_pginfo pginfo;
- struct ib_phys_buf ib_pbuf;
- u32 num_kpages;
- u32 num_hwpages;
- u64 hw_pgsize;
-
- if (!ehca_bmap) {
- ret = -EFAULT;
- goto ehca_reg_internal_maxmr_exit0;
- }
-
- e_mr = ehca_mr_new();
- if (!e_mr) {
- ehca_err(&shca->ib_device, "out of memory");
- ret = -ENOMEM;
- goto ehca_reg_internal_maxmr_exit0;
- }
- e_mr->flags |= EHCA_MR_FLAG_MAXMR;
-
- /* register internal max-MR on HCA */
- size_maxmr = ehca_mr_len;
- iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START));
- ib_pbuf.addr = 0;
- ib_pbuf.size = size_maxmr;
- num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
- PAGE_SIZE);
- hw_pgsize = ehca_get_max_hwpage_size(shca);
- num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
- hw_pgsize);
-
- memset(&pginfo, 0, sizeof(pginfo));
- pginfo.type = EHCA_MR_PGI_PHYS;
- pginfo.num_kpages = num_kpages;
- pginfo.num_hwpages = num_hwpages;
- pginfo.hwpage_size = hw_pgsize;
- pginfo.u.phy.num_phys_buf = 1;
- pginfo.u.phy.phys_buf_array = &ib_pbuf;
-
- ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
- &pginfo, &e_mr->ib.ib_mr.lkey,
- &e_mr->ib.ib_mr.rkey, EHCA_REG_BUSMAP_MR);
- if (ret) {
- ehca_err(&shca->ib_device, "reg of internal max MR failed, "
- "e_mr=%p iova_start=%p size_maxmr=%llx num_kpages=%x "
- "num_hwpages=%x", e_mr, iova_start, size_maxmr,
- num_kpages, num_hwpages);
- goto ehca_reg_internal_maxmr_exit1;
- }
-
- /* successful registration of all pages */
- e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
- e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
- e_mr->ib.ib_mr.uobject = NULL;
- atomic_inc(&(e_pd->ib_pd.usecnt));
- atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
- *e_maxmr = e_mr;
- return 0;
-
-ehca_reg_internal_maxmr_exit1:
- ehca_mr_delete(e_mr);
-ehca_reg_internal_maxmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p",
- ret, shca, e_pd, e_maxmr);
- return ret;
-} /* end ehca_reg_internal_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_reg_maxmr(struct ehca_shca *shca,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey)
-{
- u64 h_ret;
- struct ehca_mr *e_origmr = shca->maxmr;
- u32 hipz_acl;
- struct ehca_mr_hipzout_parms hipzout;
-
- ehca_mrmw_map_acl(acl, &hipz_acl);
- ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
-
- h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
- (u64)iova_start, hipz_acl, e_pd->fw_pd,
- &hipzout);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
- "e_origmr=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
- h_ret, e_origmr, shca->ipz_hca_handle.handle,
- e_origmr->ipz_mr_handle.handle,
- e_origmr->ib.ib_mr.lkey);
- return ehca2ib_return_code(h_ret);
- }
- /* successful registration */
- e_newmr->num_kpages = e_origmr->num_kpages;
- e_newmr->num_hwpages = e_origmr->num_hwpages;
- e_newmr->hwpage_size = e_origmr->hwpage_size;
- e_newmr->start = iova_start;
- e_newmr->size = e_origmr->size;
- e_newmr->acl = acl;
- e_newmr->ipz_mr_handle = hipzout.handle;
- *lkey = hipzout.lkey;
- *rkey = hipzout.rkey;
- return 0;
-} /* end ehca_reg_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
-{
- int ret;
- struct ehca_mr *e_maxmr;
- struct ib_pd *ib_pd;
-
- if (!shca->maxmr) {
- ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
- ret = -EINVAL;
- goto ehca_dereg_internal_maxmr_exit0;
- }
-
- e_maxmr = shca->maxmr;
- ib_pd = e_maxmr->ib.ib_mr.pd;
- shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
-
- ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
- if (ret) {
- ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
- "ret=%i e_maxmr=%p shca=%p lkey=%x",
- ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
- shca->maxmr = e_maxmr;
- goto ehca_dereg_internal_maxmr_exit0;
- }
-
- atomic_dec(&ib_pd->usecnt);
-
-ehca_dereg_internal_maxmr_exit0:
- if (ret)
- ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p",
- ret, shca, shca->maxmr);
- return ret;
-} /* end ehca_dereg_internal_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * check physical buffer array of MR verbs for validness and
- * calculates MR size
- */
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- u64 *iova_start,
- u64 *size)
-{
- struct ib_phys_buf *pbuf = phys_buf_array;
- u64 size_count = 0;
- u32 i;
-
- if (num_phys_buf == 0) {
- ehca_gen_err("bad phys buf array len, num_phys_buf=0");
- return -EINVAL;
- }
- /* check first buffer */
- if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
- ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
- "pbuf->addr=%llx pbuf->size=%llx",
- iova_start, pbuf->addr, pbuf->size);
- return -EINVAL;
- }
- if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
- (num_phys_buf > 1)) {
- ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx "
- "pbuf->size=%llx", pbuf->addr, pbuf->size);
- return -EINVAL;
- }
-
- for (i = 0; i < num_phys_buf; i++) {
- if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
- ehca_gen_err("bad address, i=%x pbuf->addr=%llx "
- "pbuf->size=%llx",
- i, pbuf->addr, pbuf->size);
- return -EINVAL;
- }
- if (((i > 0) && /* not 1st */
- (i < (num_phys_buf - 1)) && /* not last */
- (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
- ehca_gen_err("bad size, i=%x pbuf->size=%llx",
- i, pbuf->size);
- return -EINVAL;
- }
- size_count += pbuf->size;
- pbuf++;
- }
-
- *size = size_count;
- return 0;
-} /* end ehca_mr_chk_buf_and_calc_size() */
-
-/*----------------------------------------------------------------------*/
-
-/* check page list of map FMR verb for validness */
-int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
- u64 *page_list,
- int list_len)
-{
- u32 i;
- u64 *page;
-
- if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
- ehca_gen_err("bad list_len, list_len=%x "
- "e_fmr->fmr_max_pages=%x fmr=%p",
- list_len, e_fmr->fmr_max_pages, e_fmr);
- return -EINVAL;
- }
-
- /* each page must be aligned */
- page = page_list;
- for (i = 0; i < list_len; i++) {
- if (*page % e_fmr->fmr_page_size) {
- ehca_gen_err("bad page, i=%x *page=%llx page=%p fmr=%p "
- "fmr_page_size=%x", i, *page, page, e_fmr,
- e_fmr->fmr_page_size);
- return -EINVAL;
- }
- page++;
- }
-
- return 0;
-} /* end ehca_fmr_check_page_list() */
-
-/*----------------------------------------------------------------------*/
-
-/* PAGE_SIZE >= pginfo->hwpage_size */
-static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret = 0;
- u64 pgaddr;
- u32 j = 0;
- int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
- struct scatterlist **sg = &pginfo->u.usr.next_sg;
-
- while (*sg != NULL) {
- pgaddr = page_to_pfn(sg_page(*sg))
- << PAGE_SHIFT;
- *kpage = pgaddr + (pginfo->next_hwpage *
- pginfo->hwpage_size);
- if (!(*kpage)) {
- ehca_gen_err("pgaddr=%llx "
- "sg_dma_address=%llx "
- "entry=%llx next_hwpage=%llx",
- pgaddr, (u64)sg_dma_address(*sg),
- pginfo->u.usr.next_nmap,
- pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- (pginfo->next_hwpage)++;
- kpage++;
- if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
- (pginfo->kpage_cnt)++;
- (pginfo->u.usr.next_nmap)++;
- pginfo->next_hwpage = 0;
- *sg = sg_next(*sg);
- }
- j++;
- if (j >= number)
- break;
- }
-
- return ret;
-}
-
-/*
- * check given pages for contiguous layout
- * last page addr is returned in prev_pgaddr for further check
- */
-static int ehca_check_kpages_per_ate(struct scatterlist **sg,
- int num_pages,
- u64 *prev_pgaddr)
-{
- for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
- u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
- if (ehca_debug_level >= 3)
- ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
- *(u64 *)__va(pgaddr));
- if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
- ehca_gen_err("uncontiguous page found pgaddr=%llx "
- "prev_pgaddr=%llx entries_left_in_hwpage=%x",
- pgaddr, *prev_pgaddr, num_pages);
- return -EINVAL;
- }
- *prev_pgaddr = pgaddr;
- }
- return 0;
-}
-
-/* PAGE_SIZE < pginfo->hwpage_size */
-static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret = 0;
- u64 pgaddr, prev_pgaddr;
- u32 j = 0;
- int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
- int nr_kpages = kpages_per_hwpage;
- struct scatterlist **sg = &pginfo->u.usr.next_sg;
-
- while (*sg != NULL) {
-
- if (nr_kpages == kpages_per_hwpage) {
- pgaddr = (page_to_pfn(sg_page(*sg))
- << PAGE_SHIFT);
- *kpage = pgaddr;
- if (!(*kpage)) {
- ehca_gen_err("pgaddr=%llx entry=%llx",
- pgaddr, pginfo->u.usr.next_nmap);
- ret = -EFAULT;
- return ret;
- }
- /*
- * The first page in a hwpage must be aligned;
- * the first MR page is exempt from this rule.
- */
- if (pgaddr & (pginfo->hwpage_size - 1)) {
- if (pginfo->hwpage_cnt) {
- ehca_gen_err(
- "invalid alignment "
- "pgaddr=%llx entry=%llx "
- "mr_pgsize=%llx",
- pgaddr, pginfo->u.usr.next_nmap,
- pginfo->hwpage_size);
- ret = -EFAULT;
- return ret;
- }
- /* first MR page */
- pginfo->kpage_cnt =
- (pgaddr &
- (pginfo->hwpage_size - 1)) >>
- PAGE_SHIFT;
- nr_kpages -= pginfo->kpage_cnt;
- *kpage = pgaddr &
- ~(pginfo->hwpage_size - 1);
- }
- if (ehca_debug_level >= 3) {
- u64 val = *(u64 *)__va(pgaddr);
- ehca_gen_dbg("kpage=%llx page=%llx "
- "value=%016llx",
- *kpage, pgaddr, val);
- }
- prev_pgaddr = pgaddr;
- *sg = sg_next(*sg);
- pginfo->kpage_cnt++;
- pginfo->u.usr.next_nmap++;
- nr_kpages--;
- if (!nr_kpages)
- goto next_kpage;
- continue;
- }
-
- ret = ehca_check_kpages_per_ate(sg, nr_kpages,
- &prev_pgaddr);
- if (ret)
- return ret;
- pginfo->kpage_cnt += nr_kpages;
- pginfo->u.usr.next_nmap += nr_kpages;
-
-next_kpage:
- nr_kpages = kpages_per_hwpage;
- (pginfo->hwpage_cnt)++;
- kpage++;
- j++;
- if (j >= number)
- break;
- }
-
- return ret;
-}
-
-static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
- u32 number, u64 *kpage)
-{
- int ret = 0;
- struct ib_phys_buf *pbuf;
- u64 num_hw, offs_hw;
- u32 i = 0;
-
- /* loop over desired phys_buf_array entries */
- while (i < number) {
- pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
- num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
- pbuf->size, pginfo->hwpage_size);
- offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
- pginfo->hwpage_size;
- while (pginfo->next_hwpage < offs_hw + num_hw) {
- /* sanity check */
- if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
- (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
- ehca_gen_err("kpage_cnt >= num_kpages, "
- "kpage_cnt=%llx num_kpages=%llx "
- "hwpage_cnt=%llx "
- "num_hwpages=%llx i=%x",
- pginfo->kpage_cnt,
- pginfo->num_kpages,
- pginfo->hwpage_cnt,
- pginfo->num_hwpages, i);
- return -EFAULT;
- }
- *kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
- (pginfo->next_hwpage * pginfo->hwpage_size);
- if ( !(*kpage) && pbuf->addr ) {
- ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx "
- "next_hwpage=%llx", pbuf->addr,
- pbuf->size, pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- (pginfo->next_hwpage)++;
- if (PAGE_SIZE >= pginfo->hwpage_size) {
- if (pginfo->next_hwpage %
- (PAGE_SIZE / pginfo->hwpage_size) == 0)
- (pginfo->kpage_cnt)++;
- } else
- pginfo->kpage_cnt += pginfo->hwpage_size /
- PAGE_SIZE;
- kpage++;
- i++;
- if (i >= number) break;
- }
- if (pginfo->next_hwpage >= offs_hw + num_hw) {
- (pginfo->u.phy.next_buf)++;
- pginfo->next_hwpage = 0;
- }
- }
- return ret;
-}
-
-static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
- u32 number, u64 *kpage)
-{
- int ret = 0;
- u64 *fmrlist;
- u32 i;
-
- /* loop over desired page_list entries */
- fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
- for (i = 0; i < number; i++) {
- *kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) +
- pginfo->next_hwpage * pginfo->hwpage_size;
- if ( !(*kpage) ) {
- ehca_gen_err("*fmrlist=%llx fmrlist=%p "
- "next_listelem=%llx next_hwpage=%llx",
- *fmrlist, fmrlist,
- pginfo->u.fmr.next_listelem,
- pginfo->next_hwpage);
- return -EFAULT;
- }
- (pginfo->hwpage_cnt)++;
- if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
- if (pginfo->next_hwpage %
- (pginfo->u.fmr.fmr_pgsize /
- pginfo->hwpage_size) == 0) {
- (pginfo->kpage_cnt)++;
- (pginfo->u.fmr.next_listelem)++;
- fmrlist++;
- pginfo->next_hwpage = 0;
- } else
- (pginfo->next_hwpage)++;
- } else {
- unsigned int cnt_per_hwpage = pginfo->hwpage_size /
- pginfo->u.fmr.fmr_pgsize;
- unsigned int j;
- u64 prev = *kpage;
- /* check if adrs are contiguous */
- for (j = 1; j < cnt_per_hwpage; j++) {
- u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1);
- if (prev + pginfo->u.fmr.fmr_pgsize != p) {
- ehca_gen_err("uncontiguous fmr pages "
- "found prev=%llx p=%llx "
- "idx=%x", prev, p, i + j);
- return -EINVAL;
- }
- prev = p;
- }
- pginfo->kpage_cnt += cnt_per_hwpage;
- pginfo->u.fmr.next_listelem += cnt_per_hwpage;
- fmrlist += cnt_per_hwpage;
- }
- kpage++;
- }
- return ret;
-}
-
-/* setup page buffer from page info */
-int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage)
-{
- int ret;
-
- switch (pginfo->type) {
- case EHCA_MR_PGI_PHYS:
- ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
- break;
- case EHCA_MR_PGI_USER:
- ret = PAGE_SIZE >= pginfo->hwpage_size ?
- ehca_set_pagebuf_user1(pginfo, number, kpage) :
- ehca_set_pagebuf_user2(pginfo, number, kpage);
- break;
- case EHCA_MR_PGI_FMR:
- ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
- break;
- default:
- ehca_gen_err("bad pginfo->type=%x", pginfo->type);
- ret = -EFAULT;
- break;
- }
- return ret;
-} /* end ehca_set_pagebuf() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * check MR if it is a max-MR, i.e. uses whole memory
- * in case it's a max-MR 1 is returned, else 0
- */
-int ehca_mr_is_maxmr(u64 size,
- u64 *iova_start)
-{
- /* a MR is treated as max-MR only if it fits following: */
- if ((size == ehca_mr_len) &&
- (iova_start == (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)))) {
- ehca_gen_dbg("this is a max-MR");
- return 1;
- } else
- return 0;
-} /* end ehca_mr_is_maxmr() */
-
-/*----------------------------------------------------------------------*/
-
-/* map access control for MR/MW. This routine is used for MR and MW. */
-void ehca_mrmw_map_acl(int ib_acl,
- u32 *hipz_acl)
-{
- *hipz_acl = 0;
- if (ib_acl & IB_ACCESS_REMOTE_READ)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
- if (ib_acl & IB_ACCESS_REMOTE_WRITE)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
- if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
- *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
- if (ib_acl & IB_ACCESS_LOCAL_WRITE)
- *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
- if (ib_acl & IB_ACCESS_MW_BIND)
- *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
-} /* end ehca_mrmw_map_acl() */
-
-/*----------------------------------------------------------------------*/
-
-/* sets page size in hipz access control for MR/MW. */
-void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
-{
- *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
-} /* end ehca_mrmw_set_pgsize_hipz_acl() */
-
-/*----------------------------------------------------------------------*/
-
-/*
- * reverse map access control for MR/MW.
- * This routine is used for MR and MW.
- */
-void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
- int *ib_acl) /*OUT*/
-{
- *ib_acl = 0;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
- *ib_acl |= IB_ACCESS_REMOTE_READ;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
- *ib_acl |= IB_ACCESS_REMOTE_WRITE;
- if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
- *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
- if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
- *ib_acl |= IB_ACCESS_LOCAL_WRITE;
- if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
- *ib_acl |= IB_ACCESS_MW_BIND;
-} /* end ehca_mrmw_reverse_map_acl() */
-
-
-/*----------------------------------------------------------------------*/
-
-/*
- * MR destructor and constructor
- * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
- * except struct ib_mr and spinlock
- */
-void ehca_mr_deletenew(struct ehca_mr *mr)
-{
- mr->flags = 0;
- mr->num_kpages = 0;
- mr->num_hwpages = 0;
- mr->acl = 0;
- mr->start = NULL;
- mr->fmr_page_size = 0;
- mr->fmr_max_pages = 0;
- mr->fmr_max_maps = 0;
- mr->fmr_map_cnt = 0;
- memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
- memset(&mr->galpas, 0, sizeof(mr->galpas));
-} /* end ehca_mr_deletenew() */
-
-int ehca_init_mrmw_cache(void)
-{
- mr_cache = kmem_cache_create("ehca_cache_mr",
- sizeof(struct ehca_mr), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!mr_cache)
- return -ENOMEM;
- mw_cache = kmem_cache_create("ehca_cache_mw",
- sizeof(struct ehca_mw), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!mw_cache) {
- kmem_cache_destroy(mr_cache);
- mr_cache = NULL;
- return -ENOMEM;
- }
- return 0;
-}
-
-void ehca_cleanup_mrmw_cache(void)
-{
- if (mr_cache)
- kmem_cache_destroy(mr_cache);
- if (mw_cache)
- kmem_cache_destroy(mw_cache);
-}
-
-static inline int ehca_init_top_bmap(struct ehca_top_bmap *ehca_top_bmap,
- int dir)
-{
- if (!ehca_bmap_valid(ehca_top_bmap->dir[dir])) {
- ehca_top_bmap->dir[dir] =
- kmalloc(sizeof(struct ehca_dir_bmap), GFP_KERNEL);
- if (!ehca_top_bmap->dir[dir])
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_top_bmap->dir[dir], 0xFF, EHCA_ENT_MAP_SIZE);
- }
- return 0;
-}
-
-static inline int ehca_init_bmap(struct ehca_bmap *ehca_bmap, int top, int dir)
-{
- if (!ehca_bmap_valid(ehca_bmap->top[top])) {
- ehca_bmap->top[top] =
- kmalloc(sizeof(struct ehca_top_bmap), GFP_KERNEL);
- if (!ehca_bmap->top[top])
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_bmap->top[top], 0xFF, EHCA_DIR_MAP_SIZE);
- }
- return ehca_init_top_bmap(ehca_bmap->top[top], dir);
-}
-
-static inline int ehca_calc_index(unsigned long i, unsigned long s)
-{
- return (i >> s) & EHCA_INDEX_MASK;
-}
-
-void ehca_destroy_busmap(void)
-{
- int top, dir;
-
- if (!ehca_bmap)
- return;
-
- for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- continue;
- for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- continue;
-
- kfree(ehca_bmap->top[top]->dir[dir]);
- }
-
- kfree(ehca_bmap->top[top]);
- }
-
- kfree(ehca_bmap);
- ehca_bmap = NULL;
-}
-
-static int ehca_update_busmap(unsigned long pfn, unsigned long nr_pages)
-{
- unsigned long i, start_section, end_section;
- int top, dir, idx;
-
- if (!nr_pages)
- return 0;
-
- if (!ehca_bmap) {
- ehca_bmap = kmalloc(sizeof(struct ehca_bmap), GFP_KERNEL);
- if (!ehca_bmap)
- return -ENOMEM;
- /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
- memset(ehca_bmap, 0xFF, EHCA_TOP_MAP_SIZE);
- }
-
- start_section = (pfn * PAGE_SIZE) / EHCA_SECTSIZE;
- end_section = ((pfn + nr_pages) * PAGE_SIZE) / EHCA_SECTSIZE;
- for (i = start_section; i < end_section; i++) {
- int ret;
- top = ehca_calc_index(i, EHCA_TOP_INDEX_SHIFT);
- dir = ehca_calc_index(i, EHCA_DIR_INDEX_SHIFT);
- idx = i & EHCA_INDEX_MASK;
-
- ret = ehca_init_bmap(ehca_bmap, top, dir);
- if (ret) {
- ehca_destroy_busmap();
- return ret;
- }
- ehca_bmap->top[top]->dir[dir]->ent[idx] = ehca_mr_len;
- ehca_mr_len += EHCA_SECTSIZE;
- }
- return 0;
-}
-
-static int ehca_is_hugepage(unsigned long pfn)
-{
- int page_order;
-
- if (pfn & EHCA_HUGEPAGE_PFN_MASK)
- return 0;
-
- page_order = compound_order(pfn_to_page(pfn));
- if (page_order + PAGE_SHIFT != EHCA_HUGEPAGESHIFT)
- return 0;
-
- return 1;
-}
-
-static int ehca_create_busmap_callback(unsigned long initial_pfn,
- unsigned long total_nr_pages, void *arg)
-{
- int ret;
- unsigned long pfn, start_pfn, end_pfn, nr_pages;
-
- if ((total_nr_pages * PAGE_SIZE) < EHCA_HUGEPAGE_SIZE)
- return ehca_update_busmap(initial_pfn, total_nr_pages);
-
- /* Given chunk is >= 16GB -> check for hugepages */
- start_pfn = initial_pfn;
- end_pfn = initial_pfn + total_nr_pages;
- pfn = start_pfn;
-
- while (pfn < end_pfn) {
- if (ehca_is_hugepage(pfn)) {
- /* Add mem found in front of the hugepage */
- nr_pages = pfn - start_pfn;
- ret = ehca_update_busmap(start_pfn, nr_pages);
- if (ret)
- return ret;
- /* Skip the hugepage */
- pfn += (EHCA_HUGEPAGE_SIZE / PAGE_SIZE);
- start_pfn = pfn;
- } else
- pfn += (EHCA_SECTSIZE / PAGE_SIZE);
- }
-
- /* Add mem found behind the hugepage(s) */
- nr_pages = pfn - start_pfn;
- return ehca_update_busmap(start_pfn, nr_pages);
-}
-
-int ehca_create_busmap(void)
-{
- int ret;
-
- ehca_mr_len = 0;
- ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL,
- ehca_create_busmap_callback);
- return ret;
-}
-
-static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo)
-{
- int top;
- u64 hret, *kpage;
-
- kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!kpage) {
- ehca_err(&shca->ib_device, "kpage alloc failed");
- return -ENOMEM;
- }
- for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- continue;
- hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);
- if ((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS))
- break;
- }
-
- ehca_free_fw_ctrlblock(kpage);
-
- if (hret == H_SUCCESS)
- return 0; /* Everything is fine */
- else {
- ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, "
- "h_ret=%lli e_mr=%p top=%x lkey=%x "
- "hca_hndl=%llx mr_hndl=%llx", hret, e_mr, top,
- e_mr->ib.ib_mr.lkey,
- shca->ipz_hca_handle.handle,
- e_mr->ipz_mr_handle.handle);
- return ehca2ib_return_code(hret);
- }
-}
-
-static u64 ehca_map_vaddr(void *caddr)
-{
- int top, dir, idx;
- unsigned long abs_addr, offset;
- u64 entry;
-
- if (!ehca_bmap)
- return EHCA_INVAL_ADDR;
-
- abs_addr = __pa(caddr);
- top = ehca_calc_index(abs_addr, EHCA_TOP_INDEX_SHIFT + EHCA_SECTSHIFT);
- if (!ehca_bmap_valid(ehca_bmap->top[top]))
- return EHCA_INVAL_ADDR;
-
- dir = ehca_calc_index(abs_addr, EHCA_DIR_INDEX_SHIFT + EHCA_SECTSHIFT);
- if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
- return EHCA_INVAL_ADDR;
-
- idx = ehca_calc_index(abs_addr, EHCA_SECTSHIFT);
-
- entry = ehca_bmap->top[top]->dir[dir]->ent[idx];
- if (ehca_bmap_valid(entry)) {
- offset = (unsigned long)caddr & (EHCA_SECTSIZE - 1);
- return entry | offset;
- } else
- return EHCA_INVAL_ADDR;
-}
-
-static int ehca_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
-{
- return dma_addr == EHCA_INVAL_ADDR;
-}
-
-static u64 ehca_dma_map_single(struct ib_device *dev, void *cpu_addr,
- size_t size, enum dma_data_direction direction)
-{
- if (cpu_addr)
- return ehca_map_vaddr(cpu_addr);
- else
- return EHCA_INVAL_ADDR;
-}
-
-static void ehca_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size,
- enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static u64 ehca_dma_map_page(struct ib_device *dev, struct page *page,
- unsigned long offset, size_t size,
- enum dma_data_direction direction)
-{
- u64 addr;
-
- if (offset + size > PAGE_SIZE)
- return EHCA_INVAL_ADDR;
-
- addr = ehca_map_vaddr(page_address(page));
- if (!ehca_dma_mapping_error(dev, addr))
- addr += offset;
-
- return addr;
-}
-
-static void ehca_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size,
- enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static int ehca_dma_map_sg(struct ib_device *dev, struct scatterlist *sgl,
- int nents, enum dma_data_direction direction)
-{
- struct scatterlist *sg;
- int i;
-
- for_each_sg(sgl, sg, nents, i) {
- u64 addr;
- addr = ehca_map_vaddr(sg_virt(sg));
- if (ehca_dma_mapping_error(dev, addr))
- return 0;
-
- sg->dma_address = addr;
- sg->dma_length = sg->length;
- }
- return nents;
-}
-
-static void ehca_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg,
- int nents, enum dma_data_direction direction)
-{
- /* This is only a stub; nothing to be done here */
-}
-
-static void ehca_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr,
- size_t size,
- enum dma_data_direction dir)
-{
- dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
-}
-
-static void ehca_dma_sync_single_for_device(struct ib_device *dev, u64 addr,
- size_t size,
- enum dma_data_direction dir)
-{
- dma_sync_single_for_device(dev->dma_device, addr, size, dir);
-}
-
-static void *ehca_dma_alloc_coherent(struct ib_device *dev, size_t size,
- u64 *dma_handle, gfp_t flag)
-{
- struct page *p;
- void *addr = NULL;
- u64 dma_addr;
-
- p = alloc_pages(flag, get_order(size));
- if (p) {
- addr = page_address(p);
- dma_addr = ehca_map_vaddr(addr);
- if (ehca_dma_mapping_error(dev, dma_addr)) {
- free_pages((unsigned long)addr, get_order(size));
- return NULL;
- }
- if (dma_handle)
- *dma_handle = dma_addr;
- return addr;
- }
- return NULL;
-}
-
-static void ehca_dma_free_coherent(struct ib_device *dev, size_t size,
- void *cpu_addr, u64 dma_handle)
-{
- if (cpu_addr && size)
- free_pages((unsigned long)cpu_addr, get_order(size));
-}
-
-
-struct ib_dma_mapping_ops ehca_dma_mapping_ops = {
- .mapping_error = ehca_dma_mapping_error,
- .map_single = ehca_dma_map_single,
- .unmap_single = ehca_dma_unmap_single,
- .map_page = ehca_dma_map_page,
- .unmap_page = ehca_dma_unmap_page,
- .map_sg = ehca_dma_map_sg,
- .unmap_sg = ehca_dma_unmap_sg,
- .sync_single_for_cpu = ehca_dma_sync_single_for_cpu,
- .sync_single_for_device = ehca_dma_sync_single_for_device,
- .alloc_coherent = ehca_dma_alloc_coherent,
- .free_coherent = ehca_dma_free_coherent,
-};
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * MR/MW declarations and inline functions
- *
- * Authors: Dietmar Decker <ddecker@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _EHCA_MRMW_H_
-#define _EHCA_MRMW_H_
-
-enum ehca_reg_type {
- EHCA_REG_MR,
- EHCA_REG_BUSMAP_MR
-};
-
-int ehca_reg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int acl,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey,
- enum ehca_reg_type reg_type);
-
-int ehca_reg_mr_rpages(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- struct ehca_mr_pginfo *pginfo);
-
-int ehca_rereg_mr(struct ehca_shca *shca,
- struct ehca_mr *e_mr,
- u64 *iova_start,
- u64 size,
- int mr_access_flags,
- struct ehca_pd *e_pd,
- struct ehca_mr_pginfo *pginfo,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_unmap_one_fmr(struct ehca_shca *shca,
- struct ehca_mr *e_fmr);
-
-int ehca_reg_smr(struct ehca_shca *shca,
- struct ehca_mr *e_origmr,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_reg_internal_maxmr(struct ehca_shca *shca,
- struct ehca_pd *e_pd,
- struct ehca_mr **maxmr);
-
-int ehca_reg_maxmr(struct ehca_shca *shca,
- struct ehca_mr *e_newmr,
- u64 *iova_start,
- int acl,
- struct ehca_pd *e_pd,
- u32 *lkey,
- u32 *rkey);
-
-int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
-
-int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
- int num_phys_buf,
- u64 *iova_start,
- u64 *size);
-
-int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
- u64 *page_list,
- int list_len);
-
-int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
- u32 number,
- u64 *kpage);
-
-int ehca_mr_is_maxmr(u64 size,
- u64 *iova_start);
-
-void ehca_mrmw_map_acl(int ib_acl,
- u32 *hipz_acl);
-
-void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
-
-void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
- int *ib_acl);
-
-void ehca_mr_deletenew(struct ehca_mr *mr);
-
-int ehca_create_busmap(void);
-
-void ehca_destroy_busmap(void);
-
-extern struct ib_dma_mapping_ops ehca_dma_mapping_ops;
-#endif /*_EHCA_MRMW_H_*/
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * PD functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-
-static struct kmem_cache *pd_cache;
-
-struct ib_pd *ehca_alloc_pd(struct ib_device *device,
- struct ib_ucontext *context, struct ib_udata *udata)
-{
- struct ehca_pd *pd;
- int i;
-
- pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
- if (!pd) {
- ehca_err(device, "device=%p context=%p out of memory",
- device, context);
- return ERR_PTR(-ENOMEM);
- }
-
- for (i = 0; i < 2; i++) {
- INIT_LIST_HEAD(&pd->free[i]);
- INIT_LIST_HEAD(&pd->full[i]);
- }
- mutex_init(&pd->lock);
-
- /*
- * Kernel PD: when device = -1, 0
- * User PD: when context != -1
- */
- if (!context) {
- /*
- * Kernel PDs after init reuses always
- * the one created in ehca_shca_reopen()
- */
- struct ehca_shca *shca = container_of(device, struct ehca_shca,
- ib_device);
- pd->fw_pd.value = shca->pd->fw_pd.value;
- } else
- pd->fw_pd.value = (u64)pd;
-
- return &pd->ib_pd;
-}
-
-int ehca_dealloc_pd(struct ib_pd *pd)
-{
- struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
- int i, leftovers = 0;
- struct ipz_small_queue_page *page, *tmp;
-
- for (i = 0; i < 2; i++) {
- list_splice(&my_pd->full[i], &my_pd->free[i]);
- list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
- leftovers = 1;
- free_page(page->page);
- kmem_cache_free(small_qp_cache, page);
- }
- }
-
- if (leftovers)
- ehca_warn(pd->device,
- "Some small queue pages were not freed");
-
- kmem_cache_free(pd_cache, my_pd);
-
- return 0;
-}
-
-int ehca_init_pd_cache(void)
-{
- pd_cache = kmem_cache_create("ehca_cache_pd",
- sizeof(struct ehca_pd), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!pd_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_pd_cache(void)
-{
- if (pd_cache)
- kmem_cache_destroy(pd_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Hardware request structures
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef _EHCA_QES_H_
-#define _EHCA_QES_H_
-
-#include "ehca_tools.h"
-
-/* virtual scatter gather entry to specify remote addresses with length */
-struct ehca_vsgentry {
- u64 vaddr;
- u32 lkey;
- u32 length;
-};
-
-#define GRH_FLAG_MASK EHCA_BMASK_IBM( 7, 7)
-#define GRH_IPVERSION_MASK EHCA_BMASK_IBM( 0, 3)
-#define GRH_TCLASS_MASK EHCA_BMASK_IBM( 4, 12)
-#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13, 31)
-#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32, 47)
-#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48, 55)
-#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56, 63)
-
-/*
- * Unreliable Datagram Address Vector Format
- * see IBTA Vol1 chapter 8.3 Global Routing Header
- */
-struct ehca_ud_av {
- u8 sl;
- u8 lnh;
- u16 dlid;
- u8 reserved1;
- u8 reserved2;
- u8 reserved3;
- u8 slid_path_bits;
- u8 reserved4;
- u8 ipd;
- u8 reserved5;
- u8 pmtu;
- u32 reserved6;
- u64 reserved7;
- union {
- struct {
- u64 word_0; /* always set to 6 */
- /*should be 0x1B for IB transport */
- u64 word_1;
- u64 word_2;
- u64 word_3;
- u64 word_4;
- } grh;
- struct {
- u32 wd_0;
- u32 wd_1;
- /* DWord_1 --> SGID */
-
- u32 sgid_wd3;
- u32 sgid_wd2;
-
- u32 sgid_wd1;
- u32 sgid_wd0;
- /* DWord_3 --> DGID */
-
- u32 dgid_wd3;
- u32 dgid_wd2;
-
- u32 dgid_wd1;
- u32 dgid_wd0;
- } grh_l;
- };
-};
-
-/* maximum number of sg entries allowed in a WQE */
-#define MAX_WQE_SG_ENTRIES 252
-
-#define WQE_OPTYPE_SEND 0x80
-#define WQE_OPTYPE_RDMAREAD 0x40
-#define WQE_OPTYPE_RDMAWRITE 0x20
-#define WQE_OPTYPE_CMPSWAP 0x10
-#define WQE_OPTYPE_FETCHADD 0x08
-#define WQE_OPTYPE_BIND 0x04
-
-#define WQE_WRFLAG_REQ_SIGNAL_COM 0x80
-#define WQE_WRFLAG_FENCE 0x40
-#define WQE_WRFLAG_IMM_DATA_PRESENT 0x20
-#define WQE_WRFLAG_SOLIC_EVENT 0x10
-
-#define WQEF_CACHE_HINT 0x80
-#define WQEF_CACHE_HINT_RD_WR 0x40
-#define WQEF_TIMED_WQE 0x20
-#define WQEF_PURGE 0x08
-#define WQEF_HIGH_NIBBLE 0xF0
-
-#define MW_BIND_ACCESSCTRL_R_WRITE 0x40
-#define MW_BIND_ACCESSCTRL_R_READ 0x20
-#define MW_BIND_ACCESSCTRL_R_ATOMIC 0x10
-
-struct ehca_wqe {
- u64 work_request_id;
- u8 optype;
- u8 wr_flag;
- u16 pkeyi;
- u8 wqef;
- u8 nr_of_data_seg;
- u16 wqe_provided_slid;
- u32 destination_qp_number;
- u32 resync_psn_sqp;
- u32 local_ee_context_qkey;
- u32 immediate_data;
- union {
- struct {
- u64 remote_virtual_address;
- u32 rkey;
- u32 reserved;
- u64 atomic_1st_op_dma_len;
- u64 atomic_2nd_op;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
-
- } nud;
- struct {
- u64 ehca_ud_av_ptr;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
- } ud_avp;
- struct {
- struct ehca_ud_av ud_av;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
- 2];
- } ud_av;
- struct {
- u64 reserved0;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
- } all_rcv;
-
- struct {
- u64 reserved;
- u32 rkey;
- u32 old_rkey;
- u64 reserved1;
- u64 reserved2;
- u64 virtual_address;
- u32 reserved3;
- u32 length;
- u32 reserved4;
- u16 reserved5;
- u8 reserved6;
- u8 lr_ctl;
- u32 lkey;
- u32 reserved7;
- u64 reserved8;
- u64 reserved9;
- u64 reserved10;
- u64 reserved11;
- } bind;
- struct {
- u64 reserved12;
- u64 reserved13;
- u32 size;
- u32 start;
- } inline_data;
- } u;
-
-};
-
-#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0, 0)
-#define WC_IMM_DATA EHCA_BMASK_IBM(1, 1)
-#define WC_GRH_PRESENT EHCA_BMASK_IBM(2, 2)
-#define WC_SE_BIT EHCA_BMASK_IBM(3, 3)
-#define WC_STATUS_ERROR_BIT 0x80000000
-#define WC_STATUS_REMOTE_ERROR_FLAGS 0x0000F800
-#define WC_STATUS_PURGE_BIT 0x10
-#define WC_SEND_RECEIVE_BIT 0x80
-
-struct ehca_cqe {
- u64 work_request_id;
- u8 optype;
- u8 w_completion_flags;
- u16 reserved1;
- u32 nr_bytes_transferred;
- u32 immediate_data;
- u32 local_qp_number;
- u8 freed_resource_count;
- u8 service_level;
- u16 wqe_count;
- u32 qp_token;
- u32 qkey_ee_token;
- u32 remote_qp_number;
- u16 dlid;
- u16 rlid;
- u16 reserved2;
- u16 pkey_index;
- u32 cqe_timestamp;
- u32 wqe_timestamp;
- u8 wqe_timestamp_valid;
- u8 reserved3;
- u8 reserved4;
- u8 cqe_flags;
- u32 status;
-};
-
-struct ehca_eqe {
- u64 entry;
-};
-
-struct ehca_mrte {
- u64 starting_va;
- u64 length; /* length of memory region in bytes*/
- u32 pd;
- u8 key_instance;
- u8 pagesize;
- u8 mr_control;
- u8 local_remote_access_ctrl;
- u8 reserved[0x20 - 0x18];
- u64 at_pointer[4];
-};
-#endif /*_EHCA_QES_H_*/
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * QP functions
- *
- * Authors: Joachim Fenkes <fenkes@de.ibm.com>
- * Stefan Roscher <stefan.roscher@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-
-static struct kmem_cache *qp_cache;
-
-/*
- * attributes not supported by query qp
- */
-#define QP_ATTR_QUERY_NOT_SUPPORTED (IB_QP_ACCESS_FLAGS | \
- IB_QP_EN_SQD_ASYNC_NOTIFY)
-
-/*
- * ehca (internal) qp state values
- */
-enum ehca_qp_state {
- EHCA_QPS_RESET = 1,
- EHCA_QPS_INIT = 2,
- EHCA_QPS_RTR = 3,
- EHCA_QPS_RTS = 5,
- EHCA_QPS_SQD = 6,
- EHCA_QPS_SQE = 8,
- EHCA_QPS_ERR = 128
-};
-
-/*
- * qp state transitions as defined by IB Arch Rel 1.1 page 431
- */
-enum ib_qp_statetrans {
- IB_QPST_ANY2RESET,
- IB_QPST_ANY2ERR,
- IB_QPST_RESET2INIT,
- IB_QPST_INIT2RTR,
- IB_QPST_INIT2INIT,
- IB_QPST_RTR2RTS,
- IB_QPST_RTS2SQD,
- IB_QPST_RTS2RTS,
- IB_QPST_SQD2RTS,
- IB_QPST_SQE2RTS,
- IB_QPST_SQD2SQD,
- IB_QPST_MAX /* nr of transitions, this must be last!!! */
-};
-
-/*
- * ib2ehca_qp_state maps IB to ehca qp_state
- * returns ehca qp state corresponding to given ib qp state
- */
-static inline enum ehca_qp_state ib2ehca_qp_state(enum ib_qp_state ib_qp_state)
-{
- switch (ib_qp_state) {
- case IB_QPS_RESET:
- return EHCA_QPS_RESET;
- case IB_QPS_INIT:
- return EHCA_QPS_INIT;
- case IB_QPS_RTR:
- return EHCA_QPS_RTR;
- case IB_QPS_RTS:
- return EHCA_QPS_RTS;
- case IB_QPS_SQD:
- return EHCA_QPS_SQD;
- case IB_QPS_SQE:
- return EHCA_QPS_SQE;
- case IB_QPS_ERR:
- return EHCA_QPS_ERR;
- default:
- ehca_gen_err("invalid ib_qp_state=%x", ib_qp_state);
- return -EINVAL;
- }
-}
-
-/*
- * ehca2ib_qp_state maps ehca to IB qp_state
- * returns ib qp state corresponding to given ehca qp state
- */
-static inline enum ib_qp_state ehca2ib_qp_state(enum ehca_qp_state
- ehca_qp_state)
-{
- switch (ehca_qp_state) {
- case EHCA_QPS_RESET:
- return IB_QPS_RESET;
- case EHCA_QPS_INIT:
- return IB_QPS_INIT;
- case EHCA_QPS_RTR:
- return IB_QPS_RTR;
- case EHCA_QPS_RTS:
- return IB_QPS_RTS;
- case EHCA_QPS_SQD:
- return IB_QPS_SQD;
- case EHCA_QPS_SQE:
- return IB_QPS_SQE;
- case EHCA_QPS_ERR:
- return IB_QPS_ERR;
- default:
- ehca_gen_err("invalid ehca_qp_state=%x", ehca_qp_state);
- return -EINVAL;
- }
-}
-
-/*
- * ehca_qp_type used as index for req_attr and opt_attr of
- * struct ehca_modqp_statetrans
- */
-enum ehca_qp_type {
- QPT_RC = 0,
- QPT_UC = 1,
- QPT_UD = 2,
- QPT_SQP = 3,
- QPT_MAX
-};
-
-/*
- * ib2ehcaqptype maps Ib to ehca qp_type
- * returns ehca qp type corresponding to ib qp type
- */
-static inline enum ehca_qp_type ib2ehcaqptype(enum ib_qp_type ibqptype)
-{
- switch (ibqptype) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- return QPT_SQP;
- case IB_QPT_RC:
- return QPT_RC;
- case IB_QPT_UC:
- return QPT_UC;
- case IB_QPT_UD:
- return QPT_UD;
- default:
- ehca_gen_err("Invalid ibqptype=%x", ibqptype);
- return -EINVAL;
- }
-}
-
-static inline enum ib_qp_statetrans get_modqp_statetrans(int ib_fromstate,
- int ib_tostate)
-{
- int index = -EINVAL;
- switch (ib_tostate) {
- case IB_QPS_RESET:
- index = IB_QPST_ANY2RESET;
- break;
- case IB_QPS_INIT:
- switch (ib_fromstate) {
- case IB_QPS_RESET:
- index = IB_QPST_RESET2INIT;
- break;
- case IB_QPS_INIT:
- index = IB_QPST_INIT2INIT;
- break;
- }
- break;
- case IB_QPS_RTR:
- if (ib_fromstate == IB_QPS_INIT)
- index = IB_QPST_INIT2RTR;
- break;
- case IB_QPS_RTS:
- switch (ib_fromstate) {
- case IB_QPS_RTR:
- index = IB_QPST_RTR2RTS;
- break;
- case IB_QPS_RTS:
- index = IB_QPST_RTS2RTS;
- break;
- case IB_QPS_SQD:
- index = IB_QPST_SQD2RTS;
- break;
- case IB_QPS_SQE:
- index = IB_QPST_SQE2RTS;
- break;
- }
- break;
- case IB_QPS_SQD:
- if (ib_fromstate == IB_QPS_RTS)
- index = IB_QPST_RTS2SQD;
- break;
- case IB_QPS_SQE:
- break;
- case IB_QPS_ERR:
- index = IB_QPST_ANY2ERR;
- break;
- default:
- break;
- }
- return index;
-}
-
-/*
- * ibqptype2servicetype returns hcp service type corresponding to given
- * ib qp type used by create_qp()
- */
-static inline int ibqptype2servicetype(enum ib_qp_type ibqptype)
-{
- switch (ibqptype) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- return ST_UD;
- case IB_QPT_RC:
- return ST_RC;
- case IB_QPT_UC:
- return ST_UC;
- case IB_QPT_UD:
- return ST_UD;
- case IB_QPT_RAW_IPV6:
- return -EINVAL;
- case IB_QPT_RAW_ETHERTYPE:
- return -EINVAL;
- default:
- ehca_gen_err("Invalid ibqptype=%x", ibqptype);
- return -EINVAL;
- }
-}
-
-/*
- * init userspace queue info from ipz_queue data
- */
-static inline void queue2resp(struct ipzu_queue_resp *resp,
- struct ipz_queue *queue)
-{
- resp->qe_size = queue->qe_size;
- resp->act_nr_of_sg = queue->act_nr_of_sg;
- resp->queue_length = queue->queue_length;
- resp->pagesize = queue->pagesize;
- resp->toggle_state = queue->toggle_state;
- resp->offset = queue->offset;
-}
-
-/*
- * init_qp_queue initializes/constructs r/squeue and registers queue pages.
- */
-static inline int init_qp_queue(struct ehca_shca *shca,
- struct ehca_pd *pd,
- struct ehca_qp *my_qp,
- struct ipz_queue *queue,
- int q_type,
- u64 expected_hret,
- struct ehca_alloc_queue_parms *parms,
- int wqe_size)
-{
- int ret, cnt, ipz_rc, nr_q_pages;
- void *vpage;
- u64 rpage, h_ret;
- struct ib_device *ib_dev = &shca->ib_device;
- struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
-
- if (!parms->queue_size)
- return 0;
-
- if (parms->is_small) {
- nr_q_pages = 1;
- ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
- 128 << parms->page_size,
- wqe_size, parms->act_nr_sges, 1);
- } else {
- nr_q_pages = parms->queue_size;
- ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
- EHCA_PAGESIZE, wqe_size,
- parms->act_nr_sges, 0);
- }
-
- if (!ipz_rc) {
- ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%i",
- ipz_rc);
- return -EBUSY;
- }
-
- /* register queue pages */
- for (cnt = 0; cnt < nr_q_pages; cnt++) {
- vpage = ipz_qpageit_get_inc(queue);
- if (!vpage) {
- ehca_err(ib_dev, "ipz_qpageit_get_inc() "
- "failed p_vpage= %p", vpage);
- ret = -EINVAL;
- goto init_qp_queue1;
- }
- rpage = __pa(vpage);
-
- h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
- my_qp->ipz_qp_handle,
- NULL, 0, q_type,
- rpage, parms->is_small ? 0 : 1,
- my_qp->galpas.kernel);
- if (cnt == (nr_q_pages - 1)) { /* last page! */
- if (h_ret != expected_hret) {
- ehca_err(ib_dev, "hipz_qp_register_rpage() "
- "h_ret=%lli", h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto init_qp_queue1;
- }
- vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
- if (vpage) {
- ehca_err(ib_dev, "ipz_qpageit_get_inc() "
- "should not succeed vpage=%p", vpage);
- ret = -EINVAL;
- goto init_qp_queue1;
- }
- } else {
- if (h_ret != H_PAGE_REGISTERED) {
- ehca_err(ib_dev, "hipz_qp_register_rpage() "
- "h_ret=%lli", h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto init_qp_queue1;
- }
- }
- }
-
- ipz_qeit_reset(queue);
-
- return 0;
-
-init_qp_queue1:
- ipz_queue_dtor(pd, queue);
- return ret;
-}
-
-static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
-{
- if (is_llqp)
- return 128 << act_nr_sge;
- else
- return offsetof(struct ehca_wqe,
- u.nud.sg_list[act_nr_sge]);
-}
-
-static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
- int req_nr_sge, int is_llqp)
-{
- u32 wqe_size, q_size;
- int act_nr_sge = req_nr_sge;
-
- if (!is_llqp)
- /* round up #SGEs so WQE size is a power of 2 */
- for (act_nr_sge = 4; act_nr_sge <= 252;
- act_nr_sge = 4 + 2 * act_nr_sge)
- if (act_nr_sge >= req_nr_sge)
- break;
-
- wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
- q_size = wqe_size * (queue->max_wr + 1);
-
- if (q_size <= 512)
- queue->page_size = 2;
- else if (q_size <= 1024)
- queue->page_size = 3;
- else
- queue->page_size = 0;
-
- queue->is_small = (queue->page_size != 0);
-}
-
-/* needs to be called with cq->spinlock held */
-void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq)
-{
- struct list_head *list, *node;
-
- /* TODO: support low latency QPs */
- if (qp->ext_type == EQPT_LLQP)
- return;
-
- if (on_sq) {
- list = &qp->send_cq->sqp_err_list;
- node = &qp->sq_err_node;
- } else {
- list = &qp->recv_cq->rqp_err_list;
- node = &qp->rq_err_node;
- }
-
- if (list_empty(node))
- list_add_tail(node, list);
-
- return;
-}
-
-static void del_from_err_list(struct ehca_cq *cq, struct list_head *node)
-{
- unsigned long flags;
-
- spin_lock_irqsave(&cq->spinlock, flags);
-
- if (!list_empty(node))
- list_del_init(node);
-
- spin_unlock_irqrestore(&cq->spinlock, flags);
-}
-
-static void reset_queue_map(struct ehca_queue_map *qmap)
-{
- int i;
-
- qmap->tail = qmap->entries - 1;
- qmap->left_to_poll = 0;
- qmap->next_wqe_idx = 0;
- for (i = 0; i < qmap->entries; i++) {
- qmap->map[i].reported = 1;
- qmap->map[i].cqe_req = 0;
- }
-}
-
-/*
- * Create an ib_qp struct that is either a QP or an SRQ, depending on
- * the value of the is_srq parameter. If init_attr and srq_init_attr share
- * fields, the field out of init_attr is used.
- */
-static struct ehca_qp *internal_create_qp(
- struct ib_pd *pd,
- struct ib_qp_init_attr *init_attr,
- struct ib_srq_init_attr *srq_init_attr,
- struct ib_udata *udata, int is_srq)
-{
- struct ehca_qp *my_qp, *my_srq = NULL;
- struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
- struct ib_ucontext *context = NULL;
- u64 h_ret;
- int is_llqp = 0, has_srq = 0, is_user = 0;
- int qp_type, max_send_sge, max_recv_sge, ret;
-
- /* h_call's out parameters */
- struct ehca_alloc_qp_parms parms;
- u32 swqe_size = 0, rwqe_size = 0, ib_qp_num;
- unsigned long flags;
-
- if (!atomic_add_unless(&shca->num_qps, 1, shca->max_num_qps)) {
- ehca_err(pd->device, "Unable to create QP, max number of %i "
- "QPs reached.", shca->max_num_qps);
- ehca_err(pd->device, "To increase the maximum number of QPs "
- "use the number_of_qps module parameter.\n");
- return ERR_PTR(-ENOSPC);
- }
-
- if (init_attr->create_flags) {
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- memset(&parms, 0, sizeof(parms));
- qp_type = init_attr->qp_type;
-
- if (init_attr->sq_sig_type != IB_SIGNAL_REQ_WR &&
- init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
- ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
- init_attr->sq_sig_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- /* save LLQP info */
- if (qp_type & 0x80) {
- is_llqp = 1;
- parms.ext_type = EQPT_LLQP;
- parms.ll_comp_flags = qp_type & LLQP_COMP_MASK;
- }
- qp_type &= 0x1F;
- init_attr->qp_type &= 0x1F;
-
- /* handle SRQ base QPs */
- if (init_attr->srq) {
- my_srq = container_of(init_attr->srq, struct ehca_qp, ib_srq);
-
- if (qp_type == IB_QPT_UC) {
- ehca_err(pd->device, "UC with SRQ not supported");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- has_srq = 1;
- parms.ext_type = EQPT_SRQBASE;
- parms.srq_qpn = my_srq->real_qp_num;
- }
-
- if (is_llqp && has_srq) {
- ehca_err(pd->device, "LLQPs can't have an SRQ");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- /* handle SRQs */
- if (is_srq) {
- parms.ext_type = EQPT_SRQ;
- parms.srq_limit = srq_init_attr->attr.srq_limit;
- if (init_attr->cap.max_recv_sge > 3) {
- ehca_err(pd->device, "no more than three SGEs "
- "supported for SRQ pd=%p max_sge=%x",
- pd, init_attr->cap.max_recv_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- }
-
- /* check QP type */
- if (qp_type != IB_QPT_UD &&
- qp_type != IB_QPT_UC &&
- qp_type != IB_QPT_RC &&
- qp_type != IB_QPT_SMI &&
- qp_type != IB_QPT_GSI) {
- ehca_err(pd->device, "wrong QP Type=%x", qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
-
- if (is_llqp) {
- switch (qp_type) {
- case IB_QPT_RC:
- if ((init_attr->cap.max_send_wr > 255) ||
- (init_attr->cap.max_recv_wr > 255)) {
- ehca_err(pd->device,
- "Invalid Number of max_sq_wr=%x "
- "or max_rq_wr=%x for RC LLQP",
- init_attr->cap.max_send_wr,
- init_attr->cap.max_recv_wr);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- break;
- case IB_QPT_UD:
- if (!EHCA_BMASK_GET(HCA_CAP_UD_LL_QP, shca->hca_cap)) {
- ehca_err(pd->device, "UD LLQP not supported "
- "by this adapter");
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-ENOSYS);
- }
- if (!(init_attr->cap.max_send_sge <= 5
- && init_attr->cap.max_send_sge >= 1
- && init_attr->cap.max_recv_sge <= 5
- && init_attr->cap.max_recv_sge >= 1)) {
- ehca_err(pd->device,
- "Invalid Number of max_send_sge=%x "
- "or max_recv_sge=%x for UD LLQP",
- init_attr->cap.max_send_sge,
- init_attr->cap.max_recv_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- } else if (init_attr->cap.max_send_wr > 255) {
- ehca_err(pd->device,
- "Invalid Number of "
- "max_send_wr=%x for UD QP_TYPE=%x",
- init_attr->cap.max_send_wr, qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- break;
- default:
- ehca_err(pd->device, "unsupported LL QP Type=%x",
- qp_type);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- } else {
- int max_sge = (qp_type == IB_QPT_UD || qp_type == IB_QPT_SMI
- || qp_type == IB_QPT_GSI) ? 250 : 252;
-
- if (init_attr->cap.max_send_sge > max_sge
- || init_attr->cap.max_recv_sge > max_sge) {
- ehca_err(pd->device, "Invalid number of SGEs requested "
- "send_sge=%x recv_sge=%x max_sge=%x",
- init_attr->cap.max_send_sge,
- init_attr->cap.max_recv_sge, max_sge);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-EINVAL);
- }
- }
-
- my_qp = kmem_cache_zalloc(qp_cache, GFP_KERNEL);
- if (!my_qp) {
- ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(-ENOMEM);
- }
-
- if (pd->uobject && udata) {
- is_user = 1;
- context = pd->uobject->context;
- }
-
- atomic_set(&my_qp->nr_events, 0);
- init_waitqueue_head(&my_qp->wait_completion);
- spin_lock_init(&my_qp->spinlock_s);
- spin_lock_init(&my_qp->spinlock_r);
- my_qp->qp_type = qp_type;
- my_qp->ext_type = parms.ext_type;
- my_qp->state = IB_QPS_RESET;
-
- if (init_attr->recv_cq)
- my_qp->recv_cq =
- container_of(init_attr->recv_cq, struct ehca_cq, ib_cq);
- if (init_attr->send_cq)
- my_qp->send_cq =
- container_of(init_attr->send_cq, struct ehca_cq, ib_cq);
-
- idr_preload(GFP_KERNEL);
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
-
- ret = idr_alloc(&ehca_qp_idr, my_qp, 0, 0x2000000, GFP_NOWAIT);
- if (ret >= 0)
- my_qp->token = ret;
-
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
- idr_preload_end();
- if (ret < 0) {
- if (ret == -ENOSPC) {
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid number of qp");
- } else {
- ret = -ENOMEM;
- ehca_err(pd->device, "Can't allocate new idr entry.");
- }
- goto create_qp_exit0;
- }
-
- if (has_srq)
- parms.srq_token = my_qp->token;
-
- parms.servicetype = ibqptype2servicetype(qp_type);
- if (parms.servicetype < 0) {
- ret = -EINVAL;
- ehca_err(pd->device, "Invalid qp_type=%x", qp_type);
- goto create_qp_exit1;
- }
-
- /* Always signal by WQE so we can hide circ. WQEs */
- parms.sigtype = HCALL_SIGT_BY_WQE;
-
- /* UD_AV CIRCUMVENTION */
- max_send_sge = init_attr->cap.max_send_sge;
- max_recv_sge = init_attr->cap.max_recv_sge;
- if (parms.servicetype == ST_UD && !is_llqp) {
- max_send_sge += 2;
- max_recv_sge += 2;
- }
-
- parms.token = my_qp->token;
- parms.eq_handle = shca->eq.ipz_eq_handle;
- parms.pd = my_pd->fw_pd;
- if (my_qp->send_cq)
- parms.send_cq_handle = my_qp->send_cq->ipz_cq_handle;
- if (my_qp->recv_cq)
- parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
-
- parms.squeue.max_wr = init_attr->cap.max_send_wr;
- parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
- parms.squeue.max_sge = max_send_sge;
- parms.rqueue.max_sge = max_recv_sge;
-
- /* RC QPs need one more SWQE for unsolicited ack circumvention */
- if (qp_type == IB_QPT_RC)
- parms.squeue.max_wr++;
-
- if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)) {
- if (HAS_SQ(my_qp))
- ehca_determine_small_queue(
- &parms.squeue, max_send_sge, is_llqp);
- if (HAS_RQ(my_qp))
- ehca_determine_small_queue(
- &parms.rqueue, max_recv_sge, is_llqp);
- parms.qp_storage =
- (parms.squeue.is_small || parms.rqueue.is_small);
- }
-
- h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms, is_user);
- if (h_ret != H_SUCCESS) {
- ehca_err(pd->device, "h_alloc_resource_qp() failed h_ret=%lli",
- h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto create_qp_exit1;
- }
-
- ib_qp_num = my_qp->real_qp_num = parms.real_qp_num;
- my_qp->ipz_qp_handle = parms.qp_handle;
- my_qp->galpas = parms.galpas;
-
- swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
- rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
-
- switch (qp_type) {
- case IB_QPT_RC:
- if (is_llqp) {
- parms.squeue.act_nr_sges = 1;
- parms.rqueue.act_nr_sges = 1;
- }
- /* hide the extra WQE */
- parms.squeue.act_nr_wqes--;
- break;
- case IB_QPT_UD:
- case IB_QPT_GSI:
- case IB_QPT_SMI:
- /* UD circumvention */
- if (is_llqp) {
- parms.squeue.act_nr_sges = 1;
- parms.rqueue.act_nr_sges = 1;
- } else {
- parms.squeue.act_nr_sges -= 2;
- parms.rqueue.act_nr_sges -= 2;
- }
-
- if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
- parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
- parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
- parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
- parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
- ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
- }
-
- break;
-
- default:
- break;
- }
-
- /* initialize r/squeue and register queue pages */
- if (HAS_SQ(my_qp)) {
- ret = init_qp_queue(
- shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
- HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
- &parms.squeue, swqe_size);
- if (ret) {
- ehca_err(pd->device, "Couldn't initialize squeue "
- "and pages ret=%i", ret);
- goto create_qp_exit2;
- }
-
- if (!is_user) {
- my_qp->sq_map.entries = my_qp->ipz_squeue.queue_length /
- my_qp->ipz_squeue.qe_size;
- my_qp->sq_map.map = vmalloc(my_qp->sq_map.entries *
- sizeof(struct ehca_qmap_entry));
- if (!my_qp->sq_map.map) {
- ehca_err(pd->device, "Couldn't allocate squeue "
- "map ret=%i", ret);
- goto create_qp_exit3;
- }
- INIT_LIST_HEAD(&my_qp->sq_err_node);
- /* to avoid the generation of bogus flush CQEs */
- reset_queue_map(&my_qp->sq_map);
- }
- }
-
- if (HAS_RQ(my_qp)) {
- ret = init_qp_queue(
- shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
- H_SUCCESS, &parms.rqueue, rwqe_size);
- if (ret) {
- ehca_err(pd->device, "Couldn't initialize rqueue "
- "and pages ret=%i", ret);
- goto create_qp_exit4;
- }
- if (!is_user) {
- my_qp->rq_map.entries = my_qp->ipz_rqueue.queue_length /
- my_qp->ipz_rqueue.qe_size;
- my_qp->rq_map.map = vmalloc(my_qp->rq_map.entries *
- sizeof(struct ehca_qmap_entry));
- if (!my_qp->rq_map.map) {
- ehca_err(pd->device, "Couldn't allocate squeue "
- "map ret=%i", ret);
- goto create_qp_exit5;
- }
- INIT_LIST_HEAD(&my_qp->rq_err_node);
- /* to avoid the generation of bogus flush CQEs */
- reset_queue_map(&my_qp->rq_map);
- }
- } else if (init_attr->srq && !is_user) {
- /* this is a base QP, use the queue map of the SRQ */
- my_qp->rq_map = my_srq->rq_map;
- INIT_LIST_HEAD(&my_qp->rq_err_node);
-
- my_qp->ipz_rqueue = my_srq->ipz_rqueue;
- }
-
- if (is_srq) {
- my_qp->ib_srq.pd = &my_pd->ib_pd;
- my_qp->ib_srq.device = my_pd->ib_pd.device;
-
- my_qp->ib_srq.srq_context = init_attr->qp_context;
- my_qp->ib_srq.event_handler = init_attr->event_handler;
- } else {
- my_qp->ib_qp.qp_num = ib_qp_num;
- my_qp->ib_qp.pd = &my_pd->ib_pd;
- my_qp->ib_qp.device = my_pd->ib_pd.device;
-
- my_qp->ib_qp.recv_cq = init_attr->recv_cq;
- my_qp->ib_qp.send_cq = init_attr->send_cq;
-
- my_qp->ib_qp.qp_type = qp_type;
- my_qp->ib_qp.srq = init_attr->srq;
-
- my_qp->ib_qp.qp_context = init_attr->qp_context;
- my_qp->ib_qp.event_handler = init_attr->event_handler;
- }
-
- init_attr->cap.max_inline_data = 0; /* not supported yet */
- init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
- init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
- init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
- init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
- my_qp->init_attr = *init_attr;
-
- if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
- shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
- &my_qp->ib_qp;
- if (ehca_nr_ports < 0) {
- /* alloc array to cache subsequent modify qp parms
- * for autodetect mode
- */
- my_qp->mod_qp_parm =
- kzalloc(EHCA_MOD_QP_PARM_MAX *
- sizeof(*my_qp->mod_qp_parm),
- GFP_KERNEL);
- if (!my_qp->mod_qp_parm) {
- ehca_err(pd->device,
- "Could not alloc mod_qp_parm");
- goto create_qp_exit5;
- }
- }
- }
-
- /* NOTE: define_apq0() not supported yet */
- if (qp_type == IB_QPT_GSI) {
- h_ret = ehca_define_sqp(shca, my_qp, init_attr);
- if (h_ret != H_SUCCESS) {
- kfree(my_qp->mod_qp_parm);
- my_qp->mod_qp_parm = NULL;
- /* the QP pointer is no longer valid */
- shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
- NULL;
- ret = ehca2ib_return_code(h_ret);
- goto create_qp_exit6;
- }
- }
-
- if (my_qp->send_cq) {
- ret = ehca_cq_assign_qp(my_qp->send_cq, my_qp);
- if (ret) {
- ehca_err(pd->device,
- "Couldn't assign qp to send_cq ret=%i", ret);
- goto create_qp_exit7;
- }
- }
-
- /* copy queues, galpa data to user space */
- if (context && udata) {
- struct ehca_create_qp_resp resp;
- memset(&resp, 0, sizeof(resp));
-
- resp.qp_num = my_qp->real_qp_num;
- resp.token = my_qp->token;
- resp.qp_type = my_qp->qp_type;
- resp.ext_type = my_qp->ext_type;
- resp.qkey = my_qp->qkey;
- resp.real_qp_num = my_qp->real_qp_num;
-
- if (HAS_SQ(my_qp))
- queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
- if (HAS_RQ(my_qp))
- queue2resp(&resp.ipz_rqueue, &my_qp->ipz_rqueue);
- resp.fw_handle_ofs = (u32)
- (my_qp->galpas.user.fw_handle & (PAGE_SIZE - 1));
-
- if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
- ehca_err(pd->device, "Copy to udata failed");
- ret = -EINVAL;
- goto create_qp_exit8;
- }
- }
-
- return my_qp;
-
-create_qp_exit8:
- ehca_cq_unassign_qp(my_qp->send_cq, my_qp->real_qp_num);
-
-create_qp_exit7:
- kfree(my_qp->mod_qp_parm);
-
-create_qp_exit6:
- if (HAS_RQ(my_qp) && !is_user)
- vfree(my_qp->rq_map.map);
-
-create_qp_exit5:
- if (HAS_RQ(my_qp))
- ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
-
-create_qp_exit4:
- if (HAS_SQ(my_qp) && !is_user)
- vfree(my_qp->sq_map.map);
-
-create_qp_exit3:
- if (HAS_SQ(my_qp))
- ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
-
-create_qp_exit2:
- hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
-
-create_qp_exit1:
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
- idr_remove(&ehca_qp_idr, my_qp->token);
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
-
-create_qp_exit0:
- kmem_cache_free(qp_cache, my_qp);
- atomic_dec(&shca->num_qps);
- return ERR_PTR(ret);
-}
-
-struct ib_qp *ehca_create_qp(struct ib_pd *pd,
- struct ib_qp_init_attr *qp_init_attr,
- struct ib_udata *udata)
-{
- struct ehca_qp *ret;
-
- ret = internal_create_qp(pd, qp_init_attr, NULL, udata, 0);
- return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
-}
-
-static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject);
-
-struct ib_srq *ehca_create_srq(struct ib_pd *pd,
- struct ib_srq_init_attr *srq_init_attr,
- struct ib_udata *udata)
-{
- struct ib_qp_init_attr qp_init_attr;
- struct ehca_qp *my_qp;
- struct ib_srq *ret;
- struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
- ib_device);
- struct hcp_modify_qp_control_block *mqpcb;
- u64 hret, update_mask;
-
- if (srq_init_attr->srq_type != IB_SRQT_BASIC)
- return ERR_PTR(-ENOSYS);
-
- /* For common attributes, internal_create_qp() takes its info
- * out of qp_init_attr, so copy all common attrs there.
- */
- memset(&qp_init_attr, 0, sizeof(qp_init_attr));
- qp_init_attr.event_handler = srq_init_attr->event_handler;
- qp_init_attr.qp_context = srq_init_attr->srq_context;
- qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
- qp_init_attr.qp_type = IB_QPT_RC;
- qp_init_attr.cap.max_recv_wr = srq_init_attr->attr.max_wr;
- qp_init_attr.cap.max_recv_sge = srq_init_attr->attr.max_sge;
-
- my_qp = internal_create_qp(pd, &qp_init_attr, srq_init_attr, udata, 1);
- if (IS_ERR(my_qp))
- return (struct ib_srq *)my_qp;
-
- /* copy back return values */
- srq_init_attr->attr.max_wr = qp_init_attr.cap.max_recv_wr;
- srq_init_attr->attr.max_sge = 3;
-
- /* drive SRQ into RTR state */
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!mqpcb) {
- ehca_err(pd->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
- ret = ERR_PTR(-ENOMEM);
- goto create_srq1;
- }
-
- mqpcb->qp_state = EHCA_QPS_INIT;
- mqpcb->prim_phys_port = 1;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not modify SRQ to INIT "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- mqpcb->qp_enable = 1;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not enable SRQ "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- mqpcb->qp_state = EHCA_QPS_RTR;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- hret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
- if (hret != H_SUCCESS) {
- ehca_err(pd->device, "Could not modify SRQ to RTR "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, hret);
- goto create_srq2;
- }
-
- ehca_free_fw_ctrlblock(mqpcb);
-
- return &my_qp->ib_srq;
-
-create_srq2:
- ret = ERR_PTR(ehca2ib_return_code(hret));
- ehca_free_fw_ctrlblock(mqpcb);
-
-create_srq1:
- internal_destroy_qp(pd->device, my_qp, my_qp->ib_srq.uobject);
-
- return ret;
-}
-
-/*
- * prepare_sqe_rts called by internal_modify_qp() at trans sqe -> rts
- * set purge bit of bad wqe and subsequent wqes to avoid reentering sqe
- * returns total number of bad wqes in bad_wqe_cnt
- */
-static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
- int *bad_wqe_cnt)
-{
- u64 h_ret;
- struct ipz_queue *squeue;
- void *bad_send_wqe_p, *bad_send_wqe_v;
- u64 q_ofs;
- struct ehca_wqe *wqe;
- int qp_num = my_qp->ib_qp.qp_num;
-
- /* get send wqe pointer */
- h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle, &my_qp->pf,
- &bad_send_wqe_p, NULL, 2);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "hipz_h_disable_and_get_wqe() failed"
- " ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp_num, h_ret);
- return ehca2ib_return_code(h_ret);
- }
- bad_send_wqe_p = (void *)((u64)bad_send_wqe_p & (~(1L << 63)));
- ehca_dbg(&shca->ib_device, "qp_num=%x bad_send_wqe_p=%p",
- qp_num, bad_send_wqe_p);
- /* convert wqe pointer to vadr */
- bad_send_wqe_v = __va((u64)bad_send_wqe_p);
- if (ehca_debug_level >= 2)
- ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
- squeue = &my_qp->ipz_squeue;
- if (ipz_queue_abs_to_offset(squeue, (u64)bad_send_wqe_p, &q_ofs)) {
- ehca_err(&shca->ib_device, "failed to get wqe offset qp_num=%x"
- " bad_send_wqe_p=%p", qp_num, bad_send_wqe_p);
- return -EFAULT;
- }
-
- /* loop sets wqe's purge bit */
- wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
- *bad_wqe_cnt = 0;
- while (wqe->optype != 0xff && wqe->wqef != 0xff) {
- if (ehca_debug_level >= 2)
- ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
- wqe->nr_of_data_seg = 0; /* suppress data access */
- wqe->wqef = WQEF_PURGE; /* WQE to be purged */
- q_ofs = ipz_queue_advance_offset(squeue, q_ofs);
- wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
- *bad_wqe_cnt = (*bad_wqe_cnt)+1;
- }
- /*
- * bad wqe will be reprocessed and ignored when pol_cq() is called,
- * i.e. nr of wqes with flush error status is one less
- */
- ehca_dbg(&shca->ib_device, "qp_num=%x flusherr_wqe_cnt=%x",
- qp_num, (*bad_wqe_cnt)-1);
- wqe->wqef = 0;
-
- return 0;
-}
-
-static int calc_left_cqes(u64 wqe_p, struct ipz_queue *ipz_queue,
- struct ehca_queue_map *qmap)
-{
- void *wqe_v;
- u64 q_ofs;
- u32 wqe_idx;
- unsigned int tail_idx;
-
- /* convert real to abs address */
- wqe_p = wqe_p & (~(1UL << 63));
-
- wqe_v = __va(wqe_p);
-
- if (ipz_queue_abs_to_offset(ipz_queue, wqe_p, &q_ofs)) {
- ehca_gen_err("Invalid offset for calculating left cqes "
- "wqe_p=%#llx wqe_v=%p\n", wqe_p, wqe_v);
- return -EFAULT;
- }
-
- tail_idx = next_index(qmap->tail, qmap->entries);
- wqe_idx = q_ofs / ipz_queue->qe_size;
-
- /* check all processed wqes, whether a cqe is requested or not */
- while (tail_idx != wqe_idx) {
- if (qmap->map[tail_idx].cqe_req)
- qmap->left_to_poll++;
- tail_idx = next_index(tail_idx, qmap->entries);
- }
- /* save index in queue, where we have to start flushing */
- qmap->next_wqe_idx = wqe_idx;
- return 0;
-}
-
-static int check_for_left_cqes(struct ehca_qp *my_qp, struct ehca_shca *shca)
-{
- u64 h_ret;
- void *send_wqe_p, *recv_wqe_p;
- int ret;
- unsigned long flags;
- int qp_num = my_qp->ib_qp.qp_num;
-
- /* this hcall is not supported on base QPs */
- if (my_qp->ext_type != EQPT_SRQBASE) {
- /* get send and receive wqe pointer */
- h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle, &my_qp->pf,
- &send_wqe_p, &recv_wqe_p, 4);
- if (h_ret != H_SUCCESS) {
- ehca_err(&shca->ib_device, "disable_and_get_wqe() "
- "failed ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp_num, h_ret);
- return ehca2ib_return_code(h_ret);
- }
-
- /*
- * acquire lock to ensure that nobody is polling the cq which
- * could mean that the qmap->tail pointer is in an
- * inconsistent state.
- */
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- ret = calc_left_cqes((u64)send_wqe_p, &my_qp->ipz_squeue,
- &my_qp->sq_map);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
- if (ret)
- return ret;
-
-
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- ret = calc_left_cqes((u64)recv_wqe_p, &my_qp->ipz_rqueue,
- &my_qp->rq_map);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
- if (ret)
- return ret;
- } else {
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- my_qp->sq_map.left_to_poll = 0;
- my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
- my_qp->sq_map.entries);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
-
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- my_qp->rq_map.left_to_poll = 0;
- my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
- my_qp->rq_map.entries);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
- }
-
- /* this assures flush cqes being generated only for pending wqes */
- if ((my_qp->sq_map.left_to_poll == 0) &&
- (my_qp->rq_map.left_to_poll == 0)) {
- spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
- ehca_add_to_err_list(my_qp, 1);
- spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
-
- if (HAS_RQ(my_qp)) {
- spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
- ehca_add_to_err_list(my_qp, 0);
- spin_unlock_irqrestore(&my_qp->recv_cq->spinlock,
- flags);
- }
- }
-
- return 0;
-}
-
-/*
- * internal_modify_qp with circumvention to handle aqp0 properly
- * smi_reset2init indicates if this is an internal reset-to-init-call for
- * smi. This flag must always be zero if called from ehca_modify_qp()!
- * This internal func was intorduced to avoid recursion of ehca_modify_qp()!
- */
-static int internal_modify_qp(struct ib_qp *ibqp,
- struct ib_qp_attr *attr,
- int attr_mask, int smi_reset2init)
-{
- enum ib_qp_state qp_cur_state, qp_new_state;
- int cnt, qp_attr_idx, ret = 0;
- enum ib_qp_statetrans statetrans;
- struct hcp_modify_qp_control_block *mqpcb;
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca =
- container_of(ibqp->pd->device, struct ehca_shca, ib_device);
- u64 update_mask;
- u64 h_ret;
- int bad_wqe_cnt = 0;
- int is_user = 0;
- int squeue_locked = 0;
- unsigned long flags = 0;
-
- /* do query_qp to obtain current attr values */
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
- if (!mqpcb) {
- ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- mqpcb, my_qp->galpas.kernel);
- if (h_ret != H_SUCCESS) {
- ehca_err(ibqp->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, ibqp->qp_num, h_ret);
- ret = ehca2ib_return_code(h_ret);
- goto modify_qp_exit1;
- }
- if (ibqp->uobject)
- is_user = 1;
-
- qp_cur_state = ehca2ib_qp_state(mqpcb->qp_state);
-
- if (qp_cur_state == -EINVAL) { /* invalid qp state */
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid current ehca_qp_state=%x "
- "ehca_qp=%p qp_num=%x",
- mqpcb->qp_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
- /*
- * circumvention to set aqp0 initial state to init
- * as expected by IB spec
- */
- if (smi_reset2init == 0 &&
- ibqp->qp_type == IB_QPT_SMI &&
- qp_cur_state == IB_QPS_RESET &&
- (attr_mask & IB_QP_STATE) &&
- attr->qp_state == IB_QPS_INIT) { /* RESET -> INIT */
- struct ib_qp_attr smiqp_attr = {
- .qp_state = IB_QPS_INIT,
- .port_num = my_qp->init_attr.port_num,
- .pkey_index = 0,
- .qkey = 0
- };
- int smiqp_attr_mask = IB_QP_STATE | IB_QP_PORT |
- IB_QP_PKEY_INDEX | IB_QP_QKEY;
- int smirc = internal_modify_qp(
- ibqp, &smiqp_attr, smiqp_attr_mask, 1);
- if (smirc) {
- ehca_err(ibqp->device, "SMI RESET -> INIT failed. "
- "ehca_modify_qp() rc=%i", smirc);
- ret = H_PARAMETER;
- goto modify_qp_exit1;
- }
- qp_cur_state = IB_QPS_INIT;
- ehca_dbg(ibqp->device, "SMI RESET -> INIT succeeded");
- }
- /* is transmitted current state equal to "real" current state */
- if ((attr_mask & IB_QP_CUR_STATE) &&
- qp_cur_state != attr->cur_qp_state) {
- ret = -EINVAL;
- ehca_err(ibqp->device,
- "Invalid IB_QP_CUR_STATE attr->curr_qp_state=%x <>"
- " actual cur_qp_state=%x. ehca_qp=%p qp_num=%x",
- attr->cur_qp_state, qp_cur_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- ehca_dbg(ibqp->device, "ehca_qp=%p qp_num=%x current qp_state=%x "
- "new qp_state=%x attribute_mask=%x",
- my_qp, ibqp->qp_num, qp_cur_state, attr->qp_state, attr_mask);
-
- qp_new_state = attr_mask & IB_QP_STATE ? attr->qp_state : qp_cur_state;
- if (!smi_reset2init &&
- !ib_modify_qp_is_ok(qp_cur_state, qp_new_state, ibqp->qp_type,
- attr_mask, IB_LINK_LAYER_UNSPECIFIED)) {
- ret = -EINVAL;
- ehca_err(ibqp->device,
- "Invalid qp transition new_state=%x cur_state=%x "
- "ehca_qp=%p qp_num=%x attr_mask=%x", qp_new_state,
- qp_cur_state, my_qp, ibqp->qp_num, attr_mask);
- goto modify_qp_exit1;
- }
-
- mqpcb->qp_state = ib2ehca_qp_state(qp_new_state);
- if (mqpcb->qp_state)
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
- else {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid new qp state=%x "
- "ehca_qp=%p qp_num=%x",
- qp_new_state, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- /* retrieve state transition struct to get req and opt attrs */
- statetrans = get_modqp_statetrans(qp_cur_state, qp_new_state);
- if (statetrans < 0) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "<INVALID STATE CHANGE> qp_cur_state=%x "
- "new_qp_state=%x State_xsition=%x ehca_qp=%p "
- "qp_num=%x", qp_cur_state, qp_new_state,
- statetrans, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- qp_attr_idx = ib2ehcaqptype(ibqp->qp_type);
-
- if (qp_attr_idx < 0) {
- ret = qp_attr_idx;
- ehca_err(ibqp->device,
- "Invalid QP type=%x ehca_qp=%p qp_num=%x",
- ibqp->qp_type, my_qp, ibqp->qp_num);
- goto modify_qp_exit1;
- }
-
- ehca_dbg(ibqp->device,
- "ehca_qp=%p qp_num=%x <VALID STATE CHANGE> qp_state_xsit=%x",
- my_qp, ibqp->qp_num, statetrans);
-
- /* eHCA2 rev2 and higher require the SEND_GRH_FLAG to be set
- * in non-LL UD QPs.
- */
- if ((my_qp->qp_type == IB_QPT_UD) &&
- (my_qp->ext_type != EQPT_LLQP) &&
- (statetrans == IB_QPST_INIT2RTR) &&
- (shca->hw_level >= 0x22)) {
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
- mqpcb->send_grh_flag = 1;
- }
-
- /* sqe -> rts: set purge bit of bad wqe before actual trans */
- if ((my_qp->qp_type == IB_QPT_UD ||
- my_qp->qp_type == IB_QPT_GSI ||
- my_qp->qp_type == IB_QPT_SMI) &&
- statetrans == IB_QPST_SQE2RTS) {
- /* mark next free wqe if kernel */
- if (!ibqp->uobject) {
- struct ehca_wqe *wqe;
- /* lock send queue */
- spin_lock_irqsave(&my_qp->spinlock_s, flags);
- squeue_locked = 1;
- /* mark next free wqe */
- wqe = (struct ehca_wqe *)
- ipz_qeit_get(&my_qp->ipz_squeue);
- wqe->optype = wqe->wqef = 0xff;
- ehca_dbg(ibqp->device, "qp_num=%x next_free_wqe=%p",
- ibqp->qp_num, wqe);
- }
- ret = prepare_sqe_rts(my_qp, shca, &bad_wqe_cnt);
- if (ret) {
- ehca_err(ibqp->device, "prepare_sqe_rts() failed "
- "ehca_qp=%p qp_num=%x ret=%i",
- my_qp, ibqp->qp_num, ret);
- goto modify_qp_exit2;
- }
- }
-
- /*
- * enable RDMA_Atomic_Control if reset->init und reliable con
- * this is necessary since gen2 does not provide that flag,
- * but pHyp requires it
- */
- if (statetrans == IB_QPST_RESET2INIT &&
- (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_UC)) {
- mqpcb->rdma_atomic_ctrl = 3;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RDMA_ATOMIC_CTRL, 1);
- }
- /* circ. pHyp requires #RDMA/Atomic Resp Res for UC INIT -> RTR */
- if (statetrans == IB_QPST_INIT2RTR &&
- (ibqp->qp_type == IB_QPT_UC) &&
- !(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)) {
- mqpcb->rdma_nr_atomic_resp_res = 1; /* default to 1 */
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
- }
-
- if (attr_mask & IB_QP_PKEY_INDEX) {
- if (attr->pkey_index >= 16) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid pkey_index=%x. "
- "ehca_qp=%p qp_num=%x max_pkey_index=f",
- attr->pkey_index, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- mqpcb->prim_p_key_idx = attr->pkey_index;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
- }
- if (attr_mask & IB_QP_PORT) {
- struct ehca_sport *sport;
- struct ehca_qp *aqp1;
- if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid port=%x. "
- "ehca_qp=%p qp_num=%x num_ports=%x",
- attr->port_num, my_qp, ibqp->qp_num,
- shca->num_ports);
- goto modify_qp_exit2;
- }
- sport = &shca->sport[attr->port_num - 1];
- if (!sport->ibqp_sqp[IB_QPT_GSI]) {
- /* should not occur */
- ret = -EFAULT;
- ehca_err(ibqp->device, "AQP1 was not created for "
- "port=%x", attr->port_num);
- goto modify_qp_exit2;
- }
- aqp1 = container_of(sport->ibqp_sqp[IB_QPT_GSI],
- struct ehca_qp, ib_qp);
- if (ibqp->qp_type != IB_QPT_GSI &&
- ibqp->qp_type != IB_QPT_SMI &&
- aqp1->mod_qp_parm) {
- /*
- * firmware will reject this modify_qp() because
- * port is not activated/initialized fully
- */
- ret = -EFAULT;
- ehca_warn(ibqp->device, "Couldn't modify qp port=%x: "
- "either port is being activated (try again) "
- "or cabling issue", attr->port_num);
- goto modify_qp_exit2;
- }
- mqpcb->prim_phys_port = attr->port_num;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
- }
- if (attr_mask & IB_QP_QKEY) {
- mqpcb->qkey = attr->qkey;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_QKEY, 1);
- }
- if (attr_mask & IB_QP_AV) {
- mqpcb->dlid = attr->ah_attr.dlid;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID, 1);
- mqpcb->source_path_bits = attr->ah_attr.src_path_bits;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS, 1);
- mqpcb->service_level = attr->ah_attr.sl;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL, 1);
-
- if (ehca_calc_ipd(shca, mqpcb->prim_phys_port,
- attr->ah_attr.static_rate,
- &mqpcb->max_static_rate)) {
- ret = -EINVAL;
- goto modify_qp_exit2;
- }
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE, 1);
-
- /*
- * Always supply the GRH flag, even if it's zero, to give the
- * hypervisor a clear "yes" or "no" instead of a "perhaps"
- */
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
-
- /*
- * only if GRH is TRUE we might consider SOURCE_GID_IDX
- * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
- */
- if (attr->ah_attr.ah_flags == IB_AH_GRH) {
- mqpcb->send_grh_flag = 1;
-
- mqpcb->source_gid_idx = attr->ah_attr.grh.sgid_index;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX, 1);
-
- for (cnt = 0; cnt < 16; cnt++)
- mqpcb->dest_gid.byte[cnt] =
- attr->ah_attr.grh.dgid.raw[cnt];
-
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_GID, 1);
- mqpcb->flow_label = attr->ah_attr.grh.flow_label;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL, 1);
- mqpcb->hop_limit = attr->ah_attr.grh.hop_limit;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT, 1);
- mqpcb->traffic_class = attr->ah_attr.grh.traffic_class;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS, 1);
- }
- }
-
- if (attr_mask & IB_QP_PATH_MTU) {
- /* store ld(MTU) */
- my_qp->mtu_shift = attr->path_mtu + 7;
- mqpcb->path_mtu = attr->path_mtu;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PATH_MTU, 1);
- }
- if (attr_mask & IB_QP_TIMEOUT) {
- mqpcb->timeout = attr->timeout;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT, 1);
- }
- if (attr_mask & IB_QP_RETRY_CNT) {
- mqpcb->retry_count = attr->retry_cnt;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT, 1);
- }
- if (attr_mask & IB_QP_RNR_RETRY) {
- mqpcb->rnr_retry_count = attr->rnr_retry;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT, 1);
- }
- if (attr_mask & IB_QP_RQ_PSN) {
- mqpcb->receive_psn = attr->rq_psn;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RECEIVE_PSN, 1);
- }
- if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
- mqpcb->rdma_nr_atomic_resp_res = attr->max_dest_rd_atomic < 3 ?
- attr->max_dest_rd_atomic : 2;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
- }
- if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
- mqpcb->rdma_atomic_outst_dest_qp = attr->max_rd_atomic < 3 ?
- attr->max_rd_atomic : 2;
- update_mask |=
- EHCA_BMASK_SET
- (MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP, 1);
- }
- if (attr_mask & IB_QP_ALT_PATH) {
- if (attr->alt_port_num < 1
- || attr->alt_port_num > shca->num_ports) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid alt_port=%x. "
- "ehca_qp=%p qp_num=%x num_ports=%x",
- attr->alt_port_num, my_qp, ibqp->qp_num,
- shca->num_ports);
- goto modify_qp_exit2;
- }
- mqpcb->alt_phys_port = attr->alt_port_num;
-
- if (attr->alt_pkey_index >= 16) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid alt_pkey_index=%x. "
- "ehca_qp=%p qp_num=%x max_pkey_index=f",
- attr->pkey_index, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- mqpcb->alt_p_key_idx = attr->alt_pkey_index;
-
- mqpcb->timeout_al = attr->alt_timeout;
- mqpcb->dlid_al = attr->alt_ah_attr.dlid;
- mqpcb->source_path_bits_al = attr->alt_ah_attr.src_path_bits;
- mqpcb->service_level_al = attr->alt_ah_attr.sl;
-
- if (ehca_calc_ipd(shca, mqpcb->alt_phys_port,
- attr->alt_ah_attr.static_rate,
- &mqpcb->max_static_rate_al)) {
- ret = -EINVAL;
- goto modify_qp_exit2;
- }
-
- /* OpenIB doesn't support alternate retry counts - copy them */
- mqpcb->retry_count_al = mqpcb->retry_count;
- mqpcb->rnr_retry_count_al = mqpcb->rnr_retry_count;
-
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_ALT_PHYS_PORT, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_ALT_P_KEY_IDX, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_DLID_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT_AL, 1);
-
- /*
- * Always supply the GRH flag, even if it's zero, to give the
- * hypervisor a clear "yes" or "no" instead of a "perhaps"
- */
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG_AL, 1);
-
- /*
- * only if GRH is TRUE we might consider SOURCE_GID_IDX
- * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
- */
- if (attr->alt_ah_attr.ah_flags == IB_AH_GRH) {
- mqpcb->send_grh_flag_al = 1;
-
- for (cnt = 0; cnt < 16; cnt++)
- mqpcb->dest_gid_al.byte[cnt] =
- attr->alt_ah_attr.grh.dgid.raw[cnt];
- mqpcb->source_gid_idx_al =
- attr->alt_ah_attr.grh.sgid_index;
- mqpcb->flow_label_al = attr->alt_ah_attr.grh.flow_label;
- mqpcb->hop_limit_al = attr->alt_ah_attr.grh.hop_limit;
- mqpcb->traffic_class_al =
- attr->alt_ah_attr.grh.traffic_class;
-
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_DEST_GID_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL_AL, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT_AL, 1) |
- EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS_AL, 1);
- }
- }
-
- if (attr_mask & IB_QP_MIN_RNR_TIMER) {
- mqpcb->min_rnr_nak_timer_field = attr->min_rnr_timer;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD, 1);
- }
-
- if (attr_mask & IB_QP_SQ_PSN) {
- mqpcb->send_psn = attr->sq_psn;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_PSN, 1);
- }
-
- if (attr_mask & IB_QP_DEST_QPN) {
- mqpcb->dest_qp_nr = attr->dest_qp_num;
- update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_QP_NR, 1);
- }
-
- if (attr_mask & IB_QP_PATH_MIG_STATE) {
- if (attr->path_mig_state != IB_MIG_REARM
- && attr->path_mig_state != IB_MIG_MIGRATED) {
- ret = -EINVAL;
- ehca_err(ibqp->device, "Invalid mig_state=%x",
- attr->path_mig_state);
- goto modify_qp_exit2;
- }
- mqpcb->path_migration_state = attr->path_mig_state + 1;
- if (attr->path_mig_state == IB_MIG_REARM)
- my_qp->mig_armed = 1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
- }
-
- if (attr_mask & IB_QP_CAP) {
- mqpcb->max_nr_outst_send_wr = attr->cap.max_send_wr+1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_SEND_WR, 1);
- mqpcb->max_nr_outst_recv_wr = attr->cap.max_recv_wr+1;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_RECV_WR, 1);
- /* no support for max_send/recv_sge yet */
- }
-
- if (ehca_debug_level >= 2)
- ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibqp->device, "hipz_h_modify_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x", h_ret, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
-
- if ((my_qp->qp_type == IB_QPT_UD ||
- my_qp->qp_type == IB_QPT_GSI ||
- my_qp->qp_type == IB_QPT_SMI) &&
- statetrans == IB_QPST_SQE2RTS) {
- /* doorbell to reprocessing wqes */
- iosync(); /* serialize GAL register access */
- hipz_update_sqa(my_qp, bad_wqe_cnt-1);
- ehca_gen_dbg("doorbell for %x wqes", bad_wqe_cnt);
- }
-
- if (statetrans == IB_QPST_RESET2INIT ||
- statetrans == IB_QPST_INIT2INIT) {
- mqpcb->qp_enable = 1;
- mqpcb->qp_state = EHCA_QPS_INIT;
- update_mask = 0;
- update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- update_mask,
- mqpcb,
- my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibqp->device, "ENABLE in context of "
- "RESET_2_INIT failed! Maybe you didn't get "
- "a LID h_ret=%lli ehca_qp=%p qp_num=%x",
- h_ret, my_qp, ibqp->qp_num);
- goto modify_qp_exit2;
- }
- }
- if ((qp_new_state == IB_QPS_ERR) && (qp_cur_state != IB_QPS_ERR)
- && !is_user) {
- ret = check_for_left_cqes(my_qp, shca);
- if (ret)
- goto modify_qp_exit2;
- }
-
- if (statetrans == IB_QPST_ANY2RESET) {
- ipz_qeit_reset(&my_qp->ipz_rqueue);
- ipz_qeit_reset(&my_qp->ipz_squeue);
-
- if (qp_cur_state == IB_QPS_ERR && !is_user) {
- del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
-
- if (HAS_RQ(my_qp))
- del_from_err_list(my_qp->recv_cq,
- &my_qp->rq_err_node);
- }
- if (!is_user)
- reset_queue_map(&my_qp->sq_map);
-
- if (HAS_RQ(my_qp) && !is_user)
- reset_queue_map(&my_qp->rq_map);
- }
-
- if (attr_mask & IB_QP_QKEY)
- my_qp->qkey = attr->qkey;
-
-modify_qp_exit2:
- if (squeue_locked) { /* this means: sqe -> rts */
- spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
- my_qp->sqerr_purgeflag = 1;
- }
-
-modify_qp_exit1:
- ehca_free_fw_ctrlblock(mqpcb);
-
- return ret;
-}
-
-int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
- struct ib_udata *udata)
-{
- int ret = 0;
-
- struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
- ib_device);
- struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
-
- /* The if-block below caches qp_attr to be modified for GSI and SMI
- * qps during the initialization by ib_mad. When the respective port
- * is activated, ie we got an event PORT_ACTIVE, we'll replay the
- * cached modify calls sequence, see ehca_recover_sqs() below.
- * Why that is required:
- * 1) If one port is connected, older code requires that port one
- * to be connected and module option nr_ports=1 to be given by
- * user, which is very inconvenient for end user.
- * 2) Firmware accepts modify_qp() only if respective port has become
- * active. Older code had a wait loop of 30sec create_qp()/
- * define_aqp1(), which is not appropriate in practice. This
- * code now removes that wait loop, see define_aqp1(), and always
- * reports all ports to ib_mad resp. users. Only activated ports
- * will then usable for the users.
- */
- if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) {
- int port = my_qp->init_attr.port_num;
- struct ehca_sport *sport = &shca->sport[port - 1];
- unsigned long flags;
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
- /* cache qp_attr only during init */
- if (my_qp->mod_qp_parm) {
- struct ehca_mod_qp_parm *p;
- if (my_qp->mod_qp_parm_idx >= EHCA_MOD_QP_PARM_MAX) {
- ehca_err(&shca->ib_device,
- "mod_qp_parm overflow state=%x port=%x"
- " type=%x", attr->qp_state,
- my_qp->init_attr.port_num,
- ibqp->qp_type);
- spin_unlock_irqrestore(&sport->mod_sqp_lock,
- flags);
- return -EINVAL;
- }
- p = &my_qp->mod_qp_parm[my_qp->mod_qp_parm_idx];
- p->mask = attr_mask;
- p->attr = *attr;
- my_qp->mod_qp_parm_idx++;
- ehca_dbg(&shca->ib_device,
- "Saved qp_attr for state=%x port=%x type=%x",
- attr->qp_state, my_qp->init_attr.port_num,
- ibqp->qp_type);
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- goto out;
- }
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- }
-
- ret = internal_modify_qp(ibqp, attr, attr_mask, 0);
-
-out:
- if ((ret == 0) && (attr_mask & IB_QP_STATE))
- my_qp->state = attr->qp_state;
-
- return ret;
-}
-
-void ehca_recover_sqp(struct ib_qp *sqp)
-{
- struct ehca_qp *my_sqp = container_of(sqp, struct ehca_qp, ib_qp);
- int port = my_sqp->init_attr.port_num;
- struct ib_qp_attr attr;
- struct ehca_mod_qp_parm *qp_parm;
- int i, qp_parm_idx, ret;
- unsigned long flags, wr_cnt;
-
- if (!my_sqp->mod_qp_parm)
- return;
- ehca_dbg(sqp->device, "SQP port=%x qp_num=%x", port, sqp->qp_num);
-
- qp_parm = my_sqp->mod_qp_parm;
- qp_parm_idx = my_sqp->mod_qp_parm_idx;
- for (i = 0; i < qp_parm_idx; i++) {
- attr = qp_parm[i].attr;
- ret = internal_modify_qp(sqp, &attr, qp_parm[i].mask, 0);
- if (ret) {
- ehca_err(sqp->device, "Could not modify SQP port=%x "
- "qp_num=%x ret=%x", port, sqp->qp_num, ret);
- goto free_qp_parm;
- }
- ehca_dbg(sqp->device, "SQP port=%x qp_num=%x in state=%x",
- port, sqp->qp_num, attr.qp_state);
- }
-
- /* re-trigger posted recv wrs */
- wr_cnt = my_sqp->ipz_rqueue.current_q_offset /
- my_sqp->ipz_rqueue.qe_size;
- if (wr_cnt) {
- spin_lock_irqsave(&my_sqp->spinlock_r, flags);
- hipz_update_rqa(my_sqp, wr_cnt);
- spin_unlock_irqrestore(&my_sqp->spinlock_r, flags);
- ehca_dbg(sqp->device, "doorbell port=%x qp_num=%x wr_cnt=%lx",
- port, sqp->qp_num, wr_cnt);
- }
-
-free_qp_parm:
- kfree(qp_parm);
- /* this prevents subsequent calls to modify_qp() to cache qp_attr */
- my_sqp->mod_qp_parm = NULL;
-}
-
-int ehca_query_qp(struct ib_qp *qp,
- struct ib_qp_attr *qp_attr,
- int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
- struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- struct hcp_modify_qp_control_block *qpcb;
- int cnt, ret = 0;
- u64 h_ret;
-
- if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
- ehca_err(qp->device, "Invalid attribute mask "
- "ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
- my_qp, qp->qp_num, qp_attr_mask);
- return -EINVAL;
- }
-
- qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!qpcb) {
- ehca_err(qp->device, "Out of memory for qpcb "
- "ehca_qp=%p qp_num=%x", my_qp, qp->qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(adapter_handle,
- my_qp->ipz_qp_handle,
- &my_qp->pf,
- qpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(qp->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, qp->qp_num, h_ret);
- goto query_qp_exit1;
- }
-
- qp_attr->cur_qp_state = ehca2ib_qp_state(qpcb->qp_state);
- qp_attr->qp_state = qp_attr->cur_qp_state;
-
- if (qp_attr->cur_qp_state == -EINVAL) {
- ret = -EINVAL;
- ehca_err(qp->device, "Got invalid ehca_qp_state=%x "
- "ehca_qp=%p qp_num=%x",
- qpcb->qp_state, my_qp, qp->qp_num);
- goto query_qp_exit1;
- }
-
- if (qp_attr->qp_state == IB_QPS_SQD)
- qp_attr->sq_draining = 1;
-
- qp_attr->qkey = qpcb->qkey;
- qp_attr->path_mtu = qpcb->path_mtu;
- qp_attr->path_mig_state = qpcb->path_migration_state - 1;
- qp_attr->rq_psn = qpcb->receive_psn;
- qp_attr->sq_psn = qpcb->send_psn;
- qp_attr->min_rnr_timer = qpcb->min_rnr_nak_timer_field;
- qp_attr->cap.max_send_wr = qpcb->max_nr_outst_send_wr-1;
- qp_attr->cap.max_recv_wr = qpcb->max_nr_outst_recv_wr-1;
- /* UD_AV CIRCUMVENTION */
- if (my_qp->qp_type == IB_QPT_UD) {
- qp_attr->cap.max_send_sge =
- qpcb->actual_nr_sges_in_sq_wqe - 2;
- qp_attr->cap.max_recv_sge =
- qpcb->actual_nr_sges_in_rq_wqe - 2;
- } else {
- qp_attr->cap.max_send_sge =
- qpcb->actual_nr_sges_in_sq_wqe;
- qp_attr->cap.max_recv_sge =
- qpcb->actual_nr_sges_in_rq_wqe;
- }
-
- qp_attr->cap.max_inline_data = my_qp->sq_max_inline_data_size;
- qp_attr->dest_qp_num = qpcb->dest_qp_nr;
-
- qp_attr->pkey_index = qpcb->prim_p_key_idx;
- qp_attr->port_num = qpcb->prim_phys_port;
- qp_attr->timeout = qpcb->timeout;
- qp_attr->retry_cnt = qpcb->retry_count;
- qp_attr->rnr_retry = qpcb->rnr_retry_count;
-
- qp_attr->alt_pkey_index = qpcb->alt_p_key_idx;
- qp_attr->alt_port_num = qpcb->alt_phys_port;
- qp_attr->alt_timeout = qpcb->timeout_al;
-
- qp_attr->max_dest_rd_atomic = qpcb->rdma_nr_atomic_resp_res;
- qp_attr->max_rd_atomic = qpcb->rdma_atomic_outst_dest_qp;
-
- /* primary av */
- qp_attr->ah_attr.sl = qpcb->service_level;
-
- if (qpcb->send_grh_flag) {
- qp_attr->ah_attr.ah_flags = IB_AH_GRH;
- }
-
- qp_attr->ah_attr.static_rate = qpcb->max_static_rate;
- qp_attr->ah_attr.dlid = qpcb->dlid;
- qp_attr->ah_attr.src_path_bits = qpcb->source_path_bits;
- qp_attr->ah_attr.port_num = qp_attr->port_num;
-
- /* primary GRH */
- qp_attr->ah_attr.grh.traffic_class = qpcb->traffic_class;
- qp_attr->ah_attr.grh.hop_limit = qpcb->hop_limit;
- qp_attr->ah_attr.grh.sgid_index = qpcb->source_gid_idx;
- qp_attr->ah_attr.grh.flow_label = qpcb->flow_label;
-
- for (cnt = 0; cnt < 16; cnt++)
- qp_attr->ah_attr.grh.dgid.raw[cnt] =
- qpcb->dest_gid.byte[cnt];
-
- /* alternate AV */
- qp_attr->alt_ah_attr.sl = qpcb->service_level_al;
- if (qpcb->send_grh_flag_al) {
- qp_attr->alt_ah_attr.ah_flags = IB_AH_GRH;
- }
-
- qp_attr->alt_ah_attr.static_rate = qpcb->max_static_rate_al;
- qp_attr->alt_ah_attr.dlid = qpcb->dlid_al;
- qp_attr->alt_ah_attr.src_path_bits = qpcb->source_path_bits_al;
-
- /* alternate GRH */
- qp_attr->alt_ah_attr.grh.traffic_class = qpcb->traffic_class_al;
- qp_attr->alt_ah_attr.grh.hop_limit = qpcb->hop_limit_al;
- qp_attr->alt_ah_attr.grh.sgid_index = qpcb->source_gid_idx_al;
- qp_attr->alt_ah_attr.grh.flow_label = qpcb->flow_label_al;
-
- for (cnt = 0; cnt < 16; cnt++)
- qp_attr->alt_ah_attr.grh.dgid.raw[cnt] =
- qpcb->dest_gid_al.byte[cnt];
-
- /* return init attributes given in ehca_create_qp */
- if (qp_init_attr)
- *qp_init_attr = my_qp->init_attr;
-
- if (ehca_debug_level >= 2)
- ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
-
-query_qp_exit1:
- ehca_free_fw_ctrlblock(qpcb);
-
- return ret;
-}
-
-int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
- enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
-{
- struct ehca_qp *my_qp =
- container_of(ibsrq, struct ehca_qp, ib_srq);
- struct ehca_shca *shca =
- container_of(ibsrq->pd->device, struct ehca_shca, ib_device);
- struct hcp_modify_qp_control_block *mqpcb;
- u64 update_mask;
- u64 h_ret;
- int ret = 0;
-
- mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!mqpcb) {
- ehca_err(ibsrq->device, "Could not get zeroed page for mqpcb "
- "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
- return -ENOMEM;
- }
-
- update_mask = 0;
- if (attr_mask & IB_SRQ_LIMIT) {
- attr_mask &= ~IB_SRQ_LIMIT;
- update_mask |=
- EHCA_BMASK_SET(MQPCB_MASK_CURR_SRQ_LIMIT, 1)
- | EHCA_BMASK_SET(MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG, 1);
- mqpcb->curr_srq_limit = attr->srq_limit;
- mqpcb->qp_aff_asyn_ev_log_reg =
- EHCA_BMASK_SET(QPX_AAELOG_RESET_SRQ_LIMIT, 1);
- }
-
- /* by now, all bits in attr_mask should have been cleared */
- if (attr_mask) {
- ehca_err(ibsrq->device, "invalid attribute mask bits set "
- "attr_mask=%x", attr_mask);
- ret = -EINVAL;
- goto modify_srq_exit0;
- }
-
- if (ehca_debug_level >= 2)
- ehca_dmp(mqpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
-
- h_ret = hipz_h_modify_qp(shca->ipz_hca_handle, my_qp->ipz_qp_handle,
- NULL, update_mask, mqpcb,
- my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(ibsrq->device, "hipz_h_modify_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x",
- h_ret, my_qp, my_qp->real_qp_num);
- }
-
-modify_srq_exit0:
- ehca_free_fw_ctrlblock(mqpcb);
-
- return ret;
-}
-
-int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
-{
- struct ehca_qp *my_qp = container_of(srq, struct ehca_qp, ib_srq);
- struct ehca_shca *shca = container_of(srq->device, struct ehca_shca,
- ib_device);
- struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
- struct hcp_modify_qp_control_block *qpcb;
- int ret = 0;
- u64 h_ret;
-
- qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
- if (!qpcb) {
- ehca_err(srq->device, "Out of memory for qpcb "
- "ehca_qp=%p qp_num=%x", my_qp, my_qp->real_qp_num);
- return -ENOMEM;
- }
-
- h_ret = hipz_h_query_qp(adapter_handle, my_qp->ipz_qp_handle,
- NULL, qpcb, my_qp->galpas.kernel);
-
- if (h_ret != H_SUCCESS) {
- ret = ehca2ib_return_code(h_ret);
- ehca_err(srq->device, "hipz_h_query_qp() failed "
- "ehca_qp=%p qp_num=%x h_ret=%lli",
- my_qp, my_qp->real_qp_num, h_ret);
- goto query_srq_exit1;
- }
-
- srq_attr->max_wr = qpcb->max_nr_outst_recv_wr - 1;
- srq_attr->max_sge = 3;
- srq_attr->srq_limit = qpcb->curr_srq_limit;
-
- if (ehca_debug_level >= 2)
- ehca_dmp(qpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
-
-query_srq_exit1:
- ehca_free_fw_ctrlblock(qpcb);
-
- return ret;
-}
-
-static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
- struct ib_uobject *uobject)
-{
- struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
- struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
- ib_pd);
- struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
- u32 qp_num = my_qp->real_qp_num;
- int ret;
- u64 h_ret;
- u8 port_num;
- int is_user = 0;
- enum ib_qp_type qp_type;
- unsigned long flags;
-
- if (uobject) {
- is_user = 1;
- if (my_qp->mm_count_galpa ||
- my_qp->mm_count_rqueue || my_qp->mm_count_squeue) {
- ehca_err(dev, "Resources still referenced in "
- "user space qp_num=%x", qp_num);
- return -EINVAL;
- }
- }
-
- if (my_qp->send_cq) {
- ret = ehca_cq_unassign_qp(my_qp->send_cq, qp_num);
- if (ret) {
- ehca_err(dev, "Couldn't unassign qp from "
- "send_cq ret=%i qp_num=%x cq_num=%x", ret,
- qp_num, my_qp->send_cq->cq_number);
- return ret;
- }
- }
-
- write_lock_irqsave(&ehca_qp_idr_lock, flags);
- idr_remove(&ehca_qp_idr, my_qp->token);
- write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
-
- /*
- * SRQs will never get into an error list and do not have a recv_cq,
- * so we need to skip them here.
- */
- if (HAS_RQ(my_qp) && !IS_SRQ(my_qp) && !is_user)
- del_from_err_list(my_qp->recv_cq, &my_qp->rq_err_node);
-
- if (HAS_SQ(my_qp) && !is_user)
- del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
-
- /* now wait until all pending events have completed */
- wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
-
- h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
- if (h_ret != H_SUCCESS) {
- ehca_err(dev, "hipz_h_destroy_qp() failed h_ret=%lli "
- "ehca_qp=%p qp_num=%x", h_ret, my_qp, qp_num);
- return ehca2ib_return_code(h_ret);
- }
-
- port_num = my_qp->init_attr.port_num;
- qp_type = my_qp->init_attr.qp_type;
-
- if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
- spin_lock_irqsave(&sport->mod_sqp_lock, flags);
- kfree(my_qp->mod_qp_parm);
- my_qp->mod_qp_parm = NULL;
- shca->sport[port_num - 1].ibqp_sqp[qp_type] = NULL;
- spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
- }
-
- /* no support for IB_QPT_SMI yet */
- if (qp_type == IB_QPT_GSI) {
- struct ib_event event;
- ehca_info(dev, "device %s: port %x is inactive.",
- shca->ib_device.name, port_num);
- event.device = &shca->ib_device;
- event.event = IB_EVENT_PORT_ERR;
- event.element.port_num = port_num;
- shca->sport[port_num - 1].port_state = IB_PORT_DOWN;
- ib_dispatch_event(&event);
- }
-
- if (HAS_RQ(my_qp)) {
- ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
- if (!is_user)
- vfree(my_qp->rq_map.map);
- }
- if (HAS_SQ(my_qp)) {
- ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
- if (!is_user)
- vfree(my_qp->sq_map.map);
- }
- kmem_cache_free(qp_cache, my_qp);
- atomic_dec(&shca->num_qps);
- return 0;
-}
-
-int ehca_destroy_qp(struct ib_qp *qp)
-{
- return internal_destroy_qp(qp->device,
- container_of(qp, struct ehca_qp, ib_qp),
- qp->uobject);
-}
-
-int ehca_destroy_srq(struct ib_srq *srq)
-{
- return internal_destroy_qp(srq->device,
- container_of(srq, struct ehca_qp, ib_srq),
- srq->uobject);
-}
-
-int ehca_init_qp_cache(void)
-{
- qp_cache = kmem_cache_create("ehca_cache_qp",
- sizeof(struct ehca_qp), 0,
- SLAB_HWCACHE_ALIGN,
- NULL);
- if (!qp_cache)
- return -ENOMEM;
- return 0;
-}
-
-void ehca_cleanup_qp_cache(void)
-{
- if (qp_cache)
- kmem_cache_destroy(qp_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * post_send/recv, poll_cq, req_notify
- *
- * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-#include "hipz_fns.h"
-
-/* in RC traffic, insert an empty RDMA READ every this many packets */
-#define ACK_CIRC_THRESHOLD 2000000
-
-static u64 replace_wr_id(u64 wr_id, u16 idx)
-{
- u64 ret;
-
- ret = wr_id & ~QMAP_IDX_MASK;
- ret |= idx & QMAP_IDX_MASK;
-
- return ret;
-}
-
-static u16 get_app_wr_id(u64 wr_id)
-{
- return wr_id & QMAP_IDX_MASK;
-}
-
-static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
- struct ehca_wqe *wqe_p,
- struct ib_recv_wr *recv_wr,
- u32 rq_map_idx)
-{
- u8 cnt_ds;
- if (unlikely((recv_wr->num_sge < 0) ||
- (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
- ehca_gen_err("Invalid number of WQE SGE. "
- "num_sqe=%x max_nr_of_sg=%x",
- recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
- return -EINVAL; /* invalid SG list length */
- }
-
- /* clear wqe header until sglist */
- memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
-
- wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
- wqe_p->nr_of_data_seg = recv_wr->num_sge;
-
- for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
- wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
- recv_wr->sg_list[cnt_ds].addr;
- wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
- recv_wr->sg_list[cnt_ds].lkey;
- wqe_p->u.all_rcv.sg_list[cnt_ds].length =
- recv_wr->sg_list[cnt_ds].length;
- }
-
- if (ehca_debug_level >= 3) {
- ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
- ipz_rqueue);
- ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
- }
-
- return 0;
-}
-
-#if defined(DEBUG_GSI_SEND_WR)
-
-/* need ib_mad struct */
-#include <rdma/ib_mad.h>
-
-static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
-{
- int idx;
- int j;
- while (send_wr) {
- struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
- struct ib_sge *sge = send_wr->sg_list;
- ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
- "send_flags=%x opcode=%x", idx, send_wr->wr_id,
- send_wr->num_sge, send_wr->send_flags,
- send_wr->opcode);
- if (mad_hdr) {
- ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
- "mgmt_class=%x class_version=%x method=%x "
- "status=%x class_specific=%x tid=%lx "
- "attr_id=%x resv=%x attr_mod=%x",
- idx, mad_hdr->base_version,
- mad_hdr->mgmt_class,
- mad_hdr->class_version, mad_hdr->method,
- mad_hdr->status, mad_hdr->class_specific,
- mad_hdr->tid, mad_hdr->attr_id,
- mad_hdr->resv,
- mad_hdr->attr_mod);
- }
- for (j = 0; j < send_wr->num_sge; j++) {
- u8 *data = __va(sge->addr);
- ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
- "lkey=%x",
- idx, j, data, sge->length, sge->lkey);
- /* assume length is n*16 */
- ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
- idx, j);
- sge++;
- } /* eof for j */
- idx++;
- send_wr = send_wr->next;
- } /* eof while send_wr */
-}
-
-#endif /* DEBUG_GSI_SEND_WR */
-
-static inline int ehca_write_swqe(struct ehca_qp *qp,
- struct ehca_wqe *wqe_p,
- const struct ib_send_wr *send_wr,
- u32 sq_map_idx,
- int hidden)
-{
- u32 idx;
- u64 dma_length;
- struct ehca_av *my_av;
- u32 remote_qkey = send_wr->wr.ud.remote_qkey;
- struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
-
- if (unlikely((send_wr->num_sge < 0) ||
- (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
- ehca_gen_err("Invalid number of WQE SGE. "
- "num_sqe=%x max_nr_of_sg=%x",
- send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
- return -EINVAL; /* invalid SG list length */
- }
-
- /* clear wqe header until sglist */
- memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
-
- wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
-
- qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
- qmap_entry->reported = 0;
- qmap_entry->cqe_req = 0;
-
- switch (send_wr->opcode) {
- case IB_WR_SEND:
- case IB_WR_SEND_WITH_IMM:
- wqe_p->optype = WQE_OPTYPE_SEND;
- break;
- case IB_WR_RDMA_WRITE:
- case IB_WR_RDMA_WRITE_WITH_IMM:
- wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
- break;
- case IB_WR_RDMA_READ:
- wqe_p->optype = WQE_OPTYPE_RDMAREAD;
- break;
- default:
- ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
- return -EINVAL; /* invalid opcode */
- }
-
- wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
-
- wqe_p->wr_flag = 0;
-
- if ((send_wr->send_flags & IB_SEND_SIGNALED ||
- qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
- && !hidden) {
- wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
- qmap_entry->cqe_req = 1;
- }
-
- if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
- send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
- /* this might not work as long as HW does not support it */
- wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
- wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
- }
-
- wqe_p->nr_of_data_seg = send_wr->num_sge;
-
- switch (qp->qp_type) {
- case IB_QPT_SMI:
- case IB_QPT_GSI:
- /* no break is intential here */
- case IB_QPT_UD:
- /* IB 1.2 spec C10-15 compliance */
- if (send_wr->wr.ud.remote_qkey & 0x80000000)
- remote_qkey = qp->qkey;
-
- wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
- wqe_p->local_ee_context_qkey = remote_qkey;
- if (unlikely(!send_wr->wr.ud.ah)) {
- ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
- return -EINVAL;
- }
- if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
- ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
- return -EINVAL;
- }
- my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
- wqe_p->u.ud_av.ud_av = my_av->av;
-
- /*
- * omitted check of IB_SEND_INLINE
- * since HW does not support it
- */
- for (idx = 0; idx < send_wr->num_sge; idx++) {
- wqe_p->u.ud_av.sg_list[idx].vaddr =
- send_wr->sg_list[idx].addr;
- wqe_p->u.ud_av.sg_list[idx].lkey =
- send_wr->sg_list[idx].lkey;
- wqe_p->u.ud_av.sg_list[idx].length =
- send_wr->sg_list[idx].length;
- } /* eof for idx */
- if (qp->qp_type == IB_QPT_SMI ||
- qp->qp_type == IB_QPT_GSI)
- wqe_p->u.ud_av.ud_av.pmtu = 1;
- if (qp->qp_type == IB_QPT_GSI) {
- wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
-#ifdef DEBUG_GSI_SEND_WR
- trace_send_wr_ud(send_wr);
-#endif /* DEBUG_GSI_SEND_WR */
- }
- break;
-
- case IB_QPT_UC:
- if (send_wr->send_flags & IB_SEND_FENCE)
- wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
- /* no break is intentional here */
- case IB_QPT_RC:
- /* TODO: atomic not implemented */
- wqe_p->u.nud.remote_virtual_address =
- send_wr->wr.rdma.remote_addr;
- wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
-
- /*
- * omitted checking of IB_SEND_INLINE
- * since HW does not support it
- */
- dma_length = 0;
- for (idx = 0; idx < send_wr->num_sge; idx++) {
- wqe_p->u.nud.sg_list[idx].vaddr =
- send_wr->sg_list[idx].addr;
- wqe_p->u.nud.sg_list[idx].lkey =
- send_wr->sg_list[idx].lkey;
- wqe_p->u.nud.sg_list[idx].length =
- send_wr->sg_list[idx].length;
- dma_length += send_wr->sg_list[idx].length;
- } /* eof idx */
- wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
-
- /* unsolicited ack circumvention */
- if (send_wr->opcode == IB_WR_RDMA_READ) {
- /* on RDMA read, switch on and reset counters */
- qp->message_count = qp->packet_count = 0;
- qp->unsol_ack_circ = 1;
- } else
- /* else estimate #packets */
- qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
-
- break;
-
- default:
- ehca_gen_err("Invalid qptype=%x", qp->qp_type);
- return -EINVAL;
- }
-
- if (ehca_debug_level >= 3) {
- ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
- ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
- }
- return 0;
-}
-
-/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
-static inline void map_ib_wc_status(u32 cqe_status,
- enum ib_wc_status *wc_status)
-{
- if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
- switch (cqe_status & 0x3F) {
- case 0x01:
- case 0x21:
- *wc_status = IB_WC_LOC_LEN_ERR;
- break;
- case 0x02:
- case 0x22:
- *wc_status = IB_WC_LOC_QP_OP_ERR;
- break;
- case 0x03:
- case 0x23:
- *wc_status = IB_WC_LOC_EEC_OP_ERR;
- break;
- case 0x04:
- case 0x24:
- *wc_status = IB_WC_LOC_PROT_ERR;
- break;
- case 0x05:
- case 0x25:
- *wc_status = IB_WC_WR_FLUSH_ERR;
- break;
- case 0x06:
- *wc_status = IB_WC_MW_BIND_ERR;
- break;
- case 0x07: /* remote error - look into bits 20:24 */
- switch ((cqe_status
- & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
- case 0x0:
- /*
- * PSN Sequence Error!
- * couldn't find a matching status!
- */
- *wc_status = IB_WC_GENERAL_ERR;
- break;
- case 0x1:
- *wc_status = IB_WC_REM_INV_REQ_ERR;
- break;
- case 0x2:
- *wc_status = IB_WC_REM_ACCESS_ERR;
- break;
- case 0x3:
- *wc_status = IB_WC_REM_OP_ERR;
- break;
- case 0x4:
- *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
- break;
- }
- break;
- case 0x08:
- *wc_status = IB_WC_RETRY_EXC_ERR;
- break;
- case 0x09:
- *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
- break;
- case 0x0A:
- case 0x2D:
- *wc_status = IB_WC_REM_ABORT_ERR;
- break;
- case 0x0B:
- case 0x2E:
- *wc_status = IB_WC_INV_EECN_ERR;
- break;
- case 0x0C:
- case 0x2F:
- *wc_status = IB_WC_INV_EEC_STATE_ERR;
- break;
- case 0x0D:
- *wc_status = IB_WC_BAD_RESP_ERR;
- break;
- case 0x10:
- /* WQE purged */
- *wc_status = IB_WC_WR_FLUSH_ERR;
- break;
- default:
- *wc_status = IB_WC_FATAL_ERR;
-
- }
- } else
- *wc_status = IB_WC_SUCCESS;
-}
-
-static inline int post_one_send(struct ehca_qp *my_qp,
- struct ib_send_wr *cur_send_wr,
- int hidden)
-{
- struct ehca_wqe *wqe_p;
- int ret;
- u32 sq_map_idx;
- u64 start_offset = my_qp->ipz_squeue.current_q_offset;
-
- /* get pointer next to free WQE */
- wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
- if (unlikely(!wqe_p)) {
- /* too many posted work requests: queue overflow */
- ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
- "qp_num=%x", my_qp->ib_qp.qp_num);
- return -ENOMEM;
- }
-
- /*
- * Get the index of the WQE in the send queue. The same index is used
- * for writing into the sq_map.
- */
- sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
-
- /* write a SEND WQE into the QUEUE */
- ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
- /*
- * if something failed,
- * reset the free entry pointer to the start value
- */
- if (unlikely(ret)) {
- my_qp->ipz_squeue.current_q_offset = start_offset;
- ehca_err(my_qp->ib_qp.device, "Could not write WQE "
- "qp_num=%x", my_qp->ib_qp.qp_num);
- return -EINVAL;
- }
-
- return 0;
-}
-
-int ehca_post_send(struct ib_qp *qp,
- struct ib_send_wr *send_wr,
- struct ib_send_wr **bad_send_wr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
- int wqe_cnt = 0;
- int ret = 0;
- unsigned long flags;
-
- /* Reject WR if QP is in RESET, INIT or RTR state */
- if (unlikely(my_qp->state < IB_QPS_RTS)) {
- ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
- my_qp->state, qp->qp_num);
- ret = -EINVAL;
- goto out;
- }
-
- /* LOCK the QUEUE */
- spin_lock_irqsave(&my_qp->spinlock_s, flags);
-
- /* Send an empty extra RDMA read if:
- * 1) there has been an RDMA read on this connection before
- * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
- * 3) we can be sure that any previous extra RDMA read has been
- * processed so we don't overflow the SQ
- */
- if (unlikely(my_qp->unsol_ack_circ &&
- my_qp->packet_count > ACK_CIRC_THRESHOLD &&
- my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
- /* insert an empty RDMA READ to fix up the remote QP state */
- struct ib_send_wr circ_wr;
- memset(&circ_wr, 0, sizeof(circ_wr));
- circ_wr.opcode = IB_WR_RDMA_READ;
- post_one_send(my_qp, &circ_wr, 1); /* ignore retcode */
- wqe_cnt++;
- ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
- my_qp->message_count = my_qp->packet_count = 0;
- }
-
- /* loop processes list of send reqs */
- while (send_wr) {
- ret = post_one_send(my_qp, send_wr, 0);
- if (unlikely(ret)) {
- goto post_send_exit0;
- }
- wqe_cnt++;
- send_wr = send_wr->next;
- }
-
-post_send_exit0:
- iosync(); /* serialize GAL register access */
- hipz_update_sqa(my_qp, wqe_cnt);
- if (unlikely(ret || ehca_debug_level >= 2))
- ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
- my_qp, qp->qp_num, wqe_cnt, ret);
- my_qp->message_count += wqe_cnt;
- spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
-
-out:
- if (ret)
- *bad_send_wr = send_wr;
- return ret;
-}
-
-static int internal_post_recv(struct ehca_qp *my_qp,
- struct ib_device *dev,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- struct ehca_wqe *wqe_p;
- int wqe_cnt = 0;
- int ret = 0;
- u32 rq_map_idx;
- unsigned long flags;
- struct ehca_qmap_entry *qmap_entry;
-
- if (unlikely(!HAS_RQ(my_qp))) {
- ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
- my_qp, my_qp->real_qp_num, my_qp->ext_type);
- ret = -ENODEV;
- goto out;
- }
-
- /* LOCK the QUEUE */
- spin_lock_irqsave(&my_qp->spinlock_r, flags);
-
- /* loop processes list of recv reqs */
- while (recv_wr) {
- u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
- /* get pointer next to free WQE */
- wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
- if (unlikely(!wqe_p)) {
- /* too many posted work requests: queue overflow */
- ret = -ENOMEM;
- ehca_err(dev, "Too many posted WQEs "
- "qp_num=%x", my_qp->real_qp_num);
- goto post_recv_exit0;
- }
- /*
- * Get the index of the WQE in the recv queue. The same index
- * is used for writing into the rq_map.
- */
- rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
-
- /* write a RECV WQE into the QUEUE */
- ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, recv_wr,
- rq_map_idx);
- /*
- * if something failed,
- * reset the free entry pointer to the start value
- */
- if (unlikely(ret)) {
- my_qp->ipz_rqueue.current_q_offset = start_offset;
- ret = -EINVAL;
- ehca_err(dev, "Could not write WQE "
- "qp_num=%x", my_qp->real_qp_num);
- goto post_recv_exit0;
- }
-
- qmap_entry = &my_qp->rq_map.map[rq_map_idx];
- qmap_entry->app_wr_id = get_app_wr_id(recv_wr->wr_id);
- qmap_entry->reported = 0;
- qmap_entry->cqe_req = 1;
-
- wqe_cnt++;
- recv_wr = recv_wr->next;
- } /* eof for recv_wr */
-
-post_recv_exit0:
- iosync(); /* serialize GAL register access */
- hipz_update_rqa(my_qp, wqe_cnt);
- if (unlikely(ret || ehca_debug_level >= 2))
- ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
- my_qp, my_qp->real_qp_num, wqe_cnt, ret);
- spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
-
-out:
- if (ret)
- *bad_recv_wr = recv_wr;
-
- return ret;
-}
-
-int ehca_post_recv(struct ib_qp *qp,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
-
- /* Reject WR if QP is in RESET state */
- if (unlikely(my_qp->state == IB_QPS_RESET)) {
- ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
- my_qp->state, qp->qp_num);
- *bad_recv_wr = recv_wr;
- return -EINVAL;
- }
-
- return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
-}
-
-int ehca_post_srq_recv(struct ib_srq *srq,
- struct ib_recv_wr *recv_wr,
- struct ib_recv_wr **bad_recv_wr)
-{
- return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
- srq->device, recv_wr, bad_recv_wr);
-}
-
-/*
- * ib_wc_opcode table converts ehca wc opcode to ib
- * Since we use zero to indicate invalid opcode, the actual ib opcode must
- * be decremented!!!
- */
-static const u8 ib_wc_opcode[255] = {
- [0x01] = IB_WC_RECV+1,
- [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
- [0x04] = IB_WC_BIND_MW+1,
- [0x08] = IB_WC_FETCH_ADD+1,
- [0x10] = IB_WC_COMP_SWAP+1,
- [0x20] = IB_WC_RDMA_WRITE+1,
- [0x40] = IB_WC_RDMA_READ+1,
- [0x80] = IB_WC_SEND+1
-};
-
-/* internal function to poll one entry of cq */
-static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
-{
- int ret = 0, qmap_tail_idx;
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- struct ehca_cqe *cqe;
- struct ehca_qp *my_qp;
- struct ehca_qmap_entry *qmap_entry;
- struct ehca_queue_map *qmap;
- int cqe_count = 0, is_error;
-
-repoll:
- cqe = (struct ehca_cqe *)
- ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
- if (!cqe) {
- ret = -EAGAIN;
- if (ehca_debug_level >= 3)
- ehca_dbg(cq->device, "Completion queue is empty "
- "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
- goto poll_cq_one_exit0;
- }
-
- /* prevents loads being reordered across this point */
- rmb();
-
- cqe_count++;
- if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
- struct ehca_qp *qp;
- int purgeflag;
- unsigned long flags;
-
- qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
- if (!qp) {
- ehca_err(cq->device, "cq_num=%x qp_num=%x "
- "could not find qp -> ignore cqe",
- my_cq->cq_number, cqe->local_qp_number);
- ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
- my_cq->cq_number, cqe->local_qp_number);
- /* ignore this purged cqe */
- goto repoll;
- }
- spin_lock_irqsave(&qp->spinlock_s, flags);
- purgeflag = qp->sqerr_purgeflag;
- spin_unlock_irqrestore(&qp->spinlock_s, flags);
-
- if (purgeflag) {
- ehca_dbg(cq->device,
- "Got CQE with purged bit qp_num=%x src_qp=%x",
- cqe->local_qp_number, cqe->remote_qp_number);
- if (ehca_debug_level >= 2)
- ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
- cqe->local_qp_number,
- cqe->remote_qp_number);
- /*
- * ignore this to avoid double cqes of bad wqe
- * that caused sqe and turn off purge flag
- */
- qp->sqerr_purgeflag = 0;
- goto repoll;
- }
- }
-
- is_error = cqe->status & WC_STATUS_ERROR_BIT;
-
- /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
- if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
- ehca_dbg(cq->device,
- "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
- is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
- ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
- my_cq, my_cq->cq_number);
- ehca_dbg(cq->device,
- "ehca_cq=%p cq_num=%x -------------------------",
- my_cq, my_cq->cq_number);
- }
-
- read_lock(&ehca_qp_idr_lock);
- my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
- read_unlock(&ehca_qp_idr_lock);
- if (!my_qp)
- goto repoll;
- wc->qp = &my_qp->ib_qp;
-
- qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
- if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
- /* We got a send completion. */
- qmap = &my_qp->sq_map;
- else
- /* We got a receive completion. */
- qmap = &my_qp->rq_map;
-
- /* advance the tail pointer */
- qmap->tail = qmap_tail_idx;
-
- if (is_error) {
- /*
- * set left_to_poll to 0 because in error state, we will not
- * get any additional CQEs
- */
- my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
- my_qp->sq_map.entries);
- my_qp->sq_map.left_to_poll = 0;
- ehca_add_to_err_list(my_qp, 1);
-
- my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
- my_qp->rq_map.entries);
- my_qp->rq_map.left_to_poll = 0;
- if (HAS_RQ(my_qp))
- ehca_add_to_err_list(my_qp, 0);
- }
-
- qmap_entry = &qmap->map[qmap_tail_idx];
- if (qmap_entry->reported) {
- ehca_warn(cq->device, "Double cqe on qp_num=%#x",
- my_qp->real_qp_num);
- /* found a double cqe, discard it and read next one */
- goto repoll;
- }
-
- wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
- qmap_entry->reported = 1;
-
- /* if left_to_poll is decremented to 0, add the QP to the error list */
- if (qmap->left_to_poll > 0) {
- qmap->left_to_poll--;
- if ((my_qp->sq_map.left_to_poll == 0) &&
- (my_qp->rq_map.left_to_poll == 0)) {
- ehca_add_to_err_list(my_qp, 1);
- if (HAS_RQ(my_qp))
- ehca_add_to_err_list(my_qp, 0);
- }
- }
-
- /* eval ib_wc_opcode */
- wc->opcode = ib_wc_opcode[cqe->optype]-1;
- if (unlikely(wc->opcode == -1)) {
- ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
- "ehca_cq=%p cq_num=%x",
- cqe->optype, cqe->status, my_cq, my_cq->cq_number);
- /* dump cqe for other infos */
- ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
- my_cq, my_cq->cq_number);
- /* update also queue adder to throw away this entry!!! */
- goto repoll;
- }
-
- /* eval ib_wc_status */
- if (unlikely(is_error)) {
- /* complete with errors */
- map_ib_wc_status(cqe->status, &wc->status);
- wc->vendor_err = wc->status;
- } else
- wc->status = IB_WC_SUCCESS;
-
- wc->byte_len = cqe->nr_bytes_transferred;
- wc->pkey_index = cqe->pkey_index;
- wc->slid = cqe->rlid;
- wc->dlid_path_bits = cqe->dlid;
- wc->src_qp = cqe->remote_qp_number;
- /*
- * HW has "Immed data present" and "GRH present" in bits 6 and 5.
- * SW defines those in bits 1 and 0, so we can just shift and mask.
- */
- wc->wc_flags = (cqe->w_completion_flags >> 5) & 3;
- wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
- wc->sl = cqe->service_level;
-
-poll_cq_one_exit0:
- if (cqe_count > 0)
- hipz_update_feca(my_cq, cqe_count);
-
- return ret;
-}
-
-static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
- struct ib_wc *wc, int num_entries,
- struct ipz_queue *ipz_queue, int on_sq)
-{
- int nr = 0;
- struct ehca_wqe *wqe;
- u64 offset;
- struct ehca_queue_map *qmap;
- struct ehca_qmap_entry *qmap_entry;
-
- if (on_sq)
- qmap = &my_qp->sq_map;
- else
- qmap = &my_qp->rq_map;
-
- qmap_entry = &qmap->map[qmap->next_wqe_idx];
-
- while ((nr < num_entries) && (qmap_entry->reported == 0)) {
- /* generate flush CQE */
-
- memset(wc, 0, sizeof(*wc));
-
- offset = qmap->next_wqe_idx * ipz_queue->qe_size;
- wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
- if (!wqe) {
- ehca_err(cq->device, "Invalid wqe offset=%#llx on "
- "qp_num=%#x", offset, my_qp->real_qp_num);
- return nr;
- }
-
- wc->wr_id = replace_wr_id(wqe->work_request_id,
- qmap_entry->app_wr_id);
-
- if (on_sq) {
- switch (wqe->optype) {
- case WQE_OPTYPE_SEND:
- wc->opcode = IB_WC_SEND;
- break;
- case WQE_OPTYPE_RDMAWRITE:
- wc->opcode = IB_WC_RDMA_WRITE;
- break;
- case WQE_OPTYPE_RDMAREAD:
- wc->opcode = IB_WC_RDMA_READ;
- break;
- default:
- ehca_err(cq->device, "Invalid optype=%x",
- wqe->optype);
- return nr;
- }
- } else
- wc->opcode = IB_WC_RECV;
-
- if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
- wc->ex.imm_data = wqe->immediate_data;
- wc->wc_flags |= IB_WC_WITH_IMM;
- }
-
- wc->status = IB_WC_WR_FLUSH_ERR;
-
- wc->qp = &my_qp->ib_qp;
-
- /* mark as reported and advance next_wqe pointer */
- qmap_entry->reported = 1;
- qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
- qmap->entries);
- qmap_entry = &qmap->map[qmap->next_wqe_idx];
-
- wc++; nr++;
- }
-
- return nr;
-
-}
-
-int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
-{
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int nr;
- struct ehca_qp *err_qp;
- struct ib_wc *current_wc = wc;
- int ret = 0;
- unsigned long flags;
- int entries_left = num_entries;
-
- if (num_entries < 1) {
- ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
- "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
- ret = -EINVAL;
- goto poll_cq_exit0;
- }
-
- spin_lock_irqsave(&my_cq->spinlock, flags);
-
- /* generate flush cqes for send queues */
- list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
- nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
- &err_qp->ipz_squeue, 1);
- entries_left -= nr;
- current_wc += nr;
-
- if (entries_left == 0)
- break;
- }
-
- /* generate flush cqes for receive queues */
- list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
- nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
- &err_qp->ipz_rqueue, 0);
- entries_left -= nr;
- current_wc += nr;
-
- if (entries_left == 0)
- break;
- }
-
- for (nr = 0; nr < entries_left; nr++) {
- ret = ehca_poll_cq_one(cq, current_wc);
- if (ret)
- break;
- current_wc++;
- } /* eof for nr */
- entries_left -= nr;
-
- spin_unlock_irqrestore(&my_cq->spinlock, flags);
- if (ret == -EAGAIN || !ret)
- ret = num_entries - entries_left;
-
-poll_cq_exit0:
- return ret;
-}
-
-int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
-{
- struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
- int ret = 0;
-
- switch (notify_flags & IB_CQ_SOLICITED_MASK) {
- case IB_CQ_SOLICITED:
- hipz_set_cqx_n0(my_cq, 1);
- break;
- case IB_CQ_NEXT_COMP:
- hipz_set_cqx_n1(my_cq, 1);
- break;
- default:
- return -EINVAL;
- }
-
- if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
- unsigned long spl_flags;
- spin_lock_irqsave(&my_cq->spinlock, spl_flags);
- ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
- spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
- }
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * SQP functions
- *
- * Authors: Khadija Souissi <souissi@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <rdma/ib_mad.h>
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "ehca_iverbs.h"
-#include "hcp_if.h"
-
-#define IB_MAD_STATUS_REDIRECT cpu_to_be16(0x0002)
-#define IB_MAD_STATUS_UNSUP_VERSION cpu_to_be16(0x0004)
-#define IB_MAD_STATUS_UNSUP_METHOD cpu_to_be16(0x0008)
-
-#define IB_PMA_CLASS_PORT_INFO cpu_to_be16(0x0001)
-
-/**
- * ehca_define_sqp - Defines special queue pair 1 (GSI QP). When special queue
- * pair is created successfully, the corresponding port gets active.
- *
- * Define Special Queue pair 0 (SMI QP) is still not supported.
- *
- * @qp_init_attr: Queue pair init attributes with port and queue pair type
- */
-
-u64 ehca_define_sqp(struct ehca_shca *shca,
- struct ehca_qp *ehca_qp,
- struct ib_qp_init_attr *qp_init_attr)
-{
- u32 pma_qp_nr, bma_qp_nr;
- u64 ret;
- u8 port = qp_init_attr->port_num;
- int counter;
-
- shca->sport[port - 1].port_state = IB_PORT_DOWN;
-
- switch (qp_init_attr->qp_type) {
- case IB_QPT_SMI:
- /* function not supported yet */
- break;
- case IB_QPT_GSI:
- ret = hipz_h_define_aqp1(shca->ipz_hca_handle,
- ehca_qp->ipz_qp_handle,
- ehca_qp->galpas.kernel,
- (u32) qp_init_attr->port_num,
- &pma_qp_nr, &bma_qp_nr);
-
- if (ret != H_SUCCESS) {
- ehca_err(&shca->ib_device,
- "Can't define AQP1 for port %x. h_ret=%lli",
- port, ret);
- return ret;
- }
- shca->sport[port - 1].pma_qp_nr = pma_qp_nr;
- ehca_dbg(&shca->ib_device, "port=%x pma_qp_nr=%x",
- port, pma_qp_nr);
- break;
- default:
- ehca_err(&shca->ib_device, "invalid qp_type=%x",
- qp_init_attr->qp_type);
- return H_PARAMETER;
- }
-
- if (ehca_nr_ports < 0) /* autodetect mode */
- return H_SUCCESS;
-
- for (counter = 0;
- shca->sport[port - 1].port_state != IB_PORT_ACTIVE &&
- counter < ehca_port_act_time;
- counter++) {
- ehca_dbg(&shca->ib_device, "... wait until port %x is active",
- port);
- msleep_interruptible(1000);
- }
-
- if (counter == ehca_port_act_time) {
- ehca_err(&shca->ib_device, "Port %x is not active.", port);
- return H_HARDWARE;
- }
-
- return H_SUCCESS;
-}
-
-struct ib_perf {
- struct ib_mad_hdr mad_hdr;
- u8 reserved[40];
- u8 data[192];
-} __attribute__ ((packed));
-
-/* TC/SL/FL packed into 32 bits, as in ClassPortInfo */
-struct tcslfl {
- u32 tc:8;
- u32 sl:4;
- u32 fl:20;
-} __attribute__ ((packed));
-
-/* IP Version/TC/FL packed into 32 bits, as in GRH */
-struct vertcfl {
- u32 ver:4;
- u32 tc:8;
- u32 fl:20;
-} __attribute__ ((packed));
-
-static int ehca_process_perf(struct ib_device *ibdev, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad *in_mad, struct ib_mad *out_mad)
-{
- const struct ib_perf *in_perf = (const struct ib_perf *)in_mad;
- struct ib_perf *out_perf = (struct ib_perf *)out_mad;
- struct ib_class_port_info *poi =
- (struct ib_class_port_info *)out_perf->data;
- struct tcslfl *tcslfl =
- (struct tcslfl *)&poi->redirect_tcslfl;
- struct ehca_shca *shca =
- container_of(ibdev, struct ehca_shca, ib_device);
- struct ehca_sport *sport = &shca->sport[port_num - 1];
-
- ehca_dbg(ibdev, "method=%x", in_perf->mad_hdr.method);
-
- *out_mad = *in_mad;
-
- if (in_perf->mad_hdr.class_version != 1) {
- ehca_warn(ibdev, "Unsupported class_version=%x",
- in_perf->mad_hdr.class_version);
- out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_VERSION;
- goto perf_reply;
- }
-
- switch (in_perf->mad_hdr.method) {
- case IB_MGMT_METHOD_GET:
- case IB_MGMT_METHOD_SET:
- /* set class port info for redirection */
- out_perf->mad_hdr.attr_id = IB_PMA_CLASS_PORT_INFO;
- out_perf->mad_hdr.status = IB_MAD_STATUS_REDIRECT;
- memset(poi, 0, sizeof(*poi));
- poi->base_version = 1;
- poi->class_version = 1;
- poi->resp_time_value = 18;
-
- /* copy local routing information from WC where applicable */
- tcslfl->sl = in_wc->sl;
- poi->redirect_lid =
- sport->saved_attr.lid | in_wc->dlid_path_bits;
- poi->redirect_qp = sport->pma_qp_nr;
- poi->redirect_qkey = IB_QP1_QKEY;
-
- ehca_query_pkey(ibdev, port_num, in_wc->pkey_index,
- &poi->redirect_pkey);
-
- /* if request was globally routed, copy route info */
- if (in_grh) {
- const struct vertcfl *vertcfl =
- (const struct vertcfl *)&in_grh->version_tclass_flow;
- memcpy(poi->redirect_gid, in_grh->dgid.raw,
- sizeof(poi->redirect_gid));
- tcslfl->tc = vertcfl->tc;
- tcslfl->fl = vertcfl->fl;
- } else
- /* else only fill in default GID */
- ehca_query_gid(ibdev, port_num, 0,
- (union ib_gid *)&poi->redirect_gid);
-
- ehca_dbg(ibdev, "ehca_pma_lid=%x ehca_pma_qp=%x",
- sport->saved_attr.lid, sport->pma_qp_nr);
- break;
-
- case IB_MGMT_METHOD_GET_RESP:
- return IB_MAD_RESULT_FAILURE;
-
- default:
- out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_METHOD;
- break;
- }
-
-perf_reply:
- out_perf->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
-
- return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
-}
-
-int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
- const struct ib_wc *in_wc, const struct ib_grh *in_grh,
- const struct ib_mad_hdr *in, size_t in_mad_size,
- struct ib_mad_hdr *out, size_t *out_mad_size,
- u16 *out_mad_pkey_index)
-{
- int ret;
- const struct ib_mad *in_mad = (const struct ib_mad *)in;
- struct ib_mad *out_mad = (struct ib_mad *)out;
-
- if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
- *out_mad_size != sizeof(*out_mad)))
- return IB_MAD_RESULT_FAILURE;
-
- if (!port_num || port_num > ibdev->phys_port_cnt || !in_wc)
- return IB_MAD_RESULT_FAILURE;
-
- /* accept only pma request */
- if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
- return IB_MAD_RESULT_SUCCESS;
-
- ehca_dbg(ibdev, "port_num=%x src_qp=%x", port_num, in_wc->src_qp);
- ret = ehca_process_perf(ibdev, port_num, in_wc, in_grh,
- in_mad, out_mad);
-
- return ret;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * auxiliary functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Khadija Souissi <souissik@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-
-#ifndef EHCA_TOOLS_H
-#define EHCA_TOOLS_H
-
-#include <linux/kernel.h>
-#include <linux/spinlock.h>
-#include <linux/delay.h>
-#include <linux/idr.h>
-#include <linux/kthread.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/vmalloc.h>
-#include <linux/notifier.h>
-#include <linux/cpu.h>
-#include <linux/device.h>
-
-#include <linux/atomic.h>
-#include <asm/ibmebus.h>
-#include <asm/io.h>
-#include <asm/pgtable.h>
-#include <asm/hvcall.h>
-
-extern int ehca_debug_level;
-
-#define ehca_dbg(ib_dev, format, arg...) \
- do { \
- if (unlikely(ehca_debug_level)) \
- dev_printk(KERN_DEBUG, (ib_dev)->dma_device, \
- "PU%04x EHCA_DBG:%s " format "\n", \
- raw_smp_processor_id(), __func__, \
- ## arg); \
- } while (0)
-
-#define ehca_info(ib_dev, format, arg...) \
- dev_info((ib_dev)->dma_device, "PU%04x EHCA_INFO:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_warn(ib_dev, format, arg...) \
- dev_warn((ib_dev)->dma_device, "PU%04x EHCA_WARN:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_err(ib_dev, format, arg...) \
- dev_err((ib_dev)->dma_device, "PU%04x EHCA_ERR:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-/* use this one only if no ib_dev available */
-#define ehca_gen_dbg(format, arg...) \
- do { \
- if (unlikely(ehca_debug_level)) \
- printk(KERN_DEBUG "PU%04x EHCA_DBG:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg); \
- } while (0)
-
-#define ehca_gen_warn(format, arg...) \
- printk(KERN_INFO "PU%04x EHCA_WARN:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-#define ehca_gen_err(format, arg...) \
- printk(KERN_ERR "PU%04x EHCA_ERR:%s " format "\n", \
- raw_smp_processor_id(), __func__, ## arg)
-
-/**
- * ehca_dmp - printk a memory block, whose length is n*8 bytes.
- * Each line has the following layout:
- * <format string> adr=X ofs=Y <8 bytes hex> <8 bytes hex>
- */
-#define ehca_dmp(adr, len, format, args...) \
- do { \
- unsigned int x; \
- unsigned int l = (unsigned int)(len); \
- unsigned char *deb = (unsigned char *)(adr); \
- for (x = 0; x < l; x += 16) { \
- printk(KERN_INFO "EHCA_DMP:%s " format \
- " adr=%p ofs=%04x %016llx %016llx\n", \
- __func__, ##args, deb, x, \
- *((u64 *)&deb[0]), *((u64 *)&deb[8])); \
- deb += 16; \
- } \
- } while (0)
-
-/* define a bitmask, little endian version */
-#define EHCA_BMASK(pos, length) (((pos) << 16) + (length))
-
-/* define a bitmask, the ibm way... */
-#define EHCA_BMASK_IBM(from, to) (((63 - to) << 16) + ((to) - (from) + 1))
-
-/* internal function, don't use */
-#define EHCA_BMASK_SHIFTPOS(mask) (((mask) >> 16) & 0xffff)
-
-/* internal function, don't use */
-#define EHCA_BMASK_MASK(mask) (~0ULL >> ((64 - (mask)) & 0xffff))
-
-/**
- * EHCA_BMASK_SET - return value shifted and masked by mask
- * variable|=EHCA_BMASK_SET(MY_MASK,0x4711) ORs the bits in variable
- * variable&=~EHCA_BMASK_SET(MY_MASK,-1) clears the bits from the mask
- * in variable
- */
-#define EHCA_BMASK_SET(mask, value) \
- ((EHCA_BMASK_MASK(mask) & ((u64)(value))) << EHCA_BMASK_SHIFTPOS(mask))
-
-/**
- * EHCA_BMASK_GET - extract a parameter from value by mask
- */
-#define EHCA_BMASK_GET(mask, value) \
- (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
-
-/* Converts ehca to ib return code */
-int ehca2ib_return_code(u64 ehca_rc);
-
-#endif /* EHCA_TOOLS_H */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * userspace support verbs
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_classes.h"
-#include "ehca_iverbs.h"
-#include "ehca_mrmw.h"
-#include "ehca_tools.h"
-#include "hcp_if.h"
-
-struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
- struct ib_udata *udata)
-{
- struct ehca_ucontext *my_context;
-
- my_context = kzalloc(sizeof *my_context, GFP_KERNEL);
- if (!my_context) {
- ehca_err(device, "Out of memory device=%p", device);
- return ERR_PTR(-ENOMEM);
- }
-
- return &my_context->ib_ucontext;
-}
-
-int ehca_dealloc_ucontext(struct ib_ucontext *context)
-{
- kfree(container_of(context, struct ehca_ucontext, ib_ucontext));
- return 0;
-}
-
-static void ehca_mm_open(struct vm_area_struct *vma)
-{
- u32 *count = (u32 *)vma->vm_private_data;
- if (!count) {
- ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- return;
- }
- (*count)++;
- if (!(*count))
- ehca_gen_err("Use count overflow vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
- vma->vm_start, vma->vm_end, *count);
-}
-
-static void ehca_mm_close(struct vm_area_struct *vma)
-{
- u32 *count = (u32 *)vma->vm_private_data;
- if (!count) {
- ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
- vma->vm_start, vma->vm_end);
- return;
- }
- (*count)--;
- ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
- vma->vm_start, vma->vm_end, *count);
-}
-
-static const struct vm_operations_struct vm_ops = {
- .open = ehca_mm_open,
- .close = ehca_mm_close,
-};
-
-static int ehca_mmap_fw(struct vm_area_struct *vma, struct h_galpas *galpas,
- u32 *mm_count)
-{
- int ret;
- u64 vsize, physical;
-
- vsize = vma->vm_end - vma->vm_start;
- if (vsize < EHCA_PAGESIZE) {
- ehca_gen_err("invalid vsize=%lx", vma->vm_end - vma->vm_start);
- return -EINVAL;
- }
-
- physical = galpas->user.fw_handle;
- vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
- ehca_gen_dbg("vsize=%llx physical=%llx", vsize, physical);
- /* VM_IO | VM_DONTEXPAND | VM_DONTDUMP are set by remap_pfn_range() */
- ret = remap_4k_pfn(vma, vma->vm_start, physical >> EHCA_PAGESHIFT,
- vma->vm_page_prot);
- if (unlikely(ret)) {
- ehca_gen_err("remap_pfn_range() failed ret=%i", ret);
- return -ENOMEM;
- }
-
- vma->vm_private_data = mm_count;
- (*mm_count)++;
- vma->vm_ops = &vm_ops;
-
- return 0;
-}
-
-static int ehca_mmap_queue(struct vm_area_struct *vma, struct ipz_queue *queue,
- u32 *mm_count)
-{
- int ret;
- u64 start, ofs;
- struct page *page;
-
- vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
- start = vma->vm_start;
- for (ofs = 0; ofs < queue->queue_length; ofs += PAGE_SIZE) {
- u64 virt_addr = (u64)ipz_qeit_calc(queue, ofs);
- page = virt_to_page(virt_addr);
- ret = vm_insert_page(vma, start, page);
- if (unlikely(ret)) {
- ehca_gen_err("vm_insert_page() failed rc=%i", ret);
- return ret;
- }
- start += PAGE_SIZE;
- }
- vma->vm_private_data = mm_count;
- (*mm_count)++;
- vma->vm_ops = &vm_ops;
-
- return 0;
-}
-
-static int ehca_mmap_cq(struct vm_area_struct *vma, struct ehca_cq *cq,
- u32 rsrc_type)
-{
- int ret;
-
- switch (rsrc_type) {
- case 0: /* galpa fw handle */
- ehca_dbg(cq->ib_cq.device, "cq_num=%x fw", cq->cq_number);
- ret = ehca_mmap_fw(vma, &cq->galpas, &cq->mm_count_galpa);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_fw() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- case 1: /* cq queue_addr */
- ehca_dbg(cq->ib_cq.device, "cq_num=%x queue", cq->cq_number);
- ret = ehca_mmap_queue(vma, &cq->ipz_queue, &cq->mm_count_queue);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_queue() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- default:
- ehca_err(cq->ib_cq.device, "bad resource type=%x cq_num=%x",
- rsrc_type, cq->cq_number);
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
- u32 rsrc_type)
-{
- int ret;
-
- switch (rsrc_type) {
- case 0: /* galpa fw handle */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x fw", qp->ib_qp.qp_num);
- ret = ehca_mmap_fw(vma, &qp->galpas, &qp->mm_count_galpa);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "remap_pfn_range() failed ret=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return -ENOMEM;
- }
- break;
-
- case 1: /* qp rqueue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x rq", qp->ib_qp.qp_num);
- ret = ehca_mmap_queue(vma, &qp->ipz_rqueue,
- &qp->mm_count_rqueue);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_queue(rq) failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- case 2: /* qp squeue_addr */
- ehca_dbg(qp->ib_qp.device, "qp_num=%x sq", qp->ib_qp.qp_num);
- ret = ehca_mmap_queue(vma, &qp->ipz_squeue,
- &qp->mm_count_squeue);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_queue(sq) failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- default:
- ehca_err(qp->ib_qp.device, "bad resource type=%x qp=num=%x",
- rsrc_type, qp->ib_qp.qp_num);
- return -EINVAL;
- }
-
- return 0;
-}
-
-int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
-{
- u64 fileoffset = vma->vm_pgoff;
- u32 idr_handle = fileoffset & 0x1FFFFFF;
- u32 q_type = (fileoffset >> 27) & 0x1; /* CQ, QP,... */
- u32 rsrc_type = (fileoffset >> 25) & 0x3; /* sq,rq,cmnd_window */
- u32 ret;
- struct ehca_cq *cq;
- struct ehca_qp *qp;
- struct ib_uobject *uobject;
-
- switch (q_type) {
- case 0: /* CQ */
- read_lock(&ehca_cq_idr_lock);
- cq = idr_find(&ehca_cq_idr, idr_handle);
- read_unlock(&ehca_cq_idr_lock);
-
- /* make sure this mmap really belongs to the authorized user */
- if (!cq)
- return -EINVAL;
-
- if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
- return -EINVAL;
-
- ret = ehca_mmap_cq(vma, cq, rsrc_type);
- if (unlikely(ret)) {
- ehca_err(cq->ib_cq.device,
- "ehca_mmap_cq() failed rc=%i cq_num=%x",
- ret, cq->cq_number);
- return ret;
- }
- break;
-
- case 1: /* QP */
- read_lock(&ehca_qp_idr_lock);
- qp = idr_find(&ehca_qp_idr, idr_handle);
- read_unlock(&ehca_qp_idr_lock);
-
- /* make sure this mmap really belongs to the authorized user */
- if (!qp)
- return -EINVAL;
-
- uobject = IS_SRQ(qp) ? qp->ib_srq.uobject : qp->ib_qp.uobject;
- if (!uobject || uobject->context != context)
- return -EINVAL;
-
- ret = ehca_mmap_qp(vma, qp, rsrc_type);
- if (unlikely(ret)) {
- ehca_err(qp->ib_qp.device,
- "ehca_mmap_qp() failed rc=%i qp_num=%x",
- ret, qp->ib_qp.qp_num);
- return ret;
- }
- break;
-
- default:
- ehca_gen_err("bad queue type %x", q_type);
- return -EINVAL;
- }
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware Infiniband Interface code for POWER
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Joachim Fenkes <fenkes@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <asm/hvcall.h>
-#include "ehca_tools.h"
-#include "hcp_if.h"
-#include "hcp_phyp.h"
-#include "hipz_fns.h"
-#include "ipz_pt_fn.h"
-
-#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
-#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
-#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
-#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
-#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
-#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
-#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
-#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
-#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
-#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
-#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
-
-#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
-#define H_ALL_RES_QP_MAX_OUTST_RECV_WR EHCA_BMASK_IBM(16, 31)
-#define H_ALL_RES_QP_MAX_SEND_SGE EHCA_BMASK_IBM(32, 39)
-#define H_ALL_RES_QP_MAX_RECV_SGE EHCA_BMASK_IBM(40, 47)
-
-#define H_ALL_RES_QP_UD_AV_LKEY EHCA_BMASK_IBM(32, 63)
-#define H_ALL_RES_QP_SRQ_QP_TOKEN EHCA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_SRQ_QP_HANDLE EHCA_BMASK_IBM(0, 64)
-#define H_ALL_RES_QP_SRQ_LIMIT EHCA_BMASK_IBM(48, 63)
-#define H_ALL_RES_QP_SRQ_QPN EHCA_BMASK_IBM(40, 63)
-
-#define H_ALL_RES_QP_ACT_OUTST_SEND_WR EHCA_BMASK_IBM(16, 31)
-#define H_ALL_RES_QP_ACT_OUTST_RECV_WR EHCA_BMASK_IBM(48, 63)
-#define H_ALL_RES_QP_ACT_SEND_SGE EHCA_BMASK_IBM(8, 15)
-#define H_ALL_RES_QP_ACT_RECV_SGE EHCA_BMASK_IBM(24, 31)
-
-#define H_ALL_RES_QP_SQUEUE_SIZE_PAGES EHCA_BMASK_IBM(0, 31)
-#define H_ALL_RES_QP_RQUEUE_SIZE_PAGES EHCA_BMASK_IBM(32, 63)
-
-#define H_MP_INIT_TYPE EHCA_BMASK_IBM(44, 47)
-#define H_MP_SHUTDOWN EHCA_BMASK_IBM(48, 48)
-#define H_MP_RESET_QKEY_CTR EHCA_BMASK_IBM(49, 49)
-
-#define HCALL4_REGS_FORMAT "r4=%lx r5=%lx r6=%lx r7=%lx"
-#define HCALL7_REGS_FORMAT HCALL4_REGS_FORMAT " r8=%lx r9=%lx r10=%lx"
-#define HCALL9_REGS_FORMAT HCALL7_REGS_FORMAT " r11=%lx r12=%lx"
-
-static DEFINE_SPINLOCK(hcall_lock);
-
-static long ehca_plpar_hcall_norets(unsigned long opcode,
- unsigned long arg1,
- unsigned long arg2,
- unsigned long arg3,
- unsigned long arg4,
- unsigned long arg5,
- unsigned long arg6,
- unsigned long arg7)
-{
- long ret;
- int i, sleep_msecs;
- unsigned long flags = 0;
-
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
- opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
-
- for (i = 0; i < 5; i++) {
- /* serialize hCalls to work around firmware issue */
- if (ehca_lock_hcalls)
- spin_lock_irqsave(&hcall_lock, flags);
-
- ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
- arg5, arg6, arg7);
-
- if (ehca_lock_hcalls)
- spin_unlock_irqrestore(&hcall_lock, flags);
-
- if (H_IS_LONG_BUSY(ret)) {
- sleep_msecs = get_longbusy_msecs(ret);
- msleep_interruptible(sleep_msecs);
- continue;
- }
-
- if (ret < H_SUCCESS)
- ehca_gen_err("opcode=%lx ret=%li " HCALL7_REGS_FORMAT,
- opcode, ret, arg1, arg2, arg3,
- arg4, arg5, arg6, arg7);
- else
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
-
- return ret;
- }
-
- return H_BUSY;
-}
-
-static long ehca_plpar_hcall9(unsigned long opcode,
- unsigned long *outs, /* array of 9 outputs */
- unsigned long arg1,
- unsigned long arg2,
- unsigned long arg3,
- unsigned long arg4,
- unsigned long arg5,
- unsigned long arg6,
- unsigned long arg7,
- unsigned long arg8,
- unsigned long arg9)
-{
- long ret;
- int i, sleep_msecs;
- unsigned long flags = 0;
-
- if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
- arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
-
- for (i = 0; i < 5; i++) {
- /* serialize hCalls to work around firmware issue */
- if (ehca_lock_hcalls)
- spin_lock_irqsave(&hcall_lock, flags);
-
- ret = plpar_hcall9(opcode, outs,
- arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
-
- if (ehca_lock_hcalls)
- spin_unlock_irqrestore(&hcall_lock, flags);
-
- if (H_IS_LONG_BUSY(ret)) {
- sleep_msecs = get_longbusy_msecs(ret);
- msleep_interruptible(sleep_msecs);
- continue;
- }
-
- if (ret < H_SUCCESS) {
- ehca_gen_err("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT,
- opcode, arg1, arg2, arg3, arg4, arg5,
- arg6, arg7, arg8, arg9);
- ehca_gen_err("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
- ret, outs[0], outs[1], outs[2], outs[3],
- outs[4], outs[5], outs[6], outs[7],
- outs[8]);
- } else if (unlikely(ehca_debug_level >= 2))
- ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
- ret, outs[0], outs[1], outs[2], outs[3],
- outs[4], outs[5], outs[6], outs[7],
- outs[8]);
- return ret;
- }
-
- return H_BUSY;
-}
-
-u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_pfeq *pfeq,
- const u32 neq_control,
- const u32 number_of_entries,
- struct ipz_eq_handle *eq_handle,
- u32 *act_nr_of_entries,
- u32 *act_pages,
- u32 *eq_ist)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
- u64 allocate_controls;
-
- /* resource type */
- allocate_controls = 3ULL;
-
- /* ISN is associated */
- if (neq_control != 1)
- allocate_controls = (1ULL << (63 - 7)) | allocate_controls;
- else /* notification event queue */
- allocate_controls = (1ULL << 63) | allocate_controls;
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- allocate_controls, /* r5 */
- number_of_entries, /* r6 */
- 0, 0, 0, 0, 0, 0);
- eq_handle->handle = outs[0];
- *act_nr_of_entries = (u32)outs[3];
- *act_pages = (u32)outs[4];
- *eq_ist = (u32)outs[5];
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resource - ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
- struct ipz_eq_handle eq_handle,
- const u64 event_mask)
-{
- return ehca_plpar_hcall_norets(H_RESET_EVENTS,
- adapter_handle.handle, /* r4 */
- eq_handle.handle, /* r5 */
- event_mask, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- struct ehca_alloc_cq_parms *param)
-{
- int rc;
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 2, /* r5 */
- param->eq_handle.handle, /* r6 */
- cq->token, /* r7 */
- param->nr_cqe, /* r8 */
- 0, 0, 0, 0);
- cq->ipz_cq_handle.handle = outs[0];
- param->act_nr_of_entries = (u32)outs[3];
- param->act_pages = (u32)outs[4];
-
- if (ret == H_SUCCESS) {
- rc = hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]);
- if (rc) {
- ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
- rc, outs[5]);
-
- ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- cq->ipz_cq_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
- ret = H_NO_MEM;
- }
- }
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_alloc_qp_parms *parms, int is_user)
-{
- int rc;
- u64 ret;
- u64 allocate_controls, max_r10_reg, r11, r12;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- allocate_controls =
- EHCA_BMASK_SET(H_ALL_RES_QP_ENHANCED_OPS, parms->ext_type)
- | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
- | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
- parms->squeue.page_size)
- | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
- parms->rqueue.page_size)
- | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
- !!(parms->ll_comp_flags & LLQP_RECV_COMP))
- | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
- !!(parms->ll_comp_flags & LLQP_SEND_COMP))
- | EHCA_BMASK_SET(H_ALL_RES_QP_UD_AV_LKEY_CTRL,
- parms->ud_av_l_key_ctl)
- | EHCA_BMASK_SET(H_ALL_RES_QP_RESOURCE_TYPE, 1);
-
- max_r10_reg =
- EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
- parms->squeue.max_wr + 1)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
- parms->rqueue.max_wr + 1)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
- parms->squeue.max_sge)
- | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
- parms->rqueue.max_sge);
-
- r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
-
- if (parms->ext_type == EQPT_SRQ)
- r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_LIMIT, parms->srq_limit);
- else
- r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QPN, parms->srq_qpn);
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- allocate_controls, /* r5 */
- parms->send_cq_handle.handle,
- parms->recv_cq_handle.handle,
- parms->eq_handle.handle,
- ((u64)parms->token << 32) | parms->pd.value,
- max_r10_reg, r11, r12);
-
- parms->qp_handle.handle = outs[0];
- parms->real_qp_num = (u32)outs[1];
- parms->squeue.act_nr_wqes =
- (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
- parms->rqueue.act_nr_wqes =
- (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
- parms->squeue.act_nr_sges =
- (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
- parms->rqueue.act_nr_sges =
- (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
- parms->squeue.queue_size =
- (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
- parms->rqueue.queue_size =
- (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
-
- if (ret == H_SUCCESS) {
- rc = hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]);
- if (rc) {
- ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
- rc, outs[6]);
-
- ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- parms->qp_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
- ret = H_NO_MEM;
- }
- }
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id,
- struct hipz_query_port *query_port_response_block)
-{
- u64 ret;
- u64 r_cb = __pa(query_port_response_block);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("response block not page aligned");
- return H_PARAMETER;
- }
-
- ret = ehca_plpar_hcall_norets(H_QUERY_PORT,
- adapter_handle.handle, /* r4 */
- port_id, /* r5 */
- r_cb, /* r6 */
- 0, 0, 0, 0);
-
- if (ehca_debug_level >= 2)
- ehca_dmp(query_port_response_block, 64, "response_block");
-
- return ret;
-}
-
-u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id, const u32 port_cap,
- const u8 init_type, const int modify_mask)
-{
- u64 port_attributes = port_cap;
-
- if (modify_mask & IB_PORT_SHUTDOWN)
- port_attributes |= EHCA_BMASK_SET(H_MP_SHUTDOWN, 1);
- if (modify_mask & IB_PORT_INIT_TYPE)
- port_attributes |= EHCA_BMASK_SET(H_MP_INIT_TYPE, init_type);
- if (modify_mask & IB_PORT_RESET_QKEY_CNTR)
- port_attributes |= EHCA_BMASK_SET(H_MP_RESET_QKEY_CTR, 1);
-
- return ehca_plpar_hcall_norets(H_MODIFY_PORT,
- adapter_handle.handle, /* r4 */
- port_id, /* r5 */
- port_attributes, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
- struct hipz_query_hca *query_hca_rblock)
-{
- u64 r_cb = __pa(query_hca_rblock);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("response_block=%p not page aligned",
- query_hca_rblock);
- return H_PARAMETER;
- }
-
- return ehca_plpar_hcall_norets(H_QUERY_HCA,
- adapter_handle.handle, /* r4 */
- r_cb, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
- const u8 pagesize,
- const u8 queue_type,
- const u64 resource_handle,
- const u64 logical_address_of_page,
- u64 count)
-{
- return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
- adapter_handle.handle, /* r4 */
- (u64)queue_type | ((u64)pagesize) << 8,
- /* r5 */
- resource_handle, /* r6 */
- logical_address_of_page, /* r7 */
- count, /* r8 */
- 0, 0);
-}
-
-u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_eq_handle eq_handle,
- struct ehca_pfeq *pfeq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count)
-{
- if (count != 1) {
- ehca_gen_err("Ppage counter=%llx", count);
- return H_PARAMETER;
- }
- return hipz_h_register_rpage(adapter_handle,
- pagesize,
- queue_type,
- eq_handle.handle,
- logical_address_of_page, count);
-}
-
-u64 hipz_h_query_int_state(const struct ipz_adapter_handle adapter_handle,
- u32 ist)
-{
- u64 ret;
- ret = ehca_plpar_hcall_norets(H_QUERY_INT_STATE,
- adapter_handle.handle, /* r4 */
- ist, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret != H_SUCCESS && ret != H_BUSY)
- ehca_gen_err("Could not query interrupt state.");
-
- return ret;
-}
-
-u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_cq_handle cq_handle,
- struct ehca_pfcq *pfcq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa gal)
-{
- if (count != 1) {
- ehca_gen_err("Page counter=%llx", count);
- return H_PARAMETER;
- }
-
- return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
- cq_handle.handle, logical_address_of_page,
- count);
-}
-
-u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa galpa)
-{
- if (count > 1) {
- ehca_gen_err("Page counter=%llx", count);
- return H_PARAMETER;
- }
-
- return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
- qp_handle.handle, logical_address_of_page,
- count);
-}
-
-u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- void **log_addr_next_sq_wqe2processed,
- void **log_addr_next_rq_wqe2processed,
- int dis_and_get_function_code)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
- adapter_handle.handle, /* r4 */
- dis_and_get_function_code, /* r5 */
- qp_handle.handle, /* r6 */
- 0, 0, 0, 0, 0, 0);
- if (log_addr_next_sq_wqe2processed)
- *log_addr_next_sq_wqe2processed = (void *)outs[0];
- if (log_addr_next_rq_wqe2processed)
- *log_addr_next_rq_wqe2processed = (void *)outs[1];
-
- return ret;
-}
-
-u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u64 update_mask,
- struct hcp_modify_qp_control_block *mqpcb,
- struct h_galpa gal)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
- ret = ehca_plpar_hcall9(H_MODIFY_QP, outs,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- update_mask, /* r6 */
- __pa(mqpcb), /* r7 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Insufficient resources ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- struct hcp_modify_qp_control_block *qqpcb,
- struct h_galpa gal)
-{
- return ehca_plpar_hcall_norets(H_QUERY_QP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- __pa(qqpcb), /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_qp *qp)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = hcp_galpas_dtor(&qp->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct qp->galpas");
- return H_RESOURCE;
- }
- ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
- adapter_handle.handle, /* r4 */
- /* function code */
- 1, /* r5 */
- qp->ipz_qp_handle.handle, /* r6 */
- 0, 0, 0, 0, 0, 0);
- if (ret == H_HARDWARE)
- ehca_gen_err("HCA not operational. ret=%lli", ret);
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- qp->ipz_qp_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("Resource still in use. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port)
-{
- return ehca_plpar_hcall_norets(H_DEFINE_AQP0,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- port, /* r6 */
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port, u32 * pma_qp_nr,
- u32 * bma_qp_nr)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_DEFINE_AQP1, outs,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- port, /* r6 */
- 0, 0, 0, 0, 0, 0);
- *pma_qp_nr = (u32)outs[0];
- *bma_qp_nr = (u32)outs[1];
-
- if (ret == H_ALIAS_EXIST)
- ehca_gen_err("AQP1 already exists. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id)
-{
- u64 ret;
-
- ret = ehca_plpar_hcall_norets(H_ATTACH_MCQP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- mcg_dlid, /* r6 */
- interface_id, /* r7 */
- subnet_prefix, /* r8 */
- 0, 0);
-
- if (ret == H_NOT_ENOUGH_RESOURCES)
- ehca_gen_err("Not enough resources. ret=%lli", ret);
-
- return ret;
-}
-
-u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id)
-{
- return ehca_plpar_hcall_norets(H_DETACH_MCQP,
- adapter_handle.handle, /* r4 */
- qp_handle.handle, /* r5 */
- mcg_dlid, /* r6 */
- interface_id, /* r7 */
- subnet_prefix, /* r8 */
- 0, 0);
-}
-
-u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- u8 force_flag)
-{
- u64 ret;
-
- ret = hcp_galpas_dtor(&cq->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct cp->galpas");
- return H_RESOURCE;
- }
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- cq->ipz_cq_handle.handle, /* r5 */
- force_flag != 0 ? 1L : 0L, /* r6 */
- 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("H_FREE_RESOURCE failed ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_eq *eq)
-{
- u64 ret;
-
- ret = hcp_galpas_dtor(&eq->galpas);
- if (ret) {
- ehca_gen_err("Could not destruct eq->galpas");
- return H_RESOURCE;
- }
-
- ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- eq->ipz_eq_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-
- if (ret == H_RESOURCE)
- ehca_gen_err("Resource in use. ret=%lli ", ret);
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 5, /* r5 */
- vaddr, /* r6 */
- length, /* r7 */
- (((u64)access_ctrl) << 32ULL), /* r8 */
- pd.value, /* r9 */
- 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count)
-{
- u64 ret;
-
- if (unlikely(ehca_debug_level >= 3)) {
- if (count > 1) {
- u64 *kpage;
- int i;
- kpage = __va(logical_address_of_page);
- for (i = 0; i < count; i++)
- ehca_gen_dbg("kpage[%d]=%p",
- i, (void *)kpage[i]);
- } else
- ehca_gen_dbg("kpage=%p",
- (void *)logical_address_of_page);
- }
-
- if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
- ehca_gen_err("logical_address_of_page not on a 4k boundary "
- "adapter_handle=%llx mr=%p mr_handle=%llx "
- "pagesize=%x queue_type=%x "
- "logical_address_of_page=%llx count=%llx",
- adapter_handle.handle, mr,
- mr->ipz_mr_handle.handle, pagesize, queue_type,
- logical_address_of_page, count);
- ret = H_PARAMETER;
- } else
- ret = hipz_h_register_rpage(adapter_handle, pagesize,
- queue_type,
- mr->ipz_mr_handle.handle,
- logical_address_of_page, count);
- return ret;
-}
-
-u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_QUERY_MR, outs,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- 0, 0, 0, 0, 0, 0, 0);
- outparms->len = outs[0];
- outparms->vaddr = outs[1];
- outparms->acl = outs[4] >> 32;
- outparms->lkey = (u32)(outs[5] >> 32);
- outparms->rkey = (u32)(outs[5] & (0xffffffff));
-
- return ret;
-}
-
-u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr)
-{
- return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr_in,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- const u64 mr_addr_cb,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_REREGISTER_PMR, outs,
- adapter_handle.handle, /* r4 */
- mr->ipz_mr_handle.handle, /* r5 */
- vaddr_in, /* r6 */
- length, /* r7 */
- /* r8 */
- ((((u64)access_ctrl) << 32ULL) | pd.value),
- mr_addr_cb, /* r9 */
- 0, 0, 0);
- outparms->vaddr = outs[1];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const struct ehca_mr *orig_mr,
- const u64 vaddr_in,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_REGISTER_SMR, outs,
- adapter_handle.handle, /* r4 */
- orig_mr->ipz_mr_handle.handle, /* r5 */
- vaddr_in, /* r6 */
- (((u64)access_ctrl) << 32ULL), /* r7 */
- pd.value, /* r8 */
- 0, 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->lkey = (u32)outs[2];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- const struct ipz_pd pd,
- struct ehca_mw_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
- adapter_handle.handle, /* r4 */
- 6, /* r5 */
- pd.value, /* r6 */
- 0, 0, 0, 0, 0, 0);
- outparms->handle.handle = outs[0];
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- struct ehca_mw_hipzout_parms *outparms)
-{
- u64 ret;
- unsigned long outs[PLPAR_HCALL9_BUFSIZE];
-
- ret = ehca_plpar_hcall9(H_QUERY_MW, outs,
- adapter_handle.handle, /* r4 */
- mw->ipz_mw_handle.handle, /* r5 */
- 0, 0, 0, 0, 0, 0, 0);
- outparms->rkey = (u32)outs[3];
-
- return ret;
-}
-
-u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw)
-{
- return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
- adapter_handle.handle, /* r4 */
- mw->ipz_mw_handle.handle, /* r5 */
- 0, 0, 0, 0, 0);
-}
-
-u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
- const u64 ressource_handle,
- void *rblock,
- unsigned long *byte_count)
-{
- u64 r_cb = __pa(rblock);
-
- if (r_cb & (EHCA_PAGESIZE-1)) {
- ehca_gen_err("rblock not page aligned.");
- return H_PARAMETER;
- }
-
- return ehca_plpar_hcall_norets(H_ERROR_DATA,
- adapter_handle.handle,
- ressource_handle,
- r_cb,
- 0, 0, 0, 0);
-}
-
-u64 hipz_h_eoi(int irq)
-{
- unsigned long xirr;
-
- iosync();
- xirr = (0xffULL << 24) | irq;
-
- return plpar_hcall_norets(H_EOI, xirr);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware Infiniband Interface code for POWER
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HCP_IF_H__
-#define __HCP_IF_H__
-
-#include "ehca_classes.h"
-#include "ehca_tools.h"
-#include "hipz_hw.h"
-
-/*
- * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initialize
- * resources, create the empty EQPT (ring).
- */
-u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_pfeq *pfeq,
- const u32 neq_control,
- const u32 number_of_entries,
- struct ipz_eq_handle *eq_handle,
- u32 * act_nr_of_entries,
- u32 * act_pages,
- u32 * eq_ist);
-
-u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
- struct ipz_eq_handle eq_handle,
- const u64 event_mask);
-/*
- * hipz_h_allocate_resource_cq allocates CQ resources in HW and FW, initialize
- * resources, create the empty CQPT (ring).
- */
-u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- struct ehca_alloc_cq_parms *param);
-
-
-/*
- * hipz_h_alloc_resource_qp allocates QP resources in HW and FW,
- * initialize resources, create empty QPPTs (2 rings).
- */
-u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_alloc_qp_parms *parms, int is_user);
-
-u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id,
- struct hipz_query_port *query_port_response_block);
-
-u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
- const u8 port_id, const u32 port_cap,
- const u8 init_type, const int modify_mask);
-
-u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
- struct hipz_query_hca *query_hca_rblock);
-
-/*
- * hipz_h_register_rpage internal function in hcp_if.h for all
- * hcp_H_REGISTER_RPAGE calls.
- */
-u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
- const u8 pagesize,
- const u8 queue_type,
- const u64 resource_handle,
- const u64 logical_address_of_page,
- u64 count);
-
-u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_eq_handle eq_handle,
- struct ehca_pfeq *pfeq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count);
-
-u64 hipz_h_query_int_state(const struct ipz_adapter_handle
- hcp_adapter_handle,
- u32 ist);
-
-u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_cq_handle cq_handle,
- struct ehca_pfcq *pfcq,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa gal);
-
-u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count,
- const struct h_galpa galpa);
-
-u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- void **log_addr_next_sq_wqe_tb_processed,
- void **log_addr_next_rq_wqe_tb_processed,
- int dis_and_get_function_code);
-enum hcall_sigt {
- HCALL_SIGT_NO_CQE = 0,
- HCALL_SIGT_BY_WQE = 1,
- HCALL_SIGT_EVERY = 2
-};
-
-u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- const u64 update_mask,
- struct hcp_modify_qp_control_block *mqpcb,
- struct h_galpa gal);
-
-u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct ehca_pfqp *pfqp,
- struct hcp_modify_qp_control_block *qqpcb,
- struct h_galpa gal);
-
-u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
- struct ehca_qp *qp);
-
-u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port);
-
-u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u32 port, u32 * pma_qp_nr,
- u32 * bma_qp_nr);
-
-u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id);
-
-u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
- const struct ipz_qp_handle qp_handle,
- struct h_galpa gal,
- u16 mcg_dlid,
- u64 subnet_prefix, u64 interface_id);
-
-u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_cq *cq,
- u8 force_flag);
-
-u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
- struct ehca_eq *eq);
-
-/*
- * hipz_h_alloc_resource_mr allocates MR resources in HW and FW, initialize
- * resources.
- */
-u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_register_rpage_mr registers MR resource pages in HW and FW */
-u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u8 pagesize,
- const u8 queue_type,
- const u64 logical_address_of_page,
- const u64 count);
-
-/* hipz_h_query_mr queries MR in HW and FW */
-u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_free_resource_mr frees MR resources in HW and FW */
-u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr);
-
-/* hipz_h_reregister_pmr reregisters MR in HW and FW */
-u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const u64 vaddr_in,
- const u64 length,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- const u64 mr_addr_cb,
- struct ehca_mr_hipzout_parms *outparms);
-
-/* hipz_h_register_smr register shared MR in HW and FW */
-u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mr *mr,
- const struct ehca_mr *orig_mr,
- const u64 vaddr_in,
- const u32 access_ctrl,
- const struct ipz_pd pd,
- struct ehca_mr_hipzout_parms *outparms);
-
-/*
- * hipz_h_alloc_resource_mw allocates MW resources in HW and FW, initialize
- * resources.
- */
-u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- const struct ipz_pd pd,
- struct ehca_mw_hipzout_parms *outparms);
-
-/* hipz_h_query_mw queries MW in HW and FW */
-u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw,
- struct ehca_mw_hipzout_parms *outparms);
-
-/* hipz_h_free_resource_mw frees MW resources in HW and FW */
-u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
- const struct ehca_mw *mw);
-
-u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
- const u64 ressource_handle,
- void *rblock,
- unsigned long *byte_count);
-u64 hipz_h_eoi(int irq);
-
-#endif /* __HCP_IF_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * load store abstraction for ehca register access with tracing
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "ehca_classes.h"
-#include "hipz_hw.h"
-
-u64 hcall_map_page(u64 physaddr)
-{
- return (u64)ioremap(physaddr, EHCA_PAGESIZE);
-}
-
-int hcall_unmap_page(u64 mapaddr)
-{
- iounmap((volatile void __iomem *) mapaddr);
- return 0;
-}
-
-int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
- u64 paddr_kernel, u64 paddr_user)
-{
- if (!is_user) {
- galpas->kernel.fw_handle = hcall_map_page(paddr_kernel);
- if (!galpas->kernel.fw_handle)
- return -ENOMEM;
- } else
- galpas->kernel.fw_handle = 0;
-
- galpas->user.fw_handle = paddr_user;
-
- return 0;
-}
-
-int hcp_galpas_dtor(struct h_galpas *galpas)
-{
- if (galpas->kernel.fw_handle) {
- int ret = hcall_unmap_page(galpas->kernel.fw_handle);
- if (ret)
- return ret;
- }
-
- galpas->user.fw_handle = galpas->kernel.fw_handle = 0;
-
- return 0;
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * Firmware calls
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Waleri Fomin <fomin@de.ibm.com>
- * Gerd Bayer <gerd.bayer@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HCP_PHYP_H__
-#define __HCP_PHYP_H__
-
-
-/*
- * eHCA page (mapped into memory)
- * resource to access eHCA register pages in CPU address space
-*/
-struct h_galpa {
- u64 fw_handle;
- /* for pSeries this is a 64bit memory address where
- I/O memory is mapped into CPU address space (kv) */
-};
-
-/*
- * resource to access eHCA address space registers, all types
- */
-struct h_galpas {
- u32 pid; /*PID of userspace galpa checking */
- struct h_galpa user; /* user space accessible resource,
- set to 0 if unused */
- struct h_galpa kernel; /* kernel space accessible resource,
- set to 0 if unused */
-};
-
-static inline u64 hipz_galpa_load(struct h_galpa galpa, u32 offset)
-{
- u64 addr = galpa.fw_handle + offset;
- return *(volatile u64 __force *)addr;
-}
-
-static inline void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value)
-{
- u64 addr = galpa.fw_handle + offset;
- *(volatile u64 __force *)addr = value;
-}
-
-int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
- u64 paddr_kernel, u64 paddr_user);
-
-int hcp_galpas_dtor(struct h_galpas *galpas);
-
-u64 hcall_map_page(u64 physaddr);
-
-int hcall_unmap_page(u64 mapaddr);
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HW abstraction register functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_FNS_H__
-#define __HIPZ_FNS_H__
-
-#include "ehca_classes.h"
-#include "hipz_hw.h"
-
-#include "hipz_fns_core.h"
-
-#define hipz_galpa_store_eq(gal, offset, value) \
- hipz_galpa_store(gal, EQTEMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_eq(gal, offset) \
- hipz_galpa_load(gal, EQTEMM_OFFSET(offset))
-
-#define hipz_galpa_store_qped(gal, offset, value) \
- hipz_galpa_store(gal, QPEDMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_qped(gal, offset) \
- hipz_galpa_load(gal, QPEDMM_OFFSET(offset))
-
-#define hipz_galpa_store_mrmw(gal, offset, value) \
- hipz_galpa_store(gal, MRMWMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_mrmw(gal, offset) \
- hipz_galpa_load(gal, MRMWMM_OFFSET(offset))
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * HW abstraction register functions
- *
- * Authors: Christoph Raisch <raisch@de.ibm.com>
- * Heiko J Schick <schickhj@de.ibm.com>
- * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_FNS_CORE_H__
-#define __HIPZ_FNS_CORE_H__
-
-#include "hcp_phyp.h"
-#include "hipz_hw.h"
-
-#define hipz_galpa_store_cq(gal, offset, value) \
- hipz_galpa_store(gal, CQTEMM_OFFSET(offset), value)
-
-#define hipz_galpa_load_cq(gal, offset) \
- hipz_galpa_load(gal, CQTEMM_OFFSET(offset))
-
-#define hipz_galpa_store_qp(gal, offset, value) \
- hipz_galpa_store(gal, QPTEMM_OFFSET(offset), value)
-#define hipz_galpa_load_qp(gal, offset) \
- hipz_galpa_load(gal, QPTEMM_OFFSET(offset))
-
-static inline void hipz_update_sqa(struct ehca_qp *qp, u16 nr_wqes)
-{
- /* ringing doorbell :-) */
- hipz_galpa_store_qp(qp->galpas.kernel, qpx_sqa,
- EHCA_BMASK_SET(QPX_SQADDER, nr_wqes));
-}
-
-static inline void hipz_update_rqa(struct ehca_qp *qp, u16 nr_wqes)
-{
- /* ringing doorbell :-) */
- hipz_galpa_store_qp(qp->galpas.kernel, qpx_rqa,
- EHCA_BMASK_SET(QPX_RQADDER, nr_wqes));
-}
-
-static inline void hipz_update_feca(struct ehca_cq *cq, u32 nr_cqes)
-{
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_feca,
- EHCA_BMASK_SET(CQX_FECADDER, nr_cqes));
-}
-
-static inline void hipz_set_cqx_n0(struct ehca_cq *cq, u32 value)
-{
- u64 cqx_n0_reg;
-
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_n0,
- EHCA_BMASK_SET(CQX_N0_GENERATE_SOLICITED_COMP_EVENT,
- value));
- cqx_n0_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n0);
-}
-
-static inline void hipz_set_cqx_n1(struct ehca_cq *cq, u32 value)
-{
- u64 cqx_n1_reg;
-
- hipz_galpa_store_cq(cq->galpas.kernel, cqx_n1,
- EHCA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, value));
- cqx_n1_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n1);
-}
-
-#endif /* __HIPZ_FNC_CORE_H__ */
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * eHCA register definitions
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __HIPZ_HW_H__
-#define __HIPZ_HW_H__
-
-#include "ehca_tools.h"
-
-#define EHCA_MAX_MTU 4
-
-/* QP Table Entry Memory Map */
-struct hipz_qptemm {
- u64 qpx_hcr;
- u64 qpx_c;
- u64 qpx_herr;
- u64 qpx_aer;
-/* 0x20*/
- u64 qpx_sqa;
- u64 qpx_sqc;
- u64 qpx_rqa;
- u64 qpx_rqc;
-/* 0x40*/
- u64 qpx_st;
- u64 qpx_pmstate;
- u64 qpx_pmfa;
- u64 qpx_pkey;
-/* 0x60*/
- u64 qpx_pkeya;
- u64 qpx_pkeyb;
- u64 qpx_pkeyc;
- u64 qpx_pkeyd;
-/* 0x80*/
- u64 qpx_qkey;
- u64 qpx_dqp;
- u64 qpx_dlidp;
- u64 qpx_portp;
-/* 0xa0*/
- u64 qpx_slidp;
- u64 qpx_slidpp;
- u64 qpx_dlida;
- u64 qpx_porta;
-/* 0xc0*/
- u64 qpx_slida;
- u64 qpx_slidpa;
- u64 qpx_slvl;
- u64 qpx_ipd;
-/* 0xe0*/
- u64 qpx_mtu;
- u64 qpx_lato;
- u64 qpx_rlimit;
- u64 qpx_rnrlimit;
-/* 0x100*/
- u64 qpx_t;
- u64 qpx_sqhp;
- u64 qpx_sqptp;
- u64 qpx_nspsn;
-/* 0x120*/
- u64 qpx_nspsnhwm;
- u64 reserved1;
- u64 qpx_sdsi;
- u64 qpx_sdsbc;
-/* 0x140*/
- u64 qpx_sqwsize;
- u64 qpx_sqwts;
- u64 qpx_lsn;
- u64 qpx_nssn;
-/* 0x160 */
- u64 qpx_mor;
- u64 qpx_cor;
- u64 qpx_sqsize;
- u64 qpx_erc;
-/* 0x180*/
- u64 qpx_rnrrc;
- u64 qpx_ernrwt;
- u64 qpx_rnrresp;
- u64 qpx_lmsna;
-/* 0x1a0 */
- u64 qpx_sqhpc;
- u64 qpx_sqcptp;
- u64 qpx_sigt;
- u64 qpx_wqecnt;
-/* 0x1c0*/
- u64 qpx_rqhp;
- u64 qpx_rqptp;
- u64 qpx_rqsize;
- u64 qpx_nrr;
-/* 0x1e0*/
- u64 qpx_rdmac;
- u64 qpx_nrpsn;
- u64 qpx_lapsn;
- u64 qpx_lcr;
-/* 0x200*/
- u64 qpx_rwc;
- u64 qpx_rwva;
- u64 qpx_rdsi;
- u64 qpx_rdsbc;
-/* 0x220*/
- u64 qpx_rqwsize;
- u64 qpx_crmsn;
- u64 qpx_rdd;
- u64 qpx_larpsn;
-/* 0x240*/
- u64 qpx_pd;
- u64 qpx_scqn;
- u64 qpx_rcqn;
- u64 qpx_aeqn;
-/* 0x260*/
- u64 qpx_aaelog;
- u64 qpx_ram;
- u64 qpx_rdmaqe0;
- u64 qpx_rdmaqe1;
-/* 0x280*/
- u64 qpx_rdmaqe2;
- u64 qpx_rdmaqe3;
- u64 qpx_nrpsnhwm;
-/* 0x298*/
- u64 reserved[(0x400 - 0x298) / 8];
-/* 0x400 extended data */
- u64 reserved_ext[(0x500 - 0x400) / 8];
-/* 0x500 */
- u64 reserved2[(0x1000 - 0x500) / 8];
-/* 0x1000 */
-};
-
-#define QPX_SQADDER EHCA_BMASK_IBM(48, 63)
-#define QPX_RQADDER EHCA_BMASK_IBM(48, 63)
-#define QPX_AAELOG_RESET_SRQ_LIMIT EHCA_BMASK_IBM(3, 3)
-
-#define QPTEMM_OFFSET(x) offsetof(struct hipz_qptemm, x)
-
-/* MRMWPT Entry Memory Map */
-struct hipz_mrmwmm {
- /* 0x00 */
- u64 mrx_hcr;
-
- u64 mrx_c;
- u64 mrx_herr;
- u64 mrx_aer;
- /* 0x20 */
- u64 mrx_pp;
- u64 reserved1;
- u64 reserved2;
- u64 reserved3;
- /* 0x40 */
- u64 reserved4[(0x200 - 0x40) / 8];
- /* 0x200 */
- u64 mrx_ctl[64];
-
-};
-
-#define MRMWMM_OFFSET(x) offsetof(struct hipz_mrmwmm, x)
-
-struct hipz_qpedmm {
- /* 0x00 */
- u64 reserved0[(0x400) / 8];
- /* 0x400 */
- u64 qpedx_phh;
- u64 qpedx_ppsgp;
- /* 0x410 */
- u64 qpedx_ppsgu;
- u64 qpedx_ppdgp;
- /* 0x420 */
- u64 qpedx_ppdgu;
- u64 qpedx_aph;
- /* 0x430 */
- u64 qpedx_apsgp;
- u64 qpedx_apsgu;
- /* 0x440 */
- u64 qpedx_apdgp;
- u64 qpedx_apdgu;
- /* 0x450 */
- u64 qpedx_apav;
- u64 qpedx_apsav;
- /* 0x460 */
- u64 qpedx_hcr;
- u64 reserved1[4];
- /* 0x488 */
- u64 qpedx_rrl0;
- /* 0x490 */
- u64 qpedx_rrrkey0;
- u64 qpedx_rrva0;
- /* 0x4a0 */
- u64 reserved2;
- u64 qpedx_rrl1;
- /* 0x4b0 */
- u64 qpedx_rrrkey1;
- u64 qpedx_rrva1;
- /* 0x4c0 */
- u64 reserved3;
- u64 qpedx_rrl2;
- /* 0x4d0 */
- u64 qpedx_rrrkey2;
- u64 qpedx_rrva2;
- /* 0x4e0 */
- u64 reserved4;
- u64 qpedx_rrl3;
- /* 0x4f0 */
- u64 qpedx_rrrkey3;
- u64 qpedx_rrva3;
-};
-
-#define QPEDMM_OFFSET(x) offsetof(struct hipz_qpedmm, x)
-
-/* CQ Table Entry Memory Map */
-struct hipz_cqtemm {
- u64 cqx_hcr;
- u64 cqx_c;
- u64 cqx_herr;
- u64 cqx_aer;
-/* 0x20 */
- u64 cqx_ptp;
- u64 cqx_tp;
- u64 cqx_fec;
- u64 cqx_feca;
-/* 0x40 */
- u64 cqx_ep;
- u64 cqx_eq;
-/* 0x50 */
- u64 reserved1;
- u64 cqx_n0;
-/* 0x60 */
- u64 cqx_n1;
- u64 reserved2[(0x1000 - 0x60) / 8];
-/* 0x1000 */
-};
-
-#define CQX_FEC_CQE_CNT EHCA_BMASK_IBM(32, 63)
-#define CQX_FECADDER EHCA_BMASK_IBM(32, 63)
-#define CQX_N0_GENERATE_SOLICITED_COMP_EVENT EHCA_BMASK_IBM(0, 0)
-#define CQX_N1_GENERATE_COMP_EVENT EHCA_BMASK_IBM(0, 0)
-
-#define CQTEMM_OFFSET(x) offsetof(struct hipz_cqtemm, x)
-
-/* EQ Table Entry Memory Map */
-struct hipz_eqtemm {
- u64 eqx_hcr;
- u64 eqx_c;
-
- u64 eqx_herr;
- u64 eqx_aer;
-/* 0x20 */
- u64 eqx_ptp;
- u64 eqx_tp;
- u64 eqx_ssba;
- u64 eqx_psba;
-
-/* 0x40 */
- u64 eqx_cec;
- u64 eqx_meql;
- u64 eqx_xisbi;
- u64 eqx_xisc;
-/* 0x60 */
- u64 eqx_it;
-
-};
-
-#define EQTEMM_OFFSET(x) offsetof(struct hipz_eqtemm, x)
-
-/* access control defines for MR/MW */
-#define HIPZ_ACCESSCTRL_L_WRITE 0x00800000
-#define HIPZ_ACCESSCTRL_R_WRITE 0x00400000
-#define HIPZ_ACCESSCTRL_R_READ 0x00200000
-#define HIPZ_ACCESSCTRL_R_ATOMIC 0x00100000
-#define HIPZ_ACCESSCTRL_MW_BIND 0x00080000
-
-/* query hca response block */
-struct hipz_query_hca {
- u32 cur_reliable_dg;
- u32 cur_qp;
- u32 cur_cq;
- u32 cur_eq;
- u32 cur_mr;
- u32 cur_mw;
- u32 cur_ee_context;
- u32 cur_mcast_grp;
- u32 cur_qp_attached_mcast_grp;
- u32 reserved1;
- u32 cur_ipv6_qp;
- u32 cur_eth_qp;
- u32 cur_hp_mr;
- u32 reserved2[3];
- u32 max_rd_domain;
- u32 max_qp;
- u32 max_cq;
- u32 max_eq;
- u32 max_mr;
- u32 max_hp_mr;
- u32 max_mw;
- u32 max_mrwpte;
- u32 max_special_mrwpte;
- u32 max_rd_ee_context;
- u32 max_mcast_grp;
- u32 max_total_mcast_qp_attach;
- u32 max_mcast_qp_attach;
- u32 max_raw_ipv6_qp;
- u32 max_raw_ethy_qp;
- u32 internal_clock_frequency;
- u32 max_pd;
- u32 max_ah;
- u32 max_cqe;
- u32 max_wqes_wq;
- u32 max_partitions;
- u32 max_rr_ee_context;
- u32 max_rr_qp;
- u32 max_rr_hca;
- u32 max_act_wqs_ee_context;
- u32 max_act_wqs_qp;
- u32 max_sge;
- u32 max_sge_rd;
- u32 memory_page_size_supported;
- u64 max_mr_size;
- u32 local_ca_ack_delay;
- u32 num_ports;
- u32 vendor_id;
- u32 vendor_part_id;
- u32 hw_ver;
- u64 node_guid;
- u64 hca_cap_indicators;
- u32 data_counter_register_size;
- u32 max_shared_rq;
- u32 max_isns_eq;
- u32 max_neq;
-} __attribute__ ((packed));
-
-#define HCA_CAP_AH_PORT_NR_CHECK EHCA_BMASK_IBM( 0, 0)
-#define HCA_CAP_ATOMIC EHCA_BMASK_IBM( 1, 1)
-#define HCA_CAP_AUTO_PATH_MIG EHCA_BMASK_IBM( 2, 2)
-#define HCA_CAP_BAD_P_KEY_CTR EHCA_BMASK_IBM( 3, 3)
-#define HCA_CAP_SQD_RTS_PORT_CHANGE EHCA_BMASK_IBM( 4, 4)
-#define HCA_CAP_CUR_QP_STATE_MOD EHCA_BMASK_IBM( 5, 5)
-#define HCA_CAP_INIT_TYPE EHCA_BMASK_IBM( 6, 6)
-#define HCA_CAP_PORT_ACTIVE_EVENT EHCA_BMASK_IBM( 7, 7)
-#define HCA_CAP_Q_KEY_VIOL_CTR EHCA_BMASK_IBM( 8, 8)
-#define HCA_CAP_WQE_RESIZE EHCA_BMASK_IBM( 9, 9)
-#define HCA_CAP_RAW_PACKET_MCAST EHCA_BMASK_IBM(10, 10)
-#define HCA_CAP_SHUTDOWN_PORT EHCA_BMASK_IBM(11, 11)
-#define HCA_CAP_RC_LL_QP EHCA_BMASK_IBM(12, 12)
-#define HCA_CAP_SRQ EHCA_BMASK_IBM(13, 13)
-#define HCA_CAP_UD_LL_QP EHCA_BMASK_IBM(16, 16)
-#define HCA_CAP_RESIZE_MR EHCA_BMASK_IBM(17, 17)
-#define HCA_CAP_MINI_QP EHCA_BMASK_IBM(18, 18)
-#define HCA_CAP_H_ALLOC_RES_SYNC EHCA_BMASK_IBM(19, 19)
-
-/* query port response block */
-struct hipz_query_port {
- u32 state;
- u32 bad_pkey_cntr;
- u32 lmc;
- u32 lid;
- u32 subnet_timeout;
- u32 qkey_viol_cntr;
- u32 sm_sl;
- u32 sm_lid;
- u32 capability_mask;
- u32 init_type_reply;
- u32 pkey_tbl_len;
- u32 gid_tbl_len;
- u64 gid_prefix;
- u32 port_nr;
- u16 pkey_entries[16];
- u8 reserved1[32];
- u32 trent_size;
- u32 trbuf_size;
- u64 max_msg_sz;
- u32 max_mtu;
- u32 vl_cap;
- u32 phys_pstate;
- u32 phys_state;
- u32 phys_speed;
- u32 phys_width;
- u8 reserved2[1884];
- u64 guid_entries[255];
-} __attribute__ ((packed));
-
-#endif
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * internal queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <linux/slab.h>
-
-#include "ehca_tools.h"
-#include "ipz_pt_fn.h"
-#include "ehca_classes.h"
-
-#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
-
-struct kmem_cache *small_qp_cache;
-
-void *ipz_qpageit_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- queue->current_q_offset += queue->pagesize;
- if (queue->current_q_offset > queue->queue_length) {
- queue->current_q_offset -= queue->pagesize;
- ret = NULL;
- }
- if (((u64)ret) % queue->pagesize) {
- ehca_gen_err("ERROR!! not at PAGE-Boundary");
- return NULL;
- }
- return ret;
-}
-
-void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u64 last_entry_in_q = queue->queue_length - queue->qe_size;
-
- queue->current_q_offset += queue->qe_size;
- if (queue->current_q_offset > last_entry_in_q) {
- queue->current_q_offset = 0;
- queue->toggle_state = (~queue->toggle_state) & 1;
- }
-
- return ret;
-}
-
-int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
-{
- int i;
- for (i = 0; i < queue->queue_length / queue->pagesize; i++) {
- u64 page = __pa(queue->queue_pages[i]);
- if (addr >= page && addr < page + queue->pagesize) {
- *q_offset = addr - page + i * queue->pagesize;
- return 0;
- }
- }
- return -EINVAL;
-}
-
-#if PAGE_SHIFT < EHCA_PAGESHIFT
-#error Kernel pages must be at least as large than eHCA pages (4K) !
-#endif
-
-/*
- * allocate pages for queue:
- * outer loop allocates whole kernel pages (page aligned) and
- * inner loop divides a kernel page into smaller hca queue pages
- */
-static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
-{
- int k, f = 0;
- u8 *kpage;
-
- while (f < nr_of_pages) {
- kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
- if (!kpage)
- goto out;
-
- for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
- queue->queue_pages[f] = (struct ipz_page *)kpage;
- kpage += EHCA_PAGESIZE;
- f++;
- }
- }
- return 1;
-
-out:
- for (f = 0; f < nr_of_pages && queue->queue_pages[f];
- f += PAGES_PER_KPAGE)
- free_page((unsigned long)(queue->queue_pages)[f]);
- return 0;
-}
-
-static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
-{
- int order = ilog2(queue->pagesize) - 9;
- struct ipz_small_queue_page *page;
- unsigned long bit;
-
- mutex_lock(&pd->lock);
-
- if (!list_empty(&pd->free[order]))
- page = list_entry(pd->free[order].next,
- struct ipz_small_queue_page, list);
- else {
- page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
- if (!page)
- goto out;
-
- page->page = get_zeroed_page(GFP_KERNEL);
- if (!page->page) {
- kmem_cache_free(small_qp_cache, page);
- goto out;
- }
-
- list_add(&page->list, &pd->free[order]);
- }
-
- bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
- __set_bit(bit, page->bitmap);
- page->fill++;
-
- if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
- list_move(&page->list, &pd->full[order]);
-
- mutex_unlock(&pd->lock);
-
- queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
- queue->small_page = page;
- queue->offset = bit << (order + 9);
- return 1;
-
-out:
- ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
- mutex_unlock(&pd->lock);
- return 0;
-}
-
-static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
-{
- int order = ilog2(queue->pagesize) - 9;
- struct ipz_small_queue_page *page = queue->small_page;
- unsigned long bit;
- int free_page = 0;
-
- bit = ((unsigned long)queue->queue_pages[0] & ~PAGE_MASK)
- >> (order + 9);
-
- mutex_lock(&pd->lock);
-
- __clear_bit(bit, page->bitmap);
- page->fill--;
-
- if (page->fill == 0) {
- list_del(&page->list);
- free_page = 1;
- }
-
- if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
- /* the page was full until we freed the chunk */
- list_move_tail(&page->list, &pd->free[order]);
-
- mutex_unlock(&pd->lock);
-
- if (free_page) {
- free_page(page->page);
- kmem_cache_free(small_qp_cache, page);
- }
-}
-
-int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
- const u32 nr_of_pages, const u32 pagesize,
- const u32 qe_size, const u32 nr_of_sg,
- int is_small)
-{
- if (pagesize > PAGE_SIZE) {
- ehca_gen_err("FATAL ERROR: pagesize=%x "
- "is greater than kernel page size", pagesize);
- return 0;
- }
-
- /* init queue fields */
- queue->queue_length = nr_of_pages * pagesize;
- queue->pagesize = pagesize;
- queue->qe_size = qe_size;
- queue->act_nr_of_sg = nr_of_sg;
- queue->current_q_offset = 0;
- queue->toggle_state = 1;
- queue->small_page = NULL;
-
- /* allocate queue page pointers */
- queue->queue_pages = kzalloc(nr_of_pages * sizeof(void *),
- GFP_KERNEL | __GFP_NOWARN);
- if (!queue->queue_pages) {
- queue->queue_pages = vzalloc(nr_of_pages * sizeof(void *));
- if (!queue->queue_pages) {
- ehca_gen_err("Couldn't allocate queue page list");
- return 0;
- }
- }
-
- /* allocate actual queue pages */
- if (is_small) {
- if (!alloc_small_queue_page(queue, pd))
- goto ipz_queue_ctor_exit0;
- } else
- if (!alloc_queue_pages(queue, nr_of_pages))
- goto ipz_queue_ctor_exit0;
-
- return 1;
-
-ipz_queue_ctor_exit0:
- ehca_gen_err("Couldn't alloc pages queue=%p "
- "nr_of_pages=%x", queue, nr_of_pages);
- kvfree(queue->queue_pages);
-
- return 0;
-}
-
-int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
-{
- int i, nr_pages;
-
- if (!queue || !queue->queue_pages) {
- ehca_gen_dbg("queue or queue_pages is NULL");
- return 0;
- }
-
- if (queue->small_page)
- free_small_queue_page(queue, pd);
- else {
- nr_pages = queue->queue_length / queue->pagesize;
- for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
- free_page((unsigned long)queue->queue_pages[i]);
- }
-
- kvfree(queue->queue_pages);
-
- return 1;
-}
-
-int ehca_init_small_qp_cache(void)
-{
- small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
- sizeof(struct ipz_small_queue_page),
- 0, SLAB_HWCACHE_ALIGN, NULL);
- if (!small_qp_cache)
- return -ENOMEM;
-
- return 0;
-}
-
-void ehca_cleanup_small_qp_cache(void)
-{
- kmem_cache_destroy(small_qp_cache);
-}
+++ /dev/null
-/*
- * IBM eServer eHCA Infiniband device driver for Linux on POWER
- *
- * internal queue handling
- *
- * Authors: Waleri Fomin <fomin@de.ibm.com>
- * Reinhard Ernst <rernst@de.ibm.com>
- * Christoph Raisch <raisch@de.ibm.com>
- *
- * Copyright (c) 2005 IBM Corporation
- *
- * All rights reserved.
- *
- * This source code is distributed under a dual license of GPL v2.0 and OpenIB
- * BSD.
- *
- * OpenIB BSD License
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- * Redistributions of source code must retain the above copyright notice, this
- * list of conditions and the following disclaimer.
- *
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials
- * provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
- * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef __IPZ_PT_FN_H__
-#define __IPZ_PT_FN_H__
-
-#define EHCA_PAGESHIFT 12
-#define EHCA_PAGESIZE 4096UL
-#define EHCA_PAGEMASK (~(EHCA_PAGESIZE-1))
-#define EHCA_PT_ENTRIES 512UL
-
-#include "ehca_tools.h"
-#include "ehca_qes.h"
-
-struct ehca_pd;
-struct ipz_small_queue_page;
-
-extern struct kmem_cache *small_qp_cache;
-
-/* struct generic ehca page */
-struct ipz_page {
- u8 entries[EHCA_PAGESIZE];
-};
-
-#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
-
-struct ipz_small_queue_page {
- unsigned long page;
- unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
- int fill;
- void *mapped_addr;
- u32 mmap_count;
- struct list_head list;
-};
-
-/* struct generic queue in linux kernel virtual memory (kv) */
-struct ipz_queue {
- u64 current_q_offset; /* current queue entry */
-
- struct ipz_page **queue_pages; /* array of pages belonging to queue */
- u32 qe_size; /* queue entry size */
- u32 act_nr_of_sg;
- u32 queue_length; /* queue length allocated in bytes */
- u32 pagesize;
- u32 toggle_state; /* toggle flag - per page */
- u32 offset; /* save offset within page for small_qp */
- struct ipz_small_queue_page *small_page;
-};
-
-/*
- * return current Queue Entry for a certain q_offset
- * returns address (kv) of Queue Entry
- */
-static inline void *ipz_qeit_calc(struct ipz_queue *queue, u64 q_offset)
-{
- struct ipz_page *current_page;
- if (q_offset >= queue->queue_length)
- return NULL;
- current_page = (queue->queue_pages)[q_offset >> EHCA_PAGESHIFT];
- return ¤t_page->entries[q_offset & (EHCA_PAGESIZE - 1)];
-}
-
-/*
- * return current Queue Entry
- * returns address (kv) of Queue Entry
- */
-static inline void *ipz_qeit_get(struct ipz_queue *queue)
-{
- return ipz_qeit_calc(queue, queue->current_q_offset);
-}
-
-/*
- * return current Queue Page , increment Queue Page iterator from
- * page to page in struct ipz_queue, last increment will return 0! and
- * NOT wrap
- * returns address (kv) of Queue Page
- * warning don't use in parallel with ipz_QE_get_inc()
- */
-void *ipz_qpageit_get_inc(struct ipz_queue *queue);
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * warning don't use in parallel with ipz_qpageit_get_inc()
- */
-static inline void *ipz_qeit_get_inc(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- queue->current_q_offset += queue->qe_size;
- if (queue->current_q_offset >= queue->queue_length) {
- queue->current_q_offset = 0;
- /* toggle the valid flag */
- queue->toggle_state = (~queue->toggle_state) & 1;
- }
-
- return ret;
-}
-
-/*
- * return a bool indicating whether current Queue Entry is valid
- */
-static inline int ipz_qeit_is_valid(struct ipz_queue *queue)
-{
- struct ehca_cqe *cqe = ipz_qeit_get(queue);
- return ((cqe->cqe_flags >> 7) == (queue->toggle_state & 1));
-}
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * returns 0 and does not increment, if wrong valid state
- * warning don't use in parallel with ipz_qpageit_get_inc()
- */
-static inline void *ipz_qeit_get_inc_valid(struct ipz_queue *queue)
-{
- return ipz_qeit_is_valid(queue) ? ipz_qeit_get_inc(queue) : NULL;
-}
-
-/*
- * returns and resets Queue Entry iterator
- * returns address (kv) of first Queue Entry
- */
-static inline void *ipz_qeit_reset(struct ipz_queue *queue)
-{
- queue->current_q_offset = 0;
- return ipz_qeit_get(queue);
-}
-
-/*
- * return the q_offset corresponding to an absolute address
- */
-int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset);
-
-/*
- * return the next queue offset. don't modify the queue.
- */
-static inline u64 ipz_queue_advance_offset(struct ipz_queue *queue, u64 offset)
-{
- offset += queue->qe_size;
- if (offset >= queue->queue_length) offset = 0;
- return offset;
-}
-
-/* struct generic page table */
-struct ipz_pt {
- u64 entries[EHCA_PT_ENTRIES];
-};
-
-/* struct page table for a queue, only to be used in pf */
-struct ipz_qpt {
- /* queue page tables (kv), use u64 because we know the element length */
- u64 *qpts;
- u32 n_qpts;
- u32 n_ptes; /* number of page table entries */
- u64 *current_pte_addr;
-};
-
-/*
- * constructor for a ipz_queue_t, placement new for ipz_queue_t,
- * new for all dependent datastructors
- * all QP Tables are the same
- * flow:
- * allocate+pin queue
- * see ipz_qpt_ctor()
- * returns true if ok, false if out of memory
- */
-int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
- const u32 nr_of_pages, const u32 pagesize,
- const u32 qe_size, const u32 nr_of_sg,
- int is_small);
-
-/*
- * destructor for a ipz_queue_t
- * -# free queue
- * see ipz_queue_ctor()
- * returns true if ok, false if queue was NULL-ptr of free failed
- */
-int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
-
-/*
- * constructor for a ipz_qpt_t,
- * placement new for struct ipz_queue, new for all dependent datastructors
- * all QP Tables are the same,
- * flow:
- * -# allocate+pin queue
- * -# initialise ptcb
- * -# allocate+pin PTs
- * -# link PTs to a ring, according to HCA Arch, set bit62 id needed
- * -# the ring must have room for exactly nr_of_PTEs
- * see ipz_qpt_ctor()
- */
-void ipz_qpt_ctor(struct ipz_qpt *qpt,
- const u32 nr_of_qes,
- const u32 pagesize,
- const u32 qe_size,
- const u8 lowbyte, const u8 toggle,
- u32 * act_nr_of_QEs, u32 * act_nr_of_pages);
-
-/*
- * return current Queue Entry, increment Queue Entry iterator by one
- * step in struct ipz_queue, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * warning don't use in parallel with ipz_qpageit_get_inc()
- * warning unpredictable results may occur if steps>act_nr_of_queue_entries
- * fix EQ page problems
- */
-void *ipz_qeit_eq_get_inc(struct ipz_queue *queue);
-
-/*
- * return current Event Queue Entry, increment Queue Entry iterator
- * by one step in struct ipz_queue if valid, will wrap in ringbuffer
- * returns address (kv) of Queue Entry BEFORE increment
- * returns 0 and does not increment, if wrong valid state
- * warning don't use in parallel with ipz_queue_QPageit_get_inc()
- * warning unpredictable results may occur if steps>act_nr_of_queue_entries
- */
-static inline void *ipz_eqit_eq_get_inc_valid(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u32 qe = *(u8 *)ret;
- if ((qe >> 7) != (queue->toggle_state & 1))
- return NULL;
- ipz_qeit_eq_get_inc(queue); /* this is a good one */
- return ret;
-}
-
-static inline void *ipz_eqit_eq_peek_valid(struct ipz_queue *queue)
-{
- void *ret = ipz_qeit_get(queue);
- u32 qe = *(u8 *)ret;
- if ((qe >> 7) != (queue->toggle_state & 1))
- return NULL;
- return ret;
-}
-
-/* returns address (GX) of first queue entry */
-static inline u64 ipz_qpt_get_firstpage(struct ipz_qpt *qpt)
-{
- return be64_to_cpu(qpt->qpts[0]);
-}
-
-/* returns address (kv) of first page of queue page table */
-static inline void *ipz_qpt_get_qpt(struct ipz_qpt *qpt)
-{
- return qpt->qpts;
-}
-
-#endif /* __IPZ_PT_FN_H__ */
source "drivers/staging/rdma/amso1100/Kconfig"
+source "drivers/staging/rdma/ehca/Kconfig"
+
source "drivers/staging/rdma/hfi1/Kconfig"
source "drivers/staging/rdma/ipath/Kconfig"
# Entries for RDMA_STAGING tree
obj-$(CONFIG_INFINIBAND_AMSO1100) += amso1100/
+obj-$(CONFIG_INFINIBAND_EHCA) += ehca/
obj-$(CONFIG_INFINIBAND_HFI1) += hfi1/
obj-$(CONFIG_INFINIBAND_IPATH) += ipath/
--- /dev/null
+config INFINIBAND_EHCA
+ tristate "eHCA support"
+ depends on IBMEBUS
+ ---help---
+ This driver supports the deprecated IBM pSeries eHCA InfiniBand
+ adapter.
+
+ To compile the driver as a module, choose M here. The module
+ will be called ib_ehca.
+
--- /dev/null
+# Authors: Heiko J Schick <schickhj@de.ibm.com>
+# Christoph Raisch <raisch@de.ibm.com>
+# Joachim Fenkes <fenkes@de.ibm.com>
+#
+# Copyright (c) 2005 IBM Corporation
+#
+# All rights reserved.
+#
+# This source code is distributed under a dual license of GPL v2.0 and OpenIB BSD.
+
+obj-$(CONFIG_INFINIBAND_EHCA) += ib_ehca.o
+
+ib_ehca-objs = ehca_main.o ehca_hca.o ehca_mcast.o ehca_pd.o ehca_av.o ehca_eq.o \
+ ehca_cq.o ehca_qp.o ehca_sqp.o ehca_mrmw.o ehca_reqs.o ehca_irq.o \
+ ehca_uverbs.o ipz_pt_fn.o hcp_if.o hcp_phyp.o
+
--- /dev/null
+9/2015
+
+The ehca driver has been deprecated and moved to drivers/staging/rdma.
+It will be removed in the 4.6 merge window.
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * address vector functions
+ *
+ * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Khadija Souissi <souissik@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *av_cache;
+
+int ehca_calc_ipd(struct ehca_shca *shca, int port,
+ enum ib_rate path_rate, u32 *ipd)
+{
+ int path = ib_rate_to_mult(path_rate);
+ int link, ret;
+ struct ib_port_attr pa;
+
+ if (path_rate == IB_RATE_PORT_CURRENT) {
+ *ipd = 0;
+ return 0;
+ }
+
+ if (unlikely(path < 0)) {
+ ehca_err(&shca->ib_device, "Invalid static rate! path_rate=%x",
+ path_rate);
+ return -EINVAL;
+ }
+
+ ret = ehca_query_port(&shca->ib_device, port, &pa);
+ if (unlikely(ret < 0)) {
+ ehca_err(&shca->ib_device, "Failed to query port ret=%i", ret);
+ return ret;
+ }
+
+ link = ib_width_enum_to_int(pa.active_width) * pa.active_speed;
+
+ if (path >= link)
+ /* no need to throttle if path faster than link */
+ *ipd = 0;
+ else
+ /* IPD = round((link / path) - 1) */
+ *ipd = ((link + (path >> 1)) / path) - 1;
+
+ return 0;
+}
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+{
+ int ret;
+ struct ehca_av *av;
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+
+ av = kmem_cache_alloc(av_cache, GFP_KERNEL);
+ if (!av) {
+ ehca_err(pd->device, "Out of memory pd=%p ah_attr=%p",
+ pd, ah_attr);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ av->av.sl = ah_attr->sl;
+ av->av.dlid = ah_attr->dlid;
+ av->av.slid_path_bits = ah_attr->src_path_bits;
+
+ if (ehca_static_rate < 0) {
+ u32 ipd;
+ if (ehca_calc_ipd(shca, ah_attr->port_num,
+ ah_attr->static_rate, &ipd)) {
+ ret = -EINVAL;
+ goto create_ah_exit1;
+ }
+ av->av.ipd = ipd;
+ } else
+ av->av.ipd = ehca_static_rate;
+
+ av->av.lnh = ah_attr->ah_flags;
+ av->av.grh.word_0 = EHCA_BMASK_SET(GRH_IPVERSION_MASK, 6);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_TCLASS_MASK,
+ ah_attr->grh.traffic_class);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+ ah_attr->grh.flow_label);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+ ah_attr->grh.hop_limit);
+ av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1B);
+ /* set sgid in grh.word_1 */
+ if (ah_attr->ah_flags & IB_AH_GRH) {
+ int rc;
+ struct ib_port_attr port_attr;
+ union ib_gid gid;
+ memset(&port_attr, 0, sizeof(port_attr));
+ rc = ehca_query_port(pd->device, ah_attr->port_num,
+ &port_attr);
+ if (rc) { /* invalid port number */
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid port number "
+ "ehca_query_port() returned %x "
+ "pd=%p ah_attr=%p", rc, pd, ah_attr);
+ goto create_ah_exit1;
+ }
+ memset(&gid, 0, sizeof(gid));
+ rc = ehca_query_gid(pd->device,
+ ah_attr->port_num,
+ ah_attr->grh.sgid_index, &gid);
+ if (rc) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Failed to retrieve sgid "
+ "ehca_query_gid() returned %x "
+ "pd=%p ah_attr=%p", rc, pd, ah_attr);
+ goto create_ah_exit1;
+ }
+ memcpy(&av->av.grh.word_1, &gid, sizeof(gid));
+ }
+ av->av.pmtu = shca->max_mtu;
+
+ /* dgid comes in grh.word_3 */
+ memcpy(&av->av.grh.word_3, &ah_attr->grh.dgid,
+ sizeof(ah_attr->grh.dgid));
+
+ return &av->ib_ah;
+
+create_ah_exit1:
+ kmem_cache_free(av_cache, av);
+
+ return ERR_PTR(ret);
+}
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+ struct ehca_av *av;
+ struct ehca_ud_av new_ehca_av;
+ struct ehca_shca *shca = container_of(ah->pd->device, struct ehca_shca,
+ ib_device);
+
+ memset(&new_ehca_av, 0, sizeof(new_ehca_av));
+ new_ehca_av.sl = ah_attr->sl;
+ new_ehca_av.dlid = ah_attr->dlid;
+ new_ehca_av.slid_path_bits = ah_attr->src_path_bits;
+ new_ehca_av.ipd = ah_attr->static_rate;
+ new_ehca_av.lnh = EHCA_BMASK_SET(GRH_FLAG_MASK,
+ (ah_attr->ah_flags & IB_AH_GRH) > 0);
+ new_ehca_av.grh.word_0 = EHCA_BMASK_SET(GRH_TCLASS_MASK,
+ ah_attr->grh.traffic_class);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+ ah_attr->grh.flow_label);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+ ah_attr->grh.hop_limit);
+ new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1b);
+
+ /* set sgid in grh.word_1 */
+ if (ah_attr->ah_flags & IB_AH_GRH) {
+ int rc;
+ struct ib_port_attr port_attr;
+ union ib_gid gid;
+ memset(&port_attr, 0, sizeof(port_attr));
+ rc = ehca_query_port(ah->device, ah_attr->port_num,
+ &port_attr);
+ if (rc) { /* invalid port number */
+ ehca_err(ah->device, "Invalid port number "
+ "ehca_query_port() returned %x "
+ "ah=%p ah_attr=%p port_num=%x",
+ rc, ah, ah_attr, ah_attr->port_num);
+ return -EINVAL;
+ }
+ memset(&gid, 0, sizeof(gid));
+ rc = ehca_query_gid(ah->device,
+ ah_attr->port_num,
+ ah_attr->grh.sgid_index, &gid);
+ if (rc) {
+ ehca_err(ah->device, "Failed to retrieve sgid "
+ "ehca_query_gid() returned %x "
+ "ah=%p ah_attr=%p port_num=%x "
+ "sgid_index=%x",
+ rc, ah, ah_attr, ah_attr->port_num,
+ ah_attr->grh.sgid_index);
+ return -EINVAL;
+ }
+ memcpy(&new_ehca_av.grh.word_1, &gid, sizeof(gid));
+ }
+
+ new_ehca_av.pmtu = shca->max_mtu;
+
+ memcpy(&new_ehca_av.grh.word_3, &ah_attr->grh.dgid,
+ sizeof(ah_attr->grh.dgid));
+
+ av = container_of(ah, struct ehca_av, ib_ah);
+ av->av = new_ehca_av;
+
+ return 0;
+}
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+ struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
+
+ memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
+ sizeof(ah_attr->grh.dgid));
+ ah_attr->sl = av->av.sl;
+
+ ah_attr->dlid = av->av.dlid;
+
+ ah_attr->src_path_bits = av->av.slid_path_bits;
+ ah_attr->static_rate = av->av.ipd;
+ ah_attr->ah_flags = EHCA_BMASK_GET(GRH_FLAG_MASK, av->av.lnh);
+ ah_attr->grh.traffic_class = EHCA_BMASK_GET(GRH_TCLASS_MASK,
+ av->av.grh.word_0);
+ ah_attr->grh.hop_limit = EHCA_BMASK_GET(GRH_HOPLIMIT_MASK,
+ av->av.grh.word_0);
+ ah_attr->grh.flow_label = EHCA_BMASK_GET(GRH_FLOWLABEL_MASK,
+ av->av.grh.word_0);
+
+ return 0;
+}
+
+int ehca_destroy_ah(struct ib_ah *ah)
+{
+ kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
+
+ return 0;
+}
+
+int ehca_init_av_cache(void)
+{
+ av_cache = kmem_cache_create("ehca_cache_av",
+ sizeof(struct ehca_av), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!av_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_av_cache(void)
+{
+ if (av_cache)
+ kmem_cache_destroy(av_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Struct definition for eHCA internal structures
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_H__
+#define __EHCA_CLASSES_H__
+
+struct ehca_module;
+struct ehca_qp;
+struct ehca_cq;
+struct ehca_eq;
+struct ehca_mr;
+struct ehca_mw;
+struct ehca_pd;
+struct ehca_av;
+
+#include <linux/wait.h>
+#include <linux/mutex.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#ifdef CONFIG_PPC64
+#include "ehca_classes_pSeries.h"
+#endif
+#include "ipz_pt_fn.h"
+#include "ehca_qes.h"
+#include "ehca_irq.h"
+
+#define EHCA_EQE_CACHE_SIZE 20
+#define EHCA_MAX_NUM_QUEUES 0xffff
+
+struct ehca_eqe_cache_entry {
+ struct ehca_eqe *eqe;
+ struct ehca_cq *cq;
+};
+
+struct ehca_eq {
+ u32 length;
+ struct ipz_queue ipz_queue;
+ struct ipz_eq_handle ipz_eq_handle;
+ struct work_struct work;
+ struct h_galpas galpas;
+ int is_initialized;
+ struct ehca_pfeq pf;
+ spinlock_t spinlock;
+ struct tasklet_struct interrupt_task;
+ u32 ist;
+ spinlock_t irq_spinlock;
+ struct ehca_eqe_cache_entry eqe_cache[EHCA_EQE_CACHE_SIZE];
+};
+
+struct ehca_sma_attr {
+ u16 lid, lmc, sm_sl, sm_lid;
+ u16 pkey_tbl_len, pkeys[16];
+};
+
+struct ehca_sport {
+ struct ib_cq *ibcq_aqp1;
+ struct ib_qp *ibqp_sqp[2];
+ /* lock to serialze modify_qp() calls for sqp in normal
+ * and irq path (when event PORT_ACTIVE is received first time)
+ */
+ spinlock_t mod_sqp_lock;
+ enum ib_port_state port_state;
+ struct ehca_sma_attr saved_attr;
+ u32 pma_qp_nr;
+};
+
+#define HCA_CAP_MR_PGSIZE_4K 0x80000000
+#define HCA_CAP_MR_PGSIZE_64K 0x40000000
+#define HCA_CAP_MR_PGSIZE_1M 0x20000000
+#define HCA_CAP_MR_PGSIZE_16M 0x10000000
+
+struct ehca_shca {
+ struct ib_device ib_device;
+ struct platform_device *ofdev;
+ u8 num_ports;
+ int hw_level;
+ struct list_head shca_list;
+ struct ipz_adapter_handle ipz_hca_handle;
+ struct ehca_sport sport[2];
+ struct ehca_eq eq;
+ struct ehca_eq neq;
+ struct ehca_mr *maxmr;
+ struct ehca_pd *pd;
+ struct h_galpas galpas;
+ struct mutex modify_mutex;
+ u64 hca_cap;
+ /* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
+ u32 hca_cap_mr_pgsize;
+ int max_mtu;
+ int max_num_qps;
+ int max_num_cqs;
+ atomic_t num_cqs;
+ atomic_t num_qps;
+};
+
+struct ehca_pd {
+ struct ib_pd ib_pd;
+ struct ipz_pd fw_pd;
+ /* small queue mgmt */
+ struct mutex lock;
+ struct list_head free[2];
+ struct list_head full[2];
+};
+
+enum ehca_ext_qp_type {
+ EQPT_NORMAL = 0,
+ EQPT_LLQP = 1,
+ EQPT_SRQBASE = 2,
+ EQPT_SRQ = 3,
+};
+
+/* struct to cache modify_qp()'s parms for GSI/SMI qp */
+struct ehca_mod_qp_parm {
+ int mask;
+ struct ib_qp_attr attr;
+};
+
+#define EHCA_MOD_QP_PARM_MAX 4
+
+#define QMAP_IDX_MASK 0xFFFFULL
+
+/* struct for tracking if cqes have been reported to the application */
+struct ehca_qmap_entry {
+ u16 app_wr_id;
+ u8 reported;
+ u8 cqe_req;
+};
+
+struct ehca_queue_map {
+ struct ehca_qmap_entry *map;
+ unsigned int entries;
+ unsigned int tail;
+ unsigned int left_to_poll;
+ unsigned int next_wqe_idx; /* Idx to first wqe to be flushed */
+};
+
+/* function to calculate the next index for the qmap */
+static inline unsigned int next_index(unsigned int cur_index, unsigned int limit)
+{
+ unsigned int temp = cur_index + 1;
+ return (temp == limit) ? 0 : temp;
+}
+
+struct ehca_qp {
+ union {
+ struct ib_qp ib_qp;
+ struct ib_srq ib_srq;
+ };
+ u32 qp_type;
+ enum ehca_ext_qp_type ext_type;
+ enum ib_qp_state state;
+ struct ipz_queue ipz_squeue;
+ struct ehca_queue_map sq_map;
+ struct ipz_queue ipz_rqueue;
+ struct ehca_queue_map rq_map;
+ struct h_galpas galpas;
+ u32 qkey;
+ u32 real_qp_num;
+ u32 token;
+ spinlock_t spinlock_s;
+ spinlock_t spinlock_r;
+ u32 sq_max_inline_data_size;
+ struct ipz_qp_handle ipz_qp_handle;
+ struct ehca_pfqp pf;
+ struct ib_qp_init_attr init_attr;
+ struct ehca_cq *send_cq;
+ struct ehca_cq *recv_cq;
+ unsigned int sqerr_purgeflag;
+ struct hlist_node list_entries;
+ /* array to cache modify_qp()'s parms for GSI/SMI qp */
+ struct ehca_mod_qp_parm *mod_qp_parm;
+ int mod_qp_parm_idx;
+ /* mmap counter for resources mapped into user space */
+ u32 mm_count_squeue;
+ u32 mm_count_rqueue;
+ u32 mm_count_galpa;
+ /* unsolicited ack circumvention */
+ int unsol_ack_circ;
+ int mtu_shift;
+ u32 message_count;
+ u32 packet_count;
+ atomic_t nr_events; /* events seen */
+ wait_queue_head_t wait_completion;
+ int mig_armed;
+ struct list_head sq_err_node;
+ struct list_head rq_err_node;
+};
+
+#define IS_SRQ(qp) (qp->ext_type == EQPT_SRQ)
+#define HAS_SQ(qp) (qp->ext_type != EQPT_SRQ)
+#define HAS_RQ(qp) (qp->ext_type != EQPT_SRQBASE)
+
+/* must be power of 2 */
+#define QP_HASHTAB_LEN 8
+
+struct ehca_cq {
+ struct ib_cq ib_cq;
+ struct ipz_queue ipz_queue;
+ struct h_galpas galpas;
+ spinlock_t spinlock;
+ u32 cq_number;
+ u32 token;
+ u32 nr_of_entries;
+ struct ipz_cq_handle ipz_cq_handle;
+ struct ehca_pfcq pf;
+ spinlock_t cb_lock;
+ struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
+ struct list_head entry;
+ u32 nr_callbacks; /* #events assigned to cpu by scaling code */
+ atomic_t nr_events; /* #events seen */
+ wait_queue_head_t wait_completion;
+ spinlock_t task_lock;
+ /* mmap counter for resources mapped into user space */
+ u32 mm_count_queue;
+ u32 mm_count_galpa;
+ struct list_head sqp_err_list;
+ struct list_head rqp_err_list;
+};
+
+enum ehca_mr_flag {
+ EHCA_MR_FLAG_FMR = 0x80000000, /* FMR, created with ehca_alloc_fmr */
+ EHCA_MR_FLAG_MAXMR = 0x40000000, /* max-MR */
+};
+
+struct ehca_mr {
+ union {
+ struct ib_mr ib_mr; /* must always be first in ehca_mr */
+ struct ib_fmr ib_fmr; /* must always be first in ehca_mr */
+ } ib;
+ struct ib_umem *umem;
+ spinlock_t mrlock;
+
+ enum ehca_mr_flag flags;
+ u32 num_kpages; /* number of kernel pages */
+ u32 num_hwpages; /* number of hw pages to form MR */
+ u64 hwpage_size; /* hw page size used for this MR */
+ int acl; /* ACL (stored here for usage in reregister) */
+ u64 *start; /* virtual start address (stored here for */
+ /* usage in reregister) */
+ u64 size; /* size (stored here for usage in reregister) */
+ u32 fmr_page_size; /* page size for FMR */
+ u32 fmr_max_pages; /* max pages for FMR */
+ u32 fmr_max_maps; /* max outstanding maps for FMR */
+ u32 fmr_map_cnt; /* map counter for FMR */
+ /* fw specific data */
+ struct ipz_mrmw_handle ipz_mr_handle; /* MR handle for h-calls */
+ struct h_galpas galpas;
+};
+
+struct ehca_mw {
+ struct ib_mw ib_mw; /* gen2 mw, must always be first in ehca_mw */
+ spinlock_t mwlock;
+
+ u8 never_bound; /* indication MW was never bound */
+ struct ipz_mrmw_handle ipz_mw_handle; /* MW handle for h-calls */
+ struct h_galpas galpas;
+};
+
+enum ehca_mr_pgi_type {
+ EHCA_MR_PGI_PHYS = 1, /* type of ehca_reg_phys_mr,
+ * ehca_rereg_phys_mr,
+ * ehca_reg_internal_maxmr */
+ EHCA_MR_PGI_USER = 2, /* type of ehca_reg_user_mr */
+ EHCA_MR_PGI_FMR = 3 /* type of ehca_map_phys_fmr */
+};
+
+struct ehca_mr_pginfo {
+ enum ehca_mr_pgi_type type;
+ u64 num_kpages;
+ u64 kpage_cnt;
+ u64 hwpage_size; /* hw page size used for this MR */
+ u64 num_hwpages; /* number of hw pages */
+ u64 hwpage_cnt; /* counter for hw pages */
+ u64 next_hwpage; /* next hw page in buffer/chunk/listelem */
+
+ union {
+ struct { /* type EHCA_MR_PGI_PHYS section */
+ int num_phys_buf;
+ struct ib_phys_buf *phys_buf_array;
+ u64 next_buf;
+ } phy;
+ struct { /* type EHCA_MR_PGI_USER section */
+ struct ib_umem *region;
+ struct scatterlist *next_sg;
+ u64 next_nmap;
+ } usr;
+ struct { /* type EHCA_MR_PGI_FMR section */
+ u64 fmr_pgsize;
+ u64 *page_list;
+ u64 next_listelem;
+ } fmr;
+ } u;
+};
+
+/* output parameters for MR/FMR hipz calls */
+struct ehca_mr_hipzout_parms {
+ struct ipz_mrmw_handle handle;
+ u32 lkey;
+ u32 rkey;
+ u64 len;
+ u64 vaddr;
+ u32 acl;
+};
+
+/* output parameters for MW hipz calls */
+struct ehca_mw_hipzout_parms {
+ struct ipz_mrmw_handle handle;
+ u32 rkey;
+};
+
+struct ehca_av {
+ struct ib_ah ib_ah;
+ struct ehca_ud_av av;
+};
+
+struct ehca_ucontext {
+ struct ib_ucontext ib_ucontext;
+};
+
+int ehca_init_pd_cache(void);
+void ehca_cleanup_pd_cache(void);
+int ehca_init_cq_cache(void);
+void ehca_cleanup_cq_cache(void);
+int ehca_init_qp_cache(void);
+void ehca_cleanup_qp_cache(void);
+int ehca_init_av_cache(void);
+void ehca_cleanup_av_cache(void);
+int ehca_init_mrmw_cache(void);
+void ehca_cleanup_mrmw_cache(void);
+int ehca_init_small_qp_cache(void);
+void ehca_cleanup_small_qp_cache(void);
+
+extern rwlock_t ehca_qp_idr_lock;
+extern rwlock_t ehca_cq_idr_lock;
+extern struct idr ehca_qp_idr;
+extern struct idr ehca_cq_idr;
+extern spinlock_t shca_list_lock;
+
+extern int ehca_static_rate;
+extern int ehca_port_act_time;
+extern bool ehca_use_hp_mr;
+extern bool ehca_scaling_code;
+extern int ehca_lock_hcalls;
+extern int ehca_nr_ports;
+extern int ehca_max_cq;
+extern int ehca_max_qp;
+
+struct ipzu_queue_resp {
+ u32 qe_size; /* queue entry size */
+ u32 act_nr_of_sg;
+ u32 queue_length; /* queue length allocated in bytes */
+ u32 pagesize;
+ u32 toggle_state;
+ u32 offset; /* save offset within a page for small_qp */
+};
+
+struct ehca_create_cq_resp {
+ u32 cq_number;
+ u32 token;
+ struct ipzu_queue_resp ipz_queue;
+ u32 fw_handle_ofs;
+ u32 dummy;
+};
+
+struct ehca_create_qp_resp {
+ u32 qp_num;
+ u32 token;
+ u32 qp_type;
+ u32 ext_type;
+ u32 qkey;
+ /* qp_num assigned by ehca: sqp0/1 may have got different numbers */
+ u32 real_qp_num;
+ u32 fw_handle_ofs;
+ u32 dummy;
+ struct ipzu_queue_resp ipz_squeue;
+ struct ipzu_queue_resp ipz_rqueue;
+};
+
+struct ehca_alloc_cq_parms {
+ u32 nr_cqe;
+ u32 act_nr_of_entries;
+ u32 act_pages;
+ struct ipz_eq_handle eq_handle;
+};
+
+enum ehca_service_type {
+ ST_RC = 0,
+ ST_UC = 1,
+ ST_RD = 2,
+ ST_UD = 3,
+};
+
+enum ehca_ll_comp_flags {
+ LLQP_SEND_COMP = 0x20,
+ LLQP_RECV_COMP = 0x40,
+ LLQP_COMP_MASK = 0x60,
+};
+
+struct ehca_alloc_queue_parms {
+ /* input parameters */
+ int max_wr;
+ int max_sge;
+ int page_size;
+ int is_small;
+
+ /* output parameters */
+ u16 act_nr_wqes;
+ u8 act_nr_sges;
+ u32 queue_size; /* bytes for small queues, pages otherwise */
+};
+
+struct ehca_alloc_qp_parms {
+ struct ehca_alloc_queue_parms squeue;
+ struct ehca_alloc_queue_parms rqueue;
+
+ /* input parameters */
+ enum ehca_service_type servicetype;
+ int qp_storage;
+ int sigtype;
+ enum ehca_ext_qp_type ext_type;
+ enum ehca_ll_comp_flags ll_comp_flags;
+ int ud_av_l_key_ctl;
+
+ u32 token;
+ struct ipz_eq_handle eq_handle;
+ struct ipz_pd pd;
+ struct ipz_cq_handle send_cq_handle, recv_cq_handle;
+
+ u32 srq_qpn, srq_token, srq_limit;
+
+ /* output parameters */
+ u32 real_qp_num;
+ struct ipz_qp_handle qp_handle;
+ struct h_galpas galpas;
+};
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int qp_num);
+struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int qp_num);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * pSeries interface definitions
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_PSERIES_H__
+#define __EHCA_CLASSES_PSERIES_H__
+
+#include "hcp_phyp.h"
+#include "ipz_pt_fn.h"
+
+
+struct ehca_pfqp {
+ struct ipz_qpt sqpt;
+ struct ipz_qpt rqpt;
+};
+
+struct ehca_pfcq {
+ struct ipz_qpt qpt;
+ u32 cqnr;
+};
+
+struct ehca_pfeq {
+ struct ipz_qpt qpt;
+ struct h_galpa galpa;
+ u32 eqnr;
+};
+
+struct ipz_adapter_handle {
+ u64 handle;
+};
+
+struct ipz_cq_handle {
+ u64 handle;
+};
+
+struct ipz_eq_handle {
+ u64 handle;
+};
+
+struct ipz_qp_handle {
+ u64 handle;
+};
+struct ipz_mrmw_handle {
+ u64 handle;
+};
+
+struct ipz_pd {
+ u32 value;
+};
+
+struct hcp_modify_qp_control_block {
+ u32 qkey; /* 00 */
+ u32 rdd; /* reliable datagram domain */
+ u32 send_psn; /* 02 */
+ u32 receive_psn; /* 03 */
+ u32 prim_phys_port; /* 04 */
+ u32 alt_phys_port; /* 05 */
+ u32 prim_p_key_idx; /* 06 */
+ u32 alt_p_key_idx; /* 07 */
+ u32 rdma_atomic_ctrl; /* 08 */
+ u32 qp_state; /* 09 */
+ u32 reserved_10; /* 10 */
+ u32 rdma_nr_atomic_resp_res; /* 11 */
+ u32 path_migration_state; /* 12 */
+ u32 rdma_atomic_outst_dest_qp; /* 13 */
+ u32 dest_qp_nr; /* 14 */
+ u32 min_rnr_nak_timer_field; /* 15 */
+ u32 service_level; /* 16 */
+ u32 send_grh_flag; /* 17 */
+ u32 retry_count; /* 18 */
+ u32 timeout; /* 19 */
+ u32 path_mtu; /* 20 */
+ u32 max_static_rate; /* 21 */
+ u32 dlid; /* 22 */
+ u32 rnr_retry_count; /* 23 */
+ u32 source_path_bits; /* 24 */
+ u32 traffic_class; /* 25 */
+ u32 hop_limit; /* 26 */
+ u32 source_gid_idx; /* 27 */
+ u32 flow_label; /* 28 */
+ u32 reserved_29; /* 29 */
+ union { /* 30 */
+ u64 dw[2];
+ u8 byte[16];
+ } dest_gid;
+ u32 service_level_al; /* 34 */
+ u32 send_grh_flag_al; /* 35 */
+ u32 retry_count_al; /* 36 */
+ u32 timeout_al; /* 37 */
+ u32 max_static_rate_al; /* 38 */
+ u32 dlid_al; /* 39 */
+ u32 rnr_retry_count_al; /* 40 */
+ u32 source_path_bits_al; /* 41 */
+ u32 traffic_class_al; /* 42 */
+ u32 hop_limit_al; /* 43 */
+ u32 source_gid_idx_al; /* 44 */
+ u32 flow_label_al; /* 45 */
+ u32 reserved_46; /* 46 */
+ u32 reserved_47; /* 47 */
+ union { /* 48 */
+ u64 dw[2];
+ u8 byte[16];
+ } dest_gid_al;
+ u32 max_nr_outst_send_wr; /* 52 */
+ u32 max_nr_outst_recv_wr; /* 53 */
+ u32 disable_ete_credit_check; /* 54 */
+ u32 qp_number; /* 55 */
+ u64 send_queue_handle; /* 56 */
+ u64 recv_queue_handle; /* 58 */
+ u32 actual_nr_sges_in_sq_wqe; /* 60 */
+ u32 actual_nr_sges_in_rq_wqe; /* 61 */
+ u32 qp_enable; /* 62 */
+ u32 curr_srq_limit; /* 63 */
+ u64 qp_aff_asyn_ev_log_reg; /* 64 */
+ u64 shared_rq_hndl; /* 66 */
+ u64 trigg_doorbell_qp_hndl; /* 68 */
+ u32 reserved_70_127[58]; /* 70 */
+};
+
+#define MQPCB_MASK_QKEY EHCA_BMASK_IBM( 0, 0)
+#define MQPCB_MASK_SEND_PSN EHCA_BMASK_IBM( 2, 2)
+#define MQPCB_MASK_RECEIVE_PSN EHCA_BMASK_IBM( 3, 3)
+#define MQPCB_MASK_PRIM_PHYS_PORT EHCA_BMASK_IBM( 4, 4)
+#define MQPCB_PRIM_PHYS_PORT EHCA_BMASK_IBM(24, 31)
+#define MQPCB_MASK_ALT_PHYS_PORT EHCA_BMASK_IBM( 5, 5)
+#define MQPCB_MASK_PRIM_P_KEY_IDX EHCA_BMASK_IBM( 6, 6)
+#define MQPCB_PRIM_P_KEY_IDX EHCA_BMASK_IBM(24, 31)
+#define MQPCB_MASK_ALT_P_KEY_IDX EHCA_BMASK_IBM( 7, 7)
+#define MQPCB_MASK_RDMA_ATOMIC_CTRL EHCA_BMASK_IBM( 8, 8)
+#define MQPCB_MASK_QP_STATE EHCA_BMASK_IBM( 9, 9)
+#define MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES EHCA_BMASK_IBM(11, 11)
+#define MQPCB_MASK_PATH_MIGRATION_STATE EHCA_BMASK_IBM(12, 12)
+#define MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP EHCA_BMASK_IBM(13, 13)
+#define MQPCB_MASK_DEST_QP_NR EHCA_BMASK_IBM(14, 14)
+#define MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD EHCA_BMASK_IBM(15, 15)
+#define MQPCB_MASK_SERVICE_LEVEL EHCA_BMASK_IBM(16, 16)
+#define MQPCB_MASK_SEND_GRH_FLAG EHCA_BMASK_IBM(17, 17)
+#define MQPCB_MASK_RETRY_COUNT EHCA_BMASK_IBM(18, 18)
+#define MQPCB_MASK_TIMEOUT EHCA_BMASK_IBM(19, 19)
+#define MQPCB_MASK_PATH_MTU EHCA_BMASK_IBM(20, 20)
+#define MQPCB_MASK_MAX_STATIC_RATE EHCA_BMASK_IBM(21, 21)
+#define MQPCB_MASK_DLID EHCA_BMASK_IBM(22, 22)
+#define MQPCB_MASK_RNR_RETRY_COUNT EHCA_BMASK_IBM(23, 23)
+#define MQPCB_MASK_SOURCE_PATH_BITS EHCA_BMASK_IBM(24, 24)
+#define MQPCB_MASK_TRAFFIC_CLASS EHCA_BMASK_IBM(25, 25)
+#define MQPCB_MASK_HOP_LIMIT EHCA_BMASK_IBM(26, 26)
+#define MQPCB_MASK_SOURCE_GID_IDX EHCA_BMASK_IBM(27, 27)
+#define MQPCB_MASK_FLOW_LABEL EHCA_BMASK_IBM(28, 28)
+#define MQPCB_MASK_DEST_GID EHCA_BMASK_IBM(30, 30)
+#define MQPCB_MASK_SERVICE_LEVEL_AL EHCA_BMASK_IBM(31, 31)
+#define MQPCB_MASK_SEND_GRH_FLAG_AL EHCA_BMASK_IBM(32, 32)
+#define MQPCB_MASK_RETRY_COUNT_AL EHCA_BMASK_IBM(33, 33)
+#define MQPCB_MASK_TIMEOUT_AL EHCA_BMASK_IBM(34, 34)
+#define MQPCB_MASK_MAX_STATIC_RATE_AL EHCA_BMASK_IBM(35, 35)
+#define MQPCB_MASK_DLID_AL EHCA_BMASK_IBM(36, 36)
+#define MQPCB_MASK_RNR_RETRY_COUNT_AL EHCA_BMASK_IBM(37, 37)
+#define MQPCB_MASK_SOURCE_PATH_BITS_AL EHCA_BMASK_IBM(38, 38)
+#define MQPCB_MASK_TRAFFIC_CLASS_AL EHCA_BMASK_IBM(39, 39)
+#define MQPCB_MASK_HOP_LIMIT_AL EHCA_BMASK_IBM(40, 40)
+#define MQPCB_MASK_SOURCE_GID_IDX_AL EHCA_BMASK_IBM(41, 41)
+#define MQPCB_MASK_FLOW_LABEL_AL EHCA_BMASK_IBM(42, 42)
+#define MQPCB_MASK_DEST_GID_AL EHCA_BMASK_IBM(44, 44)
+#define MQPCB_MASK_MAX_NR_OUTST_SEND_WR EHCA_BMASK_IBM(45, 45)
+#define MQPCB_MASK_MAX_NR_OUTST_RECV_WR EHCA_BMASK_IBM(46, 46)
+#define MQPCB_MASK_DISABLE_ETE_CREDIT_CHECK EHCA_BMASK_IBM(47, 47)
+#define MQPCB_MASK_QP_ENABLE EHCA_BMASK_IBM(48, 48)
+#define MQPCB_MASK_CURR_SRQ_LIMIT EHCA_BMASK_IBM(49, 49)
+#define MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG EHCA_BMASK_IBM(50, 50)
+#define MQPCB_MASK_SHARED_RQ_HNDL EHCA_BMASK_IBM(51, 51)
+
+#endif /* __EHCA_CLASSES_PSERIES_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Completion queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *cq_cache;
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
+{
+ unsigned int qp_num = qp->real_qp_num;
+ unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+ hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
+ cq->cq_number, qp_num);
+
+ return 0;
+}
+
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
+{
+ int ret = -EINVAL;
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+ struct hlist_node *iter;
+ struct ehca_qp *qp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+ hlist_for_each(iter, &cq->qp_hashtab[key]) {
+ qp = hlist_entry(iter, struct ehca_qp, list_entries);
+ if (qp->real_qp_num == real_qp_num) {
+ hlist_del(iter);
+ ehca_dbg(cq->ib_cq.device,
+ "removed qp from cq .cq_num=%x real_qp_num=%x",
+ cq->cq_number, real_qp_num);
+ ret = 0;
+ break;
+ }
+ }
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+ if (ret)
+ ehca_err(cq->ib_cq.device,
+ "qp not found cq_num=%x real_qp_num=%x",
+ cq->cq_number, real_qp_num);
+
+ return ret;
+}
+
+struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
+{
+ struct ehca_qp *ret = NULL;
+ unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+ struct hlist_node *iter;
+ struct ehca_qp *qp;
+ hlist_for_each(iter, &cq->qp_hashtab[key]) {
+ qp = hlist_entry(iter, struct ehca_qp, list_entries);
+ if (qp->real_qp_num == real_qp_num) {
+ ret = qp;
+ break;
+ }
+ }
+ return ret;
+}
+
+struct ib_cq *ehca_create_cq(struct ib_device *device,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ int cqe = attr->cqe;
+ static const u32 additional_cqe = 20;
+ struct ib_cq *cq;
+ struct ehca_cq *my_cq;
+ struct ehca_shca *shca =
+ container_of(device, struct ehca_shca, ib_device);
+ struct ipz_adapter_handle adapter_handle;
+ struct ehca_alloc_cq_parms param; /* h_call's out parameters */
+ struct h_galpa gal;
+ void *vpage;
+ u32 counter;
+ u64 rpage, cqx_fec, h_ret;
+ int ipz_rc, i;
+ unsigned long flags;
+
+ if (attr->flags)
+ return ERR_PTR(-EINVAL);
+
+ if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
+ return ERR_PTR(-EINVAL);
+
+ if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
+ ehca_err(device, "Unable to create CQ, max number of %i "
+ "CQs reached.", shca->max_num_cqs);
+ ehca_err(device, "To increase the maximum number of CQs "
+ "use the number_of_cqs module parameter.\n");
+ return ERR_PTR(-ENOSPC);
+ }
+
+ my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
+ if (!my_cq) {
+ ehca_err(device, "Out of memory for ehca_cq struct device=%p",
+ device);
+ atomic_dec(&shca->num_cqs);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ memset(¶m, 0, sizeof(struct ehca_alloc_cq_parms));
+
+ spin_lock_init(&my_cq->spinlock);
+ spin_lock_init(&my_cq->cb_lock);
+ spin_lock_init(&my_cq->task_lock);
+ atomic_set(&my_cq->nr_events, 0);
+ init_waitqueue_head(&my_cq->wait_completion);
+
+ cq = &my_cq->ib_cq;
+
+ adapter_handle = shca->ipz_hca_handle;
+ param.eq_handle = shca->eq.ipz_eq_handle;
+
+ idr_preload(GFP_KERNEL);
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ my_cq->token = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+ idr_preload_end();
+
+ if (my_cq->token < 0) {
+ cq = ERR_PTR(-ENOMEM);
+ ehca_err(device, "Can't allocate new idr entry. device=%p",
+ device);
+ goto create_cq_exit1;
+ }
+
+ /*
+ * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
+ * for receiving errors CQEs.
+ */
+ param.nr_cqe = cqe + additional_cqe;
+ h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, ¶m);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(device, "hipz_h_alloc_resource_cq() failed "
+ "h_ret=%lli device=%p", h_ret, device);
+ cq = ERR_PTR(ehca2ib_return_code(h_ret));
+ goto create_cq_exit2;
+ }
+
+ ipz_rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
+ if (!ipz_rc) {
+ ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
+ ipz_rc, device);
+ cq = ERR_PTR(-EINVAL);
+ goto create_cq_exit3;
+ }
+
+ for (counter = 0; counter < param.act_pages; counter++) {
+ vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+ if (!vpage) {
+ ehca_err(device, "ipz_qpageit_get_inc() "
+ "returns NULL device=%p", device);
+ cq = ERR_PTR(-EAGAIN);
+ goto create_cq_exit4;
+ }
+ rpage = __pa(vpage);
+
+ h_ret = hipz_h_register_rpage_cq(adapter_handle,
+ my_cq->ipz_cq_handle,
+ &my_cq->pf,
+ 0,
+ 0,
+ rpage,
+ 1,
+ my_cq->galpas.
+ kernel);
+
+ if (h_ret < H_SUCCESS) {
+ ehca_err(device, "hipz_h_register_rpage_cq() failed "
+ "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
+ "act_pages=%i", my_cq, my_cq->cq_number,
+ h_ret, counter, param.act_pages);
+ cq = ERR_PTR(-EINVAL);
+ goto create_cq_exit4;
+ }
+
+ if (counter == (param.act_pages - 1)) {
+ vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+ if ((h_ret != H_SUCCESS) || vpage) {
+ ehca_err(device, "Registration of pages not "
+ "complete ehca_cq=%p cq_num=%x "
+ "h_ret=%lli", my_cq, my_cq->cq_number,
+ h_ret);
+ cq = ERR_PTR(-EAGAIN);
+ goto create_cq_exit4;
+ }
+ } else {
+ if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(device, "Registration of page failed "
+ "ehca_cq=%p cq_num=%x h_ret=%lli "
+ "counter=%i act_pages=%i",
+ my_cq, my_cq->cq_number,
+ h_ret, counter, param.act_pages);
+ cq = ERR_PTR(-ENOMEM);
+ goto create_cq_exit4;
+ }
+ }
+ }
+
+ ipz_qeit_reset(&my_cq->ipz_queue);
+
+ gal = my_cq->galpas.kernel;
+ cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
+ ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
+ my_cq, my_cq->cq_number, cqx_fec);
+
+ my_cq->ib_cq.cqe = my_cq->nr_of_entries =
+ param.act_nr_of_entries - additional_cqe;
+ my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
+
+ for (i = 0; i < QP_HASHTAB_LEN; i++)
+ INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
+
+ INIT_LIST_HEAD(&my_cq->sqp_err_list);
+ INIT_LIST_HEAD(&my_cq->rqp_err_list);
+
+ if (context) {
+ struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
+ struct ehca_create_cq_resp resp;
+ memset(&resp, 0, sizeof(resp));
+ resp.cq_number = my_cq->cq_number;
+ resp.token = my_cq->token;
+ resp.ipz_queue.qe_size = ipz_queue->qe_size;
+ resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
+ resp.ipz_queue.queue_length = ipz_queue->queue_length;
+ resp.ipz_queue.pagesize = ipz_queue->pagesize;
+ resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
+ resp.fw_handle_ofs = (u32)
+ (my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
+ if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
+ ehca_err(device, "Copy to udata failed.");
+ cq = ERR_PTR(-EFAULT);
+ goto create_cq_exit4;
+ }
+ }
+
+ return cq;
+
+create_cq_exit4:
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
+
+create_cq_exit3:
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+ if (h_ret != H_SUCCESS)
+ ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
+ "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
+
+create_cq_exit2:
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ idr_remove(&ehca_cq_idr, my_cq->token);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+create_cq_exit1:
+ kmem_cache_free(cq_cache, my_cq);
+
+ atomic_dec(&shca->num_cqs);
+ return cq;
+}
+
+int ehca_destroy_cq(struct ib_cq *cq)
+{
+ u64 h_ret;
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int cq_num = my_cq->cq_number;
+ struct ib_device *device = cq->device;
+ struct ehca_shca *shca = container_of(device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ unsigned long flags;
+
+ if (cq->uobject) {
+ if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
+ ehca_err(device, "Resources still referenced in "
+ "user space cq_num=%x", my_cq->cq_number);
+ return -EINVAL;
+ }
+ }
+
+ /*
+ * remove the CQ from the idr first to make sure
+ * no more interrupt tasklets will touch this CQ
+ */
+ write_lock_irqsave(&ehca_cq_idr_lock, flags);
+ idr_remove(&ehca_cq_idr, my_cq->token);
+ write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+ /* now wait until all pending events have completed */
+ wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
+
+ /* nobody's using our CQ any longer -- we can destroy it */
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
+ if (h_ret == H_R_STATE) {
+ /* cq in err: read err data and destroy it forcibly */
+ ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
+ "state. Try to delete it forcibly.",
+ my_cq, cq_num, my_cq->ipz_cq_handle.handle);
+ ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
+ h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+ if (h_ret == H_SUCCESS)
+ ehca_dbg(device, "cq_num=%x deleted successfully.",
+ cq_num);
+ }
+ if (h_ret != H_SUCCESS) {
+ ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
+ "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
+ return ehca2ib_return_code(h_ret);
+ }
+ ipz_queue_dtor(NULL, &my_cq->ipz_queue);
+ kmem_cache_free(cq_cache, my_cq);
+
+ atomic_dec(&shca->num_cqs);
+ return 0;
+}
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
+{
+ /* TODO: proper resize needs to be done */
+ ehca_err(cq->device, "not implemented yet");
+
+ return -EFAULT;
+}
+
+int ehca_init_cq_cache(void)
+{
+ cq_cache = kmem_cache_create("ehca_cache_cq",
+ sizeof(struct ehca_cq), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!cq_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_cq_cache(void)
+{
+ if (cq_cache)
+ kmem_cache_destroy(cq_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Event queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_qes.h"
+#include "hcp_if.h"
+#include "ipz_pt_fn.h"
+
+int ehca_create_eq(struct ehca_shca *shca,
+ struct ehca_eq *eq,
+ const enum ehca_eq_type type, const u32 length)
+{
+ int ret;
+ u64 h_ret;
+ u32 nr_pages;
+ u32 i;
+ void *vpage;
+ struct ib_device *ib_dev = &shca->ib_device;
+
+ spin_lock_init(&eq->spinlock);
+ spin_lock_init(&eq->irq_spinlock);
+ eq->is_initialized = 0;
+
+ if (type != EHCA_EQ && type != EHCA_NEQ) {
+ ehca_err(ib_dev, "Invalid EQ type %x. eq=%p", type, eq);
+ return -EINVAL;
+ }
+ if (!length) {
+ ehca_err(ib_dev, "EQ length must not be zero. eq=%p", eq);
+ return -EINVAL;
+ }
+
+ h_ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
+ &eq->pf,
+ type,
+ length,
+ &eq->ipz_eq_handle,
+ &eq->length,
+ &nr_pages, &eq->ist);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
+ return -EINVAL;
+ }
+
+ ret = ipz_queue_ctor(NULL, &eq->ipz_queue, nr_pages,
+ EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0, 0);
+ if (!ret) {
+ ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
+ goto create_eq_exit1;
+ }
+
+ for (i = 0; i < nr_pages; i++) {
+ u64 rpage;
+
+ vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
+ if (!vpage)
+ goto create_eq_exit2;
+
+ rpage = __pa(vpage);
+ h_ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
+ eq->ipz_eq_handle,
+ &eq->pf,
+ 0, 0, rpage, 1);
+
+ if (i == (nr_pages - 1)) {
+ /* last page */
+ vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
+ if (h_ret != H_SUCCESS || vpage)
+ goto create_eq_exit2;
+ } else {
+ if (h_ret != H_PAGE_REGISTERED)
+ goto create_eq_exit2;
+ }
+ }
+
+ ipz_qeit_reset(&eq->ipz_queue);
+
+ /* register interrupt handlers and initialize work queues */
+ if (type == EHCA_EQ) {
+ tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca);
+
+ ret = ibmebus_request_irq(eq->ist, ehca_interrupt_eq,
+ 0, "ehca_eq",
+ (void *)shca);
+ if (ret < 0)
+ ehca_err(ib_dev, "Can't map interrupt handler.");
+ } else if (type == EHCA_NEQ) {
+ tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca);
+
+ ret = ibmebus_request_irq(eq->ist, ehca_interrupt_neq,
+ 0, "ehca_neq",
+ (void *)shca);
+ if (ret < 0)
+ ehca_err(ib_dev, "Can't map interrupt handler.");
+ }
+
+ eq->is_initialized = 1;
+
+ return 0;
+
+create_eq_exit2:
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
+
+create_eq_exit1:
+ hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+ return -EINVAL;
+}
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+ unsigned long flags;
+ void *eqe;
+
+ spin_lock_irqsave(&eq->spinlock, flags);
+ eqe = ipz_eqit_eq_get_inc_valid(&eq->ipz_queue);
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+
+ return eqe;
+}
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+ unsigned long flags;
+ u64 h_ret;
+
+ ibmebus_free_irq(eq->ist, (void *)shca);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ eq->is_initialized = 0;
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ tasklet_kill(&eq->interrupt_task);
+
+ h_ret = hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't free EQ resources.");
+ return -EINVAL;
+ }
+ ipz_queue_dtor(NULL, &eq->ipz_queue);
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HCA query functions
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/gfp.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+static unsigned int limit_uint(unsigned int value)
+{
+ return min_t(unsigned int, value, INT_MAX);
+}
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw)
+{
+ int i, ret = 0;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_hca *rblock;
+
+ static const u32 cap_mapping[] = {
+ IB_DEVICE_RESIZE_MAX_WR, HCA_CAP_WQE_RESIZE,
+ IB_DEVICE_BAD_PKEY_CNTR, HCA_CAP_BAD_P_KEY_CTR,
+ IB_DEVICE_BAD_QKEY_CNTR, HCA_CAP_Q_KEY_VIOL_CTR,
+ IB_DEVICE_RAW_MULTI, HCA_CAP_RAW_PACKET_MCAST,
+ IB_DEVICE_AUTO_PATH_MIG, HCA_CAP_AUTO_PATH_MIG,
+ IB_DEVICE_CHANGE_PHY_PORT, HCA_CAP_SQD_RTS_PORT_CHANGE,
+ IB_DEVICE_UD_AV_PORT_ENFORCE, HCA_CAP_AH_PORT_NR_CHECK,
+ IB_DEVICE_CURR_QP_STATE_MOD, HCA_CAP_CUR_QP_STATE_MOD,
+ IB_DEVICE_SHUTDOWN_PORT, HCA_CAP_SHUTDOWN_PORT,
+ IB_DEVICE_INIT_TYPE, HCA_CAP_INIT_TYPE,
+ IB_DEVICE_PORT_ACTIVE_EVENT, HCA_CAP_PORT_ACTIVE_EVENT,
+ };
+
+ if (uhw->inlen || uhw->outlen)
+ return -EINVAL;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query device properties");
+ ret = -EINVAL;
+ goto query_device1;
+ }
+
+ memset(props, 0, sizeof(struct ib_device_attr));
+ props->page_size_cap = shca->hca_cap_mr_pgsize;
+ props->fw_ver = rblock->hw_ver;
+ props->max_mr_size = rblock->max_mr_size;
+ props->vendor_id = rblock->vendor_id >> 8;
+ props->vendor_part_id = rblock->vendor_part_id >> 16;
+ props->hw_ver = rblock->hw_ver;
+ props->max_qp = limit_uint(rblock->max_qp);
+ props->max_qp_wr = limit_uint(rblock->max_wqes_wq);
+ props->max_sge = limit_uint(rblock->max_sge);
+ props->max_sge_rd = limit_uint(rblock->max_sge_rd);
+ props->max_cq = limit_uint(rblock->max_cq);
+ props->max_cqe = limit_uint(rblock->max_cqe);
+ props->max_mr = limit_uint(rblock->max_mr);
+ props->max_mw = limit_uint(rblock->max_mw);
+ props->max_pd = limit_uint(rblock->max_pd);
+ props->max_ah = limit_uint(rblock->max_ah);
+ props->max_ee = limit_uint(rblock->max_rd_ee_context);
+ props->max_rdd = limit_uint(rblock->max_rd_domain);
+ props->max_fmr = limit_uint(rblock->max_mr);
+ props->max_qp_rd_atom = limit_uint(rblock->max_rr_qp);
+ props->max_ee_rd_atom = limit_uint(rblock->max_rr_ee_context);
+ props->max_res_rd_atom = limit_uint(rblock->max_rr_hca);
+ props->max_qp_init_rd_atom = limit_uint(rblock->max_act_wqs_qp);
+ props->max_ee_init_rd_atom = limit_uint(rblock->max_act_wqs_ee_context);
+
+ if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
+ props->max_srq = limit_uint(props->max_qp);
+ props->max_srq_wr = limit_uint(props->max_qp_wr);
+ props->max_srq_sge = 3;
+ }
+
+ props->max_pkeys = 16;
+ /* Some FW versions say 0 here; insert sensible value in that case */
+ props->local_ca_ack_delay = rblock->local_ca_ack_delay ?
+ min_t(u8, rblock->local_ca_ack_delay, 255) : 12;
+ props->max_raw_ipv6_qp = limit_uint(rblock->max_raw_ipv6_qp);
+ props->max_raw_ethy_qp = limit_uint(rblock->max_raw_ethy_qp);
+ props->max_mcast_grp = limit_uint(rblock->max_mcast_grp);
+ props->max_mcast_qp_attach = limit_uint(rblock->max_mcast_qp_attach);
+ props->max_total_mcast_qp_attach
+ = limit_uint(rblock->max_total_mcast_qp_attach);
+
+ /* translate device capabilities */
+ props->device_cap_flags = IB_DEVICE_SYS_IMAGE_GUID |
+ IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_N_NOTIFY_CQ;
+ for (i = 0; i < ARRAY_SIZE(cap_mapping); i += 2)
+ if (rblock->hca_cap_indicators & cap_mapping[i + 1])
+ props->device_cap_flags |= cap_mapping[i];
+
+query_device1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+static enum ib_mtu map_mtu(struct ehca_shca *shca, u32 fw_mtu)
+{
+ switch (fw_mtu) {
+ case 0x1:
+ return IB_MTU_256;
+ case 0x2:
+ return IB_MTU_512;
+ case 0x3:
+ return IB_MTU_1024;
+ case 0x4:
+ return IB_MTU_2048;
+ case 0x5:
+ return IB_MTU_4096;
+ default:
+ ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
+ fw_mtu);
+ return 0;
+ }
+}
+
+static u8 map_number_of_vls(struct ehca_shca *shca, u32 vl_cap)
+{
+ switch (vl_cap) {
+ case 0x1:
+ return 1;
+ case 0x2:
+ return 2;
+ case 0x3:
+ return 4;
+ case 0x4:
+ return 8;
+ case 0x5:
+ return 15;
+ default:
+ ehca_err(&shca->ib_device, "invalid Vl Capability: %x.",
+ vl_cap);
+ return 0;
+ }
+}
+
+int ehca_query_port(struct ib_device *ibdev,
+ u8 port, struct ib_port_attr *props)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_port *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_port1;
+ }
+
+ memset(props, 0, sizeof(struct ib_port_attr));
+
+ props->active_mtu = props->max_mtu = map_mtu(shca, rblock->max_mtu);
+ props->port_cap_flags = rblock->capability_mask;
+ props->gid_tbl_len = rblock->gid_tbl_len;
+ if (rblock->max_msg_sz)
+ props->max_msg_sz = rblock->max_msg_sz;
+ else
+ props->max_msg_sz = 0x1 << 31;
+ props->bad_pkey_cntr = rblock->bad_pkey_cntr;
+ props->qkey_viol_cntr = rblock->qkey_viol_cntr;
+ props->pkey_tbl_len = rblock->pkey_tbl_len;
+ props->lid = rblock->lid;
+ props->sm_lid = rblock->sm_lid;
+ props->lmc = rblock->lmc;
+ props->sm_sl = rblock->sm_sl;
+ props->subnet_timeout = rblock->subnet_timeout;
+ props->init_type_reply = rblock->init_type_reply;
+ props->max_vl_num = map_number_of_vls(shca, rblock->vl_cap);
+
+ if (rblock->state && rblock->phys_width) {
+ props->phys_state = rblock->phys_pstate;
+ props->state = rblock->phys_state;
+ props->active_width = rblock->phys_width;
+ props->active_speed = rblock->phys_speed;
+ } else {
+ /* old firmware releases don't report physical
+ * port info, so use default values
+ */
+ props->phys_state = 5;
+ props->state = rblock->state;
+ props->active_width = IB_WIDTH_12X;
+ props->active_speed = IB_SPEED_SDR;
+ }
+
+query_port1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_sma_attr(struct ehca_shca *shca,
+ u8 port, struct ehca_sma_attr *attr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct hipz_query_port *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_sma_attr1;
+ }
+
+ memset(attr, 0, sizeof(struct ehca_sma_attr));
+
+ attr->lid = rblock->lid;
+ attr->lmc = rblock->lmc;
+ attr->sm_sl = rblock->sm_sl;
+ attr->sm_lid = rblock->sm_lid;
+
+ attr->pkey_tbl_len = rblock->pkey_tbl_len;
+ memcpy(attr->pkeys, rblock->pkey_entries, sizeof(attr->pkeys));
+
+query_sma_attr1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+ if (index > 16) {
+ ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+ return -EINVAL;
+ }
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_pkey1;
+ }
+
+ memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
+
+query_pkey1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port,
+ int index, union ib_gid *gid)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+ ib_device);
+ struct hipz_query_port *rblock;
+
+ if (index < 0 || index > 255) {
+ ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+ return -EINVAL;
+ }
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto query_gid1;
+ }
+
+ memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
+ memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
+
+query_gid1:
+ ehca_free_fw_ctrlblock(rblock);
+
+ return ret;
+}
+
+static const u32 allowed_port_caps = (
+ IB_PORT_SM | IB_PORT_LED_INFO_SUP | IB_PORT_CM_SUP |
+ IB_PORT_SNMP_TUNNEL_SUP | IB_PORT_DEVICE_MGMT_SUP |
+ IB_PORT_VENDOR_CLASS_SUP);
+
+int ehca_modify_port(struct ib_device *ibdev,
+ u8 port, int port_modify_mask,
+ struct ib_port_modify *props)
+{
+ int ret = 0;
+ struct ehca_shca *shca;
+ struct hipz_query_port *rblock;
+ u32 cap;
+ u64 hret;
+
+ shca = container_of(ibdev, struct ehca_shca, ib_device);
+ if ((props->set_port_cap_mask | props->clr_port_cap_mask)
+ & ~allowed_port_caps) {
+ ehca_err(&shca->ib_device, "Non-changeable bits set in masks "
+ "set=%x clr=%x allowed=%x", props->set_port_cap_mask,
+ props->clr_port_cap_mask, allowed_port_caps);
+ return -EINVAL;
+ }
+
+ if (mutex_lock_interruptible(&shca->modify_mutex))
+ return -ERESTARTSYS;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ ret = -ENOMEM;
+ goto modify_port1;
+ }
+
+ hret = hipz_h_query_port(shca->ipz_hca_handle, port, rblock);
+ if (hret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query port properties");
+ ret = -EINVAL;
+ goto modify_port2;
+ }
+
+ cap = (rblock->capability_mask | props->set_port_cap_mask)
+ & ~props->clr_port_cap_mask;
+
+ hret = hipz_h_modify_port(shca->ipz_hca_handle, port,
+ cap, props->init_type, port_modify_mask);
+ if (hret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Modify port failed h_ret=%lli",
+ hret);
+ ret = -EINVAL;
+ }
+
+modify_port2:
+ ehca_free_fw_ctrlblock(rblock);
+
+modify_port1:
+ mutex_unlock(&shca->modify_mutex);
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Functions for EQs, NEQs and interrupts
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+#include <linux/smpboot.h>
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+#include "ipz_pt_fn.h"
+
+#define EQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
+#define EQE_CQ_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_EE_IDENTIFIER EHCA_BMASK_IBM( 2, 7)
+#define EQE_CQ_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_QP_NUMBER EHCA_BMASK_IBM( 8, 31)
+#define EQE_QP_TOKEN EHCA_BMASK_IBM(32, 63)
+#define EQE_CQ_TOKEN EHCA_BMASK_IBM(32, 63)
+
+#define NEQE_COMPLETION_EVENT EHCA_BMASK_IBM( 1, 1)
+#define NEQE_EVENT_CODE EHCA_BMASK_IBM( 2, 7)
+#define NEQE_PORT_NUMBER EHCA_BMASK_IBM( 8, 15)
+#define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16, 16)
+#define NEQE_DISRUPTIVE EHCA_BMASK_IBM(16, 16)
+#define NEQE_SPECIFIC_EVENT EHCA_BMASK_IBM(16, 23)
+
+#define ERROR_DATA_LENGTH EHCA_BMASK_IBM(52, 63)
+#define ERROR_DATA_TYPE EHCA_BMASK_IBM( 0, 7)
+
+static void queue_comp_task(struct ehca_cq *__cq);
+
+static struct ehca_comp_pool *pool;
+
+static inline void comp_event_callback(struct ehca_cq *cq)
+{
+ if (!cq->ib_cq.comp_handler)
+ return;
+
+ spin_lock(&cq->cb_lock);
+ cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context);
+ spin_unlock(&cq->cb_lock);
+
+ return;
+}
+
+static void print_error_data(struct ehca_shca *shca, void *data,
+ u64 *rblock, int length)
+{
+ u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
+ u64 resource = rblock[1];
+
+ switch (type) {
+ case 0x1: /* Queue Pair */
+ {
+ struct ehca_qp *qp = (struct ehca_qp *)data;
+
+ /* only print error data if AER is set */
+ if (rblock[6] == 0)
+ return;
+
+ ehca_err(&shca->ib_device,
+ "QP 0x%x (resource=%llx) has errors.",
+ qp->ib_qp.qp_num, resource);
+ break;
+ }
+ case 0x4: /* Completion Queue */
+ {
+ struct ehca_cq *cq = (struct ehca_cq *)data;
+
+ ehca_err(&shca->ib_device,
+ "CQ 0x%x (resource=%llx) has errors.",
+ cq->cq_number, resource);
+ break;
+ }
+ default:
+ ehca_err(&shca->ib_device,
+ "Unknown error type: %llx on %s.",
+ type, shca->ib_device.name);
+ break;
+ }
+
+ ehca_err(&shca->ib_device, "Error data is available: %llx.", resource);
+ ehca_err(&shca->ib_device, "EHCA ----- error data begin "
+ "---------------------------------------------------");
+ ehca_dmp(rblock, length, "resource=%llx", resource);
+ ehca_err(&shca->ib_device, "EHCA ----- error data end "
+ "----------------------------------------------------");
+
+ return;
+}
+
+int ehca_error_data(struct ehca_shca *shca, void *data,
+ u64 resource)
+{
+
+ unsigned long ret;
+ u64 *rblock;
+ unsigned long block_count;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Cannot allocate rblock memory.");
+ ret = -ENOMEM;
+ goto error_data1;
+ }
+
+ /* rblock must be 4K aligned and should be 4K large */
+ ret = hipz_h_error_data(shca->ipz_hca_handle,
+ resource,
+ rblock,
+ &block_count);
+
+ if (ret == H_R_STATE)
+ ehca_err(&shca->ib_device,
+ "No error data is available: %llx.", resource);
+ else if (ret == H_SUCCESS) {
+ int length;
+
+ length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]);
+
+ if (length > EHCA_PAGESIZE)
+ length = EHCA_PAGESIZE;
+
+ print_error_data(shca, data, rblock, length);
+ } else
+ ehca_err(&shca->ib_device,
+ "Error data could not be fetched: %llx", resource);
+
+ ehca_free_fw_ctrlblock(rblock);
+
+error_data1:
+ return ret;
+
+}
+
+static void dispatch_qp_event(struct ehca_shca *shca, struct ehca_qp *qp,
+ enum ib_event_type event_type)
+{
+ struct ib_event event;
+
+ /* PATH_MIG without the QP ever having been armed is false alarm */
+ if (event_type == IB_EVENT_PATH_MIG && !qp->mig_armed)
+ return;
+
+ event.device = &shca->ib_device;
+ event.event = event_type;
+
+ if (qp->ext_type == EQPT_SRQ) {
+ if (!qp->ib_srq.event_handler)
+ return;
+
+ event.element.srq = &qp->ib_srq;
+ qp->ib_srq.event_handler(&event, qp->ib_srq.srq_context);
+ } else {
+ if (!qp->ib_qp.event_handler)
+ return;
+
+ event.element.qp = &qp->ib_qp;
+ qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+ }
+}
+
+static void qp_event_callback(struct ehca_shca *shca, u64 eqe,
+ enum ib_event_type event_type, int fatal)
+{
+ struct ehca_qp *qp;
+ u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe);
+
+ read_lock(&ehca_qp_idr_lock);
+ qp = idr_find(&ehca_qp_idr, token);
+ if (qp)
+ atomic_inc(&qp->nr_events);
+ read_unlock(&ehca_qp_idr_lock);
+
+ if (!qp)
+ return;
+
+ if (fatal)
+ ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
+
+ dispatch_qp_event(shca, qp, fatal && qp->ext_type == EQPT_SRQ ?
+ IB_EVENT_SRQ_ERR : event_type);
+
+ /*
+ * eHCA only processes one WQE at a time for SRQ base QPs,
+ * so the last WQE has been processed as soon as the QP enters
+ * error state.
+ */
+ if (fatal && qp->ext_type == EQPT_SRQBASE)
+ dispatch_qp_event(shca, qp, IB_EVENT_QP_LAST_WQE_REACHED);
+
+ if (atomic_dec_and_test(&qp->nr_events))
+ wake_up(&qp->wait_completion);
+ return;
+}
+
+static void cq_event_callback(struct ehca_shca *shca,
+ u64 eqe)
+{
+ struct ehca_cq *cq;
+ u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe);
+
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, token);
+ if (cq)
+ atomic_inc(&cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+
+ if (!cq)
+ return;
+
+ ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
+
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+
+ return;
+}
+
+static void parse_identifier(struct ehca_shca *shca, u64 eqe)
+{
+ u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe);
+
+ switch (identifier) {
+ case 0x02: /* path migrated */
+ qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG, 0);
+ break;
+ case 0x03: /* communication established */
+ qp_event_callback(shca, eqe, IB_EVENT_COMM_EST, 0);
+ break;
+ case 0x04: /* send queue drained */
+ qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED, 0);
+ break;
+ case 0x05: /* QP error */
+ case 0x06: /* QP error */
+ qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL, 1);
+ break;
+ case 0x07: /* CQ error */
+ case 0x08: /* CQ error */
+ cq_event_callback(shca, eqe);
+ break;
+ case 0x09: /* MRMWPTE error */
+ ehca_err(&shca->ib_device, "MRMWPTE error.");
+ break;
+ case 0x0A: /* port event */
+ ehca_err(&shca->ib_device, "Port event.");
+ break;
+ case 0x0B: /* MR access error */
+ ehca_err(&shca->ib_device, "MR access error.");
+ break;
+ case 0x0C: /* EQ error */
+ ehca_err(&shca->ib_device, "EQ error.");
+ break;
+ case 0x0D: /* P/Q_Key mismatch */
+ ehca_err(&shca->ib_device, "P/Q_Key mismatch.");
+ break;
+ case 0x10: /* sampling complete */
+ ehca_err(&shca->ib_device, "Sampling complete.");
+ break;
+ case 0x11: /* unaffiliated access error */
+ ehca_err(&shca->ib_device, "Unaffiliated access error.");
+ break;
+ case 0x12: /* path migrating */
+ ehca_err(&shca->ib_device, "Path migrating.");
+ break;
+ case 0x13: /* interface trace stopped */
+ ehca_err(&shca->ib_device, "Interface trace stopped.");
+ break;
+ case 0x14: /* first error capture info available */
+ ehca_info(&shca->ib_device, "First error capture available");
+ break;
+ case 0x15: /* SRQ limit reached */
+ qp_event_callback(shca, eqe, IB_EVENT_SRQ_LIMIT_REACHED, 0);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
+ identifier, shca->ib_device.name);
+ break;
+ }
+
+ return;
+}
+
+static void dispatch_port_event(struct ehca_shca *shca, int port_num,
+ enum ib_event_type type, const char *msg)
+{
+ struct ib_event event;
+
+ ehca_info(&shca->ib_device, "port %d %s.", port_num, msg);
+ event.device = &shca->ib_device;
+ event.event = type;
+ event.element.port_num = port_num;
+ ib_dispatch_event(&event);
+}
+
+static void notify_port_conf_change(struct ehca_shca *shca, int port_num)
+{
+ struct ehca_sma_attr new_attr;
+ struct ehca_sma_attr *old_attr = &shca->sport[port_num - 1].saved_attr;
+
+ ehca_query_sma_attr(shca, port_num, &new_attr);
+
+ if (new_attr.sm_sl != old_attr->sm_sl ||
+ new_attr.sm_lid != old_attr->sm_lid)
+ dispatch_port_event(shca, port_num, IB_EVENT_SM_CHANGE,
+ "SM changed");
+
+ if (new_attr.lid != old_attr->lid ||
+ new_attr.lmc != old_attr->lmc)
+ dispatch_port_event(shca, port_num, IB_EVENT_LID_CHANGE,
+ "LID changed");
+
+ if (new_attr.pkey_tbl_len != old_attr->pkey_tbl_len ||
+ memcmp(new_attr.pkeys, old_attr->pkeys,
+ sizeof(u16) * new_attr.pkey_tbl_len))
+ dispatch_port_event(shca, port_num, IB_EVENT_PKEY_CHANGE,
+ "P_Key changed");
+
+ *old_attr = new_attr;
+}
+
+/* replay modify_qp for sqps -- return 0 if all is well, 1 if AQP1 destroyed */
+static int replay_modify_qp(struct ehca_sport *sport)
+{
+ int aqp1_destroyed;
+ unsigned long flags;
+
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+
+ aqp1_destroyed = !sport->ibqp_sqp[IB_QPT_GSI];
+
+ if (sport->ibqp_sqp[IB_QPT_SMI])
+ ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_SMI]);
+ if (!aqp1_destroyed)
+ ehca_recover_sqp(sport->ibqp_sqp[IB_QPT_GSI]);
+
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+
+ return aqp1_destroyed;
+}
+
+static void parse_ec(struct ehca_shca *shca, u64 eqe)
+{
+ u8 ec = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe);
+ u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe);
+ u8 spec_event;
+ struct ehca_sport *sport = &shca->sport[port - 1];
+
+ switch (ec) {
+ case 0x30: /* port availability change */
+ if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) {
+ /* only replay modify_qp calls in autodetect mode;
+ * if AQP1 was destroyed, the port is already down
+ * again and we can drop the event.
+ */
+ if (ehca_nr_ports < 0)
+ if (replay_modify_qp(sport))
+ break;
+
+ sport->port_state = IB_PORT_ACTIVE;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
+ "is active");
+ ehca_query_sma_attr(shca, port, &sport->saved_attr);
+ } else {
+ sport->port_state = IB_PORT_DOWN;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
+ "is inactive");
+ }
+ break;
+ case 0x31:
+ /* port configuration change
+ * disruptive change is caused by
+ * LID, PKEY or SM change
+ */
+ if (EHCA_BMASK_GET(NEQE_DISRUPTIVE, eqe)) {
+ ehca_warn(&shca->ib_device, "disruptive port "
+ "%d configuration change", port);
+
+ sport->port_state = IB_PORT_DOWN;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ERR,
+ "is inactive");
+
+ sport->port_state = IB_PORT_ACTIVE;
+ dispatch_port_event(shca, port, IB_EVENT_PORT_ACTIVE,
+ "is active");
+ ehca_query_sma_attr(shca, port,
+ &sport->saved_attr);
+ } else
+ notify_port_conf_change(shca, port);
+ break;
+ case 0x32: /* adapter malfunction */
+ ehca_err(&shca->ib_device, "Adapter malfunction.");
+ break;
+ case 0x33: /* trace stopped */
+ ehca_err(&shca->ib_device, "Traced stopped.");
+ break;
+ case 0x34: /* util async event */
+ spec_event = EHCA_BMASK_GET(NEQE_SPECIFIC_EVENT, eqe);
+ if (spec_event == 0x80) /* client reregister required */
+ dispatch_port_event(shca, port,
+ IB_EVENT_CLIENT_REREGISTER,
+ "client reregister req.");
+ else
+ ehca_warn(&shca->ib_device, "Unknown util async "
+ "event %x on port %x", spec_event, port);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "Unknown event code: %x on %s.",
+ ec, shca->ib_device.name);
+ break;
+ }
+
+ return;
+}
+
+static inline void reset_eq_pending(struct ehca_cq *cq)
+{
+ u64 CQx_EP;
+ struct h_galpa gal = cq->galpas.kernel;
+
+ hipz_galpa_store_cq(gal, cqx_ep, 0x0);
+ CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep));
+
+ return;
+}
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+ tasklet_hi_schedule(&shca->neq.interrupt_task);
+
+ return IRQ_HANDLED;
+}
+
+void ehca_tasklet_neq(unsigned long data)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)data;
+ struct ehca_eqe *eqe;
+ u64 ret;
+
+ eqe = ehca_poll_eq(shca, &shca->neq);
+
+ while (eqe) {
+ if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry))
+ parse_ec(shca, eqe->entry);
+
+ eqe = ehca_poll_eq(shca, &shca->neq);
+ }
+
+ ret = hipz_h_reset_event(shca->ipz_hca_handle,
+ shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL);
+
+ if (ret != H_SUCCESS)
+ ehca_err(&shca->ib_device, "Can't clear notification events.");
+
+ return;
+}
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id)
+{
+ struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+ tasklet_hi_schedule(&shca->eq.interrupt_task);
+
+ return IRQ_HANDLED;
+}
+
+
+static inline void process_eqe(struct ehca_shca *shca, struct ehca_eqe *eqe)
+{
+ u64 eqe_value;
+ u32 token;
+ struct ehca_cq *cq;
+
+ eqe_value = eqe->entry;
+ ehca_dbg(&shca->ib_device, "eqe_value=%llx", eqe_value);
+ if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
+ ehca_dbg(&shca->ib_device, "Got completion event");
+ token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, token);
+ if (cq)
+ atomic_inc(&cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+ if (cq == NULL) {
+ ehca_err(&shca->ib_device,
+ "Invalid eqe for non-existing cq token=%x",
+ token);
+ return;
+ }
+ reset_eq_pending(cq);
+ if (ehca_scaling_code)
+ queue_comp_task(cq);
+ else {
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+ }
+ } else {
+ ehca_dbg(&shca->ib_device, "Got non completion event");
+ parse_identifier(shca, eqe_value);
+ }
+}
+
+void ehca_process_eq(struct ehca_shca *shca, int is_irq)
+{
+ struct ehca_eq *eq = &shca->eq;
+ struct ehca_eqe_cache_entry *eqe_cache = eq->eqe_cache;
+ u64 eqe_value, ret;
+ int eqe_cnt, i;
+ int eq_empty = 0;
+
+ spin_lock(&eq->irq_spinlock);
+ if (is_irq) {
+ const int max_query_cnt = 100;
+ int query_cnt = 0;
+ int int_state = 1;
+ do {
+ int_state = hipz_h_query_int_state(
+ shca->ipz_hca_handle, eq->ist);
+ query_cnt++;
+ iosync();
+ } while (int_state && query_cnt < max_query_cnt);
+ if (unlikely((query_cnt == max_query_cnt)))
+ ehca_dbg(&shca->ib_device, "int_state=%x query_cnt=%x",
+ int_state, query_cnt);
+ }
+
+ /* read out all eqes */
+ eqe_cnt = 0;
+ do {
+ u32 token;
+ eqe_cache[eqe_cnt].eqe = ehca_poll_eq(shca, eq);
+ if (!eqe_cache[eqe_cnt].eqe)
+ break;
+ eqe_value = eqe_cache[eqe_cnt].eqe->entry;
+ if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT, eqe_value)) {
+ token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe_value);
+ read_lock(&ehca_cq_idr_lock);
+ eqe_cache[eqe_cnt].cq = idr_find(&ehca_cq_idr, token);
+ if (eqe_cache[eqe_cnt].cq)
+ atomic_inc(&eqe_cache[eqe_cnt].cq->nr_events);
+ read_unlock(&ehca_cq_idr_lock);
+ if (!eqe_cache[eqe_cnt].cq) {
+ ehca_err(&shca->ib_device,
+ "Invalid eqe for non-existing cq "
+ "token=%x", token);
+ continue;
+ }
+ } else
+ eqe_cache[eqe_cnt].cq = NULL;
+ eqe_cnt++;
+ } while (eqe_cnt < EHCA_EQE_CACHE_SIZE);
+ if (!eqe_cnt) {
+ if (is_irq)
+ ehca_dbg(&shca->ib_device,
+ "No eqe found for irq event");
+ goto unlock_irq_spinlock;
+ } else if (!is_irq) {
+ ret = hipz_h_eoi(eq->ist);
+ if (ret != H_SUCCESS)
+ ehca_err(&shca->ib_device,
+ "bad return code EOI -rc = %lld\n", ret);
+ ehca_dbg(&shca->ib_device, "deadman found %x eqe", eqe_cnt);
+ }
+ if (unlikely(eqe_cnt == EHCA_EQE_CACHE_SIZE))
+ ehca_dbg(&shca->ib_device, "too many eqes for one irq event");
+ /* enable irq for new packets */
+ for (i = 0; i < eqe_cnt; i++) {
+ if (eq->eqe_cache[i].cq)
+ reset_eq_pending(eq->eqe_cache[i].cq);
+ }
+ /* check eq */
+ spin_lock(&eq->spinlock);
+ eq_empty = (!ipz_eqit_eq_peek_valid(&shca->eq.ipz_queue));
+ spin_unlock(&eq->spinlock);
+ /* call completion handler for cached eqes */
+ for (i = 0; i < eqe_cnt; i++)
+ if (eq->eqe_cache[i].cq) {
+ if (ehca_scaling_code)
+ queue_comp_task(eq->eqe_cache[i].cq);
+ else {
+ struct ehca_cq *cq = eq->eqe_cache[i].cq;
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+ }
+ } else {
+ ehca_dbg(&shca->ib_device, "Got non completion event");
+ parse_identifier(shca, eq->eqe_cache[i].eqe->entry);
+ }
+ /* poll eq if not empty */
+ if (eq_empty)
+ goto unlock_irq_spinlock;
+ do {
+ struct ehca_eqe *eqe;
+ eqe = ehca_poll_eq(shca, &shca->eq);
+ if (!eqe)
+ break;
+ process_eqe(shca, eqe);
+ } while (1);
+
+unlock_irq_spinlock:
+ spin_unlock(&eq->irq_spinlock);
+}
+
+void ehca_tasklet_eq(unsigned long data)
+{
+ ehca_process_eq((struct ehca_shca*)data, 1);
+}
+
+static int find_next_online_cpu(struct ehca_comp_pool *pool)
+{
+ int cpu;
+ unsigned long flags;
+
+ WARN_ON_ONCE(!in_interrupt());
+ if (ehca_debug_level >= 3)
+ ehca_dmp(cpu_online_mask, cpumask_size(), "");
+
+ spin_lock_irqsave(&pool->last_cpu_lock, flags);
+ do {
+ cpu = cpumask_next(pool->last_cpu, cpu_online_mask);
+ if (cpu >= nr_cpu_ids)
+ cpu = cpumask_first(cpu_online_mask);
+ pool->last_cpu = cpu;
+ } while (!per_cpu_ptr(pool->cpu_comp_tasks, cpu)->active);
+ spin_unlock_irqrestore(&pool->last_cpu_lock, flags);
+
+ return cpu;
+}
+
+static void __queue_comp_task(struct ehca_cq *__cq,
+ struct ehca_cpu_comp_task *cct,
+ struct task_struct *thread)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cct->task_lock, flags);
+ spin_lock(&__cq->task_lock);
+
+ if (__cq->nr_callbacks == 0) {
+ __cq->nr_callbacks++;
+ list_add_tail(&__cq->entry, &cct->cq_list);
+ cct->cq_jobs++;
+ wake_up_process(thread);
+ } else
+ __cq->nr_callbacks++;
+
+ spin_unlock(&__cq->task_lock);
+ spin_unlock_irqrestore(&cct->task_lock, flags);
+}
+
+static void queue_comp_task(struct ehca_cq *__cq)
+{
+ int cpu_id;
+ struct ehca_cpu_comp_task *cct;
+ struct task_struct *thread;
+ int cq_jobs;
+ unsigned long flags;
+
+ cpu_id = find_next_online_cpu(pool);
+ BUG_ON(!cpu_online(cpu_id));
+
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
+ BUG_ON(!cct || !thread);
+
+ spin_lock_irqsave(&cct->task_lock, flags);
+ cq_jobs = cct->cq_jobs;
+ spin_unlock_irqrestore(&cct->task_lock, flags);
+ if (cq_jobs > 0) {
+ cpu_id = find_next_online_cpu(pool);
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu_id);
+ BUG_ON(!cct || !thread);
+ }
+ __queue_comp_task(__cq, cct, thread);
+}
+
+static void run_comp_task(struct ehca_cpu_comp_task *cct)
+{
+ struct ehca_cq *cq;
+
+ while (!list_empty(&cct->cq_list)) {
+ cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
+ spin_unlock_irq(&cct->task_lock);
+
+ comp_event_callback(cq);
+ if (atomic_dec_and_test(&cq->nr_events))
+ wake_up(&cq->wait_completion);
+
+ spin_lock_irq(&cct->task_lock);
+ spin_lock(&cq->task_lock);
+ cq->nr_callbacks--;
+ if (!cq->nr_callbacks) {
+ list_del_init(cct->cq_list.next);
+ cct->cq_jobs--;
+ }
+ spin_unlock(&cq->task_lock);
+ }
+}
+
+static void comp_task_park(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ struct ehca_cpu_comp_task *target;
+ struct task_struct *thread;
+ struct ehca_cq *cq, *tmp;
+ LIST_HEAD(list);
+
+ spin_lock_irq(&cct->task_lock);
+ cct->cq_jobs = 0;
+ cct->active = 0;
+ list_splice_init(&cct->cq_list, &list);
+ spin_unlock_irq(&cct->task_lock);
+
+ cpu = find_next_online_cpu(pool);
+ target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
+ spin_lock_irq(&target->task_lock);
+ list_for_each_entry_safe(cq, tmp, &list, entry) {
+ list_del(&cq->entry);
+ __queue_comp_task(cq, target, thread);
+ }
+ spin_unlock_irq(&target->task_lock);
+}
+
+static void comp_task_stop(unsigned int cpu, bool online)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+
+ spin_lock_irq(&cct->task_lock);
+ cct->cq_jobs = 0;
+ cct->active = 0;
+ WARN_ON(!list_empty(&cct->cq_list));
+ spin_unlock_irq(&cct->task_lock);
+}
+
+static int comp_task_should_run(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+
+ return cct->cq_jobs;
+}
+
+static void comp_task(unsigned int cpu)
+{
+ struct ehca_cpu_comp_task *cct = this_cpu_ptr(pool->cpu_comp_tasks);
+ int cql_empty;
+
+ spin_lock_irq(&cct->task_lock);
+ cql_empty = list_empty(&cct->cq_list);
+ if (!cql_empty) {
+ __set_current_state(TASK_RUNNING);
+ run_comp_task(cct);
+ }
+ spin_unlock_irq(&cct->task_lock);
+}
+
+static struct smp_hotplug_thread comp_pool_threads = {
+ .thread_should_run = comp_task_should_run,
+ .thread_fn = comp_task,
+ .thread_comm = "ehca_comp/%u",
+ .cleanup = comp_task_stop,
+ .park = comp_task_park,
+};
+
+int ehca_create_comp_pool(void)
+{
+ int cpu, ret = -ENOMEM;
+
+ if (!ehca_scaling_code)
+ return 0;
+
+ pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL);
+ if (pool == NULL)
+ return -ENOMEM;
+
+ spin_lock_init(&pool->last_cpu_lock);
+ pool->last_cpu = cpumask_any(cpu_online_mask);
+
+ pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task);
+ if (!pool->cpu_comp_tasks)
+ goto out_pool;
+
+ pool->cpu_comp_threads = alloc_percpu(struct task_struct *);
+ if (!pool->cpu_comp_threads)
+ goto out_tasks;
+
+ for_each_present_cpu(cpu) {
+ struct ehca_cpu_comp_task *cct;
+
+ cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+ spin_lock_init(&cct->task_lock);
+ INIT_LIST_HEAD(&cct->cq_list);
+ }
+
+ comp_pool_threads.store = pool->cpu_comp_threads;
+ ret = smpboot_register_percpu_thread(&comp_pool_threads);
+ if (ret)
+ goto out_threads;
+
+ pr_info("eHCA scaling code enabled\n");
+ return ret;
+
+out_threads:
+ free_percpu(pool->cpu_comp_threads);
+out_tasks:
+ free_percpu(pool->cpu_comp_tasks);
+out_pool:
+ kfree(pool);
+ return ret;
+}
+
+void ehca_destroy_comp_pool(void)
+{
+ if (!ehca_scaling_code)
+ return;
+
+ smpboot_unregister_percpu_thread(&comp_pool_threads);
+
+ free_percpu(pool->cpu_comp_threads);
+ free_percpu(pool->cpu_comp_tasks);
+ kfree(pool);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Function definitions and structs for EQs, NEQs and interrupts
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Khadija Souissi <souissi@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IRQ_H
+#define __EHCA_IRQ_H
+
+
+struct ehca_shca;
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+
+int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id);
+void ehca_tasklet_neq(unsigned long data);
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id);
+void ehca_tasklet_eq(unsigned long data);
+void ehca_process_eq(struct ehca_shca *shca, int is_irq);
+
+struct ehca_cpu_comp_task {
+ struct list_head cq_list;
+ spinlock_t task_lock;
+ int cq_jobs;
+ int active;
+};
+
+struct ehca_comp_pool {
+ struct ehca_cpu_comp_task __percpu *cpu_comp_tasks;
+ struct task_struct * __percpu *cpu_comp_threads;
+ int last_cpu;
+ spinlock_t last_cpu_lock;
+};
+
+int ehca_create_comp_pool(void);
+void ehca_destroy_comp_pool(void);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Function definitions for internal functions
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Dietmar Decker <ddecker@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IVERBS_H__
+#define __EHCA_IVERBS_H__
+
+#include "ehca_classes.h"
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
+ struct ib_udata *uhw);
+
+int ehca_query_port(struct ib_device *ibdev, u8 port,
+ struct ib_port_attr *props);
+
+enum rdma_protocol_type
+ehca_query_protocol(struct ib_device *device, u8 port_num);
+
+int ehca_query_sma_attr(struct ehca_shca *shca, u8 port,
+ struct ehca_sma_attr *attr);
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey);
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port, int index,
+ union ib_gid *gid);
+
+int ehca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
+ struct ib_port_modify *props);
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+ struct ib_ucontext *context,
+ struct ib_udata *udata);
+
+int ehca_dealloc_pd(struct ib_pd *pd);
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_destroy_ah(struct ib_ah *ah);
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags, u64 *iova_start);
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ u64 virt, int mr_access_flags,
+ struct ib_udata *udata);
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+ int mr_rereg_mask,
+ struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf, int mr_access_flags, u64 *iova_start);
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
+
+int ehca_dereg_mr(struct ib_mr *mr);
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
+
+int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
+ struct ib_mw_bind *mw_bind);
+
+int ehca_dealloc_mw(struct ib_mw *mw);
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+ int mr_access_flags,
+ struct ib_fmr_attr *fmr_attr);
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+ u64 *page_list, int list_len, u64 iova);
+
+int ehca_unmap_fmr(struct list_head *fmr_list);
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr);
+
+enum ehca_eq_type {
+ EHCA_EQ = 0, /* Event Queue */
+ EHCA_NEQ /* Notification Event Queue */
+};
+
+int ehca_create_eq(struct ehca_shca *shca, struct ehca_eq *eq,
+ enum ehca_eq_type type, const u32 length);
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+
+struct ib_cq *ehca_create_cq(struct ib_device *device,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata);
+
+int ehca_destroy_cq(struct ib_cq *cq);
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+
+int ehca_peek_cq(struct ib_cq *cq, int wc_cnt);
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags);
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int ehca_destroy_qp(struct ib_qp *qp);
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata);
+
+int ehca_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
+ struct ib_send_wr **bad_send_wr);
+
+int ehca_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr);
+
+int ehca_post_srq_recv(struct ib_srq *srq,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr);
+
+struct ib_srq *ehca_create_srq(struct ib_pd *pd,
+ struct ib_srq_init_attr *init_attr,
+ struct ib_udata *udata);
+
+int ehca_modify_srq(struct ib_srq *srq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
+
+int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
+
+int ehca_destroy_srq(struct ib_srq *srq);
+
+u64 ehca_define_sqp(struct ehca_shca *shca, struct ehca_qp *ibqp,
+ struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+int ehca_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+ struct ib_udata *udata);
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context);
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
+int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad_hdr *in, size_t in_mad_size,
+ struct ib_mad_hdr *out, size_t *out_mad_size,
+ u16 *out_mad_pkey_index);
+
+void ehca_poll_eqs(unsigned long data);
+
+int ehca_calc_ipd(struct ehca_shca *shca, int port,
+ enum ib_rate path_rate, u32 *ipd);
+
+void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq);
+
+#ifdef CONFIG_PPC_64K_PAGES
+void *ehca_alloc_fw_ctrlblock(gfp_t flags);
+void ehca_free_fw_ctrlblock(void *ptr);
+#else
+#define ehca_alloc_fw_ctrlblock(flags) ((void *)get_zeroed_page(flags))
+#define ehca_free_fw_ctrlblock(ptr) free_page((unsigned long)(ptr))
+#endif
+
+void ehca_recover_sqp(struct ib_qp *sqp);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * module start stop, hca detection
+ *
+ * Authors: Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifdef CONFIG_PPC_64K_PAGES
+#include <linux/slab.h>
+#endif
+
+#include <linux/notifier.h>
+#include <linux/memory.h>
+#include <rdma/ib_mad.h>
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+#define HCAD_VERSION "0029"
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
+MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
+MODULE_VERSION(HCAD_VERSION);
+
+static bool ehca_open_aqp1 = 0;
+static int ehca_hw_level = 0;
+static bool ehca_poll_all_eqs = 1;
+
+int ehca_debug_level = 0;
+int ehca_nr_ports = -1;
+bool ehca_use_hp_mr = 0;
+int ehca_port_act_time = 30;
+int ehca_static_rate = -1;
+bool ehca_scaling_code = 0;
+int ehca_lock_hcalls = -1;
+int ehca_max_cq = -1;
+int ehca_max_qp = -1;
+
+module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
+module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
+module_param_named(hw_level, ehca_hw_level, int, S_IRUGO);
+module_param_named(nr_ports, ehca_nr_ports, int, S_IRUGO);
+module_param_named(use_hp_mr, ehca_use_hp_mr, bool, S_IRUGO);
+module_param_named(port_act_time, ehca_port_act_time, int, S_IRUGO);
+module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
+module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
+module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
+module_param_named(lock_hcalls, ehca_lock_hcalls, bint, S_IRUGO);
+module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
+module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);
+
+MODULE_PARM_DESC(open_aqp1,
+ "Open AQP1 on startup (default: no)");
+MODULE_PARM_DESC(debug_level,
+ "Amount of debug output (0: none (default), 1: traces, "
+ "2: some dumps, 3: lots)");
+MODULE_PARM_DESC(hw_level,
+ "Hardware level (0: autosensing (default), "
+ "0x10..0x14: eHCA, 0x20..0x23: eHCA2)");
+MODULE_PARM_DESC(nr_ports,
+ "number of connected ports (-1: autodetect (default), "
+ "1: port one only, 2: two ports)");
+MODULE_PARM_DESC(use_hp_mr,
+ "Use high performance MRs (default: no)");
+MODULE_PARM_DESC(port_act_time,
+ "Time to wait for port activation (default: 30 sec)");
+MODULE_PARM_DESC(poll_all_eqs,
+ "Poll all event queues periodically (default: yes)");
+MODULE_PARM_DESC(static_rate,
+ "Set permanent static rate (default: no static rate)");
+MODULE_PARM_DESC(scaling_code,
+ "Enable scaling code (default: no)");
+MODULE_PARM_DESC(lock_hcalls,
+ "Serialize all hCalls made by the driver "
+ "(default: autodetect)");
+MODULE_PARM_DESC(number_of_cqs,
+ "Max number of CQs which can be allocated "
+ "(default: autodetect)");
+MODULE_PARM_DESC(number_of_qps,
+ "Max number of QPs which can be allocated "
+ "(default: autodetect)");
+
+DEFINE_RWLOCK(ehca_qp_idr_lock);
+DEFINE_RWLOCK(ehca_cq_idr_lock);
+DEFINE_IDR(ehca_qp_idr);
+DEFINE_IDR(ehca_cq_idr);
+
+static LIST_HEAD(shca_list); /* list of all registered ehcas */
+DEFINE_SPINLOCK(shca_list_lock);
+
+static struct timer_list poll_eqs_timer;
+
+#ifdef CONFIG_PPC_64K_PAGES
+static struct kmem_cache *ctblk_cache;
+
+void *ehca_alloc_fw_ctrlblock(gfp_t flags)
+{
+ void *ret = kmem_cache_zalloc(ctblk_cache, flags);
+ if (!ret)
+ ehca_gen_err("Out of memory for ctblk");
+ return ret;
+}
+
+void ehca_free_fw_ctrlblock(void *ptr)
+{
+ if (ptr)
+ kmem_cache_free(ctblk_cache, ptr);
+
+}
+#endif
+
+int ehca2ib_return_code(u64 ehca_rc)
+{
+ switch (ehca_rc) {
+ case H_SUCCESS:
+ return 0;
+ case H_RESOURCE: /* Resource in use */
+ case H_BUSY:
+ return -EBUSY;
+ case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
+ case H_CONSTRAINED: /* resource constraint */
+ case H_NO_MEM:
+ return -ENOMEM;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int ehca_create_slab_caches(void)
+{
+ int ret;
+
+ ret = ehca_init_pd_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create PD SLAB cache.");
+ return ret;
+ }
+
+ ret = ehca_init_cq_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create CQ SLAB cache.");
+ goto create_slab_caches2;
+ }
+
+ ret = ehca_init_qp_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create QP SLAB cache.");
+ goto create_slab_caches3;
+ }
+
+ ret = ehca_init_av_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create AV SLAB cache.");
+ goto create_slab_caches4;
+ }
+
+ ret = ehca_init_mrmw_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create MR&MW SLAB cache.");
+ goto create_slab_caches5;
+ }
+
+ ret = ehca_init_small_qp_cache();
+ if (ret) {
+ ehca_gen_err("Cannot create small queue SLAB cache.");
+ goto create_slab_caches6;
+ }
+
+#ifdef CONFIG_PPC_64K_PAGES
+ ctblk_cache = kmem_cache_create("ehca_cache_ctblk",
+ EHCA_PAGESIZE, H_CB_ALIGNMENT,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!ctblk_cache) {
+ ehca_gen_err("Cannot create ctblk SLAB cache.");
+ ehca_cleanup_small_qp_cache();
+ ret = -ENOMEM;
+ goto create_slab_caches6;
+ }
+#endif
+ return 0;
+
+create_slab_caches6:
+ ehca_cleanup_mrmw_cache();
+
+create_slab_caches5:
+ ehca_cleanup_av_cache();
+
+create_slab_caches4:
+ ehca_cleanup_qp_cache();
+
+create_slab_caches3:
+ ehca_cleanup_cq_cache();
+
+create_slab_caches2:
+ ehca_cleanup_pd_cache();
+
+ return ret;
+}
+
+static void ehca_destroy_slab_caches(void)
+{
+ ehca_cleanup_small_qp_cache();
+ ehca_cleanup_mrmw_cache();
+ ehca_cleanup_av_cache();
+ ehca_cleanup_qp_cache();
+ ehca_cleanup_cq_cache();
+ ehca_cleanup_pd_cache();
+#ifdef CONFIG_PPC_64K_PAGES
+ if (ctblk_cache)
+ kmem_cache_destroy(ctblk_cache);
+#endif
+}
+
+#define EHCA_HCAAVER EHCA_BMASK_IBM(32, 39)
+#define EHCA_REVID EHCA_BMASK_IBM(40, 63)
+
+static struct cap_descr {
+ u64 mask;
+ char *descr;
+} hca_cap_descr[] = {
+ { HCA_CAP_AH_PORT_NR_CHECK, "HCA_CAP_AH_PORT_NR_CHECK" },
+ { HCA_CAP_ATOMIC, "HCA_CAP_ATOMIC" },
+ { HCA_CAP_AUTO_PATH_MIG, "HCA_CAP_AUTO_PATH_MIG" },
+ { HCA_CAP_BAD_P_KEY_CTR, "HCA_CAP_BAD_P_KEY_CTR" },
+ { HCA_CAP_SQD_RTS_PORT_CHANGE, "HCA_CAP_SQD_RTS_PORT_CHANGE" },
+ { HCA_CAP_CUR_QP_STATE_MOD, "HCA_CAP_CUR_QP_STATE_MOD" },
+ { HCA_CAP_INIT_TYPE, "HCA_CAP_INIT_TYPE" },
+ { HCA_CAP_PORT_ACTIVE_EVENT, "HCA_CAP_PORT_ACTIVE_EVENT" },
+ { HCA_CAP_Q_KEY_VIOL_CTR, "HCA_CAP_Q_KEY_VIOL_CTR" },
+ { HCA_CAP_WQE_RESIZE, "HCA_CAP_WQE_RESIZE" },
+ { HCA_CAP_RAW_PACKET_MCAST, "HCA_CAP_RAW_PACKET_MCAST" },
+ { HCA_CAP_SHUTDOWN_PORT, "HCA_CAP_SHUTDOWN_PORT" },
+ { HCA_CAP_RC_LL_QP, "HCA_CAP_RC_LL_QP" },
+ { HCA_CAP_SRQ, "HCA_CAP_SRQ" },
+ { HCA_CAP_UD_LL_QP, "HCA_CAP_UD_LL_QP" },
+ { HCA_CAP_RESIZE_MR, "HCA_CAP_RESIZE_MR" },
+ { HCA_CAP_MINI_QP, "HCA_CAP_MINI_QP" },
+ { HCA_CAP_H_ALLOC_RES_SYNC, "HCA_CAP_H_ALLOC_RES_SYNC" },
+};
+
+static int ehca_sense_attributes(struct ehca_shca *shca)
+{
+ int i, ret = 0;
+ u64 h_ret;
+ struct hipz_query_hca *rblock;
+ struct hipz_query_port *port;
+ const char *loc_code;
+
+ static const u32 pgsize_map[] = {
+ HCA_CAP_MR_PGSIZE_4K, 0x1000,
+ HCA_CAP_MR_PGSIZE_64K, 0x10000,
+ HCA_CAP_MR_PGSIZE_1M, 0x100000,
+ HCA_CAP_MR_PGSIZE_16M, 0x1000000,
+ };
+
+ ehca_gen_dbg("Probing adapter %s...",
+ shca->ofdev->dev.of_node->full_name);
+ loc_code = of_get_property(shca->ofdev->dev.of_node, "ibm,loc-code",
+ NULL);
+ if (loc_code)
+ ehca_gen_dbg(" ... location lode=%s", loc_code);
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_gen_err("Cannot allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
+ if (h_ret != H_SUCCESS) {
+ ehca_gen_err("Cannot query device properties. h_ret=%lli",
+ h_ret);
+ ret = -EPERM;
+ goto sense_attributes1;
+ }
+
+ if (ehca_nr_ports == 1)
+ shca->num_ports = 1;
+ else
+ shca->num_ports = (u8)rblock->num_ports;
+
+ ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
+
+ if (ehca_hw_level == 0) {
+ u32 hcaaver;
+ u32 revid;
+
+ hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
+ revid = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
+
+ ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
+
+ if (hcaaver == 1) {
+ if (revid <= 3)
+ shca->hw_level = 0x10 | (revid + 1);
+ else
+ shca->hw_level = 0x14;
+ } else if (hcaaver == 2) {
+ if (revid == 0)
+ shca->hw_level = 0x21;
+ else if (revid == 0x10)
+ shca->hw_level = 0x22;
+ else if (revid == 0x20 || revid == 0x21)
+ shca->hw_level = 0x23;
+ }
+
+ if (!shca->hw_level) {
+ ehca_gen_warn("unknown hardware version"
+ " - assuming default level");
+ shca->hw_level = 0x22;
+ }
+ } else
+ shca->hw_level = ehca_hw_level;
+ ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
+
+ shca->hca_cap = rblock->hca_cap_indicators;
+ ehca_gen_dbg(" ... HCA capabilities:");
+ for (i = 0; i < ARRAY_SIZE(hca_cap_descr); i++)
+ if (EHCA_BMASK_GET(hca_cap_descr[i].mask, shca->hca_cap))
+ ehca_gen_dbg(" %s", hca_cap_descr[i].descr);
+
+ /* Autodetect hCall locking -- the "H_ALLOC_RESOURCE synced" flag is
+ * a firmware property, so it's valid across all adapters
+ */
+ if (ehca_lock_hcalls == -1)
+ ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC,
+ shca->hca_cap);
+
+ /* translate supported MR page sizes; always support 4K */
+ shca->hca_cap_mr_pgsize = EHCA_PAGESIZE;
+ for (i = 0; i < ARRAY_SIZE(pgsize_map); i += 2)
+ if (rblock->memory_page_size_supported & pgsize_map[i])
+ shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];
+
+ /* Set maximum number of CQs and QPs to calculate EQ size */
+ if (shca->max_num_qps == -1)
+ shca->max_num_qps = min_t(int, rblock->max_qp,
+ EHCA_MAX_NUM_QUEUES);
+ else if (shca->max_num_qps < 1 || shca->max_num_qps > rblock->max_qp) {
+ ehca_gen_warn("The requested number of QPs is out of range "
+ "(1 - %i) specified by HW. Value is set to %i",
+ rblock->max_qp, rblock->max_qp);
+ shca->max_num_qps = rblock->max_qp;
+ }
+
+ if (shca->max_num_cqs == -1)
+ shca->max_num_cqs = min_t(int, rblock->max_cq,
+ EHCA_MAX_NUM_QUEUES);
+ else if (shca->max_num_cqs < 1 || shca->max_num_cqs > rblock->max_cq) {
+ ehca_gen_warn("The requested number of CQs is out of range "
+ "(1 - %i) specified by HW. Value is set to %i",
+ rblock->max_cq, rblock->max_cq);
+ }
+
+ /* query max MTU from first port -- it's the same for all ports */
+ port = (struct hipz_query_port *)rblock;
+ h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
+ if (h_ret != H_SUCCESS) {
+ ehca_gen_err("Cannot query port properties. h_ret=%lli",
+ h_ret);
+ ret = -EPERM;
+ goto sense_attributes1;
+ }
+
+ shca->max_mtu = port->max_mtu;
+
+sense_attributes1:
+ ehca_free_fw_ctrlblock(rblock);
+ return ret;
+}
+
+static int init_node_guid(struct ehca_shca *shca)
+{
+ int ret = 0;
+ struct hipz_query_hca *rblock;
+
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!rblock) {
+ ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+ return -ENOMEM;
+ }
+
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "Can't query device properties");
+ ret = -EINVAL;
+ goto init_node_guid1;
+ }
+
+ memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
+
+init_node_guid1:
+ ehca_free_fw_ctrlblock(rblock);
+ return ret;
+}
+
+static int ehca_port_immutable(struct ib_device *ibdev, u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ struct ib_port_attr attr;
+ int err;
+
+ err = ehca_query_port(ibdev, port_num, &attr);
+ if (err)
+ return err;
+
+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
+ immutable->gid_tbl_len = attr.gid_tbl_len;
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+ return 0;
+}
+
+static int ehca_init_device(struct ehca_shca *shca)
+{
+ int ret;
+
+ ret = init_node_guid(shca);
+ if (ret)
+ return ret;
+
+ strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
+ shca->ib_device.owner = THIS_MODULE;
+
+ shca->ib_device.uverbs_abi_ver = 8;
+ shca->ib_device.uverbs_cmd_mask =
+ (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+ (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+ (1ull << IB_USER_VERBS_CMD_REG_MR) |
+ (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+ (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+ (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
+ (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
+
+ shca->ib_device.node_type = RDMA_NODE_IB_CA;
+ shca->ib_device.phys_port_cnt = shca->num_ports;
+ shca->ib_device.num_comp_vectors = 1;
+ shca->ib_device.dma_device = &shca->ofdev->dev;
+ shca->ib_device.query_device = ehca_query_device;
+ shca->ib_device.query_port = ehca_query_port;
+ shca->ib_device.query_gid = ehca_query_gid;
+ shca->ib_device.query_pkey = ehca_query_pkey;
+ /* shca->in_device.modify_device = ehca_modify_device */
+ shca->ib_device.modify_port = ehca_modify_port;
+ shca->ib_device.alloc_ucontext = ehca_alloc_ucontext;
+ shca->ib_device.dealloc_ucontext = ehca_dealloc_ucontext;
+ shca->ib_device.alloc_pd = ehca_alloc_pd;
+ shca->ib_device.dealloc_pd = ehca_dealloc_pd;
+ shca->ib_device.create_ah = ehca_create_ah;
+ /* shca->ib_device.modify_ah = ehca_modify_ah; */
+ shca->ib_device.query_ah = ehca_query_ah;
+ shca->ib_device.destroy_ah = ehca_destroy_ah;
+ shca->ib_device.create_qp = ehca_create_qp;
+ shca->ib_device.modify_qp = ehca_modify_qp;
+ shca->ib_device.query_qp = ehca_query_qp;
+ shca->ib_device.destroy_qp = ehca_destroy_qp;
+ shca->ib_device.post_send = ehca_post_send;
+ shca->ib_device.post_recv = ehca_post_recv;
+ shca->ib_device.create_cq = ehca_create_cq;
+ shca->ib_device.destroy_cq = ehca_destroy_cq;
+ shca->ib_device.resize_cq = ehca_resize_cq;
+ shca->ib_device.poll_cq = ehca_poll_cq;
+ /* shca->ib_device.peek_cq = ehca_peek_cq; */
+ shca->ib_device.req_notify_cq = ehca_req_notify_cq;
+ /* shca->ib_device.req_ncomp_notif = ehca_req_ncomp_notif; */
+ shca->ib_device.get_dma_mr = ehca_get_dma_mr;
+ shca->ib_device.reg_phys_mr = ehca_reg_phys_mr;
+ shca->ib_device.reg_user_mr = ehca_reg_user_mr;
+ shca->ib_device.query_mr = ehca_query_mr;
+ shca->ib_device.dereg_mr = ehca_dereg_mr;
+ shca->ib_device.rereg_phys_mr = ehca_rereg_phys_mr;
+ shca->ib_device.alloc_mw = ehca_alloc_mw;
+ shca->ib_device.bind_mw = ehca_bind_mw;
+ shca->ib_device.dealloc_mw = ehca_dealloc_mw;
+ shca->ib_device.alloc_fmr = ehca_alloc_fmr;
+ shca->ib_device.map_phys_fmr = ehca_map_phys_fmr;
+ shca->ib_device.unmap_fmr = ehca_unmap_fmr;
+ shca->ib_device.dealloc_fmr = ehca_dealloc_fmr;
+ shca->ib_device.attach_mcast = ehca_attach_mcast;
+ shca->ib_device.detach_mcast = ehca_detach_mcast;
+ shca->ib_device.process_mad = ehca_process_mad;
+ shca->ib_device.mmap = ehca_mmap;
+ shca->ib_device.dma_ops = &ehca_dma_mapping_ops;
+ shca->ib_device.get_port_immutable = ehca_port_immutable;
+
+ if (EHCA_BMASK_GET(HCA_CAP_SRQ, shca->hca_cap)) {
+ shca->ib_device.uverbs_cmd_mask |=
+ (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
+ (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ);
+
+ shca->ib_device.create_srq = ehca_create_srq;
+ shca->ib_device.modify_srq = ehca_modify_srq;
+ shca->ib_device.query_srq = ehca_query_srq;
+ shca->ib_device.destroy_srq = ehca_destroy_srq;
+ shca->ib_device.post_srq_recv = ehca_post_srq_recv;
+ }
+
+ return ret;
+}
+
+static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
+{
+ struct ehca_sport *sport = &shca->sport[port - 1];
+ struct ib_cq *ibcq;
+ struct ib_qp *ibqp;
+ struct ib_qp_init_attr qp_init_attr;
+ struct ib_cq_init_attr cq_attr = {};
+ int ret;
+
+ if (sport->ibcq_aqp1) {
+ ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
+ return -EPERM;
+ }
+
+ cq_attr.cqe = 10;
+ ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void *)(-1),
+ &cq_attr);
+ if (IS_ERR(ibcq)) {
+ ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
+ return PTR_ERR(ibcq);
+ }
+ sport->ibcq_aqp1 = ibcq;
+
+ if (sport->ibqp_sqp[IB_QPT_GSI]) {
+ ehca_err(&shca->ib_device, "AQP1 QP is already created.");
+ ret = -EPERM;
+ goto create_aqp1;
+ }
+
+ memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
+ qp_init_attr.send_cq = ibcq;
+ qp_init_attr.recv_cq = ibcq;
+ qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
+ qp_init_attr.cap.max_send_wr = 100;
+ qp_init_attr.cap.max_recv_wr = 100;
+ qp_init_attr.cap.max_send_sge = 2;
+ qp_init_attr.cap.max_recv_sge = 1;
+ qp_init_attr.qp_type = IB_QPT_GSI;
+ qp_init_attr.port_num = port;
+ qp_init_attr.qp_context = NULL;
+ qp_init_attr.event_handler = NULL;
+ qp_init_attr.srq = NULL;
+
+ ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
+ if (IS_ERR(ibqp)) {
+ ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
+ ret = PTR_ERR(ibqp);
+ goto create_aqp1;
+ }
+ sport->ibqp_sqp[IB_QPT_GSI] = ibqp;
+
+ return 0;
+
+create_aqp1:
+ ib_destroy_cq(sport->ibcq_aqp1);
+ return ret;
+}
+
+static int ehca_destroy_aqp1(struct ehca_sport *sport)
+{
+ int ret;
+
+ ret = ib_destroy_qp(sport->ibqp_sqp[IB_QPT_GSI]);
+ if (ret) {
+ ehca_gen_err("Cannot destroy AQP1 QP. ret=%i", ret);
+ return ret;
+ }
+
+ ret = ib_destroy_cq(sport->ibcq_aqp1);
+ if (ret)
+ ehca_gen_err("Cannot destroy AQP1 CQ. ret=%i", ret);
+
+ return ret;
+}
+
+static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
+{
+ return snprintf(buf, PAGE_SIZE, "%d\n", ehca_debug_level);
+}
+
+static ssize_t ehca_store_debug_level(struct device_driver *ddp,
+ const char *buf, size_t count)
+{
+ int value = (*buf) - '0';
+ if (value >= 0 && value <= 9)
+ ehca_debug_level = value;
+ return 1;
+}
+
+static DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
+ ehca_show_debug_level, ehca_store_debug_level);
+
+static struct attribute *ehca_drv_attrs[] = {
+ &driver_attr_debug_level.attr,
+ NULL
+};
+
+static struct attribute_group ehca_drv_attr_grp = {
+ .attrs = ehca_drv_attrs
+};
+
+static const struct attribute_group *ehca_drv_attr_groups[] = {
+ &ehca_drv_attr_grp,
+ NULL,
+};
+
+#define EHCA_RESOURCE_ATTR(name) \
+static ssize_t ehca_show_##name(struct device *dev, \
+ struct device_attribute *attr, \
+ char *buf) \
+{ \
+ struct ehca_shca *shca; \
+ struct hipz_query_hca *rblock; \
+ int data; \
+ \
+ shca = dev_get_drvdata(dev); \
+ \
+ rblock = ehca_alloc_fw_ctrlblock(GFP_KERNEL); \
+ if (!rblock) { \
+ dev_err(dev, "Can't allocate rblock memory.\n"); \
+ return 0; \
+ } \
+ \
+ if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
+ dev_err(dev, "Can't query device properties\n"); \
+ ehca_free_fw_ctrlblock(rblock); \
+ return 0; \
+ } \
+ \
+ data = rblock->name; \
+ ehca_free_fw_ctrlblock(rblock); \
+ \
+ if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1)) \
+ return snprintf(buf, 256, "1\n"); \
+ else \
+ return snprintf(buf, 256, "%d\n", data); \
+ \
+} \
+static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
+
+EHCA_RESOURCE_ATTR(num_ports);
+EHCA_RESOURCE_ATTR(hw_ver);
+EHCA_RESOURCE_ATTR(max_eq);
+EHCA_RESOURCE_ATTR(cur_eq);
+EHCA_RESOURCE_ATTR(max_cq);
+EHCA_RESOURCE_ATTR(cur_cq);
+EHCA_RESOURCE_ATTR(max_qp);
+EHCA_RESOURCE_ATTR(cur_qp);
+EHCA_RESOURCE_ATTR(max_mr);
+EHCA_RESOURCE_ATTR(cur_mr);
+EHCA_RESOURCE_ATTR(max_mw);
+EHCA_RESOURCE_ATTR(cur_mw);
+EHCA_RESOURCE_ATTR(max_pd);
+EHCA_RESOURCE_ATTR(max_ah);
+
+static ssize_t ehca_show_adapter_handle(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct ehca_shca *shca = dev_get_drvdata(dev);
+
+ return sprintf(buf, "%llx\n", shca->ipz_hca_handle.handle);
+
+}
+static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
+
+static struct attribute *ehca_dev_attrs[] = {
+ &dev_attr_adapter_handle.attr,
+ &dev_attr_num_ports.attr,
+ &dev_attr_hw_ver.attr,
+ &dev_attr_max_eq.attr,
+ &dev_attr_cur_eq.attr,
+ &dev_attr_max_cq.attr,
+ &dev_attr_cur_cq.attr,
+ &dev_attr_max_qp.attr,
+ &dev_attr_cur_qp.attr,
+ &dev_attr_max_mr.attr,
+ &dev_attr_cur_mr.attr,
+ &dev_attr_max_mw.attr,
+ &dev_attr_cur_mw.attr,
+ &dev_attr_max_pd.attr,
+ &dev_attr_max_ah.attr,
+ NULL
+};
+
+static struct attribute_group ehca_dev_attr_grp = {
+ .attrs = ehca_dev_attrs
+};
+
+static int ehca_probe(struct platform_device *dev)
+{
+ struct ehca_shca *shca;
+ const u64 *handle;
+ struct ib_pd *ibpd;
+ int ret, i, eq_size;
+ unsigned long flags;
+
+ handle = of_get_property(dev->dev.of_node, "ibm,hca-handle", NULL);
+ if (!handle) {
+ ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
+ dev->dev.of_node->full_name);
+ return -ENODEV;
+ }
+
+ if (!(*handle)) {
+ ehca_gen_err("Wrong eHCA handle for adapter: %s.",
+ dev->dev.of_node->full_name);
+ return -ENODEV;
+ }
+
+ shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
+ if (!shca) {
+ ehca_gen_err("Cannot allocate shca memory.");
+ return -ENOMEM;
+ }
+
+ mutex_init(&shca->modify_mutex);
+ atomic_set(&shca->num_cqs, 0);
+ atomic_set(&shca->num_qps, 0);
+ shca->max_num_qps = ehca_max_qp;
+ shca->max_num_cqs = ehca_max_cq;
+
+ for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
+ spin_lock_init(&shca->sport[i].mod_sqp_lock);
+
+ shca->ofdev = dev;
+ shca->ipz_hca_handle.handle = *handle;
+ dev_set_drvdata(&dev->dev, shca);
+
+ ret = ehca_sense_attributes(shca);
+ if (ret < 0) {
+ ehca_gen_err("Cannot sense eHCA attributes.");
+ goto probe1;
+ }
+
+ ret = ehca_init_device(shca);
+ if (ret) {
+ ehca_gen_err("Cannot init ehca device struct");
+ goto probe1;
+ }
+
+ eq_size = 2 * shca->max_num_cqs + 4 * shca->max_num_qps;
+ /* create event queues */
+ ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create EQ.");
+ goto probe1;
+ }
+
+ ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create NEQ.");
+ goto probe3;
+ }
+
+ /* create internal protection domain */
+ ibpd = ehca_alloc_pd(&shca->ib_device, (void *)(-1), NULL);
+ if (IS_ERR(ibpd)) {
+ ehca_err(&shca->ib_device, "Cannot create internal PD.");
+ ret = PTR_ERR(ibpd);
+ goto probe4;
+ }
+
+ shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
+ shca->pd->ib_pd.device = &shca->ib_device;
+
+ /* create internal max MR */
+ ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
+
+ if (ret) {
+ ehca_err(&shca->ib_device, "Cannot create internal MR ret=%i",
+ ret);
+ goto probe5;
+ }
+
+ ret = ib_register_device(&shca->ib_device, NULL);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "ib_register_device() failed ret=%i", ret);
+ goto probe6;
+ }
+
+ /* create AQP1 for port 1 */
+ if (ehca_open_aqp1 == 1) {
+ shca->sport[0].port_state = IB_PORT_DOWN;
+ ret = ehca_create_aqp1(shca, 1);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "Cannot create AQP1 for port 1.");
+ goto probe7;
+ }
+ }
+
+ /* create AQP1 for port 2 */
+ if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
+ shca->sport[1].port_state = IB_PORT_DOWN;
+ ret = ehca_create_aqp1(shca, 2);
+ if (ret) {
+ ehca_err(&shca->ib_device,
+ "Cannot create AQP1 for port 2.");
+ goto probe8;
+ }
+ }
+
+ ret = sysfs_create_group(&dev->dev.kobj, &ehca_dev_attr_grp);
+ if (ret) /* only complain; we can live without attributes */
+ ehca_err(&shca->ib_device,
+ "Cannot create device attributes ret=%d", ret);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ list_add(&shca->shca_list, &shca_list);
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ return 0;
+
+probe8:
+ ret = ehca_destroy_aqp1(&shca->sport[0]);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy AQP1 for port 1. ret=%i", ret);
+
+probe7:
+ ib_unregister_device(&shca->ib_device);
+
+probe6:
+ ret = ehca_dereg_internal_maxmr(shca);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal MR. ret=%x", ret);
+
+probe5:
+ ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal PD. ret=%x", ret);
+
+probe4:
+ ret = ehca_destroy_eq(shca, &shca->neq);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy NEQ. ret=%x", ret);
+
+probe3:
+ ret = ehca_destroy_eq(shca, &shca->eq);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy EQ. ret=%x", ret);
+
+probe1:
+ ib_dealloc_device(&shca->ib_device);
+
+ return -EINVAL;
+}
+
+static int ehca_remove(struct platform_device *dev)
+{
+ struct ehca_shca *shca = dev_get_drvdata(&dev->dev);
+ unsigned long flags;
+ int ret;
+
+ sysfs_remove_group(&dev->dev.kobj, &ehca_dev_attr_grp);
+
+ if (ehca_open_aqp1 == 1) {
+ int i;
+ for (i = 0; i < shca->num_ports; i++) {
+ ret = ehca_destroy_aqp1(&shca->sport[i]);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy AQP1 for port %x "
+ "ret=%i", ret, i);
+ }
+ }
+
+ ib_unregister_device(&shca->ib_device);
+
+ ret = ehca_dereg_internal_maxmr(shca);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal MR. ret=%i", ret);
+
+ ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+ if (ret)
+ ehca_err(&shca->ib_device,
+ "Cannot destroy internal PD. ret=%i", ret);
+
+ ret = ehca_destroy_eq(shca, &shca->eq);
+ if (ret)
+ ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%i", ret);
+
+ ret = ehca_destroy_eq(shca, &shca->neq);
+ if (ret)
+ ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%i", ret);
+
+ ib_dealloc_device(&shca->ib_device);
+
+ spin_lock_irqsave(&shca_list_lock, flags);
+ list_del(&shca->shca_list);
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+
+ return ret;
+}
+
+static struct of_device_id ehca_device_table[] =
+{
+ {
+ .name = "lhca",
+ .compatible = "IBM,lhca",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, ehca_device_table);
+
+static struct platform_driver ehca_driver = {
+ .probe = ehca_probe,
+ .remove = ehca_remove,
+ .driver = {
+ .name = "ehca",
+ .owner = THIS_MODULE,
+ .groups = ehca_drv_attr_groups,
+ .of_match_table = ehca_device_table,
+ },
+};
+
+void ehca_poll_eqs(unsigned long data)
+{
+ struct ehca_shca *shca;
+
+ spin_lock(&shca_list_lock);
+ list_for_each_entry(shca, &shca_list, shca_list) {
+ if (shca->eq.is_initialized) {
+ /* call deadman proc only if eq ptr does not change */
+ struct ehca_eq *eq = &shca->eq;
+ int max = 3;
+ volatile u64 q_ofs, q_ofs2;
+ unsigned long flags;
+ spin_lock_irqsave(&eq->spinlock, flags);
+ q_ofs = eq->ipz_queue.current_q_offset;
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+ do {
+ spin_lock_irqsave(&eq->spinlock, flags);
+ q_ofs2 = eq->ipz_queue.current_q_offset;
+ spin_unlock_irqrestore(&eq->spinlock, flags);
+ max--;
+ } while (q_ofs == q_ofs2 && max > 0);
+ if (q_ofs == q_ofs2)
+ ehca_process_eq(shca, 0);
+ }
+ }
+ mod_timer(&poll_eqs_timer, round_jiffies(jiffies + HZ));
+ spin_unlock(&shca_list_lock);
+}
+
+static int ehca_mem_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ static unsigned long ehca_dmem_warn_time;
+ unsigned long flags;
+
+ switch (action) {
+ case MEM_CANCEL_OFFLINE:
+ case MEM_CANCEL_ONLINE:
+ case MEM_ONLINE:
+ case MEM_OFFLINE:
+ return NOTIFY_OK;
+ case MEM_GOING_ONLINE:
+ case MEM_GOING_OFFLINE:
+ /* only ok if no hca is attached to the lpar */
+ spin_lock_irqsave(&shca_list_lock, flags);
+ if (list_empty(&shca_list)) {
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+ return NOTIFY_OK;
+ } else {
+ spin_unlock_irqrestore(&shca_list_lock, flags);
+ if (printk_timed_ratelimit(&ehca_dmem_warn_time,
+ 30 * 1000))
+ ehca_gen_err("DMEM operations are not allowed"
+ "in conjunction with eHCA");
+ return NOTIFY_BAD;
+ }
+ }
+ return NOTIFY_OK;
+}
+
+static struct notifier_block ehca_mem_nb = {
+ .notifier_call = ehca_mem_notifier,
+};
+
+static int __init ehca_module_init(void)
+{
+ int ret;
+
+ printk(KERN_INFO "eHCA Infiniband Device Driver "
+ "(Version " HCAD_VERSION ")\n");
+
+ ret = ehca_create_comp_pool();
+ if (ret) {
+ ehca_gen_err("Cannot create comp pool.");
+ return ret;
+ }
+
+ ret = ehca_create_slab_caches();
+ if (ret) {
+ ehca_gen_err("Cannot create SLAB caches");
+ ret = -ENOMEM;
+ goto module_init1;
+ }
+
+ ret = ehca_create_busmap();
+ if (ret) {
+ ehca_gen_err("Cannot create busmap.");
+ goto module_init2;
+ }
+
+ ret = ibmebus_register_driver(&ehca_driver);
+ if (ret) {
+ ehca_gen_err("Cannot register eHCA device driver");
+ ret = -EINVAL;
+ goto module_init3;
+ }
+
+ ret = register_memory_notifier(&ehca_mem_nb);
+ if (ret) {
+ ehca_gen_err("Failed registering memory add/remove notifier");
+ goto module_init4;
+ }
+
+ if (ehca_poll_all_eqs != 1) {
+ ehca_gen_err("WARNING!!!");
+ ehca_gen_err("It is possible to lose interrupts.");
+ } else {
+ init_timer(&poll_eqs_timer);
+ poll_eqs_timer.function = ehca_poll_eqs;
+ poll_eqs_timer.expires = jiffies + HZ;
+ add_timer(&poll_eqs_timer);
+ }
+
+ return 0;
+
+module_init4:
+ ibmebus_unregister_driver(&ehca_driver);
+
+module_init3:
+ ehca_destroy_busmap();
+
+module_init2:
+ ehca_destroy_slab_caches();
+
+module_init1:
+ ehca_destroy_comp_pool();
+ return ret;
+};
+
+static void __exit ehca_module_exit(void)
+{
+ if (ehca_poll_all_eqs == 1)
+ del_timer_sync(&poll_eqs_timer);
+
+ ibmebus_unregister_driver(&ehca_driver);
+
+ unregister_memory_notifier(&ehca_mem_nb);
+
+ ehca_destroy_busmap();
+
+ ehca_destroy_slab_caches();
+
+ ehca_destroy_comp_pool();
+
+ idr_destroy(&ehca_cq_idr);
+ idr_destroy(&ehca_qp_idr);
+};
+
+module_init(ehca_module_init);
+module_exit(ehca_module_exit);
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * mcast functions
+ *
+ * Authors: Khadija Souissi <souissik@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+#define MAX_MC_LID 0xFFFE
+#define MIN_MC_LID 0xC000 /* Multicast limits */
+#define EHCA_VALID_MULTICAST_GID(gid) ((gid)[0] == 0xFF)
+#define EHCA_VALID_MULTICAST_LID(lid) \
+ (((lid) >= MIN_MC_LID) && ((lid) <= MAX_MC_LID))
+
+int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+ ib_device);
+ union ib_gid my_gid;
+ u64 subnet_prefix, interface_id, h_ret;
+
+ if (ibqp->qp_type != IB_QPT_UD) {
+ ehca_err(ibqp->device, "invalid qp_type=%x", ibqp->qp_type);
+ return -EINVAL;
+ }
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ ehca_err(ibqp->device, "invalid mulitcast gid");
+ return -EINVAL;
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+ return -EINVAL;
+ }
+
+ memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
+
+ subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+ interface_id = be64_to_cpu(my_gid.global.interface_id);
+ h_ret = hipz_h_attach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->galpas.kernel,
+ lid, subnet_prefix, interface_id);
+ if (h_ret != H_SUCCESS)
+ ehca_err(ibqp->device,
+ "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
+ "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
+
+ return ehca2ib_return_code(h_ret);
+}
+
+int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(ibqp->pd->device,
+ struct ehca_shca, ib_device);
+ union ib_gid my_gid;
+ u64 subnet_prefix, interface_id, h_ret;
+
+ if (ibqp->qp_type != IB_QPT_UD) {
+ ehca_err(ibqp->device, "invalid qp_type %x", ibqp->qp_type);
+ return -EINVAL;
+ }
+
+ if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+ ehca_err(ibqp->device, "invalid mulitcast gid");
+ return -EINVAL;
+ } else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+ ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+ return -EINVAL;
+ }
+
+ memcpy(&my_gid, gid->raw, sizeof(union ib_gid));
+
+ subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+ interface_id = be64_to_cpu(my_gid.global.interface_id);
+ h_ret = hipz_h_detach_mcqp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ my_qp->galpas.kernel,
+ lid, subnet_prefix, interface_id);
+ if (h_ret != H_SUCCESS)
+ ehca_err(ibqp->device,
+ "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
+ "h_ret=%lli", my_qp, ibqp->qp_num, h_ret);
+
+ return ehca2ib_return_code(h_ret);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * MR/MW functions
+ *
+ * Authors: Dietmar Decker <ddecker@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+#include <rdma/ib_umem.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "hcp_if.h"
+#include "hipz_hw.h"
+
+#define NUM_CHUNKS(length, chunk_size) \
+ (((length) + (chunk_size - 1)) / (chunk_size))
+
+/* max number of rpages (per hcall register_rpages) */
+#define MAX_RPAGES 512
+
+/* DMEM toleration management */
+#define EHCA_SECTSHIFT SECTION_SIZE_BITS
+#define EHCA_SECTSIZE (1UL << EHCA_SECTSHIFT)
+#define EHCA_HUGEPAGESHIFT 34
+#define EHCA_HUGEPAGE_SIZE (1UL << EHCA_HUGEPAGESHIFT)
+#define EHCA_HUGEPAGE_PFN_MASK ((EHCA_HUGEPAGE_SIZE - 1) >> PAGE_SHIFT)
+#define EHCA_INVAL_ADDR 0xFFFFFFFFFFFFFFFFULL
+#define EHCA_DIR_INDEX_SHIFT 13 /* 8k Entries in 64k block */
+#define EHCA_TOP_INDEX_SHIFT (EHCA_DIR_INDEX_SHIFT * 2)
+#define EHCA_MAP_ENTRIES (1 << EHCA_DIR_INDEX_SHIFT)
+#define EHCA_TOP_MAP_SIZE (0x10000) /* currently fixed map size */
+#define EHCA_DIR_MAP_SIZE (0x10000)
+#define EHCA_ENT_MAP_SIZE (0x10000)
+#define EHCA_INDEX_MASK (EHCA_MAP_ENTRIES - 1)
+
+static unsigned long ehca_mr_len;
+
+/*
+ * Memory map data structures
+ */
+struct ehca_dir_bmap {
+ u64 ent[EHCA_MAP_ENTRIES];
+};
+struct ehca_top_bmap {
+ struct ehca_dir_bmap *dir[EHCA_MAP_ENTRIES];
+};
+struct ehca_bmap {
+ struct ehca_top_bmap *top[EHCA_MAP_ENTRIES];
+};
+
+static struct ehca_bmap *ehca_bmap;
+
+static struct kmem_cache *mr_cache;
+static struct kmem_cache *mw_cache;
+
+enum ehca_mr_pgsize {
+ EHCA_MR_PGSIZE4K = 0x1000L,
+ EHCA_MR_PGSIZE64K = 0x10000L,
+ EHCA_MR_PGSIZE1M = 0x100000L,
+ EHCA_MR_PGSIZE16M = 0x1000000L
+};
+
+#define EHCA_MR_PGSHIFT4K 12
+#define EHCA_MR_PGSHIFT64K 16
+#define EHCA_MR_PGSHIFT1M 20
+#define EHCA_MR_PGSHIFT16M 24
+
+static u64 ehca_map_vaddr(void *caddr);
+
+static u32 ehca_encode_hwpage_size(u32 pgsize)
+{
+ int log = ilog2(pgsize);
+ WARN_ON(log < 12 || log > 24 || log & 3);
+ return (log - 12) / 4;
+}
+
+static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)
+{
+ return rounddown_pow_of_two(shca->hca_cap_mr_pgsize);
+}
+
+static struct ehca_mr *ehca_mr_new(void)
+{
+ struct ehca_mr *me;
+
+ me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
+ if (me)
+ spin_lock_init(&me->mrlock);
+ else
+ ehca_gen_err("alloc failed");
+
+ return me;
+}
+
+static void ehca_mr_delete(struct ehca_mr *me)
+{
+ kmem_cache_free(mr_cache, me);
+}
+
+static struct ehca_mw *ehca_mw_new(void)
+{
+ struct ehca_mw *me;
+
+ me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
+ if (me)
+ spin_lock_init(&me->mwlock);
+ else
+ ehca_gen_err("alloc failed");
+
+ return me;
+}
+
+static void ehca_mw_delete(struct ehca_mw *me)
+{
+ kmem_cache_free(mw_cache, me);
+}
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
+{
+ struct ib_mr *ib_mr;
+ int ret;
+ struct ehca_mr *e_maxmr;
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+
+ if (shca->maxmr) {
+ e_maxmr = ehca_mr_new();
+ if (!e_maxmr) {
+ ehca_err(&shca->ib_device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto get_dma_mr_exit0;
+ }
+
+ ret = ehca_reg_maxmr(shca, e_maxmr,
+ (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)),
+ mr_access_flags, e_pd,
+ &e_maxmr->ib.ib_mr.lkey,
+ &e_maxmr->ib.ib_mr.rkey);
+ if (ret) {
+ ehca_mr_delete(e_maxmr);
+ ib_mr = ERR_PTR(ret);
+ goto get_dma_mr_exit0;
+ }
+ ib_mr = &e_maxmr->ib.ib_mr;
+ } else {
+ ehca_err(&shca->ib_device, "no internal max-MR exist!");
+ ib_mr = ERR_PTR(-EINVAL);
+ goto get_dma_mr_exit0;
+ }
+
+get_dma_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x",
+ PTR_ERR(ib_mr), pd, mr_access_flags);
+ return ib_mr;
+} /* end ehca_get_dma_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags,
+ u64 *iova_start)
+{
+ struct ib_mr *ib_mr;
+ int ret;
+ struct ehca_mr *e_mr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+ u64 size;
+
+ if ((num_phys_buf <= 0) || !phys_buf_array) {
+ ehca_err(pd->device, "bad input values: num_phys_buf=%x "
+ "phys_buf_array=%p", num_phys_buf, phys_buf_array);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+
+ /* check physical buffer list and calculate size */
+ ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
+ iova_start, &size);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit0;
+ }
+ if ((size == 0) ||
+ (((u64)iova_start + size) < (u64)iova_start)) {
+ ehca_err(pd->device, "bad input values: size=%llx iova_start=%p",
+ size, iova_start);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_phys_mr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(pd->device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto reg_phys_mr_exit0;
+ }
+
+ /* register MR on HCA */
+ if (ehca_mr_is_maxmr(size, iova_start)) {
+ e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+ ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
+ e_pd, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit1;
+ }
+ } else {
+ struct ehca_mr_pginfo pginfo;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hw_pgsize;
+
+ num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
+ PAGE_SIZE);
+ /* for kernel space we try most possible pgsize */
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size,
+ hw_pgsize);
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.phy.num_phys_buf = num_phys_buf;
+ pginfo.u.phy.phys_buf_array = phys_buf_array;
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
+ e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_phys_mr_exit1;
+ }
+ }
+
+ /* successful registration of all pages */
+ return &e_mr->ib.ib_mr;
+
+reg_phys_mr_exit1:
+ ehca_mr_delete(e_mr);
+reg_phys_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(pd->device, "h_ret=%li pd=%p phys_buf_array=%p "
+ "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
+ PTR_ERR(ib_mr), pd, phys_buf_array,
+ num_phys_buf, mr_access_flags, iova_start);
+ return ib_mr;
+} /* end ehca_reg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+ u64 virt, int mr_access_flags,
+ struct ib_udata *udata)
+{
+ struct ib_mr *ib_mr;
+ struct ehca_mr *e_mr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_mr_pginfo pginfo;
+ int ret, page_shift;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hwpage_size;
+
+ if (!pd) {
+ ehca_gen_err("bad pd=%p", pd);
+ return ERR_PTR(-EFAULT);
+ }
+
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit0;
+ }
+
+ if (length == 0 || virt + length < virt) {
+ ehca_err(pd->device, "bad input values: length=%llx "
+ "virt_base=%llx", length, virt);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(pd->device, "out of memory");
+ ib_mr = ERR_PTR(-ENOMEM);
+ goto reg_user_mr_exit0;
+ }
+
+ e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
+ mr_access_flags, 0);
+ if (IS_ERR(e_mr->umem)) {
+ ib_mr = (void *)e_mr->umem;
+ goto reg_user_mr_exit1;
+ }
+
+ if (e_mr->umem->page_size != PAGE_SIZE) {
+ ehca_err(pd->device, "page size not supported, "
+ "e_mr->umem->page_size=%x", e_mr->umem->page_size);
+ ib_mr = ERR_PTR(-EINVAL);
+ goto reg_user_mr_exit2;
+ }
+
+ /* determine number of MR pages */
+ num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
+ /* select proper hw_pgsize */
+ page_shift = PAGE_SHIFT;
+ if (e_mr->umem->hugetlb) {
+ /* determine page_shift, clamp between 4K and 16M */
+ page_shift = (fls64(length - 1) + 3) & ~3;
+ page_shift = min(max(page_shift, EHCA_MR_PGSHIFT4K),
+ EHCA_MR_PGSHIFT16M);
+ }
+ hwpage_size = 1UL << page_shift;
+
+ /* now that we have the desired page size, shift until it's
+ * supported, too. 4K is always supported, so this terminates.
+ */
+ while (!(hwpage_size & shca->hca_cap_mr_pgsize))
+ hwpage_size >>= 4;
+
+reg_user_mr_fallback:
+ num_hwpages = NUM_CHUNKS((virt % hwpage_size) + length, hwpage_size);
+ /* register MR on HCA */
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_USER;
+ pginfo.hwpage_size = hwpage_size;
+ pginfo.num_kpages = num_kpages;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.usr.region = e_mr->umem;
+ pginfo.next_hwpage = ib_umem_offset(e_mr->umem) / hwpage_size;
+ pginfo.u.usr.next_sg = pginfo.u.usr.region->sg_head.sgl;
+ ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
+ e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_MR);
+ if (ret == -EINVAL && pginfo.hwpage_size > PAGE_SIZE) {
+ ehca_warn(pd->device, "failed to register mr "
+ "with hwpage_size=%llx", hwpage_size);
+ ehca_info(pd->device, "try to register mr with "
+ "kpage_size=%lx", PAGE_SIZE);
+ /*
+ * this means kpages are not contiguous for a hw page
+ * try kernel page size as fallback solution
+ */
+ hwpage_size = PAGE_SIZE;
+ goto reg_user_mr_fallback;
+ }
+ if (ret) {
+ ib_mr = ERR_PTR(ret);
+ goto reg_user_mr_exit2;
+ }
+
+ /* successful registration of all pages */
+ return &e_mr->ib.ib_mr;
+
+reg_user_mr_exit2:
+ ib_umem_release(e_mr->umem);
+reg_user_mr_exit1:
+ ehca_mr_delete(e_mr);
+reg_user_mr_exit0:
+ if (IS_ERR(ib_mr))
+ ehca_err(pd->device, "rc=%li pd=%p mr_access_flags=%x udata=%p",
+ PTR_ERR(ib_mr), pd, mr_access_flags, udata);
+ return ib_mr;
+} /* end ehca_reg_user_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+ int mr_rereg_mask,
+ struct ib_pd *pd,
+ struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ int mr_access_flags,
+ u64 *iova_start)
+{
+ int ret;
+
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+ u64 new_size;
+ u64 *new_start;
+ u32 new_acl;
+ struct ehca_pd *new_pd;
+ u32 tmp_lkey, tmp_rkey;
+ unsigned long sl_flags;
+ u32 num_kpages = 0;
+ u32 num_hwpages = 0;
+ struct ehca_mr_pginfo pginfo;
+
+ if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
+ /* TODO not supported, because PHYP rereg hCall needs pages */
+ ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
+ "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ if (mr_rereg_mask & IB_MR_REREG_PD) {
+ if (!pd) {
+ ehca_err(mr->device, "rereg with bad pd, pd=%p "
+ "mr_rereg_mask=%x", pd, mr_rereg_mask);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ }
+
+ if ((mr_rereg_mask &
+ ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
+ (mr_rereg_mask == 0)) {
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ /* check other parameters */
+ if (e_mr == shca->maxmr) {
+ /* should be impossible, however reject to be sure */
+ ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
+ "shca->maxmr=%p mr->lkey=%x",
+ mr, shca->maxmr, mr->lkey);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
+ if (e_mr->flags & EHCA_MR_FLAG_FMR) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p "
+ "flags=%x", mr, e_mr->flags);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ if (!phys_buf_array || num_phys_buf <= 0) {
+ ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
+ " phys_buf_array=%p num_phys_buf=%x",
+ mr_rereg_mask, phys_buf_array, num_phys_buf);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+ }
+ if ((mr_rereg_mask & IB_MR_REREG_ACCESS) && /* change ACL */
+ (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
+ "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit0;
+ }
+
+ /* set requested values dependent on rereg request */
+ spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+ new_start = e_mr->start;
+ new_size = e_mr->size;
+ new_acl = e_mr->acl;
+ new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
+
+ if (mr_rereg_mask & IB_MR_REREG_TRANS) {
+ u64 hw_pgsize = ehca_get_max_hwpage_size(shca);
+
+ new_start = iova_start; /* change address */
+ /* check physical buffer list and calculate size */
+ ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
+ num_phys_buf, iova_start,
+ &new_size);
+ if (ret)
+ goto rereg_phys_mr_exit1;
+ if ((new_size == 0) ||
+ (((u64)iova_start + new_size) < (u64)iova_start)) {
+ ehca_err(mr->device, "bad input values: new_size=%llx "
+ "iova_start=%p", new_size, iova_start);
+ ret = -EINVAL;
+ goto rereg_phys_mr_exit1;
+ }
+ num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
+ new_size, PAGE_SIZE);
+ num_hwpages = NUM_CHUNKS(((u64)new_start % hw_pgsize) +
+ new_size, hw_pgsize);
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.u.phy.num_phys_buf = num_phys_buf;
+ pginfo.u.phy.phys_buf_array = phys_buf_array;
+ pginfo.next_hwpage =
+ ((u64)iova_start & ~PAGE_MASK) / hw_pgsize;
+ }
+ if (mr_rereg_mask & IB_MR_REREG_ACCESS)
+ new_acl = mr_access_flags;
+ if (mr_rereg_mask & IB_MR_REREG_PD)
+ new_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+ ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
+ new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+ if (ret)
+ goto rereg_phys_mr_exit1;
+
+ /* successful reregistration */
+ if (mr_rereg_mask & IB_MR_REREG_PD)
+ mr->pd = pd;
+ mr->lkey = tmp_lkey;
+ mr->rkey = tmp_rkey;
+
+rereg_phys_mr_exit1:
+ spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+rereg_phys_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p mr_rereg_mask=%x pd=%p "
+ "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
+ "iova_start=%p",
+ ret, mr, mr_rereg_mask, pd, phys_buf_array,
+ num_phys_buf, mr_access_flags, iova_start);
+ return ret;
+} /* end ehca_rereg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+ unsigned long sl_flags;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+ "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+ ret = -EINVAL;
+ goto query_mr_exit0;
+ }
+
+ memset(mr_attr, 0, sizeof(struct ib_mr_attr));
+ spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+
+ h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lli mr=%p "
+ "hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle, mr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto query_mr_exit1;
+ }
+ mr_attr->pd = mr->pd;
+ mr_attr->device_virt_addr = hipzout.vaddr;
+ mr_attr->size = hipzout.len;
+ mr_attr->lkey = hipzout.lkey;
+ mr_attr->rkey = hipzout.rkey;
+ ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
+
+query_mr_exit1:
+ spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+query_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p mr_attr=%p",
+ ret, mr, mr_attr);
+ return ret;
+} /* end ehca_query_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_mr(struct ib_mr *mr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+
+ if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+ "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+ ret = -EINVAL;
+ goto dereg_mr_exit0;
+ } else if (e_mr == shca->maxmr) {
+ /* should be impossible, however reject to be sure */
+ ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
+ "shca->maxmr=%p mr->lkey=%x",
+ mr, shca->maxmr, mr->lkey);
+ ret = -EINVAL;
+ goto dereg_mr_exit0;
+ }
+
+ /* TODO: BUSY: MR still has bound window(s) */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lli shca=%p "
+ "e_mr=%p hca_hndl=%llx mr_hndl=%llx mr->lkey=%x",
+ h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle, mr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto dereg_mr_exit0;
+ }
+
+ if (e_mr->umem)
+ ib_umem_release(e_mr->umem);
+
+ /* successful deregistration */
+ ehca_mr_delete(e_mr);
+
+dereg_mr_exit0:
+ if (ret)
+ ehca_err(mr->device, "ret=%i mr=%p", ret, mr);
+ return ret;
+} /* end ehca_dereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
+{
+ struct ib_mw *ib_mw;
+ u64 h_ret;
+ struct ehca_mw *e_mw;
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_mw_hipzout_parms hipzout;
+
+ if (type != IB_MW_TYPE_1)
+ return ERR_PTR(-EINVAL);
+
+ e_mw = ehca_mw_new();
+ if (!e_mw) {
+ ib_mw = ERR_PTR(-ENOMEM);
+ goto alloc_mw_exit0;
+ }
+
+ h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
+ e_pd->fw_pd, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lli "
+ "shca=%p hca_hndl=%llx mw=%p",
+ h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
+ ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
+ goto alloc_mw_exit1;
+ }
+ /* successful MW allocation */
+ e_mw->ipz_mw_handle = hipzout.handle;
+ e_mw->ib_mw.rkey = hipzout.rkey;
+ return &e_mw->ib_mw;
+
+alloc_mw_exit1:
+ ehca_mw_delete(e_mw);
+alloc_mw_exit0:
+ if (IS_ERR(ib_mw))
+ ehca_err(pd->device, "h_ret=%li pd=%p", PTR_ERR(ib_mw), pd);
+ return ib_mw;
+} /* end ehca_alloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_bind_mw(struct ib_qp *qp,
+ struct ib_mw *mw,
+ struct ib_mw_bind *mw_bind)
+{
+ /* TODO: not supported up to now */
+ ehca_gen_err("bind MW currently not supported by HCAD");
+
+ return -EPERM;
+} /* end ehca_bind_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_mw(struct ib_mw *mw)
+{
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(mw->device, struct ehca_shca, ib_device);
+ struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
+
+ h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lli shca=%p "
+ "mw=%p rkey=%x hca_hndl=%llx mw_hndl=%llx",
+ h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
+ e_mw->ipz_mw_handle.handle);
+ return ehca2ib_return_code(h_ret);
+ }
+ /* successful deallocation */
+ ehca_mw_delete(e_mw);
+ return 0;
+} /* end ehca_dealloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+ int mr_access_flags,
+ struct ib_fmr_attr *fmr_attr)
+{
+ struct ib_fmr *ib_fmr;
+ struct ehca_shca *shca =
+ container_of(pd->device, struct ehca_shca, ib_device);
+ struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_mr *e_fmr;
+ int ret;
+ u32 tmp_lkey, tmp_rkey;
+ struct ehca_mr_pginfo pginfo;
+ u64 hw_pgsize;
+
+ /* check other parameters */
+ if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+ ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+ !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+ /*
+ * Remote Write Access requires Local Write Access
+ * Remote Atomic Access requires Local Write Access
+ */
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+ if (mr_access_flags & IB_ACCESS_MW_BIND) {
+ ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+ mr_access_flags);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+ if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
+ ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
+ "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
+ fmr_attr->max_pages, fmr_attr->max_maps,
+ fmr_attr->page_shift);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+
+ hw_pgsize = 1 << fmr_attr->page_shift;
+ if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) {
+ ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
+ fmr_attr->page_shift);
+ ib_fmr = ERR_PTR(-EINVAL);
+ goto alloc_fmr_exit0;
+ }
+
+ e_fmr = ehca_mr_new();
+ if (!e_fmr) {
+ ib_fmr = ERR_PTR(-ENOMEM);
+ goto alloc_fmr_exit0;
+ }
+ e_fmr->flags |= EHCA_MR_FLAG_FMR;
+
+ /* register MR on HCA */
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.hwpage_size = hw_pgsize;
+ /*
+ * pginfo.num_hwpages==0, ie register_rpages() will not be called
+ * but deferred to map_phys_fmr()
+ */
+ ret = ehca_reg_mr(shca, e_fmr, NULL,
+ fmr_attr->max_pages * (1 << fmr_attr->page_shift),
+ mr_access_flags, e_pd, &pginfo,
+ &tmp_lkey, &tmp_rkey, EHCA_REG_MR);
+ if (ret) {
+ ib_fmr = ERR_PTR(ret);
+ goto alloc_fmr_exit1;
+ }
+
+ /* successful */
+ e_fmr->hwpage_size = hw_pgsize;
+ e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
+ e_fmr->fmr_max_pages = fmr_attr->max_pages;
+ e_fmr->fmr_max_maps = fmr_attr->max_maps;
+ e_fmr->fmr_map_cnt = 0;
+ return &e_fmr->ib.ib_fmr;
+
+alloc_fmr_exit1:
+ ehca_mr_delete(e_fmr);
+alloc_fmr_exit0:
+ return ib_fmr;
+} /* end ehca_alloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+ u64 *page_list,
+ int list_len,
+ u64 iova)
+{
+ int ret;
+ struct ehca_shca *shca =
+ container_of(fmr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+ struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
+ struct ehca_mr_pginfo pginfo;
+ u32 tmp_lkey, tmp_rkey;
+
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+ e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto map_phys_fmr_exit0;
+ }
+ ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
+ if (ret)
+ goto map_phys_fmr_exit0;
+ if (iova % e_fmr->fmr_page_size) {
+ /* only whole-numbered pages */
+ ehca_err(fmr->device, "bad iova, iova=%llx fmr_page_size=%x",
+ iova, e_fmr->fmr_page_size);
+ ret = -EINVAL;
+ goto map_phys_fmr_exit0;
+ }
+ if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
+ /* HCAD does not limit the maps, however trace this anyway */
+ ehca_info(fmr->device, "map limit exceeded, fmr=%p "
+ "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
+ fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
+ }
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_FMR;
+ pginfo.num_kpages = list_len;
+ pginfo.hwpage_size = e_fmr->hwpage_size;
+ pginfo.num_hwpages =
+ list_len * e_fmr->fmr_page_size / pginfo.hwpage_size;
+ pginfo.u.fmr.page_list = page_list;
+ pginfo.next_hwpage =
+ (iova & (e_fmr->fmr_page_size-1)) / pginfo.hwpage_size;
+ pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
+
+ ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
+ list_len * e_fmr->fmr_page_size,
+ e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+ if (ret)
+ goto map_phys_fmr_exit0;
+
+ /* successful reregistration */
+ e_fmr->fmr_map_cnt++;
+ e_fmr->ib.ib_fmr.lkey = tmp_lkey;
+ e_fmr->ib.ib_fmr.rkey = tmp_rkey;
+ return 0;
+
+map_phys_fmr_exit0:
+ if (ret)
+ ehca_err(fmr->device, "ret=%i fmr=%p page_list=%p list_len=%x "
+ "iova=%llx", ret, fmr, page_list, list_len, iova);
+ return ret;
+} /* end ehca_map_phys_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_fmr(struct list_head *fmr_list)
+{
+ int ret = 0;
+ struct ib_fmr *ib_fmr;
+ struct ehca_shca *shca = NULL;
+ struct ehca_shca *prev_shca;
+ struct ehca_mr *e_fmr;
+ u32 num_fmr = 0;
+ u32 unmap_fmr_cnt = 0;
+
+ /* check all FMR belong to same SHCA, and check internal flag */
+ list_for_each_entry(ib_fmr, fmr_list, list) {
+ prev_shca = shca;
+ shca = container_of(ib_fmr->device, struct ehca_shca,
+ ib_device);
+ e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+ if ((shca != prev_shca) && prev_shca) {
+ ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
+ "prev_shca=%p e_fmr=%p",
+ shca, prev_shca, e_fmr);
+ ret = -EINVAL;
+ goto unmap_fmr_exit0;
+ }
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
+ "e_fmr->flags=%x", e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto unmap_fmr_exit0;
+ }
+ num_fmr++;
+ }
+
+ /* loop over all FMRs to unmap */
+ list_for_each_entry(ib_fmr, fmr_list, list) {
+ unmap_fmr_cnt++;
+ e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+ shca = container_of(ib_fmr->device, struct ehca_shca,
+ ib_device);
+ ret = ehca_unmap_one_fmr(shca, e_fmr);
+ if (ret) {
+ /* unmap failed, stop unmapping of rest of FMRs */
+ ehca_err(&shca->ib_device, "unmap of one FMR failed, "
+ "stop rest, e_fmr=%p num_fmr=%x "
+ "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
+ unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
+ goto unmap_fmr_exit0;
+ }
+ }
+
+unmap_fmr_exit0:
+ if (ret)
+ ehca_gen_err("ret=%i fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
+ ret, fmr_list, num_fmr, unmap_fmr_cnt);
+ return ret;
+} /* end ehca_unmap_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr)
+{
+ int ret;
+ u64 h_ret;
+ struct ehca_shca *shca =
+ container_of(fmr->device, struct ehca_shca, ib_device);
+ struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+
+ if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+ ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+ e_fmr, e_fmr->flags);
+ ret = -EINVAL;
+ goto free_fmr_exit0;
+ }
+
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lli e_fmr=%p "
+ "hca_hndl=%llx fmr_hndl=%llx fmr->lkey=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle, fmr->lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto free_fmr_exit0;
+ }
+ /* successful deregistration */
+ ehca_mr_delete(e_fmr);
+ return 0;
+
+free_fmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr);
+ return ret;
+} /* end ehca_dealloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo);
+
+int ehca_reg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey, /*OUT*/
+ u32 *rkey, /*OUT*/
+ enum ehca_reg_type reg_type)
+{
+ int ret;
+ u64 h_ret;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
+ if (ehca_use_hp_mr == 1)
+ hipz_acl |= 0x00000001;
+
+ h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
+ (u64)iova_start, size, hipz_acl,
+ e_pd->fw_pd, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli "
+ "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_reg_mr_exit0;
+ }
+
+ e_mr->ipz_mr_handle = hipzout.handle;
+
+ if (reg_type == EHCA_REG_BUSMAP_MR)
+ ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);
+ else if (reg_type == EHCA_REG_MR)
+ ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
+ else
+ ret = -EINVAL;
+
+ if (ret)
+ goto ehca_reg_mr_exit1;
+
+ /* successful registration */
+ e_mr->num_kpages = pginfo->num_kpages;
+ e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
+ e_mr->start = iova_start;
+ e_mr->size = size;
+ e_mr->acl = acl;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+
+ehca_reg_mr_exit1:
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p lkey=%x "
+ "pginfo=%p num_kpages=%llx num_hwpages=%llx ret=%i",
+ h_ret, shca, e_mr, iova_start, size, acl, e_pd,
+ hipzout.lkey, pginfo, pginfo->num_kpages,
+ pginfo->num_hwpages, ret);
+ ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
+ "not recoverable");
+ }
+ehca_reg_mr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
+ "num_kpages=%llx num_hwpages=%llx",
+ ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
+ pginfo->num_kpages, pginfo->num_hwpages);
+ return ret;
+} /* end ehca_reg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ int ret = 0;
+ u64 h_ret;
+ u32 rnum;
+ u64 rpage;
+ u32 i;
+ u64 *kpage;
+
+ if (!pginfo->num_hwpages) /* in case of fmr */
+ return 0;
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ ret = -ENOMEM;
+ goto ehca_reg_mr_rpages_exit0;
+ }
+
+ /* max MAX_RPAGES ehca mr pages per register call */
+ for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
+
+ if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
+ rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
+ if (rnum == 0)
+ rnum = MAX_RPAGES; /* last shot is full */
+ } else
+ rnum = MAX_RPAGES;
+
+ ret = ehca_set_pagebuf(pginfo, rnum, kpage);
+ if (ret) {
+ ehca_err(&shca->ib_device, "ehca_set_pagebuf "
+ "bad rc, ret=%i rnum=%x kpage=%p",
+ ret, rnum, kpage);
+ goto ehca_reg_mr_rpages_exit1;
+ }
+
+ if (rnum > 1) {
+ rpage = __pa(kpage);
+ if (!rpage) {
+ ehca_err(&shca->ib_device, "kpage=%p i=%x",
+ kpage, i);
+ ret = -EFAULT;
+ goto ehca_reg_mr_rpages_exit1;
+ }
+ } else
+ rpage = *kpage;
+
+ h_ret = hipz_h_register_rpage_mr(
+ shca->ipz_hca_handle, e_mr,
+ ehca_encode_hwpage_size(pginfo->hwpage_size),
+ 0, rpage, rnum);
+
+ if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
+ /*
+ * check for 'registration complete'==H_SUCCESS
+ * and for 'page registered'==H_PAGE_REGISTERED
+ */
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "last "
+ "hipz_reg_rpage_mr failed, h_ret=%lli "
+ "e_mr=%p i=%x hca_hndl=%llx mr_hndl=%llx"
+ " lkey=%x", h_ret, e_mr, i,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ break;
+ } else
+ ret = 0;
+ } else if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
+ "h_ret=%lli e_mr=%p i=%x lkey=%x hca_hndl=%llx "
+ "mr_hndl=%llx", h_ret, e_mr, i,
+ e_mr->ib.ib_mr.lkey,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle);
+ ret = ehca2ib_return_code(h_ret);
+ break;
+ } else
+ ret = 0;
+ } /* end for(i) */
+
+
+ehca_reg_mr_rpages_exit1:
+ ehca_free_fw_ctrlblock(kpage);
+ehca_reg_mr_rpages_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p "
+ "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr,
+ pginfo, pginfo->num_kpages, pginfo->num_hwpages);
+ return ret;
+} /* end ehca_reg_mr_rpages() */
+
+/*----------------------------------------------------------------------*/
+
+inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ u32 acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey, /*OUT*/
+ u32 *rkey) /*OUT*/
+{
+ int ret;
+ u64 h_ret;
+ u32 hipz_acl;
+ u64 *kpage;
+ u64 rpage;
+ struct ehca_mr_pginfo pginfo_save;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(pginfo->hwpage_size, &hipz_acl);
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ ret = -ENOMEM;
+ goto ehca_rereg_mr_rereg1_exit0;
+ }
+
+ pginfo_save = *pginfo;
+ ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
+ if (ret) {
+ ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
+ "pginfo=%p type=%x num_kpages=%llx num_hwpages=%llx "
+ "kpage=%p", e_mr, pginfo, pginfo->type,
+ pginfo->num_kpages, pginfo->num_hwpages, kpage);
+ goto ehca_rereg_mr_rereg1_exit1;
+ }
+ rpage = __pa(kpage);
+ if (!rpage) {
+ ehca_err(&shca->ib_device, "kpage=%p", kpage);
+ ret = -EFAULT;
+ goto ehca_rereg_mr_rereg1_exit1;
+ }
+ h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
+ (u64)iova_start, size, hipz_acl,
+ e_pd->fw_pd, rpage, &hipzout);
+ if (h_ret != H_SUCCESS) {
+ /*
+ * reregistration unsuccessful, try it again with the 3 hCalls,
+ * e.g. this is required in case H_MR_CONDITION
+ * (MW bound or MR is shared)
+ */
+ ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
+ "(Rereg1), h_ret=%lli e_mr=%p", h_ret, e_mr);
+ *pginfo = pginfo_save;
+ ret = -EAGAIN;
+ } else if ((u64 *)hipzout.vaddr != iova_start) {
+ ehca_err(&shca->ib_device, "PHYP changed iova_start in "
+ "rereg_pmr, iova_start=%p iova_start_out=%llx e_mr=%p "
+ "mr_handle=%llx lkey=%x lkey_out=%x", iova_start,
+ hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey, hipzout.lkey);
+ ret = -EFAULT;
+ } else {
+ /*
+ * successful reregistration
+ * note: start and start_out are identical for eServer HCAs
+ */
+ e_mr->num_kpages = pginfo->num_kpages;
+ e_mr->num_hwpages = pginfo->num_hwpages;
+ e_mr->hwpage_size = pginfo->hwpage_size;
+ e_mr->start = iova_start;
+ e_mr->size = size;
+ e_mr->acl = acl;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ }
+
+ehca_rereg_mr_rereg1_exit1:
+ ehca_free_fw_ctrlblock(kpage);
+ehca_rereg_mr_rereg1_exit0:
+ if ( ret && (ret != -EAGAIN) )
+ ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x "
+ "pginfo=%p num_kpages=%llx num_hwpages=%llx",
+ ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
+ pginfo->num_hwpages);
+ return ret;
+} /* end ehca_rereg_mr_rereg1() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey)
+{
+ int ret = 0;
+ u64 h_ret;
+ int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
+ int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
+
+ /* first determine reregistration hCall(s) */
+ if ((pginfo->num_hwpages > MAX_RPAGES) ||
+ (e_mr->num_hwpages > MAX_RPAGES) ||
+ (pginfo->num_hwpages > e_mr->num_hwpages)) {
+ ehca_dbg(&shca->ib_device, "Rereg3 case, "
+ "pginfo->num_hwpages=%llx e_mr->num_hwpages=%x",
+ pginfo->num_hwpages, e_mr->num_hwpages);
+ rereg_1_hcall = 0;
+ rereg_3_hcall = 1;
+ }
+
+ if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
+ rereg_1_hcall = 0;
+ rereg_3_hcall = 1;
+ e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
+ ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
+ e_mr);
+ }
+
+ if (rereg_1_hcall) {
+ ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
+ acl, e_pd, pginfo, lkey, rkey);
+ if (ret) {
+ if (ret == -EAGAIN)
+ rereg_3_hcall = 1;
+ else
+ goto ehca_rereg_mr_exit0;
+ }
+ }
+
+ if (rereg_3_hcall) {
+ struct ehca_mr save_mr;
+
+ /* first deregister old MR */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+ "h_ret=%lli e_mr=%p hca_hndl=%llx mr_hndl=%llx "
+ "mr->lkey=%x",
+ h_ret, e_mr, shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle,
+ e_mr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_rereg_mr_exit0;
+ }
+ /* clean ehca_mr_t, without changing struct ib_mr and lock */
+ save_mr = *e_mr;
+ ehca_mr_deletenew(e_mr);
+
+ /* set some MR values */
+ e_mr->flags = save_mr.flags;
+ e_mr->hwpage_size = save_mr.hwpage_size;
+ e_mr->fmr_page_size = save_mr.fmr_page_size;
+ e_mr->fmr_max_pages = save_mr.fmr_max_pages;
+ e_mr->fmr_max_maps = save_mr.fmr_max_maps;
+ e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
+ e_pd, pginfo, lkey, rkey, EHCA_REG_MR);
+ if (ret) {
+ u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
+ memcpy(&e_mr->flags, &(save_mr.flags),
+ sizeof(struct ehca_mr) - offset);
+ goto ehca_rereg_mr_exit0;
+ }
+ }
+
+ehca_rereg_mr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "
+ "iova_start=%p size=%llx acl=%x e_pd=%p pginfo=%p "
+ "num_kpages=%llx lkey=%x rkey=%x rereg_1_hcall=%x "
+ "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
+ acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
+ rereg_1_hcall, rereg_3_hcall);
+ return ret;
+} /* end ehca_rereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+ struct ehca_mr *e_fmr)
+{
+ int ret = 0;
+ u64 h_ret;
+ struct ehca_pd *e_pd =
+ container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
+ struct ehca_mr save_fmr;
+ u32 tmp_lkey, tmp_rkey;
+ struct ehca_mr_pginfo pginfo;
+ struct ehca_mr_hipzout_parms hipzout;
+ struct ehca_mr save_mr;
+
+ if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
+ /*
+ * note: after using rereg hcall with len=0,
+ * rereg hcall must be used again for registering pages
+ */
+ h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
+ 0, 0, e_pd->fw_pd, 0, &hipzout);
+ if (h_ret == H_SUCCESS) {
+ /* successful reregistration */
+ e_fmr->start = NULL;
+ e_fmr->size = 0;
+ tmp_lkey = hipzout.lkey;
+ tmp_rkey = hipzout.rkey;
+ return 0;
+ }
+ /*
+ * should not happen, because length checked above,
+ * FMRs are not shared and no MW bound to FMRs
+ */
+ ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
+ "(Rereg1), h_ret=%lli e_fmr=%p hca_hndl=%llx "
+ "mr_hndl=%llx lkey=%x lkey_out=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle,
+ e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
+ /* try free and rereg */
+ }
+
+ /* first free old FMR */
+ h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+ "h_ret=%lli e_fmr=%p hca_hndl=%llx mr_hndl=%llx "
+ "lkey=%x",
+ h_ret, e_fmr, shca->ipz_hca_handle.handle,
+ e_fmr->ipz_mr_handle.handle,
+ e_fmr->ib.ib_fmr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_unmap_one_fmr_exit0;
+ }
+ /* clean ehca_mr_t, without changing lock */
+ save_fmr = *e_fmr;
+ ehca_mr_deletenew(e_fmr);
+
+ /* set some MR values */
+ e_fmr->flags = save_fmr.flags;
+ e_fmr->hwpage_size = save_fmr.hwpage_size;
+ e_fmr->fmr_page_size = save_fmr.fmr_page_size;
+ e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
+ e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
+ e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
+ e_fmr->acl = save_fmr.acl;
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_FMR;
+ ret = ehca_reg_mr(shca, e_fmr, NULL,
+ (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
+ e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
+ &tmp_rkey, EHCA_REG_MR);
+ if (ret) {
+ u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
+ memcpy(&e_fmr->flags, &(save_mr.flags),
+ sizeof(struct ehca_mr) - offset);
+ }
+
+ehca_unmap_one_fmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x "
+ "fmr_max_pages=%x",
+ ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
+ return ret;
+} /* end ehca_unmap_one_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_smr(struct ehca_shca *shca,
+ struct ehca_mr *e_origmr,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey, /*OUT*/
+ u32 *rkey) /*OUT*/
+{
+ int ret = 0;
+ u64 h_ret;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
+
+ h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+ (u64)iova_start, hipz_acl, e_pd->fw_pd,
+ &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
+ "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
+ "e_pd=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
+ shca->ipz_hca_handle.handle,
+ e_origmr->ipz_mr_handle.handle,
+ e_origmr->ib.ib_mr.lkey);
+ ret = ehca2ib_return_code(h_ret);
+ goto ehca_reg_smr_exit0;
+ }
+ /* successful registration */
+ e_newmr->num_kpages = e_origmr->num_kpages;
+ e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
+ e_newmr->start = iova_start;
+ e_newmr->size = e_origmr->size;
+ e_newmr->acl = acl;
+ e_newmr->ipz_mr_handle = hipzout.handle;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+
+ehca_reg_smr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p "
+ "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
+ ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
+ return ret;
+} /* end ehca_reg_smr() */
+
+/*----------------------------------------------------------------------*/
+static inline void *ehca_calc_sectbase(int top, int dir, int idx)
+{
+ unsigned long ret = idx;
+ ret |= dir << EHCA_DIR_INDEX_SHIFT;
+ ret |= top << EHCA_TOP_INDEX_SHIFT;
+ return __va(ret << SECTION_SIZE_BITS);
+}
+
+#define ehca_bmap_valid(entry) \
+ ((u64)entry != (u64)EHCA_INVAL_ADDR)
+
+static u64 ehca_reg_mr_section(int top, int dir, int idx, u64 *kpage,
+ struct ehca_shca *shca, struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 h_ret = 0;
+ unsigned long page = 0;
+ u64 rpage = __pa(kpage);
+ int page_count;
+
+ void *sectbase = ehca_calc_sectbase(top, dir, idx);
+ if ((unsigned long)sectbase & (pginfo->hwpage_size - 1)) {
+ ehca_err(&shca->ib_device, "reg_mr_section will probably fail:"
+ "hwpage_size does not fit to "
+ "section start address");
+ }
+ page_count = EHCA_SECTSIZE / pginfo->hwpage_size;
+
+ while (page < page_count) {
+ u64 rnum;
+ for (rnum = 0; (rnum < MAX_RPAGES) && (page < page_count);
+ rnum++) {
+ void *pg = sectbase + ((page++) * pginfo->hwpage_size);
+ kpage[rnum] = __pa(pg);
+ }
+
+ h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr,
+ ehca_encode_hwpage_size(pginfo->hwpage_size),
+ 0, rpage, rnum);
+
+ if ((h_ret != H_SUCCESS) && (h_ret != H_PAGE_REGISTERED)) {
+ ehca_err(&shca->ib_device, "register_rpage_mr failed");
+ return h_ret;
+ }
+ }
+ return h_ret;
+}
+
+static u64 ehca_reg_mr_sections(int top, int dir, u64 *kpage,
+ struct ehca_shca *shca, struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 hret = H_SUCCESS;
+ int idx;
+
+ for (idx = 0; idx < EHCA_MAP_ENTRIES; idx++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]->ent[idx]))
+ continue;
+
+ hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr,
+ pginfo);
+ if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
+ return hret;
+ }
+ return hret;
+}
+
+static u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca,
+ struct ehca_mr *mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ u64 hret = H_SUCCESS;
+ int dir;
+
+ for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ continue;
+
+ hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);
+ if ((hret != H_SUCCESS) && (hret != H_PAGE_REGISTERED))
+ return hret;
+ }
+ return hret;
+}
+
+/* register internal max-MR to internal SHCA */
+int ehca_reg_internal_maxmr(
+ struct ehca_shca *shca,
+ struct ehca_pd *e_pd,
+ struct ehca_mr **e_maxmr) /*OUT*/
+{
+ int ret;
+ struct ehca_mr *e_mr;
+ u64 *iova_start;
+ u64 size_maxmr;
+ struct ehca_mr_pginfo pginfo;
+ struct ib_phys_buf ib_pbuf;
+ u32 num_kpages;
+ u32 num_hwpages;
+ u64 hw_pgsize;
+
+ if (!ehca_bmap) {
+ ret = -EFAULT;
+ goto ehca_reg_internal_maxmr_exit0;
+ }
+
+ e_mr = ehca_mr_new();
+ if (!e_mr) {
+ ehca_err(&shca->ib_device, "out of memory");
+ ret = -ENOMEM;
+ goto ehca_reg_internal_maxmr_exit0;
+ }
+ e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+
+ /* register internal max-MR on HCA */
+ size_maxmr = ehca_mr_len;
+ iova_start = (u64 *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START));
+ ib_pbuf.addr = 0;
+ ib_pbuf.size = size_maxmr;
+ num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
+ PAGE_SIZE);
+ hw_pgsize = ehca_get_max_hwpage_size(shca);
+ num_hwpages = NUM_CHUNKS(((u64)iova_start % hw_pgsize) + size_maxmr,
+ hw_pgsize);
+
+ memset(&pginfo, 0, sizeof(pginfo));
+ pginfo.type = EHCA_MR_PGI_PHYS;
+ pginfo.num_kpages = num_kpages;
+ pginfo.num_hwpages = num_hwpages;
+ pginfo.hwpage_size = hw_pgsize;
+ pginfo.u.phy.num_phys_buf = 1;
+ pginfo.u.phy.phys_buf_array = &ib_pbuf;
+
+ ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
+ &pginfo, &e_mr->ib.ib_mr.lkey,
+ &e_mr->ib.ib_mr.rkey, EHCA_REG_BUSMAP_MR);
+ if (ret) {
+ ehca_err(&shca->ib_device, "reg of internal max MR failed, "
+ "e_mr=%p iova_start=%p size_maxmr=%llx num_kpages=%x "
+ "num_hwpages=%x", e_mr, iova_start, size_maxmr,
+ num_kpages, num_hwpages);
+ goto ehca_reg_internal_maxmr_exit1;
+ }
+
+ /* successful registration of all pages */
+ e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
+ e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
+ e_mr->ib.ib_mr.uobject = NULL;
+ atomic_inc(&(e_pd->ib_pd.usecnt));
+ atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
+ *e_maxmr = e_mr;
+ return 0;
+
+ehca_reg_internal_maxmr_exit1:
+ ehca_mr_delete(e_mr);
+ehca_reg_internal_maxmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p",
+ ret, shca, e_pd, e_maxmr);
+ return ret;
+} /* end ehca_reg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey)
+{
+ u64 h_ret;
+ struct ehca_mr *e_origmr = shca->maxmr;
+ u32 hipz_acl;
+ struct ehca_mr_hipzout_parms hipzout;
+
+ ehca_mrmw_map_acl(acl, &hipz_acl);
+ ehca_mrmw_set_pgsize_hipz_acl(e_origmr->hwpage_size, &hipz_acl);
+
+ h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+ (u64)iova_start, hipz_acl, e_pd->fw_pd,
+ &hipzout);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "
+ "e_origmr=%p hca_hndl=%llx mr_hndl=%llx lkey=%x",
+ h_ret, e_origmr, shca->ipz_hca_handle.handle,
+ e_origmr->ipz_mr_handle.handle,
+ e_origmr->ib.ib_mr.lkey);
+ return ehca2ib_return_code(h_ret);
+ }
+ /* successful registration */
+ e_newmr->num_kpages = e_origmr->num_kpages;
+ e_newmr->num_hwpages = e_origmr->num_hwpages;
+ e_newmr->hwpage_size = e_origmr->hwpage_size;
+ e_newmr->start = iova_start;
+ e_newmr->size = e_origmr->size;
+ e_newmr->acl = acl;
+ e_newmr->ipz_mr_handle = hipzout.handle;
+ *lkey = hipzout.lkey;
+ *rkey = hipzout.rkey;
+ return 0;
+} /* end ehca_reg_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
+{
+ int ret;
+ struct ehca_mr *e_maxmr;
+ struct ib_pd *ib_pd;
+
+ if (!shca->maxmr) {
+ ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
+ ret = -EINVAL;
+ goto ehca_dereg_internal_maxmr_exit0;
+ }
+
+ e_maxmr = shca->maxmr;
+ ib_pd = e_maxmr->ib.ib_mr.pd;
+ shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
+
+ ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
+ if (ret) {
+ ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
+ "ret=%i e_maxmr=%p shca=%p lkey=%x",
+ ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
+ shca->maxmr = e_maxmr;
+ goto ehca_dereg_internal_maxmr_exit0;
+ }
+
+ atomic_dec(&ib_pd->usecnt);
+
+ehca_dereg_internal_maxmr_exit0:
+ if (ret)
+ ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p",
+ ret, shca, shca->maxmr);
+ return ret;
+} /* end ehca_dereg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check physical buffer array of MR verbs for validness and
+ * calculates MR size
+ */
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ u64 *iova_start,
+ u64 *size)
+{
+ struct ib_phys_buf *pbuf = phys_buf_array;
+ u64 size_count = 0;
+ u32 i;
+
+ if (num_phys_buf == 0) {
+ ehca_gen_err("bad phys buf array len, num_phys_buf=0");
+ return -EINVAL;
+ }
+ /* check first buffer */
+ if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
+ ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
+ "pbuf->addr=%llx pbuf->size=%llx",
+ iova_start, pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+ if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
+ (num_phys_buf > 1)) {
+ ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%llx "
+ "pbuf->size=%llx", pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+
+ for (i = 0; i < num_phys_buf; i++) {
+ if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
+ ehca_gen_err("bad address, i=%x pbuf->addr=%llx "
+ "pbuf->size=%llx",
+ i, pbuf->addr, pbuf->size);
+ return -EINVAL;
+ }
+ if (((i > 0) && /* not 1st */
+ (i < (num_phys_buf - 1)) && /* not last */
+ (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
+ ehca_gen_err("bad size, i=%x pbuf->size=%llx",
+ i, pbuf->size);
+ return -EINVAL;
+ }
+ size_count += pbuf->size;
+ pbuf++;
+ }
+
+ *size = size_count;
+ return 0;
+} /* end ehca_mr_chk_buf_and_calc_size() */
+
+/*----------------------------------------------------------------------*/
+
+/* check page list of map FMR verb for validness */
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+ u64 *page_list,
+ int list_len)
+{
+ u32 i;
+ u64 *page;
+
+ if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
+ ehca_gen_err("bad list_len, list_len=%x "
+ "e_fmr->fmr_max_pages=%x fmr=%p",
+ list_len, e_fmr->fmr_max_pages, e_fmr);
+ return -EINVAL;
+ }
+
+ /* each page must be aligned */
+ page = page_list;
+ for (i = 0; i < list_len; i++) {
+ if (*page % e_fmr->fmr_page_size) {
+ ehca_gen_err("bad page, i=%x *page=%llx page=%p fmr=%p "
+ "fmr_page_size=%x", i, *page, page, e_fmr,
+ e_fmr->fmr_page_size);
+ return -EINVAL;
+ }
+ page++;
+ }
+
+ return 0;
+} /* end ehca_fmr_check_page_list() */
+
+/*----------------------------------------------------------------------*/
+
+/* PAGE_SIZE >= pginfo->hwpage_size */
+static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret = 0;
+ u64 pgaddr;
+ u32 j = 0;
+ int hwpages_per_kpage = PAGE_SIZE / pginfo->hwpage_size;
+ struct scatterlist **sg = &pginfo->u.usr.next_sg;
+
+ while (*sg != NULL) {
+ pgaddr = page_to_pfn(sg_page(*sg))
+ << PAGE_SHIFT;
+ *kpage = pgaddr + (pginfo->next_hwpage *
+ pginfo->hwpage_size);
+ if (!(*kpage)) {
+ ehca_gen_err("pgaddr=%llx "
+ "sg_dma_address=%llx "
+ "entry=%llx next_hwpage=%llx",
+ pgaddr, (u64)sg_dma_address(*sg),
+ pginfo->u.usr.next_nmap,
+ pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ (pginfo->next_hwpage)++;
+ kpage++;
+ if (pginfo->next_hwpage % hwpages_per_kpage == 0) {
+ (pginfo->kpage_cnt)++;
+ (pginfo->u.usr.next_nmap)++;
+ pginfo->next_hwpage = 0;
+ *sg = sg_next(*sg);
+ }
+ j++;
+ if (j >= number)
+ break;
+ }
+
+ return ret;
+}
+
+/*
+ * check given pages for contiguous layout
+ * last page addr is returned in prev_pgaddr for further check
+ */
+static int ehca_check_kpages_per_ate(struct scatterlist **sg,
+ int num_pages,
+ u64 *prev_pgaddr)
+{
+ for (; *sg && num_pages > 0; *sg = sg_next(*sg), num_pages--) {
+ u64 pgaddr = page_to_pfn(sg_page(*sg)) << PAGE_SHIFT;
+ if (ehca_debug_level >= 3)
+ ehca_gen_dbg("chunk_page=%llx value=%016llx", pgaddr,
+ *(u64 *)__va(pgaddr));
+ if (pgaddr - PAGE_SIZE != *prev_pgaddr) {
+ ehca_gen_err("uncontiguous page found pgaddr=%llx "
+ "prev_pgaddr=%llx entries_left_in_hwpage=%x",
+ pgaddr, *prev_pgaddr, num_pages);
+ return -EINVAL;
+ }
+ *prev_pgaddr = pgaddr;
+ }
+ return 0;
+}
+
+/* PAGE_SIZE < pginfo->hwpage_size */
+static int ehca_set_pagebuf_user2(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret = 0;
+ u64 pgaddr, prev_pgaddr;
+ u32 j = 0;
+ int kpages_per_hwpage = pginfo->hwpage_size / PAGE_SIZE;
+ int nr_kpages = kpages_per_hwpage;
+ struct scatterlist **sg = &pginfo->u.usr.next_sg;
+
+ while (*sg != NULL) {
+
+ if (nr_kpages == kpages_per_hwpage) {
+ pgaddr = (page_to_pfn(sg_page(*sg))
+ << PAGE_SHIFT);
+ *kpage = pgaddr;
+ if (!(*kpage)) {
+ ehca_gen_err("pgaddr=%llx entry=%llx",
+ pgaddr, pginfo->u.usr.next_nmap);
+ ret = -EFAULT;
+ return ret;
+ }
+ /*
+ * The first page in a hwpage must be aligned;
+ * the first MR page is exempt from this rule.
+ */
+ if (pgaddr & (pginfo->hwpage_size - 1)) {
+ if (pginfo->hwpage_cnt) {
+ ehca_gen_err(
+ "invalid alignment "
+ "pgaddr=%llx entry=%llx "
+ "mr_pgsize=%llx",
+ pgaddr, pginfo->u.usr.next_nmap,
+ pginfo->hwpage_size);
+ ret = -EFAULT;
+ return ret;
+ }
+ /* first MR page */
+ pginfo->kpage_cnt =
+ (pgaddr &
+ (pginfo->hwpage_size - 1)) >>
+ PAGE_SHIFT;
+ nr_kpages -= pginfo->kpage_cnt;
+ *kpage = pgaddr &
+ ~(pginfo->hwpage_size - 1);
+ }
+ if (ehca_debug_level >= 3) {
+ u64 val = *(u64 *)__va(pgaddr);
+ ehca_gen_dbg("kpage=%llx page=%llx "
+ "value=%016llx",
+ *kpage, pgaddr, val);
+ }
+ prev_pgaddr = pgaddr;
+ *sg = sg_next(*sg);
+ pginfo->kpage_cnt++;
+ pginfo->u.usr.next_nmap++;
+ nr_kpages--;
+ if (!nr_kpages)
+ goto next_kpage;
+ continue;
+ }
+
+ ret = ehca_check_kpages_per_ate(sg, nr_kpages,
+ &prev_pgaddr);
+ if (ret)
+ return ret;
+ pginfo->kpage_cnt += nr_kpages;
+ pginfo->u.usr.next_nmap += nr_kpages;
+
+next_kpage:
+ nr_kpages = kpages_per_hwpage;
+ (pginfo->hwpage_cnt)++;
+ kpage++;
+ j++;
+ if (j >= number)
+ break;
+ }
+
+ return ret;
+}
+
+static int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
+{
+ int ret = 0;
+ struct ib_phys_buf *pbuf;
+ u64 num_hw, offs_hw;
+ u32 i = 0;
+
+ /* loop over desired phys_buf_array entries */
+ while (i < number) {
+ pbuf = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
+ num_hw = NUM_CHUNKS((pbuf->addr % pginfo->hwpage_size) +
+ pbuf->size, pginfo->hwpage_size);
+ offs_hw = (pbuf->addr & ~(pginfo->hwpage_size - 1)) /
+ pginfo->hwpage_size;
+ while (pginfo->next_hwpage < offs_hw + num_hw) {
+ /* sanity check */
+ if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
+ (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
+ ehca_gen_err("kpage_cnt >= num_kpages, "
+ "kpage_cnt=%llx num_kpages=%llx "
+ "hwpage_cnt=%llx "
+ "num_hwpages=%llx i=%x",
+ pginfo->kpage_cnt,
+ pginfo->num_kpages,
+ pginfo->hwpage_cnt,
+ pginfo->num_hwpages, i);
+ return -EFAULT;
+ }
+ *kpage = (pbuf->addr & ~(pginfo->hwpage_size - 1)) +
+ (pginfo->next_hwpage * pginfo->hwpage_size);
+ if ( !(*kpage) && pbuf->addr ) {
+ ehca_gen_err("pbuf->addr=%llx pbuf->size=%llx "
+ "next_hwpage=%llx", pbuf->addr,
+ pbuf->size, pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ (pginfo->next_hwpage)++;
+ if (PAGE_SIZE >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (PAGE_SIZE / pginfo->hwpage_size) == 0)
+ (pginfo->kpage_cnt)++;
+ } else
+ pginfo->kpage_cnt += pginfo->hwpage_size /
+ PAGE_SIZE;
+ kpage++;
+ i++;
+ if (i >= number) break;
+ }
+ if (pginfo->next_hwpage >= offs_hw + num_hw) {
+ (pginfo->u.phy.next_buf)++;
+ pginfo->next_hwpage = 0;
+ }
+ }
+ return ret;
+}
+
+static int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
+ u32 number, u64 *kpage)
+{
+ int ret = 0;
+ u64 *fmrlist;
+ u32 i;
+
+ /* loop over desired page_list entries */
+ fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
+ for (i = 0; i < number; i++) {
+ *kpage = (*fmrlist & ~(pginfo->hwpage_size - 1)) +
+ pginfo->next_hwpage * pginfo->hwpage_size;
+ if ( !(*kpage) ) {
+ ehca_gen_err("*fmrlist=%llx fmrlist=%p "
+ "next_listelem=%llx next_hwpage=%llx",
+ *fmrlist, fmrlist,
+ pginfo->u.fmr.next_listelem,
+ pginfo->next_hwpage);
+ return -EFAULT;
+ }
+ (pginfo->hwpage_cnt)++;
+ if (pginfo->u.fmr.fmr_pgsize >= pginfo->hwpage_size) {
+ if (pginfo->next_hwpage %
+ (pginfo->u.fmr.fmr_pgsize /
+ pginfo->hwpage_size) == 0) {
+ (pginfo->kpage_cnt)++;
+ (pginfo->u.fmr.next_listelem)++;
+ fmrlist++;
+ pginfo->next_hwpage = 0;
+ } else
+ (pginfo->next_hwpage)++;
+ } else {
+ unsigned int cnt_per_hwpage = pginfo->hwpage_size /
+ pginfo->u.fmr.fmr_pgsize;
+ unsigned int j;
+ u64 prev = *kpage;
+ /* check if adrs are contiguous */
+ for (j = 1; j < cnt_per_hwpage; j++) {
+ u64 p = fmrlist[j] & ~(pginfo->hwpage_size - 1);
+ if (prev + pginfo->u.fmr.fmr_pgsize != p) {
+ ehca_gen_err("uncontiguous fmr pages "
+ "found prev=%llx p=%llx "
+ "idx=%x", prev, p, i + j);
+ return -EINVAL;
+ }
+ prev = p;
+ }
+ pginfo->kpage_cnt += cnt_per_hwpage;
+ pginfo->u.fmr.next_listelem += cnt_per_hwpage;
+ fmrlist += cnt_per_hwpage;
+ }
+ kpage++;
+ }
+ return ret;
+}
+
+/* setup page buffer from page info */
+int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage)
+{
+ int ret;
+
+ switch (pginfo->type) {
+ case EHCA_MR_PGI_PHYS:
+ ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
+ break;
+ case EHCA_MR_PGI_USER:
+ ret = PAGE_SIZE >= pginfo->hwpage_size ?
+ ehca_set_pagebuf_user1(pginfo, number, kpage) :
+ ehca_set_pagebuf_user2(pginfo, number, kpage);
+ break;
+ case EHCA_MR_PGI_FMR:
+ ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
+ break;
+ default:
+ ehca_gen_err("bad pginfo->type=%x", pginfo->type);
+ ret = -EFAULT;
+ break;
+ }
+ return ret;
+} /* end ehca_set_pagebuf() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check MR if it is a max-MR, i.e. uses whole memory
+ * in case it's a max-MR 1 is returned, else 0
+ */
+int ehca_mr_is_maxmr(u64 size,
+ u64 *iova_start)
+{
+ /* a MR is treated as max-MR only if it fits following: */
+ if ((size == ehca_mr_len) &&
+ (iova_start == (void *)ehca_map_vaddr((void *)(KERNELBASE + PHYSICAL_START)))) {
+ ehca_gen_dbg("this is a max-MR");
+ return 1;
+ } else
+ return 0;
+} /* end ehca_mr_is_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/* map access control for MR/MW. This routine is used for MR and MW. */
+void ehca_mrmw_map_acl(int ib_acl,
+ u32 *hipz_acl)
+{
+ *hipz_acl = 0;
+ if (ib_acl & IB_ACCESS_REMOTE_READ)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
+ if (ib_acl & IB_ACCESS_REMOTE_WRITE)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
+ if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
+ *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
+ if (ib_acl & IB_ACCESS_LOCAL_WRITE)
+ *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
+ if (ib_acl & IB_ACCESS_MW_BIND)
+ *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
+} /* end ehca_mrmw_map_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/* sets page size in hipz access control for MR/MW. */
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl) /*INOUT*/
+{
+ *hipz_acl |= (ehca_encode_hwpage_size(pgsize) << 24);
+} /* end ehca_mrmw_set_pgsize_hipz_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * reverse map access control for MR/MW.
+ * This routine is used for MR and MW.
+ */
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+ int *ib_acl) /*OUT*/
+{
+ *ib_acl = 0;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
+ *ib_acl |= IB_ACCESS_REMOTE_READ;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
+ *ib_acl |= IB_ACCESS_REMOTE_WRITE;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
+ *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
+ *ib_acl |= IB_ACCESS_LOCAL_WRITE;
+ if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
+ *ib_acl |= IB_ACCESS_MW_BIND;
+} /* end ehca_mrmw_reverse_map_acl() */
+
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * MR destructor and constructor
+ * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
+ * except struct ib_mr and spinlock
+ */
+void ehca_mr_deletenew(struct ehca_mr *mr)
+{
+ mr->flags = 0;
+ mr->num_kpages = 0;
+ mr->num_hwpages = 0;
+ mr->acl = 0;
+ mr->start = NULL;
+ mr->fmr_page_size = 0;
+ mr->fmr_max_pages = 0;
+ mr->fmr_max_maps = 0;
+ mr->fmr_map_cnt = 0;
+ memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
+ memset(&mr->galpas, 0, sizeof(mr->galpas));
+} /* end ehca_mr_deletenew() */
+
+int ehca_init_mrmw_cache(void)
+{
+ mr_cache = kmem_cache_create("ehca_cache_mr",
+ sizeof(struct ehca_mr), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!mr_cache)
+ return -ENOMEM;
+ mw_cache = kmem_cache_create("ehca_cache_mw",
+ sizeof(struct ehca_mw), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!mw_cache) {
+ kmem_cache_destroy(mr_cache);
+ mr_cache = NULL;
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+void ehca_cleanup_mrmw_cache(void)
+{
+ if (mr_cache)
+ kmem_cache_destroy(mr_cache);
+ if (mw_cache)
+ kmem_cache_destroy(mw_cache);
+}
+
+static inline int ehca_init_top_bmap(struct ehca_top_bmap *ehca_top_bmap,
+ int dir)
+{
+ if (!ehca_bmap_valid(ehca_top_bmap->dir[dir])) {
+ ehca_top_bmap->dir[dir] =
+ kmalloc(sizeof(struct ehca_dir_bmap), GFP_KERNEL);
+ if (!ehca_top_bmap->dir[dir])
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_top_bmap->dir[dir], 0xFF, EHCA_ENT_MAP_SIZE);
+ }
+ return 0;
+}
+
+static inline int ehca_init_bmap(struct ehca_bmap *ehca_bmap, int top, int dir)
+{
+ if (!ehca_bmap_valid(ehca_bmap->top[top])) {
+ ehca_bmap->top[top] =
+ kmalloc(sizeof(struct ehca_top_bmap), GFP_KERNEL);
+ if (!ehca_bmap->top[top])
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_bmap->top[top], 0xFF, EHCA_DIR_MAP_SIZE);
+ }
+ return ehca_init_top_bmap(ehca_bmap->top[top], dir);
+}
+
+static inline int ehca_calc_index(unsigned long i, unsigned long s)
+{
+ return (i >> s) & EHCA_INDEX_MASK;
+}
+
+void ehca_destroy_busmap(void)
+{
+ int top, dir;
+
+ if (!ehca_bmap)
+ return;
+
+ for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ continue;
+ for (dir = 0; dir < EHCA_MAP_ENTRIES; dir++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ continue;
+
+ kfree(ehca_bmap->top[top]->dir[dir]);
+ }
+
+ kfree(ehca_bmap->top[top]);
+ }
+
+ kfree(ehca_bmap);
+ ehca_bmap = NULL;
+}
+
+static int ehca_update_busmap(unsigned long pfn, unsigned long nr_pages)
+{
+ unsigned long i, start_section, end_section;
+ int top, dir, idx;
+
+ if (!nr_pages)
+ return 0;
+
+ if (!ehca_bmap) {
+ ehca_bmap = kmalloc(sizeof(struct ehca_bmap), GFP_KERNEL);
+ if (!ehca_bmap)
+ return -ENOMEM;
+ /* Set map block to 0xFF according to EHCA_INVAL_ADDR */
+ memset(ehca_bmap, 0xFF, EHCA_TOP_MAP_SIZE);
+ }
+
+ start_section = (pfn * PAGE_SIZE) / EHCA_SECTSIZE;
+ end_section = ((pfn + nr_pages) * PAGE_SIZE) / EHCA_SECTSIZE;
+ for (i = start_section; i < end_section; i++) {
+ int ret;
+ top = ehca_calc_index(i, EHCA_TOP_INDEX_SHIFT);
+ dir = ehca_calc_index(i, EHCA_DIR_INDEX_SHIFT);
+ idx = i & EHCA_INDEX_MASK;
+
+ ret = ehca_init_bmap(ehca_bmap, top, dir);
+ if (ret) {
+ ehca_destroy_busmap();
+ return ret;
+ }
+ ehca_bmap->top[top]->dir[dir]->ent[idx] = ehca_mr_len;
+ ehca_mr_len += EHCA_SECTSIZE;
+ }
+ return 0;
+}
+
+static int ehca_is_hugepage(unsigned long pfn)
+{
+ int page_order;
+
+ if (pfn & EHCA_HUGEPAGE_PFN_MASK)
+ return 0;
+
+ page_order = compound_order(pfn_to_page(pfn));
+ if (page_order + PAGE_SHIFT != EHCA_HUGEPAGESHIFT)
+ return 0;
+
+ return 1;
+}
+
+static int ehca_create_busmap_callback(unsigned long initial_pfn,
+ unsigned long total_nr_pages, void *arg)
+{
+ int ret;
+ unsigned long pfn, start_pfn, end_pfn, nr_pages;
+
+ if ((total_nr_pages * PAGE_SIZE) < EHCA_HUGEPAGE_SIZE)
+ return ehca_update_busmap(initial_pfn, total_nr_pages);
+
+ /* Given chunk is >= 16GB -> check for hugepages */
+ start_pfn = initial_pfn;
+ end_pfn = initial_pfn + total_nr_pages;
+ pfn = start_pfn;
+
+ while (pfn < end_pfn) {
+ if (ehca_is_hugepage(pfn)) {
+ /* Add mem found in front of the hugepage */
+ nr_pages = pfn - start_pfn;
+ ret = ehca_update_busmap(start_pfn, nr_pages);
+ if (ret)
+ return ret;
+ /* Skip the hugepage */
+ pfn += (EHCA_HUGEPAGE_SIZE / PAGE_SIZE);
+ start_pfn = pfn;
+ } else
+ pfn += (EHCA_SECTSIZE / PAGE_SIZE);
+ }
+
+ /* Add mem found behind the hugepage(s) */
+ nr_pages = pfn - start_pfn;
+ return ehca_update_busmap(start_pfn, nr_pages);
+}
+
+int ehca_create_busmap(void)
+{
+ int ret;
+
+ ehca_mr_len = 0;
+ ret = walk_system_ram_range(0, 1ULL << MAX_PHYSMEM_BITS, NULL,
+ ehca_create_busmap_callback);
+ return ret;
+}
+
+static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo)
+{
+ int top;
+ u64 hret, *kpage;
+
+ kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!kpage) {
+ ehca_err(&shca->ib_device, "kpage alloc failed");
+ return -ENOMEM;
+ }
+ for (top = 0; top < EHCA_MAP_ENTRIES; top++) {
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ continue;
+ hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);
+ if ((hret != H_PAGE_REGISTERED) && (hret != H_SUCCESS))
+ break;
+ }
+
+ ehca_free_fw_ctrlblock(kpage);
+
+ if (hret == H_SUCCESS)
+ return 0; /* Everything is fine */
+ else {
+ ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, "
+ "h_ret=%lli e_mr=%p top=%x lkey=%x "
+ "hca_hndl=%llx mr_hndl=%llx", hret, e_mr, top,
+ e_mr->ib.ib_mr.lkey,
+ shca->ipz_hca_handle.handle,
+ e_mr->ipz_mr_handle.handle);
+ return ehca2ib_return_code(hret);
+ }
+}
+
+static u64 ehca_map_vaddr(void *caddr)
+{
+ int top, dir, idx;
+ unsigned long abs_addr, offset;
+ u64 entry;
+
+ if (!ehca_bmap)
+ return EHCA_INVAL_ADDR;
+
+ abs_addr = __pa(caddr);
+ top = ehca_calc_index(abs_addr, EHCA_TOP_INDEX_SHIFT + EHCA_SECTSHIFT);
+ if (!ehca_bmap_valid(ehca_bmap->top[top]))
+ return EHCA_INVAL_ADDR;
+
+ dir = ehca_calc_index(abs_addr, EHCA_DIR_INDEX_SHIFT + EHCA_SECTSHIFT);
+ if (!ehca_bmap_valid(ehca_bmap->top[top]->dir[dir]))
+ return EHCA_INVAL_ADDR;
+
+ idx = ehca_calc_index(abs_addr, EHCA_SECTSHIFT);
+
+ entry = ehca_bmap->top[top]->dir[dir]->ent[idx];
+ if (ehca_bmap_valid(entry)) {
+ offset = (unsigned long)caddr & (EHCA_SECTSIZE - 1);
+ return entry | offset;
+ } else
+ return EHCA_INVAL_ADDR;
+}
+
+static int ehca_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
+{
+ return dma_addr == EHCA_INVAL_ADDR;
+}
+
+static u64 ehca_dma_map_single(struct ib_device *dev, void *cpu_addr,
+ size_t size, enum dma_data_direction direction)
+{
+ if (cpu_addr)
+ return ehca_map_vaddr(cpu_addr);
+ else
+ return EHCA_INVAL_ADDR;
+}
+
+static void ehca_dma_unmap_single(struct ib_device *dev, u64 addr, size_t size,
+ enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static u64 ehca_dma_map_page(struct ib_device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction direction)
+{
+ u64 addr;
+
+ if (offset + size > PAGE_SIZE)
+ return EHCA_INVAL_ADDR;
+
+ addr = ehca_map_vaddr(page_address(page));
+ if (!ehca_dma_mapping_error(dev, addr))
+ addr += offset;
+
+ return addr;
+}
+
+static void ehca_dma_unmap_page(struct ib_device *dev, u64 addr, size_t size,
+ enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static int ehca_dma_map_sg(struct ib_device *dev, struct scatterlist *sgl,
+ int nents, enum dma_data_direction direction)
+{
+ struct scatterlist *sg;
+ int i;
+
+ for_each_sg(sgl, sg, nents, i) {
+ u64 addr;
+ addr = ehca_map_vaddr(sg_virt(sg));
+ if (ehca_dma_mapping_error(dev, addr))
+ return 0;
+
+ sg->dma_address = addr;
+ sg->dma_length = sg->length;
+ }
+ return nents;
+}
+
+static void ehca_dma_unmap_sg(struct ib_device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction direction)
+{
+ /* This is only a stub; nothing to be done here */
+}
+
+static void ehca_dma_sync_single_for_cpu(struct ib_device *dev, u64 addr,
+ size_t size,
+ enum dma_data_direction dir)
+{
+ dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
+}
+
+static void ehca_dma_sync_single_for_device(struct ib_device *dev, u64 addr,
+ size_t size,
+ enum dma_data_direction dir)
+{
+ dma_sync_single_for_device(dev->dma_device, addr, size, dir);
+}
+
+static void *ehca_dma_alloc_coherent(struct ib_device *dev, size_t size,
+ u64 *dma_handle, gfp_t flag)
+{
+ struct page *p;
+ void *addr = NULL;
+ u64 dma_addr;
+
+ p = alloc_pages(flag, get_order(size));
+ if (p) {
+ addr = page_address(p);
+ dma_addr = ehca_map_vaddr(addr);
+ if (ehca_dma_mapping_error(dev, dma_addr)) {
+ free_pages((unsigned long)addr, get_order(size));
+ return NULL;
+ }
+ if (dma_handle)
+ *dma_handle = dma_addr;
+ return addr;
+ }
+ return NULL;
+}
+
+static void ehca_dma_free_coherent(struct ib_device *dev, size_t size,
+ void *cpu_addr, u64 dma_handle)
+{
+ if (cpu_addr && size)
+ free_pages((unsigned long)cpu_addr, get_order(size));
+}
+
+
+struct ib_dma_mapping_ops ehca_dma_mapping_ops = {
+ .mapping_error = ehca_dma_mapping_error,
+ .map_single = ehca_dma_map_single,
+ .unmap_single = ehca_dma_unmap_single,
+ .map_page = ehca_dma_map_page,
+ .unmap_page = ehca_dma_unmap_page,
+ .map_sg = ehca_dma_map_sg,
+ .unmap_sg = ehca_dma_unmap_sg,
+ .sync_single_for_cpu = ehca_dma_sync_single_for_cpu,
+ .sync_single_for_device = ehca_dma_sync_single_for_device,
+ .alloc_coherent = ehca_dma_alloc_coherent,
+ .free_coherent = ehca_dma_free_coherent,
+};
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * MR/MW declarations and inline functions
+ *
+ * Authors: Dietmar Decker <ddecker@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _EHCA_MRMW_H_
+#define _EHCA_MRMW_H_
+
+enum ehca_reg_type {
+ EHCA_REG_MR,
+ EHCA_REG_BUSMAP_MR
+};
+
+int ehca_reg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int acl,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey,
+ enum ehca_reg_type reg_type);
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ struct ehca_mr_pginfo *pginfo);
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+ struct ehca_mr *e_mr,
+ u64 *iova_start,
+ u64 size,
+ int mr_access_flags,
+ struct ehca_pd *e_pd,
+ struct ehca_mr_pginfo *pginfo,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+ struct ehca_mr *e_fmr);
+
+int ehca_reg_smr(struct ehca_shca *shca,
+ struct ehca_mr *e_origmr,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_reg_internal_maxmr(struct ehca_shca *shca,
+ struct ehca_pd *e_pd,
+ struct ehca_mr **maxmr);
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+ struct ehca_mr *e_newmr,
+ u64 *iova_start,
+ int acl,
+ struct ehca_pd *e_pd,
+ u32 *lkey,
+ u32 *rkey);
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
+
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+ int num_phys_buf,
+ u64 *iova_start,
+ u64 *size);
+
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+ u64 *page_list,
+ int list_len);
+
+int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
+ u32 number,
+ u64 *kpage);
+
+int ehca_mr_is_maxmr(u64 size,
+ u64 *iova_start);
+
+void ehca_mrmw_map_acl(int ib_acl,
+ u32 *hipz_acl);
+
+void ehca_mrmw_set_pgsize_hipz_acl(u32 pgsize, u32 *hipz_acl);
+
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+ int *ib_acl);
+
+void ehca_mr_deletenew(struct ehca_mr *mr);
+
+int ehca_create_busmap(void);
+
+void ehca_destroy_busmap(void);
+
+extern struct ib_dma_mapping_ops ehca_dma_mapping_ops;
+#endif /*_EHCA_MRMW_H_*/
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * PD functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+
+static struct kmem_cache *pd_cache;
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+ struct ib_ucontext *context, struct ib_udata *udata)
+{
+ struct ehca_pd *pd;
+ int i;
+
+ pd = kmem_cache_zalloc(pd_cache, GFP_KERNEL);
+ if (!pd) {
+ ehca_err(device, "device=%p context=%p out of memory",
+ device, context);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ for (i = 0; i < 2; i++) {
+ INIT_LIST_HEAD(&pd->free[i]);
+ INIT_LIST_HEAD(&pd->full[i]);
+ }
+ mutex_init(&pd->lock);
+
+ /*
+ * Kernel PD: when device = -1, 0
+ * User PD: when context != -1
+ */
+ if (!context) {
+ /*
+ * Kernel PDs after init reuses always
+ * the one created in ehca_shca_reopen()
+ */
+ struct ehca_shca *shca = container_of(device, struct ehca_shca,
+ ib_device);
+ pd->fw_pd.value = shca->pd->fw_pd.value;
+ } else
+ pd->fw_pd.value = (u64)pd;
+
+ return &pd->ib_pd;
+}
+
+int ehca_dealloc_pd(struct ib_pd *pd)
+{
+ struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+ int i, leftovers = 0;
+ struct ipz_small_queue_page *page, *tmp;
+
+ for (i = 0; i < 2; i++) {
+ list_splice(&my_pd->full[i], &my_pd->free[i]);
+ list_for_each_entry_safe(page, tmp, &my_pd->free[i], list) {
+ leftovers = 1;
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+ }
+
+ if (leftovers)
+ ehca_warn(pd->device,
+ "Some small queue pages were not freed");
+
+ kmem_cache_free(pd_cache, my_pd);
+
+ return 0;
+}
+
+int ehca_init_pd_cache(void)
+{
+ pd_cache = kmem_cache_create("ehca_cache_pd",
+ sizeof(struct ehca_pd), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!pd_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_pd_cache(void)
+{
+ if (pd_cache)
+ kmem_cache_destroy(pd_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Hardware request structures
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef _EHCA_QES_H_
+#define _EHCA_QES_H_
+
+#include "ehca_tools.h"
+
+/* virtual scatter gather entry to specify remote addresses with length */
+struct ehca_vsgentry {
+ u64 vaddr;
+ u32 lkey;
+ u32 length;
+};
+
+#define GRH_FLAG_MASK EHCA_BMASK_IBM( 7, 7)
+#define GRH_IPVERSION_MASK EHCA_BMASK_IBM( 0, 3)
+#define GRH_TCLASS_MASK EHCA_BMASK_IBM( 4, 12)
+#define GRH_FLOWLABEL_MASK EHCA_BMASK_IBM(13, 31)
+#define GRH_PAYLEN_MASK EHCA_BMASK_IBM(32, 47)
+#define GRH_NEXTHEADER_MASK EHCA_BMASK_IBM(48, 55)
+#define GRH_HOPLIMIT_MASK EHCA_BMASK_IBM(56, 63)
+
+/*
+ * Unreliable Datagram Address Vector Format
+ * see IBTA Vol1 chapter 8.3 Global Routing Header
+ */
+struct ehca_ud_av {
+ u8 sl;
+ u8 lnh;
+ u16 dlid;
+ u8 reserved1;
+ u8 reserved2;
+ u8 reserved3;
+ u8 slid_path_bits;
+ u8 reserved4;
+ u8 ipd;
+ u8 reserved5;
+ u8 pmtu;
+ u32 reserved6;
+ u64 reserved7;
+ union {
+ struct {
+ u64 word_0; /* always set to 6 */
+ /*should be 0x1B for IB transport */
+ u64 word_1;
+ u64 word_2;
+ u64 word_3;
+ u64 word_4;
+ } grh;
+ struct {
+ u32 wd_0;
+ u32 wd_1;
+ /* DWord_1 --> SGID */
+
+ u32 sgid_wd3;
+ u32 sgid_wd2;
+
+ u32 sgid_wd1;
+ u32 sgid_wd0;
+ /* DWord_3 --> DGID */
+
+ u32 dgid_wd3;
+ u32 dgid_wd2;
+
+ u32 dgid_wd1;
+ u32 dgid_wd0;
+ } grh_l;
+ };
+};
+
+/* maximum number of sg entries allowed in a WQE */
+#define MAX_WQE_SG_ENTRIES 252
+
+#define WQE_OPTYPE_SEND 0x80
+#define WQE_OPTYPE_RDMAREAD 0x40
+#define WQE_OPTYPE_RDMAWRITE 0x20
+#define WQE_OPTYPE_CMPSWAP 0x10
+#define WQE_OPTYPE_FETCHADD 0x08
+#define WQE_OPTYPE_BIND 0x04
+
+#define WQE_WRFLAG_REQ_SIGNAL_COM 0x80
+#define WQE_WRFLAG_FENCE 0x40
+#define WQE_WRFLAG_IMM_DATA_PRESENT 0x20
+#define WQE_WRFLAG_SOLIC_EVENT 0x10
+
+#define WQEF_CACHE_HINT 0x80
+#define WQEF_CACHE_HINT_RD_WR 0x40
+#define WQEF_TIMED_WQE 0x20
+#define WQEF_PURGE 0x08
+#define WQEF_HIGH_NIBBLE 0xF0
+
+#define MW_BIND_ACCESSCTRL_R_WRITE 0x40
+#define MW_BIND_ACCESSCTRL_R_READ 0x20
+#define MW_BIND_ACCESSCTRL_R_ATOMIC 0x10
+
+struct ehca_wqe {
+ u64 work_request_id;
+ u8 optype;
+ u8 wr_flag;
+ u16 pkeyi;
+ u8 wqef;
+ u8 nr_of_data_seg;
+ u16 wqe_provided_slid;
+ u32 destination_qp_number;
+ u32 resync_psn_sqp;
+ u32 local_ee_context_qkey;
+ u32 immediate_data;
+ union {
+ struct {
+ u64 remote_virtual_address;
+ u32 rkey;
+ u32 reserved;
+ u64 atomic_1st_op_dma_len;
+ u64 atomic_2nd_op;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+
+ } nud;
+ struct {
+ u64 ehca_ud_av_ptr;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+ } ud_avp;
+ struct {
+ struct ehca_ud_av ud_av;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
+ 2];
+ } ud_av;
+ struct {
+ u64 reserved0;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+ } all_rcv;
+
+ struct {
+ u64 reserved;
+ u32 rkey;
+ u32 old_rkey;
+ u64 reserved1;
+ u64 reserved2;
+ u64 virtual_address;
+ u32 reserved3;
+ u32 length;
+ u32 reserved4;
+ u16 reserved5;
+ u8 reserved6;
+ u8 lr_ctl;
+ u32 lkey;
+ u32 reserved7;
+ u64 reserved8;
+ u64 reserved9;
+ u64 reserved10;
+ u64 reserved11;
+ } bind;
+ struct {
+ u64 reserved12;
+ u64 reserved13;
+ u32 size;
+ u32 start;
+ } inline_data;
+ } u;
+
+};
+
+#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0, 0)
+#define WC_IMM_DATA EHCA_BMASK_IBM(1, 1)
+#define WC_GRH_PRESENT EHCA_BMASK_IBM(2, 2)
+#define WC_SE_BIT EHCA_BMASK_IBM(3, 3)
+#define WC_STATUS_ERROR_BIT 0x80000000
+#define WC_STATUS_REMOTE_ERROR_FLAGS 0x0000F800
+#define WC_STATUS_PURGE_BIT 0x10
+#define WC_SEND_RECEIVE_BIT 0x80
+
+struct ehca_cqe {
+ u64 work_request_id;
+ u8 optype;
+ u8 w_completion_flags;
+ u16 reserved1;
+ u32 nr_bytes_transferred;
+ u32 immediate_data;
+ u32 local_qp_number;
+ u8 freed_resource_count;
+ u8 service_level;
+ u16 wqe_count;
+ u32 qp_token;
+ u32 qkey_ee_token;
+ u32 remote_qp_number;
+ u16 dlid;
+ u16 rlid;
+ u16 reserved2;
+ u16 pkey_index;
+ u32 cqe_timestamp;
+ u32 wqe_timestamp;
+ u8 wqe_timestamp_valid;
+ u8 reserved3;
+ u8 reserved4;
+ u8 cqe_flags;
+ u32 status;
+};
+
+struct ehca_eqe {
+ u64 entry;
+};
+
+struct ehca_mrte {
+ u64 starting_va;
+ u64 length; /* length of memory region in bytes*/
+ u32 pd;
+ u8 key_instance;
+ u8 pagesize;
+ u8 mr_control;
+ u8 local_remote_access_ctrl;
+ u8 reserved[0x20 - 0x18];
+ u64 at_pointer[4];
+};
+#endif /*_EHCA_QES_H_*/
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * QP functions
+ *
+ * Authors: Joachim Fenkes <fenkes@de.ibm.com>
+ * Stefan Roscher <stefan.roscher@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+static struct kmem_cache *qp_cache;
+
+/*
+ * attributes not supported by query qp
+ */
+#define QP_ATTR_QUERY_NOT_SUPPORTED (IB_QP_ACCESS_FLAGS | \
+ IB_QP_EN_SQD_ASYNC_NOTIFY)
+
+/*
+ * ehca (internal) qp state values
+ */
+enum ehca_qp_state {
+ EHCA_QPS_RESET = 1,
+ EHCA_QPS_INIT = 2,
+ EHCA_QPS_RTR = 3,
+ EHCA_QPS_RTS = 5,
+ EHCA_QPS_SQD = 6,
+ EHCA_QPS_SQE = 8,
+ EHCA_QPS_ERR = 128
+};
+
+/*
+ * qp state transitions as defined by IB Arch Rel 1.1 page 431
+ */
+enum ib_qp_statetrans {
+ IB_QPST_ANY2RESET,
+ IB_QPST_ANY2ERR,
+ IB_QPST_RESET2INIT,
+ IB_QPST_INIT2RTR,
+ IB_QPST_INIT2INIT,
+ IB_QPST_RTR2RTS,
+ IB_QPST_RTS2SQD,
+ IB_QPST_RTS2RTS,
+ IB_QPST_SQD2RTS,
+ IB_QPST_SQE2RTS,
+ IB_QPST_SQD2SQD,
+ IB_QPST_MAX /* nr of transitions, this must be last!!! */
+};
+
+/*
+ * ib2ehca_qp_state maps IB to ehca qp_state
+ * returns ehca qp state corresponding to given ib qp state
+ */
+static inline enum ehca_qp_state ib2ehca_qp_state(enum ib_qp_state ib_qp_state)
+{
+ switch (ib_qp_state) {
+ case IB_QPS_RESET:
+ return EHCA_QPS_RESET;
+ case IB_QPS_INIT:
+ return EHCA_QPS_INIT;
+ case IB_QPS_RTR:
+ return EHCA_QPS_RTR;
+ case IB_QPS_RTS:
+ return EHCA_QPS_RTS;
+ case IB_QPS_SQD:
+ return EHCA_QPS_SQD;
+ case IB_QPS_SQE:
+ return EHCA_QPS_SQE;
+ case IB_QPS_ERR:
+ return EHCA_QPS_ERR;
+ default:
+ ehca_gen_err("invalid ib_qp_state=%x", ib_qp_state);
+ return -EINVAL;
+ }
+}
+
+/*
+ * ehca2ib_qp_state maps ehca to IB qp_state
+ * returns ib qp state corresponding to given ehca qp state
+ */
+static inline enum ib_qp_state ehca2ib_qp_state(enum ehca_qp_state
+ ehca_qp_state)
+{
+ switch (ehca_qp_state) {
+ case EHCA_QPS_RESET:
+ return IB_QPS_RESET;
+ case EHCA_QPS_INIT:
+ return IB_QPS_INIT;
+ case EHCA_QPS_RTR:
+ return IB_QPS_RTR;
+ case EHCA_QPS_RTS:
+ return IB_QPS_RTS;
+ case EHCA_QPS_SQD:
+ return IB_QPS_SQD;
+ case EHCA_QPS_SQE:
+ return IB_QPS_SQE;
+ case EHCA_QPS_ERR:
+ return IB_QPS_ERR;
+ default:
+ ehca_gen_err("invalid ehca_qp_state=%x", ehca_qp_state);
+ return -EINVAL;
+ }
+}
+
+/*
+ * ehca_qp_type used as index for req_attr and opt_attr of
+ * struct ehca_modqp_statetrans
+ */
+enum ehca_qp_type {
+ QPT_RC = 0,
+ QPT_UC = 1,
+ QPT_UD = 2,
+ QPT_SQP = 3,
+ QPT_MAX
+};
+
+/*
+ * ib2ehcaqptype maps Ib to ehca qp_type
+ * returns ehca qp type corresponding to ib qp type
+ */
+static inline enum ehca_qp_type ib2ehcaqptype(enum ib_qp_type ibqptype)
+{
+ switch (ibqptype) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ return QPT_SQP;
+ case IB_QPT_RC:
+ return QPT_RC;
+ case IB_QPT_UC:
+ return QPT_UC;
+ case IB_QPT_UD:
+ return QPT_UD;
+ default:
+ ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+ return -EINVAL;
+ }
+}
+
+static inline enum ib_qp_statetrans get_modqp_statetrans(int ib_fromstate,
+ int ib_tostate)
+{
+ int index = -EINVAL;
+ switch (ib_tostate) {
+ case IB_QPS_RESET:
+ index = IB_QPST_ANY2RESET;
+ break;
+ case IB_QPS_INIT:
+ switch (ib_fromstate) {
+ case IB_QPS_RESET:
+ index = IB_QPST_RESET2INIT;
+ break;
+ case IB_QPS_INIT:
+ index = IB_QPST_INIT2INIT;
+ break;
+ }
+ break;
+ case IB_QPS_RTR:
+ if (ib_fromstate == IB_QPS_INIT)
+ index = IB_QPST_INIT2RTR;
+ break;
+ case IB_QPS_RTS:
+ switch (ib_fromstate) {
+ case IB_QPS_RTR:
+ index = IB_QPST_RTR2RTS;
+ break;
+ case IB_QPS_RTS:
+ index = IB_QPST_RTS2RTS;
+ break;
+ case IB_QPS_SQD:
+ index = IB_QPST_SQD2RTS;
+ break;
+ case IB_QPS_SQE:
+ index = IB_QPST_SQE2RTS;
+ break;
+ }
+ break;
+ case IB_QPS_SQD:
+ if (ib_fromstate == IB_QPS_RTS)
+ index = IB_QPST_RTS2SQD;
+ break;
+ case IB_QPS_SQE:
+ break;
+ case IB_QPS_ERR:
+ index = IB_QPST_ANY2ERR;
+ break;
+ default:
+ break;
+ }
+ return index;
+}
+
+/*
+ * ibqptype2servicetype returns hcp service type corresponding to given
+ * ib qp type used by create_qp()
+ */
+static inline int ibqptype2servicetype(enum ib_qp_type ibqptype)
+{
+ switch (ibqptype) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ return ST_UD;
+ case IB_QPT_RC:
+ return ST_RC;
+ case IB_QPT_UC:
+ return ST_UC;
+ case IB_QPT_UD:
+ return ST_UD;
+ case IB_QPT_RAW_IPV6:
+ return -EINVAL;
+ case IB_QPT_RAW_ETHERTYPE:
+ return -EINVAL;
+ default:
+ ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+ return -EINVAL;
+ }
+}
+
+/*
+ * init userspace queue info from ipz_queue data
+ */
+static inline void queue2resp(struct ipzu_queue_resp *resp,
+ struct ipz_queue *queue)
+{
+ resp->qe_size = queue->qe_size;
+ resp->act_nr_of_sg = queue->act_nr_of_sg;
+ resp->queue_length = queue->queue_length;
+ resp->pagesize = queue->pagesize;
+ resp->toggle_state = queue->toggle_state;
+ resp->offset = queue->offset;
+}
+
+/*
+ * init_qp_queue initializes/constructs r/squeue and registers queue pages.
+ */
+static inline int init_qp_queue(struct ehca_shca *shca,
+ struct ehca_pd *pd,
+ struct ehca_qp *my_qp,
+ struct ipz_queue *queue,
+ int q_type,
+ u64 expected_hret,
+ struct ehca_alloc_queue_parms *parms,
+ int wqe_size)
+{
+ int ret, cnt, ipz_rc, nr_q_pages;
+ void *vpage;
+ u64 rpage, h_ret;
+ struct ib_device *ib_dev = &shca->ib_device;
+ struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
+
+ if (!parms->queue_size)
+ return 0;
+
+ if (parms->is_small) {
+ nr_q_pages = 1;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ 128 << parms->page_size,
+ wqe_size, parms->act_nr_sges, 1);
+ } else {
+ nr_q_pages = parms->queue_size;
+ ipz_rc = ipz_queue_ctor(pd, queue, nr_q_pages,
+ EHCA_PAGESIZE, wqe_size,
+ parms->act_nr_sges, 0);
+ }
+
+ if (!ipz_rc) {
+ ehca_err(ib_dev, "Cannot allocate page for queue. ipz_rc=%i",
+ ipz_rc);
+ return -EBUSY;
+ }
+
+ /* register queue pages */
+ for (cnt = 0; cnt < nr_q_pages; cnt++) {
+ vpage = ipz_qpageit_get_inc(queue);
+ if (!vpage) {
+ ehca_err(ib_dev, "ipz_qpageit_get_inc() "
+ "failed p_vpage= %p", vpage);
+ ret = -EINVAL;
+ goto init_qp_queue1;
+ }
+ rpage = __pa(vpage);
+
+ h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ NULL, 0, q_type,
+ rpage, parms->is_small ? 0 : 1,
+ my_qp->galpas.kernel);
+ if (cnt == (nr_q_pages - 1)) { /* last page! */
+ if (h_ret != expected_hret) {
+ ehca_err(ib_dev, "hipz_qp_register_rpage() "
+ "h_ret=%lli", h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto init_qp_queue1;
+ }
+ vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
+ if (vpage) {
+ ehca_err(ib_dev, "ipz_qpageit_get_inc() "
+ "should not succeed vpage=%p", vpage);
+ ret = -EINVAL;
+ goto init_qp_queue1;
+ }
+ } else {
+ if (h_ret != H_PAGE_REGISTERED) {
+ ehca_err(ib_dev, "hipz_qp_register_rpage() "
+ "h_ret=%lli", h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto init_qp_queue1;
+ }
+ }
+ }
+
+ ipz_qeit_reset(queue);
+
+ return 0;
+
+init_qp_queue1:
+ ipz_queue_dtor(pd, queue);
+ return ret;
+}
+
+static inline int ehca_calc_wqe_size(int act_nr_sge, int is_llqp)
+{
+ if (is_llqp)
+ return 128 << act_nr_sge;
+ else
+ return offsetof(struct ehca_wqe,
+ u.nud.sg_list[act_nr_sge]);
+}
+
+static void ehca_determine_small_queue(struct ehca_alloc_queue_parms *queue,
+ int req_nr_sge, int is_llqp)
+{
+ u32 wqe_size, q_size;
+ int act_nr_sge = req_nr_sge;
+
+ if (!is_llqp)
+ /* round up #SGEs so WQE size is a power of 2 */
+ for (act_nr_sge = 4; act_nr_sge <= 252;
+ act_nr_sge = 4 + 2 * act_nr_sge)
+ if (act_nr_sge >= req_nr_sge)
+ break;
+
+ wqe_size = ehca_calc_wqe_size(act_nr_sge, is_llqp);
+ q_size = wqe_size * (queue->max_wr + 1);
+
+ if (q_size <= 512)
+ queue->page_size = 2;
+ else if (q_size <= 1024)
+ queue->page_size = 3;
+ else
+ queue->page_size = 0;
+
+ queue->is_small = (queue->page_size != 0);
+}
+
+/* needs to be called with cq->spinlock held */
+void ehca_add_to_err_list(struct ehca_qp *qp, int on_sq)
+{
+ struct list_head *list, *node;
+
+ /* TODO: support low latency QPs */
+ if (qp->ext_type == EQPT_LLQP)
+ return;
+
+ if (on_sq) {
+ list = &qp->send_cq->sqp_err_list;
+ node = &qp->sq_err_node;
+ } else {
+ list = &qp->recv_cq->rqp_err_list;
+ node = &qp->rq_err_node;
+ }
+
+ if (list_empty(node))
+ list_add_tail(node, list);
+
+ return;
+}
+
+static void del_from_err_list(struct ehca_cq *cq, struct list_head *node)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->spinlock, flags);
+
+ if (!list_empty(node))
+ list_del_init(node);
+
+ spin_unlock_irqrestore(&cq->spinlock, flags);
+}
+
+static void reset_queue_map(struct ehca_queue_map *qmap)
+{
+ int i;
+
+ qmap->tail = qmap->entries - 1;
+ qmap->left_to_poll = 0;
+ qmap->next_wqe_idx = 0;
+ for (i = 0; i < qmap->entries; i++) {
+ qmap->map[i].reported = 1;
+ qmap->map[i].cqe_req = 0;
+ }
+}
+
+/*
+ * Create an ib_qp struct that is either a QP or an SRQ, depending on
+ * the value of the is_srq parameter. If init_attr and srq_init_attr share
+ * fields, the field out of init_attr is used.
+ */
+static struct ehca_qp *internal_create_qp(
+ struct ib_pd *pd,
+ struct ib_qp_init_attr *init_attr,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata, int is_srq)
+{
+ struct ehca_qp *my_qp, *my_srq = NULL;
+ struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+ struct ib_ucontext *context = NULL;
+ u64 h_ret;
+ int is_llqp = 0, has_srq = 0, is_user = 0;
+ int qp_type, max_send_sge, max_recv_sge, ret;
+
+ /* h_call's out parameters */
+ struct ehca_alloc_qp_parms parms;
+ u32 swqe_size = 0, rwqe_size = 0, ib_qp_num;
+ unsigned long flags;
+
+ if (!atomic_add_unless(&shca->num_qps, 1, shca->max_num_qps)) {
+ ehca_err(pd->device, "Unable to create QP, max number of %i "
+ "QPs reached.", shca->max_num_qps);
+ ehca_err(pd->device, "To increase the maximum number of QPs "
+ "use the number_of_qps module parameter.\n");
+ return ERR_PTR(-ENOSPC);
+ }
+
+ if (init_attr->create_flags) {
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ memset(&parms, 0, sizeof(parms));
+ qp_type = init_attr->qp_type;
+
+ if (init_attr->sq_sig_type != IB_SIGNAL_REQ_WR &&
+ init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
+ ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
+ init_attr->sq_sig_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ /* save LLQP info */
+ if (qp_type & 0x80) {
+ is_llqp = 1;
+ parms.ext_type = EQPT_LLQP;
+ parms.ll_comp_flags = qp_type & LLQP_COMP_MASK;
+ }
+ qp_type &= 0x1F;
+ init_attr->qp_type &= 0x1F;
+
+ /* handle SRQ base QPs */
+ if (init_attr->srq) {
+ my_srq = container_of(init_attr->srq, struct ehca_qp, ib_srq);
+
+ if (qp_type == IB_QPT_UC) {
+ ehca_err(pd->device, "UC with SRQ not supported");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ has_srq = 1;
+ parms.ext_type = EQPT_SRQBASE;
+ parms.srq_qpn = my_srq->real_qp_num;
+ }
+
+ if (is_llqp && has_srq) {
+ ehca_err(pd->device, "LLQPs can't have an SRQ");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ /* handle SRQs */
+ if (is_srq) {
+ parms.ext_type = EQPT_SRQ;
+ parms.srq_limit = srq_init_attr->attr.srq_limit;
+ if (init_attr->cap.max_recv_sge > 3) {
+ ehca_err(pd->device, "no more than three SGEs "
+ "supported for SRQ pd=%p max_sge=%x",
+ pd, init_attr->cap.max_recv_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ }
+
+ /* check QP type */
+ if (qp_type != IB_QPT_UD &&
+ qp_type != IB_QPT_UC &&
+ qp_type != IB_QPT_RC &&
+ qp_type != IB_QPT_SMI &&
+ qp_type != IB_QPT_GSI) {
+ ehca_err(pd->device, "wrong QP Type=%x", qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+
+ if (is_llqp) {
+ switch (qp_type) {
+ case IB_QPT_RC:
+ if ((init_attr->cap.max_send_wr > 255) ||
+ (init_attr->cap.max_recv_wr > 255)) {
+ ehca_err(pd->device,
+ "Invalid Number of max_sq_wr=%x "
+ "or max_rq_wr=%x for RC LLQP",
+ init_attr->cap.max_send_wr,
+ init_attr->cap.max_recv_wr);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ break;
+ case IB_QPT_UD:
+ if (!EHCA_BMASK_GET(HCA_CAP_UD_LL_QP, shca->hca_cap)) {
+ ehca_err(pd->device, "UD LLQP not supported "
+ "by this adapter");
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-ENOSYS);
+ }
+ if (!(init_attr->cap.max_send_sge <= 5
+ && init_attr->cap.max_send_sge >= 1
+ && init_attr->cap.max_recv_sge <= 5
+ && init_attr->cap.max_recv_sge >= 1)) {
+ ehca_err(pd->device,
+ "Invalid Number of max_send_sge=%x "
+ "or max_recv_sge=%x for UD LLQP",
+ init_attr->cap.max_send_sge,
+ init_attr->cap.max_recv_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ } else if (init_attr->cap.max_send_wr > 255) {
+ ehca_err(pd->device,
+ "Invalid Number of "
+ "max_send_wr=%x for UD QP_TYPE=%x",
+ init_attr->cap.max_send_wr, qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ break;
+ default:
+ ehca_err(pd->device, "unsupported LL QP Type=%x",
+ qp_type);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ } else {
+ int max_sge = (qp_type == IB_QPT_UD || qp_type == IB_QPT_SMI
+ || qp_type == IB_QPT_GSI) ? 250 : 252;
+
+ if (init_attr->cap.max_send_sge > max_sge
+ || init_attr->cap.max_recv_sge > max_sge) {
+ ehca_err(pd->device, "Invalid number of SGEs requested "
+ "send_sge=%x recv_sge=%x max_sge=%x",
+ init_attr->cap.max_send_sge,
+ init_attr->cap.max_recv_sge, max_sge);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-EINVAL);
+ }
+ }
+
+ my_qp = kmem_cache_zalloc(qp_cache, GFP_KERNEL);
+ if (!my_qp) {
+ ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ if (pd->uobject && udata) {
+ is_user = 1;
+ context = pd->uobject->context;
+ }
+
+ atomic_set(&my_qp->nr_events, 0);
+ init_waitqueue_head(&my_qp->wait_completion);
+ spin_lock_init(&my_qp->spinlock_s);
+ spin_lock_init(&my_qp->spinlock_r);
+ my_qp->qp_type = qp_type;
+ my_qp->ext_type = parms.ext_type;
+ my_qp->state = IB_QPS_RESET;
+
+ if (init_attr->recv_cq)
+ my_qp->recv_cq =
+ container_of(init_attr->recv_cq, struct ehca_cq, ib_cq);
+ if (init_attr->send_cq)
+ my_qp->send_cq =
+ container_of(init_attr->send_cq, struct ehca_cq, ib_cq);
+
+ idr_preload(GFP_KERNEL);
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+
+ ret = idr_alloc(&ehca_qp_idr, my_qp, 0, 0x2000000, GFP_NOWAIT);
+ if (ret >= 0)
+ my_qp->token = ret;
+
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+ idr_preload_end();
+ if (ret < 0) {
+ if (ret == -ENOSPC) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid number of qp");
+ } else {
+ ret = -ENOMEM;
+ ehca_err(pd->device, "Can't allocate new idr entry.");
+ }
+ goto create_qp_exit0;
+ }
+
+ if (has_srq)
+ parms.srq_token = my_qp->token;
+
+ parms.servicetype = ibqptype2servicetype(qp_type);
+ if (parms.servicetype < 0) {
+ ret = -EINVAL;
+ ehca_err(pd->device, "Invalid qp_type=%x", qp_type);
+ goto create_qp_exit1;
+ }
+
+ /* Always signal by WQE so we can hide circ. WQEs */
+ parms.sigtype = HCALL_SIGT_BY_WQE;
+
+ /* UD_AV CIRCUMVENTION */
+ max_send_sge = init_attr->cap.max_send_sge;
+ max_recv_sge = init_attr->cap.max_recv_sge;
+ if (parms.servicetype == ST_UD && !is_llqp) {
+ max_send_sge += 2;
+ max_recv_sge += 2;
+ }
+
+ parms.token = my_qp->token;
+ parms.eq_handle = shca->eq.ipz_eq_handle;
+ parms.pd = my_pd->fw_pd;
+ if (my_qp->send_cq)
+ parms.send_cq_handle = my_qp->send_cq->ipz_cq_handle;
+ if (my_qp->recv_cq)
+ parms.recv_cq_handle = my_qp->recv_cq->ipz_cq_handle;
+
+ parms.squeue.max_wr = init_attr->cap.max_send_wr;
+ parms.rqueue.max_wr = init_attr->cap.max_recv_wr;
+ parms.squeue.max_sge = max_send_sge;
+ parms.rqueue.max_sge = max_recv_sge;
+
+ /* RC QPs need one more SWQE for unsolicited ack circumvention */
+ if (qp_type == IB_QPT_RC)
+ parms.squeue.max_wr++;
+
+ if (EHCA_BMASK_GET(HCA_CAP_MINI_QP, shca->hca_cap)) {
+ if (HAS_SQ(my_qp))
+ ehca_determine_small_queue(
+ &parms.squeue, max_send_sge, is_llqp);
+ if (HAS_RQ(my_qp))
+ ehca_determine_small_queue(
+ &parms.rqueue, max_recv_sge, is_llqp);
+ parms.qp_storage =
+ (parms.squeue.is_small || parms.rqueue.is_small);
+ }
+
+ h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, &parms, is_user);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(pd->device, "h_alloc_resource_qp() failed h_ret=%lli",
+ h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto create_qp_exit1;
+ }
+
+ ib_qp_num = my_qp->real_qp_num = parms.real_qp_num;
+ my_qp->ipz_qp_handle = parms.qp_handle;
+ my_qp->galpas = parms.galpas;
+
+ swqe_size = ehca_calc_wqe_size(parms.squeue.act_nr_sges, is_llqp);
+ rwqe_size = ehca_calc_wqe_size(parms.rqueue.act_nr_sges, is_llqp);
+
+ switch (qp_type) {
+ case IB_QPT_RC:
+ if (is_llqp) {
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
+ }
+ /* hide the extra WQE */
+ parms.squeue.act_nr_wqes--;
+ break;
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ case IB_QPT_SMI:
+ /* UD circumvention */
+ if (is_llqp) {
+ parms.squeue.act_nr_sges = 1;
+ parms.rqueue.act_nr_sges = 1;
+ } else {
+ parms.squeue.act_nr_sges -= 2;
+ parms.rqueue.act_nr_sges -= 2;
+ }
+
+ if (IB_QPT_GSI == qp_type || IB_QPT_SMI == qp_type) {
+ parms.squeue.act_nr_wqes = init_attr->cap.max_send_wr;
+ parms.rqueue.act_nr_wqes = init_attr->cap.max_recv_wr;
+ parms.squeue.act_nr_sges = init_attr->cap.max_send_sge;
+ parms.rqueue.act_nr_sges = init_attr->cap.max_recv_sge;
+ ib_qp_num = (qp_type == IB_QPT_SMI) ? 0 : 1;
+ }
+
+ break;
+
+ default:
+ break;
+ }
+
+ /* initialize r/squeue and register queue pages */
+ if (HAS_SQ(my_qp)) {
+ ret = init_qp_queue(
+ shca, my_pd, my_qp, &my_qp->ipz_squeue, 0,
+ HAS_RQ(my_qp) ? H_PAGE_REGISTERED : H_SUCCESS,
+ &parms.squeue, swqe_size);
+ if (ret) {
+ ehca_err(pd->device, "Couldn't initialize squeue "
+ "and pages ret=%i", ret);
+ goto create_qp_exit2;
+ }
+
+ if (!is_user) {
+ my_qp->sq_map.entries = my_qp->ipz_squeue.queue_length /
+ my_qp->ipz_squeue.qe_size;
+ my_qp->sq_map.map = vmalloc(my_qp->sq_map.entries *
+ sizeof(struct ehca_qmap_entry));
+ if (!my_qp->sq_map.map) {
+ ehca_err(pd->device, "Couldn't allocate squeue "
+ "map ret=%i", ret);
+ goto create_qp_exit3;
+ }
+ INIT_LIST_HEAD(&my_qp->sq_err_node);
+ /* to avoid the generation of bogus flush CQEs */
+ reset_queue_map(&my_qp->sq_map);
+ }
+ }
+
+ if (HAS_RQ(my_qp)) {
+ ret = init_qp_queue(
+ shca, my_pd, my_qp, &my_qp->ipz_rqueue, 1,
+ H_SUCCESS, &parms.rqueue, rwqe_size);
+ if (ret) {
+ ehca_err(pd->device, "Couldn't initialize rqueue "
+ "and pages ret=%i", ret);
+ goto create_qp_exit4;
+ }
+ if (!is_user) {
+ my_qp->rq_map.entries = my_qp->ipz_rqueue.queue_length /
+ my_qp->ipz_rqueue.qe_size;
+ my_qp->rq_map.map = vmalloc(my_qp->rq_map.entries *
+ sizeof(struct ehca_qmap_entry));
+ if (!my_qp->rq_map.map) {
+ ehca_err(pd->device, "Couldn't allocate squeue "
+ "map ret=%i", ret);
+ goto create_qp_exit5;
+ }
+ INIT_LIST_HEAD(&my_qp->rq_err_node);
+ /* to avoid the generation of bogus flush CQEs */
+ reset_queue_map(&my_qp->rq_map);
+ }
+ } else if (init_attr->srq && !is_user) {
+ /* this is a base QP, use the queue map of the SRQ */
+ my_qp->rq_map = my_srq->rq_map;
+ INIT_LIST_HEAD(&my_qp->rq_err_node);
+
+ my_qp->ipz_rqueue = my_srq->ipz_rqueue;
+ }
+
+ if (is_srq) {
+ my_qp->ib_srq.pd = &my_pd->ib_pd;
+ my_qp->ib_srq.device = my_pd->ib_pd.device;
+
+ my_qp->ib_srq.srq_context = init_attr->qp_context;
+ my_qp->ib_srq.event_handler = init_attr->event_handler;
+ } else {
+ my_qp->ib_qp.qp_num = ib_qp_num;
+ my_qp->ib_qp.pd = &my_pd->ib_pd;
+ my_qp->ib_qp.device = my_pd->ib_pd.device;
+
+ my_qp->ib_qp.recv_cq = init_attr->recv_cq;
+ my_qp->ib_qp.send_cq = init_attr->send_cq;
+
+ my_qp->ib_qp.qp_type = qp_type;
+ my_qp->ib_qp.srq = init_attr->srq;
+
+ my_qp->ib_qp.qp_context = init_attr->qp_context;
+ my_qp->ib_qp.event_handler = init_attr->event_handler;
+ }
+
+ init_attr->cap.max_inline_data = 0; /* not supported yet */
+ init_attr->cap.max_recv_sge = parms.rqueue.act_nr_sges;
+ init_attr->cap.max_recv_wr = parms.rqueue.act_nr_wqes;
+ init_attr->cap.max_send_sge = parms.squeue.act_nr_sges;
+ init_attr->cap.max_send_wr = parms.squeue.act_nr_wqes;
+ my_qp->init_attr = *init_attr;
+
+ if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+ shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
+ &my_qp->ib_qp;
+ if (ehca_nr_ports < 0) {
+ /* alloc array to cache subsequent modify qp parms
+ * for autodetect mode
+ */
+ my_qp->mod_qp_parm =
+ kzalloc(EHCA_MOD_QP_PARM_MAX *
+ sizeof(*my_qp->mod_qp_parm),
+ GFP_KERNEL);
+ if (!my_qp->mod_qp_parm) {
+ ehca_err(pd->device,
+ "Could not alloc mod_qp_parm");
+ goto create_qp_exit5;
+ }
+ }
+ }
+
+ /* NOTE: define_apq0() not supported yet */
+ if (qp_type == IB_QPT_GSI) {
+ h_ret = ehca_define_sqp(shca, my_qp, init_attr);
+ if (h_ret != H_SUCCESS) {
+ kfree(my_qp->mod_qp_parm);
+ my_qp->mod_qp_parm = NULL;
+ /* the QP pointer is no longer valid */
+ shca->sport[init_attr->port_num - 1].ibqp_sqp[qp_type] =
+ NULL;
+ ret = ehca2ib_return_code(h_ret);
+ goto create_qp_exit6;
+ }
+ }
+
+ if (my_qp->send_cq) {
+ ret = ehca_cq_assign_qp(my_qp->send_cq, my_qp);
+ if (ret) {
+ ehca_err(pd->device,
+ "Couldn't assign qp to send_cq ret=%i", ret);
+ goto create_qp_exit7;
+ }
+ }
+
+ /* copy queues, galpa data to user space */
+ if (context && udata) {
+ struct ehca_create_qp_resp resp;
+ memset(&resp, 0, sizeof(resp));
+
+ resp.qp_num = my_qp->real_qp_num;
+ resp.token = my_qp->token;
+ resp.qp_type = my_qp->qp_type;
+ resp.ext_type = my_qp->ext_type;
+ resp.qkey = my_qp->qkey;
+ resp.real_qp_num = my_qp->real_qp_num;
+
+ if (HAS_SQ(my_qp))
+ queue2resp(&resp.ipz_squeue, &my_qp->ipz_squeue);
+ if (HAS_RQ(my_qp))
+ queue2resp(&resp.ipz_rqueue, &my_qp->ipz_rqueue);
+ resp.fw_handle_ofs = (u32)
+ (my_qp->galpas.user.fw_handle & (PAGE_SIZE - 1));
+
+ if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
+ ehca_err(pd->device, "Copy to udata failed");
+ ret = -EINVAL;
+ goto create_qp_exit8;
+ }
+ }
+
+ return my_qp;
+
+create_qp_exit8:
+ ehca_cq_unassign_qp(my_qp->send_cq, my_qp->real_qp_num);
+
+create_qp_exit7:
+ kfree(my_qp->mod_qp_parm);
+
+create_qp_exit6:
+ if (HAS_RQ(my_qp) && !is_user)
+ vfree(my_qp->rq_map.map);
+
+create_qp_exit5:
+ if (HAS_RQ(my_qp))
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
+
+create_qp_exit4:
+ if (HAS_SQ(my_qp) && !is_user)
+ vfree(my_qp->sq_map.map);
+
+create_qp_exit3:
+ if (HAS_SQ(my_qp))
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
+
+create_qp_exit2:
+ hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+
+create_qp_exit1:
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+ idr_remove(&ehca_qp_idr, my_qp->token);
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+create_qp_exit0:
+ kmem_cache_free(qp_cache, my_qp);
+ atomic_dec(&shca->num_qps);
+ return ERR_PTR(ret);
+}
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+ struct ib_qp_init_attr *qp_init_attr,
+ struct ib_udata *udata)
+{
+ struct ehca_qp *ret;
+
+ ret = internal_create_qp(pd, qp_init_attr, NULL, udata, 0);
+ return IS_ERR(ret) ? (struct ib_qp *)ret : &ret->ib_qp;
+}
+
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject);
+
+struct ib_srq *ehca_create_srq(struct ib_pd *pd,
+ struct ib_srq_init_attr *srq_init_attr,
+ struct ib_udata *udata)
+{
+ struct ib_qp_init_attr qp_init_attr;
+ struct ehca_qp *my_qp;
+ struct ib_srq *ret;
+ struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+ ib_device);
+ struct hcp_modify_qp_control_block *mqpcb;
+ u64 hret, update_mask;
+
+ if (srq_init_attr->srq_type != IB_SRQT_BASIC)
+ return ERR_PTR(-ENOSYS);
+
+ /* For common attributes, internal_create_qp() takes its info
+ * out of qp_init_attr, so copy all common attrs there.
+ */
+ memset(&qp_init_attr, 0, sizeof(qp_init_attr));
+ qp_init_attr.event_handler = srq_init_attr->event_handler;
+ qp_init_attr.qp_context = srq_init_attr->srq_context;
+ qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
+ qp_init_attr.qp_type = IB_QPT_RC;
+ qp_init_attr.cap.max_recv_wr = srq_init_attr->attr.max_wr;
+ qp_init_attr.cap.max_recv_sge = srq_init_attr->attr.max_sge;
+
+ my_qp = internal_create_qp(pd, &qp_init_attr, srq_init_attr, udata, 1);
+ if (IS_ERR(my_qp))
+ return (struct ib_srq *)my_qp;
+
+ /* copy back return values */
+ srq_init_attr->attr.max_wr = qp_init_attr.cap.max_recv_wr;
+ srq_init_attr->attr.max_sge = 3;
+
+ /* drive SRQ into RTR state */
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!mqpcb) {
+ ehca_err(pd->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
+ ret = ERR_PTR(-ENOMEM);
+ goto create_srq1;
+ }
+
+ mqpcb->qp_state = EHCA_QPS_INIT;
+ mqpcb->prim_phys_port = 1;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not modify SRQ to INIT "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ mqpcb->qp_enable = 1;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not enable SRQ "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ mqpcb->qp_state = EHCA_QPS_RTR;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ hret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+ if (hret != H_SUCCESS) {
+ ehca_err(pd->device, "Could not modify SRQ to RTR "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, hret);
+ goto create_srq2;
+ }
+
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return &my_qp->ib_srq;
+
+create_srq2:
+ ret = ERR_PTR(ehca2ib_return_code(hret));
+ ehca_free_fw_ctrlblock(mqpcb);
+
+create_srq1:
+ internal_destroy_qp(pd->device, my_qp, my_qp->ib_srq.uobject);
+
+ return ret;
+}
+
+/*
+ * prepare_sqe_rts called by internal_modify_qp() at trans sqe -> rts
+ * set purge bit of bad wqe and subsequent wqes to avoid reentering sqe
+ * returns total number of bad wqes in bad_wqe_cnt
+ */
+static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
+ int *bad_wqe_cnt)
+{
+ u64 h_ret;
+ struct ipz_queue *squeue;
+ void *bad_send_wqe_p, *bad_send_wqe_v;
+ u64 q_ofs;
+ struct ehca_wqe *wqe;
+ int qp_num = my_qp->ib_qp.qp_num;
+
+ /* get send wqe pointer */
+ h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle, &my_qp->pf,
+ &bad_send_wqe_p, NULL, 2);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "hipz_h_disable_and_get_wqe() failed"
+ " ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp_num, h_ret);
+ return ehca2ib_return_code(h_ret);
+ }
+ bad_send_wqe_p = (void *)((u64)bad_send_wqe_p & (~(1L << 63)));
+ ehca_dbg(&shca->ib_device, "qp_num=%x bad_send_wqe_p=%p",
+ qp_num, bad_send_wqe_p);
+ /* convert wqe pointer to vadr */
+ bad_send_wqe_v = __va((u64)bad_send_wqe_p);
+ if (ehca_debug_level >= 2)
+ ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
+ squeue = &my_qp->ipz_squeue;
+ if (ipz_queue_abs_to_offset(squeue, (u64)bad_send_wqe_p, &q_ofs)) {
+ ehca_err(&shca->ib_device, "failed to get wqe offset qp_num=%x"
+ " bad_send_wqe_p=%p", qp_num, bad_send_wqe_p);
+ return -EFAULT;
+ }
+
+ /* loop sets wqe's purge bit */
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
+ *bad_wqe_cnt = 0;
+ while (wqe->optype != 0xff && wqe->wqef != 0xff) {
+ if (ehca_debug_level >= 2)
+ ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
+ wqe->nr_of_data_seg = 0; /* suppress data access */
+ wqe->wqef = WQEF_PURGE; /* WQE to be purged */
+ q_ofs = ipz_queue_advance_offset(squeue, q_ofs);
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(squeue, q_ofs);
+ *bad_wqe_cnt = (*bad_wqe_cnt)+1;
+ }
+ /*
+ * bad wqe will be reprocessed and ignored when pol_cq() is called,
+ * i.e. nr of wqes with flush error status is one less
+ */
+ ehca_dbg(&shca->ib_device, "qp_num=%x flusherr_wqe_cnt=%x",
+ qp_num, (*bad_wqe_cnt)-1);
+ wqe->wqef = 0;
+
+ return 0;
+}
+
+static int calc_left_cqes(u64 wqe_p, struct ipz_queue *ipz_queue,
+ struct ehca_queue_map *qmap)
+{
+ void *wqe_v;
+ u64 q_ofs;
+ u32 wqe_idx;
+ unsigned int tail_idx;
+
+ /* convert real to abs address */
+ wqe_p = wqe_p & (~(1UL << 63));
+
+ wqe_v = __va(wqe_p);
+
+ if (ipz_queue_abs_to_offset(ipz_queue, wqe_p, &q_ofs)) {
+ ehca_gen_err("Invalid offset for calculating left cqes "
+ "wqe_p=%#llx wqe_v=%p\n", wqe_p, wqe_v);
+ return -EFAULT;
+ }
+
+ tail_idx = next_index(qmap->tail, qmap->entries);
+ wqe_idx = q_ofs / ipz_queue->qe_size;
+
+ /* check all processed wqes, whether a cqe is requested or not */
+ while (tail_idx != wqe_idx) {
+ if (qmap->map[tail_idx].cqe_req)
+ qmap->left_to_poll++;
+ tail_idx = next_index(tail_idx, qmap->entries);
+ }
+ /* save index in queue, where we have to start flushing */
+ qmap->next_wqe_idx = wqe_idx;
+ return 0;
+}
+
+static int check_for_left_cqes(struct ehca_qp *my_qp, struct ehca_shca *shca)
+{
+ u64 h_ret;
+ void *send_wqe_p, *recv_wqe_p;
+ int ret;
+ unsigned long flags;
+ int qp_num = my_qp->ib_qp.qp_num;
+
+ /* this hcall is not supported on base QPs */
+ if (my_qp->ext_type != EQPT_SRQBASE) {
+ /* get send and receive wqe pointer */
+ h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle, &my_qp->pf,
+ &send_wqe_p, &recv_wqe_p, 4);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device, "disable_and_get_wqe() "
+ "failed ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp_num, h_ret);
+ return ehca2ib_return_code(h_ret);
+ }
+
+ /*
+ * acquire lock to ensure that nobody is polling the cq which
+ * could mean that the qmap->tail pointer is in an
+ * inconsistent state.
+ */
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ ret = calc_left_cqes((u64)send_wqe_p, &my_qp->ipz_squeue,
+ &my_qp->sq_map);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+ if (ret)
+ return ret;
+
+
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ ret = calc_left_cqes((u64)recv_wqe_p, &my_qp->ipz_rqueue,
+ &my_qp->rq_map);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
+ if (ret)
+ return ret;
+ } else {
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ my_qp->sq_map.left_to_poll = 0;
+ my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
+ my_qp->sq_map.entries);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ my_qp->rq_map.left_to_poll = 0;
+ my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
+ my_qp->rq_map.entries);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock, flags);
+ }
+
+ /* this assures flush cqes being generated only for pending wqes */
+ if ((my_qp->sq_map.left_to_poll == 0) &&
+ (my_qp->rq_map.left_to_poll == 0)) {
+ spin_lock_irqsave(&my_qp->send_cq->spinlock, flags);
+ ehca_add_to_err_list(my_qp, 1);
+ spin_unlock_irqrestore(&my_qp->send_cq->spinlock, flags);
+
+ if (HAS_RQ(my_qp)) {
+ spin_lock_irqsave(&my_qp->recv_cq->spinlock, flags);
+ ehca_add_to_err_list(my_qp, 0);
+ spin_unlock_irqrestore(&my_qp->recv_cq->spinlock,
+ flags);
+ }
+ }
+
+ return 0;
+}
+
+/*
+ * internal_modify_qp with circumvention to handle aqp0 properly
+ * smi_reset2init indicates if this is an internal reset-to-init-call for
+ * smi. This flag must always be zero if called from ehca_modify_qp()!
+ * This internal func was intorduced to avoid recursion of ehca_modify_qp()!
+ */
+static int internal_modify_qp(struct ib_qp *ibqp,
+ struct ib_qp_attr *attr,
+ int attr_mask, int smi_reset2init)
+{
+ enum ib_qp_state qp_cur_state, qp_new_state;
+ int cnt, qp_attr_idx, ret = 0;
+ enum ib_qp_statetrans statetrans;
+ struct hcp_modify_qp_control_block *mqpcb;
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca =
+ container_of(ibqp->pd->device, struct ehca_shca, ib_device);
+ u64 update_mask;
+ u64 h_ret;
+ int bad_wqe_cnt = 0;
+ int is_user = 0;
+ int squeue_locked = 0;
+ unsigned long flags = 0;
+
+ /* do query_qp to obtain current attr values */
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_ATOMIC);
+ if (!mqpcb) {
+ ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ mqpcb, my_qp->galpas.kernel);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(ibqp->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, ibqp->qp_num, h_ret);
+ ret = ehca2ib_return_code(h_ret);
+ goto modify_qp_exit1;
+ }
+ if (ibqp->uobject)
+ is_user = 1;
+
+ qp_cur_state = ehca2ib_qp_state(mqpcb->qp_state);
+
+ if (qp_cur_state == -EINVAL) { /* invalid qp state */
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid current ehca_qp_state=%x "
+ "ehca_qp=%p qp_num=%x",
+ mqpcb->qp_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+ /*
+ * circumvention to set aqp0 initial state to init
+ * as expected by IB spec
+ */
+ if (smi_reset2init == 0 &&
+ ibqp->qp_type == IB_QPT_SMI &&
+ qp_cur_state == IB_QPS_RESET &&
+ (attr_mask & IB_QP_STATE) &&
+ attr->qp_state == IB_QPS_INIT) { /* RESET -> INIT */
+ struct ib_qp_attr smiqp_attr = {
+ .qp_state = IB_QPS_INIT,
+ .port_num = my_qp->init_attr.port_num,
+ .pkey_index = 0,
+ .qkey = 0
+ };
+ int smiqp_attr_mask = IB_QP_STATE | IB_QP_PORT |
+ IB_QP_PKEY_INDEX | IB_QP_QKEY;
+ int smirc = internal_modify_qp(
+ ibqp, &smiqp_attr, smiqp_attr_mask, 1);
+ if (smirc) {
+ ehca_err(ibqp->device, "SMI RESET -> INIT failed. "
+ "ehca_modify_qp() rc=%i", smirc);
+ ret = H_PARAMETER;
+ goto modify_qp_exit1;
+ }
+ qp_cur_state = IB_QPS_INIT;
+ ehca_dbg(ibqp->device, "SMI RESET -> INIT succeeded");
+ }
+ /* is transmitted current state equal to "real" current state */
+ if ((attr_mask & IB_QP_CUR_STATE) &&
+ qp_cur_state != attr->cur_qp_state) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device,
+ "Invalid IB_QP_CUR_STATE attr->curr_qp_state=%x <>"
+ " actual cur_qp_state=%x. ehca_qp=%p qp_num=%x",
+ attr->cur_qp_state, qp_cur_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ ehca_dbg(ibqp->device, "ehca_qp=%p qp_num=%x current qp_state=%x "
+ "new qp_state=%x attribute_mask=%x",
+ my_qp, ibqp->qp_num, qp_cur_state, attr->qp_state, attr_mask);
+
+ qp_new_state = attr_mask & IB_QP_STATE ? attr->qp_state : qp_cur_state;
+ if (!smi_reset2init &&
+ !ib_modify_qp_is_ok(qp_cur_state, qp_new_state, ibqp->qp_type,
+ attr_mask, IB_LINK_LAYER_UNSPECIFIED)) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device,
+ "Invalid qp transition new_state=%x cur_state=%x "
+ "ehca_qp=%p qp_num=%x attr_mask=%x", qp_new_state,
+ qp_cur_state, my_qp, ibqp->qp_num, attr_mask);
+ goto modify_qp_exit1;
+ }
+
+ mqpcb->qp_state = ib2ehca_qp_state(qp_new_state);
+ if (mqpcb->qp_state)
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+ else {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid new qp state=%x "
+ "ehca_qp=%p qp_num=%x",
+ qp_new_state, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ /* retrieve state transition struct to get req and opt attrs */
+ statetrans = get_modqp_statetrans(qp_cur_state, qp_new_state);
+ if (statetrans < 0) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "<INVALID STATE CHANGE> qp_cur_state=%x "
+ "new_qp_state=%x State_xsition=%x ehca_qp=%p "
+ "qp_num=%x", qp_cur_state, qp_new_state,
+ statetrans, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ qp_attr_idx = ib2ehcaqptype(ibqp->qp_type);
+
+ if (qp_attr_idx < 0) {
+ ret = qp_attr_idx;
+ ehca_err(ibqp->device,
+ "Invalid QP type=%x ehca_qp=%p qp_num=%x",
+ ibqp->qp_type, my_qp, ibqp->qp_num);
+ goto modify_qp_exit1;
+ }
+
+ ehca_dbg(ibqp->device,
+ "ehca_qp=%p qp_num=%x <VALID STATE CHANGE> qp_state_xsit=%x",
+ my_qp, ibqp->qp_num, statetrans);
+
+ /* eHCA2 rev2 and higher require the SEND_GRH_FLAG to be set
+ * in non-LL UD QPs.
+ */
+ if ((my_qp->qp_type == IB_QPT_UD) &&
+ (my_qp->ext_type != EQPT_LLQP) &&
+ (statetrans == IB_QPST_INIT2RTR) &&
+ (shca->hw_level >= 0x22)) {
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
+ mqpcb->send_grh_flag = 1;
+ }
+
+ /* sqe -> rts: set purge bit of bad wqe before actual trans */
+ if ((my_qp->qp_type == IB_QPT_UD ||
+ my_qp->qp_type == IB_QPT_GSI ||
+ my_qp->qp_type == IB_QPT_SMI) &&
+ statetrans == IB_QPST_SQE2RTS) {
+ /* mark next free wqe if kernel */
+ if (!ibqp->uobject) {
+ struct ehca_wqe *wqe;
+ /* lock send queue */
+ spin_lock_irqsave(&my_qp->spinlock_s, flags);
+ squeue_locked = 1;
+ /* mark next free wqe */
+ wqe = (struct ehca_wqe *)
+ ipz_qeit_get(&my_qp->ipz_squeue);
+ wqe->optype = wqe->wqef = 0xff;
+ ehca_dbg(ibqp->device, "qp_num=%x next_free_wqe=%p",
+ ibqp->qp_num, wqe);
+ }
+ ret = prepare_sqe_rts(my_qp, shca, &bad_wqe_cnt);
+ if (ret) {
+ ehca_err(ibqp->device, "prepare_sqe_rts() failed "
+ "ehca_qp=%p qp_num=%x ret=%i",
+ my_qp, ibqp->qp_num, ret);
+ goto modify_qp_exit2;
+ }
+ }
+
+ /*
+ * enable RDMA_Atomic_Control if reset->init und reliable con
+ * this is necessary since gen2 does not provide that flag,
+ * but pHyp requires it
+ */
+ if (statetrans == IB_QPST_RESET2INIT &&
+ (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_UC)) {
+ mqpcb->rdma_atomic_ctrl = 3;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RDMA_ATOMIC_CTRL, 1);
+ }
+ /* circ. pHyp requires #RDMA/Atomic Resp Res for UC INIT -> RTR */
+ if (statetrans == IB_QPST_INIT2RTR &&
+ (ibqp->qp_type == IB_QPT_UC) &&
+ !(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)) {
+ mqpcb->rdma_nr_atomic_resp_res = 1; /* default to 1 */
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+ }
+
+ if (attr_mask & IB_QP_PKEY_INDEX) {
+ if (attr->pkey_index >= 16) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid pkey_index=%x. "
+ "ehca_qp=%p qp_num=%x max_pkey_index=f",
+ attr->pkey_index, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->prim_p_key_idx = attr->pkey_index;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
+ }
+ if (attr_mask & IB_QP_PORT) {
+ struct ehca_sport *sport;
+ struct ehca_qp *aqp1;
+ if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid port=%x. "
+ "ehca_qp=%p qp_num=%x num_ports=%x",
+ attr->port_num, my_qp, ibqp->qp_num,
+ shca->num_ports);
+ goto modify_qp_exit2;
+ }
+ sport = &shca->sport[attr->port_num - 1];
+ if (!sport->ibqp_sqp[IB_QPT_GSI]) {
+ /* should not occur */
+ ret = -EFAULT;
+ ehca_err(ibqp->device, "AQP1 was not created for "
+ "port=%x", attr->port_num);
+ goto modify_qp_exit2;
+ }
+ aqp1 = container_of(sport->ibqp_sqp[IB_QPT_GSI],
+ struct ehca_qp, ib_qp);
+ if (ibqp->qp_type != IB_QPT_GSI &&
+ ibqp->qp_type != IB_QPT_SMI &&
+ aqp1->mod_qp_parm) {
+ /*
+ * firmware will reject this modify_qp() because
+ * port is not activated/initialized fully
+ */
+ ret = -EFAULT;
+ ehca_warn(ibqp->device, "Couldn't modify qp port=%x: "
+ "either port is being activated (try again) "
+ "or cabling issue", attr->port_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->prim_phys_port = attr->port_num;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
+ }
+ if (attr_mask & IB_QP_QKEY) {
+ mqpcb->qkey = attr->qkey;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_QKEY, 1);
+ }
+ if (attr_mask & IB_QP_AV) {
+ mqpcb->dlid = attr->ah_attr.dlid;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID, 1);
+ mqpcb->source_path_bits = attr->ah_attr.src_path_bits;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS, 1);
+ mqpcb->service_level = attr->ah_attr.sl;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL, 1);
+
+ if (ehca_calc_ipd(shca, mqpcb->prim_phys_port,
+ attr->ah_attr.static_rate,
+ &mqpcb->max_static_rate)) {
+ ret = -EINVAL;
+ goto modify_qp_exit2;
+ }
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE, 1);
+
+ /*
+ * Always supply the GRH flag, even if it's zero, to give the
+ * hypervisor a clear "yes" or "no" instead of a "perhaps"
+ */
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
+
+ /*
+ * only if GRH is TRUE we might consider SOURCE_GID_IDX
+ * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+ */
+ if (attr->ah_attr.ah_flags == IB_AH_GRH) {
+ mqpcb->send_grh_flag = 1;
+
+ mqpcb->source_gid_idx = attr->ah_attr.grh.sgid_index;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX, 1);
+
+ for (cnt = 0; cnt < 16; cnt++)
+ mqpcb->dest_gid.byte[cnt] =
+ attr->ah_attr.grh.dgid.raw[cnt];
+
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_GID, 1);
+ mqpcb->flow_label = attr->ah_attr.grh.flow_label;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL, 1);
+ mqpcb->hop_limit = attr->ah_attr.grh.hop_limit;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT, 1);
+ mqpcb->traffic_class = attr->ah_attr.grh.traffic_class;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS, 1);
+ }
+ }
+
+ if (attr_mask & IB_QP_PATH_MTU) {
+ /* store ld(MTU) */
+ my_qp->mtu_shift = attr->path_mtu + 7;
+ mqpcb->path_mtu = attr->path_mtu;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PATH_MTU, 1);
+ }
+ if (attr_mask & IB_QP_TIMEOUT) {
+ mqpcb->timeout = attr->timeout;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT, 1);
+ }
+ if (attr_mask & IB_QP_RETRY_CNT) {
+ mqpcb->retry_count = attr->retry_cnt;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT, 1);
+ }
+ if (attr_mask & IB_QP_RNR_RETRY) {
+ mqpcb->rnr_retry_count = attr->rnr_retry;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT, 1);
+ }
+ if (attr_mask & IB_QP_RQ_PSN) {
+ mqpcb->receive_psn = attr->rq_psn;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RECEIVE_PSN, 1);
+ }
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ mqpcb->rdma_nr_atomic_resp_res = attr->max_dest_rd_atomic < 3 ?
+ attr->max_dest_rd_atomic : 2;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+ }
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ mqpcb->rdma_atomic_outst_dest_qp = attr->max_rd_atomic < 3 ?
+ attr->max_rd_atomic : 2;
+ update_mask |=
+ EHCA_BMASK_SET
+ (MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP, 1);
+ }
+ if (attr_mask & IB_QP_ALT_PATH) {
+ if (attr->alt_port_num < 1
+ || attr->alt_port_num > shca->num_ports) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid alt_port=%x. "
+ "ehca_qp=%p qp_num=%x num_ports=%x",
+ attr->alt_port_num, my_qp, ibqp->qp_num,
+ shca->num_ports);
+ goto modify_qp_exit2;
+ }
+ mqpcb->alt_phys_port = attr->alt_port_num;
+
+ if (attr->alt_pkey_index >= 16) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid alt_pkey_index=%x. "
+ "ehca_qp=%p qp_num=%x max_pkey_index=f",
+ attr->pkey_index, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ mqpcb->alt_p_key_idx = attr->alt_pkey_index;
+
+ mqpcb->timeout_al = attr->alt_timeout;
+ mqpcb->dlid_al = attr->alt_ah_attr.dlid;
+ mqpcb->source_path_bits_al = attr->alt_ah_attr.src_path_bits;
+ mqpcb->service_level_al = attr->alt_ah_attr.sl;
+
+ if (ehca_calc_ipd(shca, mqpcb->alt_phys_port,
+ attr->alt_ah_attr.static_rate,
+ &mqpcb->max_static_rate_al)) {
+ ret = -EINVAL;
+ goto modify_qp_exit2;
+ }
+
+ /* OpenIB doesn't support alternate retry counts - copy them */
+ mqpcb->retry_count_al = mqpcb->retry_count;
+ mqpcb->rnr_retry_count_al = mqpcb->rnr_retry_count;
+
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_ALT_PHYS_PORT, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_ALT_P_KEY_IDX, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_DLID_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT_AL, 1);
+
+ /*
+ * Always supply the GRH flag, even if it's zero, to give the
+ * hypervisor a clear "yes" or "no" instead of a "perhaps"
+ */
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG_AL, 1);
+
+ /*
+ * only if GRH is TRUE we might consider SOURCE_GID_IDX
+ * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+ */
+ if (attr->alt_ah_attr.ah_flags == IB_AH_GRH) {
+ mqpcb->send_grh_flag_al = 1;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ mqpcb->dest_gid_al.byte[cnt] =
+ attr->alt_ah_attr.grh.dgid.raw[cnt];
+ mqpcb->source_gid_idx_al =
+ attr->alt_ah_attr.grh.sgid_index;
+ mqpcb->flow_label_al = attr->alt_ah_attr.grh.flow_label;
+ mqpcb->hop_limit_al = attr->alt_ah_attr.grh.hop_limit;
+ mqpcb->traffic_class_al =
+ attr->alt_ah_attr.grh.traffic_class;
+
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_DEST_GID_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL_AL, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT_AL, 1) |
+ EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS_AL, 1);
+ }
+ }
+
+ if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+ mqpcb->min_rnr_nak_timer_field = attr->min_rnr_timer;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD, 1);
+ }
+
+ if (attr_mask & IB_QP_SQ_PSN) {
+ mqpcb->send_psn = attr->sq_psn;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_PSN, 1);
+ }
+
+ if (attr_mask & IB_QP_DEST_QPN) {
+ mqpcb->dest_qp_nr = attr->dest_qp_num;
+ update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_QP_NR, 1);
+ }
+
+ if (attr_mask & IB_QP_PATH_MIG_STATE) {
+ if (attr->path_mig_state != IB_MIG_REARM
+ && attr->path_mig_state != IB_MIG_MIGRATED) {
+ ret = -EINVAL;
+ ehca_err(ibqp->device, "Invalid mig_state=%x",
+ attr->path_mig_state);
+ goto modify_qp_exit2;
+ }
+ mqpcb->path_migration_state = attr->path_mig_state + 1;
+ if (attr->path_mig_state == IB_MIG_REARM)
+ my_qp->mig_armed = 1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
+ }
+
+ if (attr_mask & IB_QP_CAP) {
+ mqpcb->max_nr_outst_send_wr = attr->cap.max_send_wr+1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_SEND_WR, 1);
+ mqpcb->max_nr_outst_recv_wr = attr->cap.max_recv_wr+1;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_RECV_WR, 1);
+ /* no support for max_send/recv_sge yet */
+ }
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibqp->device, "hipz_h_modify_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x", h_ret, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+
+ if ((my_qp->qp_type == IB_QPT_UD ||
+ my_qp->qp_type == IB_QPT_GSI ||
+ my_qp->qp_type == IB_QPT_SMI) &&
+ statetrans == IB_QPST_SQE2RTS) {
+ /* doorbell to reprocessing wqes */
+ iosync(); /* serialize GAL register access */
+ hipz_update_sqa(my_qp, bad_wqe_cnt-1);
+ ehca_gen_dbg("doorbell for %x wqes", bad_wqe_cnt);
+ }
+
+ if (statetrans == IB_QPST_RESET2INIT ||
+ statetrans == IB_QPST_INIT2INIT) {
+ mqpcb->qp_enable = 1;
+ mqpcb->qp_state = EHCA_QPS_INIT;
+ update_mask = 0;
+ update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ update_mask,
+ mqpcb,
+ my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibqp->device, "ENABLE in context of "
+ "RESET_2_INIT failed! Maybe you didn't get "
+ "a LID h_ret=%lli ehca_qp=%p qp_num=%x",
+ h_ret, my_qp, ibqp->qp_num);
+ goto modify_qp_exit2;
+ }
+ }
+ if ((qp_new_state == IB_QPS_ERR) && (qp_cur_state != IB_QPS_ERR)
+ && !is_user) {
+ ret = check_for_left_cqes(my_qp, shca);
+ if (ret)
+ goto modify_qp_exit2;
+ }
+
+ if (statetrans == IB_QPST_ANY2RESET) {
+ ipz_qeit_reset(&my_qp->ipz_rqueue);
+ ipz_qeit_reset(&my_qp->ipz_squeue);
+
+ if (qp_cur_state == IB_QPS_ERR && !is_user) {
+ del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
+
+ if (HAS_RQ(my_qp))
+ del_from_err_list(my_qp->recv_cq,
+ &my_qp->rq_err_node);
+ }
+ if (!is_user)
+ reset_queue_map(&my_qp->sq_map);
+
+ if (HAS_RQ(my_qp) && !is_user)
+ reset_queue_map(&my_qp->rq_map);
+ }
+
+ if (attr_mask & IB_QP_QKEY)
+ my_qp->qkey = attr->qkey;
+
+modify_qp_exit2:
+ if (squeue_locked) { /* this means: sqe -> rts */
+ spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
+ my_qp->sqerr_purgeflag = 1;
+ }
+
+modify_qp_exit1:
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return ret;
+}
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+ struct ib_udata *udata)
+{
+ int ret = 0;
+
+ struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+ ib_device);
+ struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+
+ /* The if-block below caches qp_attr to be modified for GSI and SMI
+ * qps during the initialization by ib_mad. When the respective port
+ * is activated, ie we got an event PORT_ACTIVE, we'll replay the
+ * cached modify calls sequence, see ehca_recover_sqs() below.
+ * Why that is required:
+ * 1) If one port is connected, older code requires that port one
+ * to be connected and module option nr_ports=1 to be given by
+ * user, which is very inconvenient for end user.
+ * 2) Firmware accepts modify_qp() only if respective port has become
+ * active. Older code had a wait loop of 30sec create_qp()/
+ * define_aqp1(), which is not appropriate in practice. This
+ * code now removes that wait loop, see define_aqp1(), and always
+ * reports all ports to ib_mad resp. users. Only activated ports
+ * will then usable for the users.
+ */
+ if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) {
+ int port = my_qp->init_attr.port_num;
+ struct ehca_sport *sport = &shca->sport[port - 1];
+ unsigned long flags;
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+ /* cache qp_attr only during init */
+ if (my_qp->mod_qp_parm) {
+ struct ehca_mod_qp_parm *p;
+ if (my_qp->mod_qp_parm_idx >= EHCA_MOD_QP_PARM_MAX) {
+ ehca_err(&shca->ib_device,
+ "mod_qp_parm overflow state=%x port=%x"
+ " type=%x", attr->qp_state,
+ my_qp->init_attr.port_num,
+ ibqp->qp_type);
+ spin_unlock_irqrestore(&sport->mod_sqp_lock,
+ flags);
+ return -EINVAL;
+ }
+ p = &my_qp->mod_qp_parm[my_qp->mod_qp_parm_idx];
+ p->mask = attr_mask;
+ p->attr = *attr;
+ my_qp->mod_qp_parm_idx++;
+ ehca_dbg(&shca->ib_device,
+ "Saved qp_attr for state=%x port=%x type=%x",
+ attr->qp_state, my_qp->init_attr.port_num,
+ ibqp->qp_type);
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ goto out;
+ }
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ }
+
+ ret = internal_modify_qp(ibqp, attr, attr_mask, 0);
+
+out:
+ if ((ret == 0) && (attr_mask & IB_QP_STATE))
+ my_qp->state = attr->qp_state;
+
+ return ret;
+}
+
+void ehca_recover_sqp(struct ib_qp *sqp)
+{
+ struct ehca_qp *my_sqp = container_of(sqp, struct ehca_qp, ib_qp);
+ int port = my_sqp->init_attr.port_num;
+ struct ib_qp_attr attr;
+ struct ehca_mod_qp_parm *qp_parm;
+ int i, qp_parm_idx, ret;
+ unsigned long flags, wr_cnt;
+
+ if (!my_sqp->mod_qp_parm)
+ return;
+ ehca_dbg(sqp->device, "SQP port=%x qp_num=%x", port, sqp->qp_num);
+
+ qp_parm = my_sqp->mod_qp_parm;
+ qp_parm_idx = my_sqp->mod_qp_parm_idx;
+ for (i = 0; i < qp_parm_idx; i++) {
+ attr = qp_parm[i].attr;
+ ret = internal_modify_qp(sqp, &attr, qp_parm[i].mask, 0);
+ if (ret) {
+ ehca_err(sqp->device, "Could not modify SQP port=%x "
+ "qp_num=%x ret=%x", port, sqp->qp_num, ret);
+ goto free_qp_parm;
+ }
+ ehca_dbg(sqp->device, "SQP port=%x qp_num=%x in state=%x",
+ port, sqp->qp_num, attr.qp_state);
+ }
+
+ /* re-trigger posted recv wrs */
+ wr_cnt = my_sqp->ipz_rqueue.current_q_offset /
+ my_sqp->ipz_rqueue.qe_size;
+ if (wr_cnt) {
+ spin_lock_irqsave(&my_sqp->spinlock_r, flags);
+ hipz_update_rqa(my_sqp, wr_cnt);
+ spin_unlock_irqrestore(&my_sqp->spinlock_r, flags);
+ ehca_dbg(sqp->device, "doorbell port=%x qp_num=%x wr_cnt=%lx",
+ port, sqp->qp_num, wr_cnt);
+ }
+
+free_qp_parm:
+ kfree(qp_parm);
+ /* this prevents subsequent calls to modify_qp() to cache qp_attr */
+ my_sqp->mod_qp_parm = NULL;
+}
+
+int ehca_query_qp(struct ib_qp *qp,
+ struct ib_qp_attr *qp_attr,
+ int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+ struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ struct hcp_modify_qp_control_block *qpcb;
+ int cnt, ret = 0;
+ u64 h_ret;
+
+ if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
+ ehca_err(qp->device, "Invalid attribute mask "
+ "ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
+ my_qp, qp->qp_num, qp_attr_mask);
+ return -EINVAL;
+ }
+
+ qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!qpcb) {
+ ehca_err(qp->device, "Out of memory for qpcb "
+ "ehca_qp=%p qp_num=%x", my_qp, qp->qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(adapter_handle,
+ my_qp->ipz_qp_handle,
+ &my_qp->pf,
+ qpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(qp->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, qp->qp_num, h_ret);
+ goto query_qp_exit1;
+ }
+
+ qp_attr->cur_qp_state = ehca2ib_qp_state(qpcb->qp_state);
+ qp_attr->qp_state = qp_attr->cur_qp_state;
+
+ if (qp_attr->cur_qp_state == -EINVAL) {
+ ret = -EINVAL;
+ ehca_err(qp->device, "Got invalid ehca_qp_state=%x "
+ "ehca_qp=%p qp_num=%x",
+ qpcb->qp_state, my_qp, qp->qp_num);
+ goto query_qp_exit1;
+ }
+
+ if (qp_attr->qp_state == IB_QPS_SQD)
+ qp_attr->sq_draining = 1;
+
+ qp_attr->qkey = qpcb->qkey;
+ qp_attr->path_mtu = qpcb->path_mtu;
+ qp_attr->path_mig_state = qpcb->path_migration_state - 1;
+ qp_attr->rq_psn = qpcb->receive_psn;
+ qp_attr->sq_psn = qpcb->send_psn;
+ qp_attr->min_rnr_timer = qpcb->min_rnr_nak_timer_field;
+ qp_attr->cap.max_send_wr = qpcb->max_nr_outst_send_wr-1;
+ qp_attr->cap.max_recv_wr = qpcb->max_nr_outst_recv_wr-1;
+ /* UD_AV CIRCUMVENTION */
+ if (my_qp->qp_type == IB_QPT_UD) {
+ qp_attr->cap.max_send_sge =
+ qpcb->actual_nr_sges_in_sq_wqe - 2;
+ qp_attr->cap.max_recv_sge =
+ qpcb->actual_nr_sges_in_rq_wqe - 2;
+ } else {
+ qp_attr->cap.max_send_sge =
+ qpcb->actual_nr_sges_in_sq_wqe;
+ qp_attr->cap.max_recv_sge =
+ qpcb->actual_nr_sges_in_rq_wqe;
+ }
+
+ qp_attr->cap.max_inline_data = my_qp->sq_max_inline_data_size;
+ qp_attr->dest_qp_num = qpcb->dest_qp_nr;
+
+ qp_attr->pkey_index = qpcb->prim_p_key_idx;
+ qp_attr->port_num = qpcb->prim_phys_port;
+ qp_attr->timeout = qpcb->timeout;
+ qp_attr->retry_cnt = qpcb->retry_count;
+ qp_attr->rnr_retry = qpcb->rnr_retry_count;
+
+ qp_attr->alt_pkey_index = qpcb->alt_p_key_idx;
+ qp_attr->alt_port_num = qpcb->alt_phys_port;
+ qp_attr->alt_timeout = qpcb->timeout_al;
+
+ qp_attr->max_dest_rd_atomic = qpcb->rdma_nr_atomic_resp_res;
+ qp_attr->max_rd_atomic = qpcb->rdma_atomic_outst_dest_qp;
+
+ /* primary av */
+ qp_attr->ah_attr.sl = qpcb->service_level;
+
+ if (qpcb->send_grh_flag) {
+ qp_attr->ah_attr.ah_flags = IB_AH_GRH;
+ }
+
+ qp_attr->ah_attr.static_rate = qpcb->max_static_rate;
+ qp_attr->ah_attr.dlid = qpcb->dlid;
+ qp_attr->ah_attr.src_path_bits = qpcb->source_path_bits;
+ qp_attr->ah_attr.port_num = qp_attr->port_num;
+
+ /* primary GRH */
+ qp_attr->ah_attr.grh.traffic_class = qpcb->traffic_class;
+ qp_attr->ah_attr.grh.hop_limit = qpcb->hop_limit;
+ qp_attr->ah_attr.grh.sgid_index = qpcb->source_gid_idx;
+ qp_attr->ah_attr.grh.flow_label = qpcb->flow_label;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ qp_attr->ah_attr.grh.dgid.raw[cnt] =
+ qpcb->dest_gid.byte[cnt];
+
+ /* alternate AV */
+ qp_attr->alt_ah_attr.sl = qpcb->service_level_al;
+ if (qpcb->send_grh_flag_al) {
+ qp_attr->alt_ah_attr.ah_flags = IB_AH_GRH;
+ }
+
+ qp_attr->alt_ah_attr.static_rate = qpcb->max_static_rate_al;
+ qp_attr->alt_ah_attr.dlid = qpcb->dlid_al;
+ qp_attr->alt_ah_attr.src_path_bits = qpcb->source_path_bits_al;
+
+ /* alternate GRH */
+ qp_attr->alt_ah_attr.grh.traffic_class = qpcb->traffic_class_al;
+ qp_attr->alt_ah_attr.grh.hop_limit = qpcb->hop_limit_al;
+ qp_attr->alt_ah_attr.grh.sgid_index = qpcb->source_gid_idx_al;
+ qp_attr->alt_ah_attr.grh.flow_label = qpcb->flow_label_al;
+
+ for (cnt = 0; cnt < 16; cnt++)
+ qp_attr->alt_ah_attr.grh.dgid.raw[cnt] =
+ qpcb->dest_gid_al.byte[cnt];
+
+ /* return init attributes given in ehca_create_qp */
+ if (qp_init_attr)
+ *qp_init_attr = my_qp->init_attr;
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
+
+query_qp_exit1:
+ ehca_free_fw_ctrlblock(qpcb);
+
+ return ret;
+}
+
+int ehca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
+{
+ struct ehca_qp *my_qp =
+ container_of(ibsrq, struct ehca_qp, ib_srq);
+ struct ehca_shca *shca =
+ container_of(ibsrq->pd->device, struct ehca_shca, ib_device);
+ struct hcp_modify_qp_control_block *mqpcb;
+ u64 update_mask;
+ u64 h_ret;
+ int ret = 0;
+
+ mqpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!mqpcb) {
+ ehca_err(ibsrq->device, "Could not get zeroed page for mqpcb "
+ "ehca_qp=%p qp_num=%x ", my_qp, my_qp->real_qp_num);
+ return -ENOMEM;
+ }
+
+ update_mask = 0;
+ if (attr_mask & IB_SRQ_LIMIT) {
+ attr_mask &= ~IB_SRQ_LIMIT;
+ update_mask |=
+ EHCA_BMASK_SET(MQPCB_MASK_CURR_SRQ_LIMIT, 1)
+ | EHCA_BMASK_SET(MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG, 1);
+ mqpcb->curr_srq_limit = attr->srq_limit;
+ mqpcb->qp_aff_asyn_ev_log_reg =
+ EHCA_BMASK_SET(QPX_AAELOG_RESET_SRQ_LIMIT, 1);
+ }
+
+ /* by now, all bits in attr_mask should have been cleared */
+ if (attr_mask) {
+ ehca_err(ibsrq->device, "invalid attribute mask bits set "
+ "attr_mask=%x", attr_mask);
+ ret = -EINVAL;
+ goto modify_srq_exit0;
+ }
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(mqpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
+
+ h_ret = hipz_h_modify_qp(shca->ipz_hca_handle, my_qp->ipz_qp_handle,
+ NULL, update_mask, mqpcb,
+ my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(ibsrq->device, "hipz_h_modify_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x",
+ h_ret, my_qp, my_qp->real_qp_num);
+ }
+
+modify_srq_exit0:
+ ehca_free_fw_ctrlblock(mqpcb);
+
+ return ret;
+}
+
+int ehca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr)
+{
+ struct ehca_qp *my_qp = container_of(srq, struct ehca_qp, ib_srq);
+ struct ehca_shca *shca = container_of(srq->device, struct ehca_shca,
+ ib_device);
+ struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+ struct hcp_modify_qp_control_block *qpcb;
+ int ret = 0;
+ u64 h_ret;
+
+ qpcb = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
+ if (!qpcb) {
+ ehca_err(srq->device, "Out of memory for qpcb "
+ "ehca_qp=%p qp_num=%x", my_qp, my_qp->real_qp_num);
+ return -ENOMEM;
+ }
+
+ h_ret = hipz_h_query_qp(adapter_handle, my_qp->ipz_qp_handle,
+ NULL, qpcb, my_qp->galpas.kernel);
+
+ if (h_ret != H_SUCCESS) {
+ ret = ehca2ib_return_code(h_ret);
+ ehca_err(srq->device, "hipz_h_query_qp() failed "
+ "ehca_qp=%p qp_num=%x h_ret=%lli",
+ my_qp, my_qp->real_qp_num, h_ret);
+ goto query_srq_exit1;
+ }
+
+ srq_attr->max_wr = qpcb->max_nr_outst_recv_wr - 1;
+ srq_attr->max_sge = 3;
+ srq_attr->srq_limit = qpcb->curr_srq_limit;
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(qpcb, 4*70, "qp_num=%x", my_qp->real_qp_num);
+
+query_srq_exit1:
+ ehca_free_fw_ctrlblock(qpcb);
+
+ return ret;
+}
+
+static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
+ struct ib_uobject *uobject)
+{
+ struct ehca_shca *shca = container_of(dev, struct ehca_shca, ib_device);
+ struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
+ ib_pd);
+ struct ehca_sport *sport = &shca->sport[my_qp->init_attr.port_num - 1];
+ u32 qp_num = my_qp->real_qp_num;
+ int ret;
+ u64 h_ret;
+ u8 port_num;
+ int is_user = 0;
+ enum ib_qp_type qp_type;
+ unsigned long flags;
+
+ if (uobject) {
+ is_user = 1;
+ if (my_qp->mm_count_galpa ||
+ my_qp->mm_count_rqueue || my_qp->mm_count_squeue) {
+ ehca_err(dev, "Resources still referenced in "
+ "user space qp_num=%x", qp_num);
+ return -EINVAL;
+ }
+ }
+
+ if (my_qp->send_cq) {
+ ret = ehca_cq_unassign_qp(my_qp->send_cq, qp_num);
+ if (ret) {
+ ehca_err(dev, "Couldn't unassign qp from "
+ "send_cq ret=%i qp_num=%x cq_num=%x", ret,
+ qp_num, my_qp->send_cq->cq_number);
+ return ret;
+ }
+ }
+
+ write_lock_irqsave(&ehca_qp_idr_lock, flags);
+ idr_remove(&ehca_qp_idr, my_qp->token);
+ write_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+ /*
+ * SRQs will never get into an error list and do not have a recv_cq,
+ * so we need to skip them here.
+ */
+ if (HAS_RQ(my_qp) && !IS_SRQ(my_qp) && !is_user)
+ del_from_err_list(my_qp->recv_cq, &my_qp->rq_err_node);
+
+ if (HAS_SQ(my_qp) && !is_user)
+ del_from_err_list(my_qp->send_cq, &my_qp->sq_err_node);
+
+ /* now wait until all pending events have completed */
+ wait_event(my_qp->wait_completion, !atomic_read(&my_qp->nr_events));
+
+ h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+ if (h_ret != H_SUCCESS) {
+ ehca_err(dev, "hipz_h_destroy_qp() failed h_ret=%lli "
+ "ehca_qp=%p qp_num=%x", h_ret, my_qp, qp_num);
+ return ehca2ib_return_code(h_ret);
+ }
+
+ port_num = my_qp->init_attr.port_num;
+ qp_type = my_qp->init_attr.qp_type;
+
+ if (qp_type == IB_QPT_SMI || qp_type == IB_QPT_GSI) {
+ spin_lock_irqsave(&sport->mod_sqp_lock, flags);
+ kfree(my_qp->mod_qp_parm);
+ my_qp->mod_qp_parm = NULL;
+ shca->sport[port_num - 1].ibqp_sqp[qp_type] = NULL;
+ spin_unlock_irqrestore(&sport->mod_sqp_lock, flags);
+ }
+
+ /* no support for IB_QPT_SMI yet */
+ if (qp_type == IB_QPT_GSI) {
+ struct ib_event event;
+ ehca_info(dev, "device %s: port %x is inactive.",
+ shca->ib_device.name, port_num);
+ event.device = &shca->ib_device;
+ event.event = IB_EVENT_PORT_ERR;
+ event.element.port_num = port_num;
+ shca->sport[port_num - 1].port_state = IB_PORT_DOWN;
+ ib_dispatch_event(&event);
+ }
+
+ if (HAS_RQ(my_qp)) {
+ ipz_queue_dtor(my_pd, &my_qp->ipz_rqueue);
+ if (!is_user)
+ vfree(my_qp->rq_map.map);
+ }
+ if (HAS_SQ(my_qp)) {
+ ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
+ if (!is_user)
+ vfree(my_qp->sq_map.map);
+ }
+ kmem_cache_free(qp_cache, my_qp);
+ atomic_dec(&shca->num_qps);
+ return 0;
+}
+
+int ehca_destroy_qp(struct ib_qp *qp)
+{
+ return internal_destroy_qp(qp->device,
+ container_of(qp, struct ehca_qp, ib_qp),
+ qp->uobject);
+}
+
+int ehca_destroy_srq(struct ib_srq *srq)
+{
+ return internal_destroy_qp(srq->device,
+ container_of(srq, struct ehca_qp, ib_srq),
+ srq->uobject);
+}
+
+int ehca_init_qp_cache(void)
+{
+ qp_cache = kmem_cache_create("ehca_cache_qp",
+ sizeof(struct ehca_qp), 0,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
+ if (!qp_cache)
+ return -ENOMEM;
+ return 0;
+}
+
+void ehca_cleanup_qp_cache(void)
+{
+ if (qp_cache)
+ kmem_cache_destroy(qp_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * post_send/recv, poll_cq, req_notify
+ *
+ * Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+/* in RC traffic, insert an empty RDMA READ every this many packets */
+#define ACK_CIRC_THRESHOLD 2000000
+
+static u64 replace_wr_id(u64 wr_id, u16 idx)
+{
+ u64 ret;
+
+ ret = wr_id & ~QMAP_IDX_MASK;
+ ret |= idx & QMAP_IDX_MASK;
+
+ return ret;
+}
+
+static u16 get_app_wr_id(u64 wr_id)
+{
+ return wr_id & QMAP_IDX_MASK;
+}
+
+static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
+ struct ehca_wqe *wqe_p,
+ struct ib_recv_wr *recv_wr,
+ u32 rq_map_idx)
+{
+ u8 cnt_ds;
+ if (unlikely((recv_wr->num_sge < 0) ||
+ (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
+ ehca_gen_err("Invalid number of WQE SGE. "
+ "num_sqe=%x max_nr_of_sg=%x",
+ recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
+ return -EINVAL; /* invalid SG list length */
+ }
+
+ /* clear wqe header until sglist */
+ memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+ wqe_p->work_request_id = replace_wr_id(recv_wr->wr_id, rq_map_idx);
+ wqe_p->nr_of_data_seg = recv_wr->num_sge;
+
+ for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
+ wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
+ recv_wr->sg_list[cnt_ds].addr;
+ wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
+ recv_wr->sg_list[cnt_ds].lkey;
+ wqe_p->u.all_rcv.sg_list[cnt_ds].length =
+ recv_wr->sg_list[cnt_ds].length;
+ }
+
+ if (ehca_debug_level >= 3) {
+ ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p",
+ ipz_rqueue);
+ ehca_dmp(wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
+ }
+
+ return 0;
+}
+
+#if defined(DEBUG_GSI_SEND_WR)
+
+/* need ib_mad struct */
+#include <rdma/ib_mad.h>
+
+static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
+{
+ int idx;
+ int j;
+ while (send_wr) {
+ struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
+ struct ib_sge *sge = send_wr->sg_list;
+ ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
+ "send_flags=%x opcode=%x", idx, send_wr->wr_id,
+ send_wr->num_sge, send_wr->send_flags,
+ send_wr->opcode);
+ if (mad_hdr) {
+ ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
+ "mgmt_class=%x class_version=%x method=%x "
+ "status=%x class_specific=%x tid=%lx "
+ "attr_id=%x resv=%x attr_mod=%x",
+ idx, mad_hdr->base_version,
+ mad_hdr->mgmt_class,
+ mad_hdr->class_version, mad_hdr->method,
+ mad_hdr->status, mad_hdr->class_specific,
+ mad_hdr->tid, mad_hdr->attr_id,
+ mad_hdr->resv,
+ mad_hdr->attr_mod);
+ }
+ for (j = 0; j < send_wr->num_sge; j++) {
+ u8 *data = __va(sge->addr);
+ ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
+ "lkey=%x",
+ idx, j, data, sge->length, sge->lkey);
+ /* assume length is n*16 */
+ ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
+ idx, j);
+ sge++;
+ } /* eof for j */
+ idx++;
+ send_wr = send_wr->next;
+ } /* eof while send_wr */
+}
+
+#endif /* DEBUG_GSI_SEND_WR */
+
+static inline int ehca_write_swqe(struct ehca_qp *qp,
+ struct ehca_wqe *wqe_p,
+ const struct ib_send_wr *send_wr,
+ u32 sq_map_idx,
+ int hidden)
+{
+ u32 idx;
+ u64 dma_length;
+ struct ehca_av *my_av;
+ u32 remote_qkey = send_wr->wr.ud.remote_qkey;
+ struct ehca_qmap_entry *qmap_entry = &qp->sq_map.map[sq_map_idx];
+
+ if (unlikely((send_wr->num_sge < 0) ||
+ (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
+ ehca_gen_err("Invalid number of WQE SGE. "
+ "num_sqe=%x max_nr_of_sg=%x",
+ send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
+ return -EINVAL; /* invalid SG list length */
+ }
+
+ /* clear wqe header until sglist */
+ memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+ wqe_p->work_request_id = replace_wr_id(send_wr->wr_id, sq_map_idx);
+
+ qmap_entry->app_wr_id = get_app_wr_id(send_wr->wr_id);
+ qmap_entry->reported = 0;
+ qmap_entry->cqe_req = 0;
+
+ switch (send_wr->opcode) {
+ case IB_WR_SEND:
+ case IB_WR_SEND_WITH_IMM:
+ wqe_p->optype = WQE_OPTYPE_SEND;
+ break;
+ case IB_WR_RDMA_WRITE:
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
+ break;
+ case IB_WR_RDMA_READ:
+ wqe_p->optype = WQE_OPTYPE_RDMAREAD;
+ break;
+ default:
+ ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
+ return -EINVAL; /* invalid opcode */
+ }
+
+ wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
+
+ wqe_p->wr_flag = 0;
+
+ if ((send_wr->send_flags & IB_SEND_SIGNALED ||
+ qp->init_attr.sq_sig_type == IB_SIGNAL_ALL_WR)
+ && !hidden) {
+ wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
+ qmap_entry->cqe_req = 1;
+ }
+
+ if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
+ send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
+ /* this might not work as long as HW does not support it */
+ wqe_p->immediate_data = be32_to_cpu(send_wr->ex.imm_data);
+ wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
+ }
+
+ wqe_p->nr_of_data_seg = send_wr->num_sge;
+
+ switch (qp->qp_type) {
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ /* no break is intential here */
+ case IB_QPT_UD:
+ /* IB 1.2 spec C10-15 compliance */
+ if (send_wr->wr.ud.remote_qkey & 0x80000000)
+ remote_qkey = qp->qkey;
+
+ wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
+ wqe_p->local_ee_context_qkey = remote_qkey;
+ if (unlikely(!send_wr->wr.ud.ah)) {
+ ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
+ return -EINVAL;
+ }
+ if (unlikely(send_wr->wr.ud.remote_qpn == 0)) {
+ ehca_gen_err("dest QP# is 0. qp=%x", qp->real_qp_num);
+ return -EINVAL;
+ }
+ my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
+ wqe_p->u.ud_av.ud_av = my_av->av;
+
+ /*
+ * omitted check of IB_SEND_INLINE
+ * since HW does not support it
+ */
+ for (idx = 0; idx < send_wr->num_sge; idx++) {
+ wqe_p->u.ud_av.sg_list[idx].vaddr =
+ send_wr->sg_list[idx].addr;
+ wqe_p->u.ud_av.sg_list[idx].lkey =
+ send_wr->sg_list[idx].lkey;
+ wqe_p->u.ud_av.sg_list[idx].length =
+ send_wr->sg_list[idx].length;
+ } /* eof for idx */
+ if (qp->qp_type == IB_QPT_SMI ||
+ qp->qp_type == IB_QPT_GSI)
+ wqe_p->u.ud_av.ud_av.pmtu = 1;
+ if (qp->qp_type == IB_QPT_GSI) {
+ wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
+#ifdef DEBUG_GSI_SEND_WR
+ trace_send_wr_ud(send_wr);
+#endif /* DEBUG_GSI_SEND_WR */
+ }
+ break;
+
+ case IB_QPT_UC:
+ if (send_wr->send_flags & IB_SEND_FENCE)
+ wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
+ /* no break is intentional here */
+ case IB_QPT_RC:
+ /* TODO: atomic not implemented */
+ wqe_p->u.nud.remote_virtual_address =
+ send_wr->wr.rdma.remote_addr;
+ wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
+
+ /*
+ * omitted checking of IB_SEND_INLINE
+ * since HW does not support it
+ */
+ dma_length = 0;
+ for (idx = 0; idx < send_wr->num_sge; idx++) {
+ wqe_p->u.nud.sg_list[idx].vaddr =
+ send_wr->sg_list[idx].addr;
+ wqe_p->u.nud.sg_list[idx].lkey =
+ send_wr->sg_list[idx].lkey;
+ wqe_p->u.nud.sg_list[idx].length =
+ send_wr->sg_list[idx].length;
+ dma_length += send_wr->sg_list[idx].length;
+ } /* eof idx */
+ wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
+
+ /* unsolicited ack circumvention */
+ if (send_wr->opcode == IB_WR_RDMA_READ) {
+ /* on RDMA read, switch on and reset counters */
+ qp->message_count = qp->packet_count = 0;
+ qp->unsol_ack_circ = 1;
+ } else
+ /* else estimate #packets */
+ qp->packet_count += (dma_length >> qp->mtu_shift) + 1;
+
+ break;
+
+ default:
+ ehca_gen_err("Invalid qptype=%x", qp->qp_type);
+ return -EINVAL;
+ }
+
+ if (ehca_debug_level >= 3) {
+ ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
+ ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
+ }
+ return 0;
+}
+
+/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
+static inline void map_ib_wc_status(u32 cqe_status,
+ enum ib_wc_status *wc_status)
+{
+ if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
+ switch (cqe_status & 0x3F) {
+ case 0x01:
+ case 0x21:
+ *wc_status = IB_WC_LOC_LEN_ERR;
+ break;
+ case 0x02:
+ case 0x22:
+ *wc_status = IB_WC_LOC_QP_OP_ERR;
+ break;
+ case 0x03:
+ case 0x23:
+ *wc_status = IB_WC_LOC_EEC_OP_ERR;
+ break;
+ case 0x04:
+ case 0x24:
+ *wc_status = IB_WC_LOC_PROT_ERR;
+ break;
+ case 0x05:
+ case 0x25:
+ *wc_status = IB_WC_WR_FLUSH_ERR;
+ break;
+ case 0x06:
+ *wc_status = IB_WC_MW_BIND_ERR;
+ break;
+ case 0x07: /* remote error - look into bits 20:24 */
+ switch ((cqe_status
+ & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
+ case 0x0:
+ /*
+ * PSN Sequence Error!
+ * couldn't find a matching status!
+ */
+ *wc_status = IB_WC_GENERAL_ERR;
+ break;
+ case 0x1:
+ *wc_status = IB_WC_REM_INV_REQ_ERR;
+ break;
+ case 0x2:
+ *wc_status = IB_WC_REM_ACCESS_ERR;
+ break;
+ case 0x3:
+ *wc_status = IB_WC_REM_OP_ERR;
+ break;
+ case 0x4:
+ *wc_status = IB_WC_REM_INV_RD_REQ_ERR;
+ break;
+ }
+ break;
+ case 0x08:
+ *wc_status = IB_WC_RETRY_EXC_ERR;
+ break;
+ case 0x09:
+ *wc_status = IB_WC_RNR_RETRY_EXC_ERR;
+ break;
+ case 0x0A:
+ case 0x2D:
+ *wc_status = IB_WC_REM_ABORT_ERR;
+ break;
+ case 0x0B:
+ case 0x2E:
+ *wc_status = IB_WC_INV_EECN_ERR;
+ break;
+ case 0x0C:
+ case 0x2F:
+ *wc_status = IB_WC_INV_EEC_STATE_ERR;
+ break;
+ case 0x0D:
+ *wc_status = IB_WC_BAD_RESP_ERR;
+ break;
+ case 0x10:
+ /* WQE purged */
+ *wc_status = IB_WC_WR_FLUSH_ERR;
+ break;
+ default:
+ *wc_status = IB_WC_FATAL_ERR;
+
+ }
+ } else
+ *wc_status = IB_WC_SUCCESS;
+}
+
+static inline int post_one_send(struct ehca_qp *my_qp,
+ struct ib_send_wr *cur_send_wr,
+ int hidden)
+{
+ struct ehca_wqe *wqe_p;
+ int ret;
+ u32 sq_map_idx;
+ u64 start_offset = my_qp->ipz_squeue.current_q_offset;
+
+ /* get pointer next to free WQE */
+ wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
+ if (unlikely(!wqe_p)) {
+ /* too many posted work requests: queue overflow */
+ ehca_err(my_qp->ib_qp.device, "Too many posted WQEs "
+ "qp_num=%x", my_qp->ib_qp.qp_num);
+ return -ENOMEM;
+ }
+
+ /*
+ * Get the index of the WQE in the send queue. The same index is used
+ * for writing into the sq_map.
+ */
+ sq_map_idx = start_offset / my_qp->ipz_squeue.qe_size;
+
+ /* write a SEND WQE into the QUEUE */
+ ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr, sq_map_idx, hidden);
+ /*
+ * if something failed,
+ * reset the free entry pointer to the start value
+ */
+ if (unlikely(ret)) {
+ my_qp->ipz_squeue.current_q_offset = start_offset;
+ ehca_err(my_qp->ib_qp.device, "Could not write WQE "
+ "qp_num=%x", my_qp->ib_qp.qp_num);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int ehca_post_send(struct ib_qp *qp,
+ struct ib_send_wr *send_wr,
+ struct ib_send_wr **bad_send_wr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+ int wqe_cnt = 0;
+ int ret = 0;
+ unsigned long flags;
+
+ /* Reject WR if QP is in RESET, INIT or RTR state */
+ if (unlikely(my_qp->state < IB_QPS_RTS)) {
+ ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
+ my_qp->state, qp->qp_num);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* LOCK the QUEUE */
+ spin_lock_irqsave(&my_qp->spinlock_s, flags);
+
+ /* Send an empty extra RDMA read if:
+ * 1) there has been an RDMA read on this connection before
+ * 2) no RDMA read occurred for ACK_CIRC_THRESHOLD link packets
+ * 3) we can be sure that any previous extra RDMA read has been
+ * processed so we don't overflow the SQ
+ */
+ if (unlikely(my_qp->unsol_ack_circ &&
+ my_qp->packet_count > ACK_CIRC_THRESHOLD &&
+ my_qp->message_count > my_qp->init_attr.cap.max_send_wr)) {
+ /* insert an empty RDMA READ to fix up the remote QP state */
+ struct ib_send_wr circ_wr;
+ memset(&circ_wr, 0, sizeof(circ_wr));
+ circ_wr.opcode = IB_WR_RDMA_READ;
+ post_one_send(my_qp, &circ_wr, 1); /* ignore retcode */
+ wqe_cnt++;
+ ehca_dbg(qp->device, "posted circ wr qp_num=%x", qp->qp_num);
+ my_qp->message_count = my_qp->packet_count = 0;
+ }
+
+ /* loop processes list of send reqs */
+ while (send_wr) {
+ ret = post_one_send(my_qp, send_wr, 0);
+ if (unlikely(ret)) {
+ goto post_send_exit0;
+ }
+ wqe_cnt++;
+ send_wr = send_wr->next;
+ }
+
+post_send_exit0:
+ iosync(); /* serialize GAL register access */
+ hipz_update_sqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, qp->qp_num, wqe_cnt, ret);
+ my_qp->message_count += wqe_cnt;
+ spin_unlock_irqrestore(&my_qp->spinlock_s, flags);
+
+out:
+ if (ret)
+ *bad_send_wr = send_wr;
+ return ret;
+}
+
+static int internal_post_recv(struct ehca_qp *my_qp,
+ struct ib_device *dev,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ struct ehca_wqe *wqe_p;
+ int wqe_cnt = 0;
+ int ret = 0;
+ u32 rq_map_idx;
+ unsigned long flags;
+ struct ehca_qmap_entry *qmap_entry;
+
+ if (unlikely(!HAS_RQ(my_qp))) {
+ ehca_err(dev, "QP has no RQ ehca_qp=%p qp_num=%x ext_type=%d",
+ my_qp, my_qp->real_qp_num, my_qp->ext_type);
+ ret = -ENODEV;
+ goto out;
+ }
+
+ /* LOCK the QUEUE */
+ spin_lock_irqsave(&my_qp->spinlock_r, flags);
+
+ /* loop processes list of recv reqs */
+ while (recv_wr) {
+ u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
+ /* get pointer next to free WQE */
+ wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
+ if (unlikely(!wqe_p)) {
+ /* too many posted work requests: queue overflow */
+ ret = -ENOMEM;
+ ehca_err(dev, "Too many posted WQEs "
+ "qp_num=%x", my_qp->real_qp_num);
+ goto post_recv_exit0;
+ }
+ /*
+ * Get the index of the WQE in the recv queue. The same index
+ * is used for writing into the rq_map.
+ */
+ rq_map_idx = start_offset / my_qp->ipz_rqueue.qe_size;
+
+ /* write a RECV WQE into the QUEUE */
+ ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, recv_wr,
+ rq_map_idx);
+ /*
+ * if something failed,
+ * reset the free entry pointer to the start value
+ */
+ if (unlikely(ret)) {
+ my_qp->ipz_rqueue.current_q_offset = start_offset;
+ ret = -EINVAL;
+ ehca_err(dev, "Could not write WQE "
+ "qp_num=%x", my_qp->real_qp_num);
+ goto post_recv_exit0;
+ }
+
+ qmap_entry = &my_qp->rq_map.map[rq_map_idx];
+ qmap_entry->app_wr_id = get_app_wr_id(recv_wr->wr_id);
+ qmap_entry->reported = 0;
+ qmap_entry->cqe_req = 1;
+
+ wqe_cnt++;
+ recv_wr = recv_wr->next;
+ } /* eof for recv_wr */
+
+post_recv_exit0:
+ iosync(); /* serialize GAL register access */
+ hipz_update_rqa(my_qp, wqe_cnt);
+ if (unlikely(ret || ehca_debug_level >= 2))
+ ehca_dbg(dev, "ehca_qp=%p qp_num=%x wqe_cnt=%d ret=%i",
+ my_qp, my_qp->real_qp_num, wqe_cnt, ret);
+ spin_unlock_irqrestore(&my_qp->spinlock_r, flags);
+
+out:
+ if (ret)
+ *bad_recv_wr = recv_wr;
+
+ return ret;
+}
+
+int ehca_post_recv(struct ib_qp *qp,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+
+ /* Reject WR if QP is in RESET state */
+ if (unlikely(my_qp->state == IB_QPS_RESET)) {
+ ehca_err(qp->device, "Invalid QP state qp_state=%d qpn=%x",
+ my_qp->state, qp->qp_num);
+ *bad_recv_wr = recv_wr;
+ return -EINVAL;
+ }
+
+ return internal_post_recv(my_qp, qp->device, recv_wr, bad_recv_wr);
+}
+
+int ehca_post_srq_recv(struct ib_srq *srq,
+ struct ib_recv_wr *recv_wr,
+ struct ib_recv_wr **bad_recv_wr)
+{
+ return internal_post_recv(container_of(srq, struct ehca_qp, ib_srq),
+ srq->device, recv_wr, bad_recv_wr);
+}
+
+/*
+ * ib_wc_opcode table converts ehca wc opcode to ib
+ * Since we use zero to indicate invalid opcode, the actual ib opcode must
+ * be decremented!!!
+ */
+static const u8 ib_wc_opcode[255] = {
+ [0x01] = IB_WC_RECV+1,
+ [0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
+ [0x04] = IB_WC_BIND_MW+1,
+ [0x08] = IB_WC_FETCH_ADD+1,
+ [0x10] = IB_WC_COMP_SWAP+1,
+ [0x20] = IB_WC_RDMA_WRITE+1,
+ [0x40] = IB_WC_RDMA_READ+1,
+ [0x80] = IB_WC_SEND+1
+};
+
+/* internal function to poll one entry of cq */
+static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
+{
+ int ret = 0, qmap_tail_idx;
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ struct ehca_cqe *cqe;
+ struct ehca_qp *my_qp;
+ struct ehca_qmap_entry *qmap_entry;
+ struct ehca_queue_map *qmap;
+ int cqe_count = 0, is_error;
+
+repoll:
+ cqe = (struct ehca_cqe *)
+ ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
+ if (!cqe) {
+ ret = -EAGAIN;
+ if (ehca_debug_level >= 3)
+ ehca_dbg(cq->device, "Completion queue is empty "
+ "my_cq=%p cq_num=%x", my_cq, my_cq->cq_number);
+ goto poll_cq_one_exit0;
+ }
+
+ /* prevents loads being reordered across this point */
+ rmb();
+
+ cqe_count++;
+ if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
+ struct ehca_qp *qp;
+ int purgeflag;
+ unsigned long flags;
+
+ qp = ehca_cq_get_qp(my_cq, cqe->local_qp_number);
+ if (!qp) {
+ ehca_err(cq->device, "cq_num=%x qp_num=%x "
+ "could not find qp -> ignore cqe",
+ my_cq->cq_number, cqe->local_qp_number);
+ ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
+ my_cq->cq_number, cqe->local_qp_number);
+ /* ignore this purged cqe */
+ goto repoll;
+ }
+ spin_lock_irqsave(&qp->spinlock_s, flags);
+ purgeflag = qp->sqerr_purgeflag;
+ spin_unlock_irqrestore(&qp->spinlock_s, flags);
+
+ if (purgeflag) {
+ ehca_dbg(cq->device,
+ "Got CQE with purged bit qp_num=%x src_qp=%x",
+ cqe->local_qp_number, cqe->remote_qp_number);
+ if (ehca_debug_level >= 2)
+ ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
+ cqe->local_qp_number,
+ cqe->remote_qp_number);
+ /*
+ * ignore this to avoid double cqes of bad wqe
+ * that caused sqe and turn off purge flag
+ */
+ qp->sqerr_purgeflag = 0;
+ goto repoll;
+ }
+ }
+
+ is_error = cqe->status & WC_STATUS_ERROR_BIT;
+
+ /* trace error CQEs if debug_level >= 1, trace all CQEs if >= 3 */
+ if (unlikely(ehca_debug_level >= 3 || (ehca_debug_level && is_error))) {
+ ehca_dbg(cq->device,
+ "Received %sCOMPLETION ehca_cq=%p cq_num=%x -----",
+ is_error ? "ERROR " : "", my_cq, my_cq->cq_number);
+ ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+ my_cq, my_cq->cq_number);
+ ehca_dbg(cq->device,
+ "ehca_cq=%p cq_num=%x -------------------------",
+ my_cq, my_cq->cq_number);
+ }
+
+ read_lock(&ehca_qp_idr_lock);
+ my_qp = idr_find(&ehca_qp_idr, cqe->qp_token);
+ read_unlock(&ehca_qp_idr_lock);
+ if (!my_qp)
+ goto repoll;
+ wc->qp = &my_qp->ib_qp;
+
+ qmap_tail_idx = get_app_wr_id(cqe->work_request_id);
+ if (!(cqe->w_completion_flags & WC_SEND_RECEIVE_BIT))
+ /* We got a send completion. */
+ qmap = &my_qp->sq_map;
+ else
+ /* We got a receive completion. */
+ qmap = &my_qp->rq_map;
+
+ /* advance the tail pointer */
+ qmap->tail = qmap_tail_idx;
+
+ if (is_error) {
+ /*
+ * set left_to_poll to 0 because in error state, we will not
+ * get any additional CQEs
+ */
+ my_qp->sq_map.next_wqe_idx = next_index(my_qp->sq_map.tail,
+ my_qp->sq_map.entries);
+ my_qp->sq_map.left_to_poll = 0;
+ ehca_add_to_err_list(my_qp, 1);
+
+ my_qp->rq_map.next_wqe_idx = next_index(my_qp->rq_map.tail,
+ my_qp->rq_map.entries);
+ my_qp->rq_map.left_to_poll = 0;
+ if (HAS_RQ(my_qp))
+ ehca_add_to_err_list(my_qp, 0);
+ }
+
+ qmap_entry = &qmap->map[qmap_tail_idx];
+ if (qmap_entry->reported) {
+ ehca_warn(cq->device, "Double cqe on qp_num=%#x",
+ my_qp->real_qp_num);
+ /* found a double cqe, discard it and read next one */
+ goto repoll;
+ }
+
+ wc->wr_id = replace_wr_id(cqe->work_request_id, qmap_entry->app_wr_id);
+ qmap_entry->reported = 1;
+
+ /* if left_to_poll is decremented to 0, add the QP to the error list */
+ if (qmap->left_to_poll > 0) {
+ qmap->left_to_poll--;
+ if ((my_qp->sq_map.left_to_poll == 0) &&
+ (my_qp->rq_map.left_to_poll == 0)) {
+ ehca_add_to_err_list(my_qp, 1);
+ if (HAS_RQ(my_qp))
+ ehca_add_to_err_list(my_qp, 0);
+ }
+ }
+
+ /* eval ib_wc_opcode */
+ wc->opcode = ib_wc_opcode[cqe->optype]-1;
+ if (unlikely(wc->opcode == -1)) {
+ ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
+ "ehca_cq=%p cq_num=%x",
+ cqe->optype, cqe->status, my_cq, my_cq->cq_number);
+ /* dump cqe for other infos */
+ ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+ my_cq, my_cq->cq_number);
+ /* update also queue adder to throw away this entry!!! */
+ goto repoll;
+ }
+
+ /* eval ib_wc_status */
+ if (unlikely(is_error)) {
+ /* complete with errors */
+ map_ib_wc_status(cqe->status, &wc->status);
+ wc->vendor_err = wc->status;
+ } else
+ wc->status = IB_WC_SUCCESS;
+
+ wc->byte_len = cqe->nr_bytes_transferred;
+ wc->pkey_index = cqe->pkey_index;
+ wc->slid = cqe->rlid;
+ wc->dlid_path_bits = cqe->dlid;
+ wc->src_qp = cqe->remote_qp_number;
+ /*
+ * HW has "Immed data present" and "GRH present" in bits 6 and 5.
+ * SW defines those in bits 1 and 0, so we can just shift and mask.
+ */
+ wc->wc_flags = (cqe->w_completion_flags >> 5) & 3;
+ wc->ex.imm_data = cpu_to_be32(cqe->immediate_data);
+ wc->sl = cqe->service_level;
+
+poll_cq_one_exit0:
+ if (cqe_count > 0)
+ hipz_update_feca(my_cq, cqe_count);
+
+ return ret;
+}
+
+static int generate_flush_cqes(struct ehca_qp *my_qp, struct ib_cq *cq,
+ struct ib_wc *wc, int num_entries,
+ struct ipz_queue *ipz_queue, int on_sq)
+{
+ int nr = 0;
+ struct ehca_wqe *wqe;
+ u64 offset;
+ struct ehca_queue_map *qmap;
+ struct ehca_qmap_entry *qmap_entry;
+
+ if (on_sq)
+ qmap = &my_qp->sq_map;
+ else
+ qmap = &my_qp->rq_map;
+
+ qmap_entry = &qmap->map[qmap->next_wqe_idx];
+
+ while ((nr < num_entries) && (qmap_entry->reported == 0)) {
+ /* generate flush CQE */
+
+ memset(wc, 0, sizeof(*wc));
+
+ offset = qmap->next_wqe_idx * ipz_queue->qe_size;
+ wqe = (struct ehca_wqe *)ipz_qeit_calc(ipz_queue, offset);
+ if (!wqe) {
+ ehca_err(cq->device, "Invalid wqe offset=%#llx on "
+ "qp_num=%#x", offset, my_qp->real_qp_num);
+ return nr;
+ }
+
+ wc->wr_id = replace_wr_id(wqe->work_request_id,
+ qmap_entry->app_wr_id);
+
+ if (on_sq) {
+ switch (wqe->optype) {
+ case WQE_OPTYPE_SEND:
+ wc->opcode = IB_WC_SEND;
+ break;
+ case WQE_OPTYPE_RDMAWRITE:
+ wc->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case WQE_OPTYPE_RDMAREAD:
+ wc->opcode = IB_WC_RDMA_READ;
+ break;
+ default:
+ ehca_err(cq->device, "Invalid optype=%x",
+ wqe->optype);
+ return nr;
+ }
+ } else
+ wc->opcode = IB_WC_RECV;
+
+ if (wqe->wr_flag & WQE_WRFLAG_IMM_DATA_PRESENT) {
+ wc->ex.imm_data = wqe->immediate_data;
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ }
+
+ wc->status = IB_WC_WR_FLUSH_ERR;
+
+ wc->qp = &my_qp->ib_qp;
+
+ /* mark as reported and advance next_wqe pointer */
+ qmap_entry->reported = 1;
+ qmap->next_wqe_idx = next_index(qmap->next_wqe_idx,
+ qmap->entries);
+ qmap_entry = &qmap->map[qmap->next_wqe_idx];
+
+ wc++; nr++;
+ }
+
+ return nr;
+
+}
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
+{
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int nr;
+ struct ehca_qp *err_qp;
+ struct ib_wc *current_wc = wc;
+ int ret = 0;
+ unsigned long flags;
+ int entries_left = num_entries;
+
+ if (num_entries < 1) {
+ ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
+ "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
+ ret = -EINVAL;
+ goto poll_cq_exit0;
+ }
+
+ spin_lock_irqsave(&my_cq->spinlock, flags);
+
+ /* generate flush cqes for send queues */
+ list_for_each_entry(err_qp, &my_cq->sqp_err_list, sq_err_node) {
+ nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
+ &err_qp->ipz_squeue, 1);
+ entries_left -= nr;
+ current_wc += nr;
+
+ if (entries_left == 0)
+ break;
+ }
+
+ /* generate flush cqes for receive queues */
+ list_for_each_entry(err_qp, &my_cq->rqp_err_list, rq_err_node) {
+ nr = generate_flush_cqes(err_qp, cq, current_wc, entries_left,
+ &err_qp->ipz_rqueue, 0);
+ entries_left -= nr;
+ current_wc += nr;
+
+ if (entries_left == 0)
+ break;
+ }
+
+ for (nr = 0; nr < entries_left; nr++) {
+ ret = ehca_poll_cq_one(cq, current_wc);
+ if (ret)
+ break;
+ current_wc++;
+ } /* eof for nr */
+ entries_left -= nr;
+
+ spin_unlock_irqrestore(&my_cq->spinlock, flags);
+ if (ret == -EAGAIN || !ret)
+ ret = num_entries - entries_left;
+
+poll_cq_exit0:
+ return ret;
+}
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags notify_flags)
+{
+ struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+ int ret = 0;
+
+ switch (notify_flags & IB_CQ_SOLICITED_MASK) {
+ case IB_CQ_SOLICITED:
+ hipz_set_cqx_n0(my_cq, 1);
+ break;
+ case IB_CQ_NEXT_COMP:
+ hipz_set_cqx_n1(my_cq, 1);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (notify_flags & IB_CQ_REPORT_MISSED_EVENTS) {
+ unsigned long spl_flags;
+ spin_lock_irqsave(&my_cq->spinlock, spl_flags);
+ ret = ipz_qeit_is_valid(&my_cq->ipz_queue);
+ spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
+ }
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * SQP functions
+ *
+ * Authors: Khadija Souissi <souissi@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rdma/ib_mad.h>
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+#define IB_MAD_STATUS_REDIRECT cpu_to_be16(0x0002)
+#define IB_MAD_STATUS_UNSUP_VERSION cpu_to_be16(0x0004)
+#define IB_MAD_STATUS_UNSUP_METHOD cpu_to_be16(0x0008)
+
+#define IB_PMA_CLASS_PORT_INFO cpu_to_be16(0x0001)
+
+/**
+ * ehca_define_sqp - Defines special queue pair 1 (GSI QP). When special queue
+ * pair is created successfully, the corresponding port gets active.
+ *
+ * Define Special Queue pair 0 (SMI QP) is still not supported.
+ *
+ * @qp_init_attr: Queue pair init attributes with port and queue pair type
+ */
+
+u64 ehca_define_sqp(struct ehca_shca *shca,
+ struct ehca_qp *ehca_qp,
+ struct ib_qp_init_attr *qp_init_attr)
+{
+ u32 pma_qp_nr, bma_qp_nr;
+ u64 ret;
+ u8 port = qp_init_attr->port_num;
+ int counter;
+
+ shca->sport[port - 1].port_state = IB_PORT_DOWN;
+
+ switch (qp_init_attr->qp_type) {
+ case IB_QPT_SMI:
+ /* function not supported yet */
+ break;
+ case IB_QPT_GSI:
+ ret = hipz_h_define_aqp1(shca->ipz_hca_handle,
+ ehca_qp->ipz_qp_handle,
+ ehca_qp->galpas.kernel,
+ (u32) qp_init_attr->port_num,
+ &pma_qp_nr, &bma_qp_nr);
+
+ if (ret != H_SUCCESS) {
+ ehca_err(&shca->ib_device,
+ "Can't define AQP1 for port %x. h_ret=%lli",
+ port, ret);
+ return ret;
+ }
+ shca->sport[port - 1].pma_qp_nr = pma_qp_nr;
+ ehca_dbg(&shca->ib_device, "port=%x pma_qp_nr=%x",
+ port, pma_qp_nr);
+ break;
+ default:
+ ehca_err(&shca->ib_device, "invalid qp_type=%x",
+ qp_init_attr->qp_type);
+ return H_PARAMETER;
+ }
+
+ if (ehca_nr_ports < 0) /* autodetect mode */
+ return H_SUCCESS;
+
+ for (counter = 0;
+ shca->sport[port - 1].port_state != IB_PORT_ACTIVE &&
+ counter < ehca_port_act_time;
+ counter++) {
+ ehca_dbg(&shca->ib_device, "... wait until port %x is active",
+ port);
+ msleep_interruptible(1000);
+ }
+
+ if (counter == ehca_port_act_time) {
+ ehca_err(&shca->ib_device, "Port %x is not active.", port);
+ return H_HARDWARE;
+ }
+
+ return H_SUCCESS;
+}
+
+struct ib_perf {
+ struct ib_mad_hdr mad_hdr;
+ u8 reserved[40];
+ u8 data[192];
+} __attribute__ ((packed));
+
+/* TC/SL/FL packed into 32 bits, as in ClassPortInfo */
+struct tcslfl {
+ u32 tc:8;
+ u32 sl:4;
+ u32 fl:20;
+} __attribute__ ((packed));
+
+/* IP Version/TC/FL packed into 32 bits, as in GRH */
+struct vertcfl {
+ u32 ver:4;
+ u32 tc:8;
+ u32 fl:20;
+} __attribute__ ((packed));
+
+static int ehca_process_perf(struct ib_device *ibdev, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+ const struct ib_perf *in_perf = (const struct ib_perf *)in_mad;
+ struct ib_perf *out_perf = (struct ib_perf *)out_mad;
+ struct ib_class_port_info *poi =
+ (struct ib_class_port_info *)out_perf->data;
+ struct tcslfl *tcslfl =
+ (struct tcslfl *)&poi->redirect_tcslfl;
+ struct ehca_shca *shca =
+ container_of(ibdev, struct ehca_shca, ib_device);
+ struct ehca_sport *sport = &shca->sport[port_num - 1];
+
+ ehca_dbg(ibdev, "method=%x", in_perf->mad_hdr.method);
+
+ *out_mad = *in_mad;
+
+ if (in_perf->mad_hdr.class_version != 1) {
+ ehca_warn(ibdev, "Unsupported class_version=%x",
+ in_perf->mad_hdr.class_version);
+ out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_VERSION;
+ goto perf_reply;
+ }
+
+ switch (in_perf->mad_hdr.method) {
+ case IB_MGMT_METHOD_GET:
+ case IB_MGMT_METHOD_SET:
+ /* set class port info for redirection */
+ out_perf->mad_hdr.attr_id = IB_PMA_CLASS_PORT_INFO;
+ out_perf->mad_hdr.status = IB_MAD_STATUS_REDIRECT;
+ memset(poi, 0, sizeof(*poi));
+ poi->base_version = 1;
+ poi->class_version = 1;
+ poi->resp_time_value = 18;
+
+ /* copy local routing information from WC where applicable */
+ tcslfl->sl = in_wc->sl;
+ poi->redirect_lid =
+ sport->saved_attr.lid | in_wc->dlid_path_bits;
+ poi->redirect_qp = sport->pma_qp_nr;
+ poi->redirect_qkey = IB_QP1_QKEY;
+
+ ehca_query_pkey(ibdev, port_num, in_wc->pkey_index,
+ &poi->redirect_pkey);
+
+ /* if request was globally routed, copy route info */
+ if (in_grh) {
+ const struct vertcfl *vertcfl =
+ (const struct vertcfl *)&in_grh->version_tclass_flow;
+ memcpy(poi->redirect_gid, in_grh->dgid.raw,
+ sizeof(poi->redirect_gid));
+ tcslfl->tc = vertcfl->tc;
+ tcslfl->fl = vertcfl->fl;
+ } else
+ /* else only fill in default GID */
+ ehca_query_gid(ibdev, port_num, 0,
+ (union ib_gid *)&poi->redirect_gid);
+
+ ehca_dbg(ibdev, "ehca_pma_lid=%x ehca_pma_qp=%x",
+ sport->saved_attr.lid, sport->pma_qp_nr);
+ break;
+
+ case IB_MGMT_METHOD_GET_RESP:
+ return IB_MAD_RESULT_FAILURE;
+
+ default:
+ out_perf->mad_hdr.status = IB_MAD_STATUS_UNSUP_METHOD;
+ break;
+ }
+
+perf_reply:
+ out_perf->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
+
+ return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+int ehca_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+ const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+ const struct ib_mad_hdr *in, size_t in_mad_size,
+ struct ib_mad_hdr *out, size_t *out_mad_size,
+ u16 *out_mad_pkey_index)
+{
+ int ret;
+ const struct ib_mad *in_mad = (const struct ib_mad *)in;
+ struct ib_mad *out_mad = (struct ib_mad *)out;
+
+ if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) ||
+ *out_mad_size != sizeof(*out_mad)))
+ return IB_MAD_RESULT_FAILURE;
+
+ if (!port_num || port_num > ibdev->phys_port_cnt || !in_wc)
+ return IB_MAD_RESULT_FAILURE;
+
+ /* accept only pma request */
+ if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
+ return IB_MAD_RESULT_SUCCESS;
+
+ ehca_dbg(ibdev, "port_num=%x src_qp=%x", port_num, in_wc->src_qp);
+ ret = ehca_process_perf(ibdev, port_num, in_wc, in_grh,
+ in_mad, out_mad);
+
+ return ret;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * auxiliary functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Khadija Souissi <souissik@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef EHCA_TOOLS_H
+#define EHCA_TOOLS_H
+
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/idr.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/vmalloc.h>
+#include <linux/notifier.h>
+#include <linux/cpu.h>
+#include <linux/device.h>
+
+#include <linux/atomic.h>
+#include <asm/ibmebus.h>
+#include <asm/io.h>
+#include <asm/pgtable.h>
+#include <asm/hvcall.h>
+
+extern int ehca_debug_level;
+
+#define ehca_dbg(ib_dev, format, arg...) \
+ do { \
+ if (unlikely(ehca_debug_level)) \
+ dev_printk(KERN_DEBUG, (ib_dev)->dma_device, \
+ "PU%04x EHCA_DBG:%s " format "\n", \
+ raw_smp_processor_id(), __func__, \
+ ## arg); \
+ } while (0)
+
+#define ehca_info(ib_dev, format, arg...) \
+ dev_info((ib_dev)->dma_device, "PU%04x EHCA_INFO:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_warn(ib_dev, format, arg...) \
+ dev_warn((ib_dev)->dma_device, "PU%04x EHCA_WARN:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_err(ib_dev, format, arg...) \
+ dev_err((ib_dev)->dma_device, "PU%04x EHCA_ERR:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+/* use this one only if no ib_dev available */
+#define ehca_gen_dbg(format, arg...) \
+ do { \
+ if (unlikely(ehca_debug_level)) \
+ printk(KERN_DEBUG "PU%04x EHCA_DBG:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg); \
+ } while (0)
+
+#define ehca_gen_warn(format, arg...) \
+ printk(KERN_INFO "PU%04x EHCA_WARN:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+#define ehca_gen_err(format, arg...) \
+ printk(KERN_ERR "PU%04x EHCA_ERR:%s " format "\n", \
+ raw_smp_processor_id(), __func__, ## arg)
+
+/**
+ * ehca_dmp - printk a memory block, whose length is n*8 bytes.
+ * Each line has the following layout:
+ * <format string> adr=X ofs=Y <8 bytes hex> <8 bytes hex>
+ */
+#define ehca_dmp(adr, len, format, args...) \
+ do { \
+ unsigned int x; \
+ unsigned int l = (unsigned int)(len); \
+ unsigned char *deb = (unsigned char *)(adr); \
+ for (x = 0; x < l; x += 16) { \
+ printk(KERN_INFO "EHCA_DMP:%s " format \
+ " adr=%p ofs=%04x %016llx %016llx\n", \
+ __func__, ##args, deb, x, \
+ *((u64 *)&deb[0]), *((u64 *)&deb[8])); \
+ deb += 16; \
+ } \
+ } while (0)
+
+/* define a bitmask, little endian version */
+#define EHCA_BMASK(pos, length) (((pos) << 16) + (length))
+
+/* define a bitmask, the ibm way... */
+#define EHCA_BMASK_IBM(from, to) (((63 - to) << 16) + ((to) - (from) + 1))
+
+/* internal function, don't use */
+#define EHCA_BMASK_SHIFTPOS(mask) (((mask) >> 16) & 0xffff)
+
+/* internal function, don't use */
+#define EHCA_BMASK_MASK(mask) (~0ULL >> ((64 - (mask)) & 0xffff))
+
+/**
+ * EHCA_BMASK_SET - return value shifted and masked by mask
+ * variable|=EHCA_BMASK_SET(MY_MASK,0x4711) ORs the bits in variable
+ * variable&=~EHCA_BMASK_SET(MY_MASK,-1) clears the bits from the mask
+ * in variable
+ */
+#define EHCA_BMASK_SET(mask, value) \
+ ((EHCA_BMASK_MASK(mask) & ((u64)(value))) << EHCA_BMASK_SHIFTPOS(mask))
+
+/**
+ * EHCA_BMASK_GET - extract a parameter from value by mask
+ */
+#define EHCA_BMASK_GET(mask, value) \
+ (EHCA_BMASK_MASK(mask) & (((u64)(value)) >> EHCA_BMASK_SHIFTPOS(mask)))
+
+/* Converts ehca to ib return code */
+int ehca2ib_return_code(u64 ehca_rc);
+
+#endif /* EHCA_TOOLS_H */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * userspace support verbs
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+ struct ib_udata *udata)
+{
+ struct ehca_ucontext *my_context;
+
+ my_context = kzalloc(sizeof *my_context, GFP_KERNEL);
+ if (!my_context) {
+ ehca_err(device, "Out of memory device=%p", device);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return &my_context->ib_ucontext;
+}
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context)
+{
+ kfree(container_of(context, struct ehca_ucontext, ib_ucontext));
+ return 0;
+}
+
+static void ehca_mm_open(struct vm_area_struct *vma)
+{
+ u32 *count = (u32 *)vma->vm_private_data;
+ if (!count) {
+ ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ return;
+ }
+ (*count)++;
+ if (!(*count))
+ ehca_gen_err("Use count overflow vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
+ vma->vm_start, vma->vm_end, *count);
+}
+
+static void ehca_mm_close(struct vm_area_struct *vma)
+{
+ u32 *count = (u32 *)vma->vm_private_data;
+ if (!count) {
+ ehca_gen_err("Invalid vma struct vm_start=%lx vm_end=%lx",
+ vma->vm_start, vma->vm_end);
+ return;
+ }
+ (*count)--;
+ ehca_gen_dbg("vm_start=%lx vm_end=%lx count=%x",
+ vma->vm_start, vma->vm_end, *count);
+}
+
+static const struct vm_operations_struct vm_ops = {
+ .open = ehca_mm_open,
+ .close = ehca_mm_close,
+};
+
+static int ehca_mmap_fw(struct vm_area_struct *vma, struct h_galpas *galpas,
+ u32 *mm_count)
+{
+ int ret;
+ u64 vsize, physical;
+
+ vsize = vma->vm_end - vma->vm_start;
+ if (vsize < EHCA_PAGESIZE) {
+ ehca_gen_err("invalid vsize=%lx", vma->vm_end - vma->vm_start);
+ return -EINVAL;
+ }
+
+ physical = galpas->user.fw_handle;
+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+ ehca_gen_dbg("vsize=%llx physical=%llx", vsize, physical);
+ /* VM_IO | VM_DONTEXPAND | VM_DONTDUMP are set by remap_pfn_range() */
+ ret = remap_4k_pfn(vma, vma->vm_start, physical >> EHCA_PAGESHIFT,
+ vma->vm_page_prot);
+ if (unlikely(ret)) {
+ ehca_gen_err("remap_pfn_range() failed ret=%i", ret);
+ return -ENOMEM;
+ }
+
+ vma->vm_private_data = mm_count;
+ (*mm_count)++;
+ vma->vm_ops = &vm_ops;
+
+ return 0;
+}
+
+static int ehca_mmap_queue(struct vm_area_struct *vma, struct ipz_queue *queue,
+ u32 *mm_count)
+{
+ int ret;
+ u64 start, ofs;
+ struct page *page;
+
+ vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
+ start = vma->vm_start;
+ for (ofs = 0; ofs < queue->queue_length; ofs += PAGE_SIZE) {
+ u64 virt_addr = (u64)ipz_qeit_calc(queue, ofs);
+ page = virt_to_page(virt_addr);
+ ret = vm_insert_page(vma, start, page);
+ if (unlikely(ret)) {
+ ehca_gen_err("vm_insert_page() failed rc=%i", ret);
+ return ret;
+ }
+ start += PAGE_SIZE;
+ }
+ vma->vm_private_data = mm_count;
+ (*mm_count)++;
+ vma->vm_ops = &vm_ops;
+
+ return 0;
+}
+
+static int ehca_mmap_cq(struct vm_area_struct *vma, struct ehca_cq *cq,
+ u32 rsrc_type)
+{
+ int ret;
+
+ switch (rsrc_type) {
+ case 0: /* galpa fw handle */
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x fw", cq->cq_number);
+ ret = ehca_mmap_fw(vma, &cq->galpas, &cq->mm_count_galpa);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_fw() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ case 1: /* cq queue_addr */
+ ehca_dbg(cq->ib_cq.device, "cq_num=%x queue", cq->cq_number);
+ ret = ehca_mmap_queue(vma, &cq->ipz_queue, &cq->mm_count_queue);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_queue() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_err(cq->ib_cq.device, "bad resource type=%x cq_num=%x",
+ rsrc_type, cq->cq_number);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int ehca_mmap_qp(struct vm_area_struct *vma, struct ehca_qp *qp,
+ u32 rsrc_type)
+{
+ int ret;
+
+ switch (rsrc_type) {
+ case 0: /* galpa fw handle */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x fw", qp->ib_qp.qp_num);
+ ret = ehca_mmap_fw(vma, &qp->galpas, &qp->mm_count_galpa);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "remap_pfn_range() failed ret=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return -ENOMEM;
+ }
+ break;
+
+ case 1: /* qp rqueue_addr */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x rq", qp->ib_qp.qp_num);
+ ret = ehca_mmap_queue(vma, &qp->ipz_rqueue,
+ &qp->mm_count_rqueue);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_queue(rq) failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ case 2: /* qp squeue_addr */
+ ehca_dbg(qp->ib_qp.device, "qp_num=%x sq", qp->ib_qp.qp_num);
+ ret = ehca_mmap_queue(vma, &qp->ipz_squeue,
+ &qp->mm_count_squeue);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_queue(sq) failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_err(qp->ib_qp.device, "bad resource type=%x qp=num=%x",
+ rsrc_type, qp->ib_qp.qp_num);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+ u64 fileoffset = vma->vm_pgoff;
+ u32 idr_handle = fileoffset & 0x1FFFFFF;
+ u32 q_type = (fileoffset >> 27) & 0x1; /* CQ, QP,... */
+ u32 rsrc_type = (fileoffset >> 25) & 0x3; /* sq,rq,cmnd_window */
+ u32 ret;
+ struct ehca_cq *cq;
+ struct ehca_qp *qp;
+ struct ib_uobject *uobject;
+
+ switch (q_type) {
+ case 0: /* CQ */
+ read_lock(&ehca_cq_idr_lock);
+ cq = idr_find(&ehca_cq_idr, idr_handle);
+ read_unlock(&ehca_cq_idr_lock);
+
+ /* make sure this mmap really belongs to the authorized user */
+ if (!cq)
+ return -EINVAL;
+
+ if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
+ return -EINVAL;
+
+ ret = ehca_mmap_cq(vma, cq, rsrc_type);
+ if (unlikely(ret)) {
+ ehca_err(cq->ib_cq.device,
+ "ehca_mmap_cq() failed rc=%i cq_num=%x",
+ ret, cq->cq_number);
+ return ret;
+ }
+ break;
+
+ case 1: /* QP */
+ read_lock(&ehca_qp_idr_lock);
+ qp = idr_find(&ehca_qp_idr, idr_handle);
+ read_unlock(&ehca_qp_idr_lock);
+
+ /* make sure this mmap really belongs to the authorized user */
+ if (!qp)
+ return -EINVAL;
+
+ uobject = IS_SRQ(qp) ? qp->ib_srq.uobject : qp->ib_qp.uobject;
+ if (!uobject || uobject->context != context)
+ return -EINVAL;
+
+ ret = ehca_mmap_qp(vma, qp, rsrc_type);
+ if (unlikely(ret)) {
+ ehca_err(qp->ib_qp.device,
+ "ehca_mmap_qp() failed rc=%i qp_num=%x",
+ ret, qp->ib_qp.qp_num);
+ return ret;
+ }
+ break;
+
+ default:
+ ehca_gen_err("bad queue type %x", q_type);
+ return -EINVAL;
+ }
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware Infiniband Interface code for POWER
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Joachim Fenkes <fenkes@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/hvcall.h>
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hcp_phyp.h"
+#include "hipz_fns.h"
+#include "ipz_pt_fn.h"
+
+#define H_ALL_RES_QP_ENHANCED_OPS EHCA_BMASK_IBM(9, 11)
+#define H_ALL_RES_QP_PTE_PIN EHCA_BMASK_IBM(12, 12)
+#define H_ALL_RES_QP_SERVICE_TYPE EHCA_BMASK_IBM(13, 15)
+#define H_ALL_RES_QP_STORAGE EHCA_BMASK_IBM(16, 17)
+#define H_ALL_RES_QP_LL_RQ_CQE_POSTING EHCA_BMASK_IBM(18, 18)
+#define H_ALL_RES_QP_LL_SQ_CQE_POSTING EHCA_BMASK_IBM(19, 21)
+#define H_ALL_RES_QP_SIGNALING_TYPE EHCA_BMASK_IBM(22, 23)
+#define H_ALL_RES_QP_UD_AV_LKEY_CTRL EHCA_BMASK_IBM(31, 31)
+#define H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE EHCA_BMASK_IBM(32, 35)
+#define H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE EHCA_BMASK_IBM(36, 39)
+#define H_ALL_RES_QP_RESOURCE_TYPE EHCA_BMASK_IBM(56, 63)
+
+#define H_ALL_RES_QP_MAX_OUTST_SEND_WR EHCA_BMASK_IBM(0, 15)
+#define H_ALL_RES_QP_MAX_OUTST_RECV_WR EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_MAX_SEND_SGE EHCA_BMASK_IBM(32, 39)
+#define H_ALL_RES_QP_MAX_RECV_SGE EHCA_BMASK_IBM(40, 47)
+
+#define H_ALL_RES_QP_UD_AV_LKEY EHCA_BMASK_IBM(32, 63)
+#define H_ALL_RES_QP_SRQ_QP_TOKEN EHCA_BMASK_IBM(0, 31)
+#define H_ALL_RES_QP_SRQ_QP_HANDLE EHCA_BMASK_IBM(0, 64)
+#define H_ALL_RES_QP_SRQ_LIMIT EHCA_BMASK_IBM(48, 63)
+#define H_ALL_RES_QP_SRQ_QPN EHCA_BMASK_IBM(40, 63)
+
+#define H_ALL_RES_QP_ACT_OUTST_SEND_WR EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_ACT_OUTST_RECV_WR EHCA_BMASK_IBM(48, 63)
+#define H_ALL_RES_QP_ACT_SEND_SGE EHCA_BMASK_IBM(8, 15)
+#define H_ALL_RES_QP_ACT_RECV_SGE EHCA_BMASK_IBM(24, 31)
+
+#define H_ALL_RES_QP_SQUEUE_SIZE_PAGES EHCA_BMASK_IBM(0, 31)
+#define H_ALL_RES_QP_RQUEUE_SIZE_PAGES EHCA_BMASK_IBM(32, 63)
+
+#define H_MP_INIT_TYPE EHCA_BMASK_IBM(44, 47)
+#define H_MP_SHUTDOWN EHCA_BMASK_IBM(48, 48)
+#define H_MP_RESET_QKEY_CTR EHCA_BMASK_IBM(49, 49)
+
+#define HCALL4_REGS_FORMAT "r4=%lx r5=%lx r6=%lx r7=%lx"
+#define HCALL7_REGS_FORMAT HCALL4_REGS_FORMAT " r8=%lx r9=%lx r10=%lx"
+#define HCALL9_REGS_FORMAT HCALL7_REGS_FORMAT " r11=%lx r12=%lx"
+
+static DEFINE_SPINLOCK(hcall_lock);
+
+static long ehca_plpar_hcall_norets(unsigned long opcode,
+ unsigned long arg1,
+ unsigned long arg2,
+ unsigned long arg3,
+ unsigned long arg4,
+ unsigned long arg5,
+ unsigned long arg6,
+ unsigned long arg7)
+{
+ long ret;
+ int i, sleep_msecs;
+ unsigned long flags = 0;
+
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx " HCALL7_REGS_FORMAT,
+ opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+
+ for (i = 0; i < 5; i++) {
+ /* serialize hCalls to work around firmware issue */
+ if (ehca_lock_hcalls)
+ spin_lock_irqsave(&hcall_lock, flags);
+
+ ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
+ arg5, arg6, arg7);
+
+ if (ehca_lock_hcalls)
+ spin_unlock_irqrestore(&hcall_lock, flags);
+
+ if (H_IS_LONG_BUSY(ret)) {
+ sleep_msecs = get_longbusy_msecs(ret);
+ msleep_interruptible(sleep_msecs);
+ continue;
+ }
+
+ if (ret < H_SUCCESS)
+ ehca_gen_err("opcode=%lx ret=%li " HCALL7_REGS_FORMAT,
+ opcode, ret, arg1, arg2, arg3,
+ arg4, arg5, arg6, arg7);
+ else
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("opcode=%lx ret=%li", opcode, ret);
+
+ return ret;
+ }
+
+ return H_BUSY;
+}
+
+static long ehca_plpar_hcall9(unsigned long opcode,
+ unsigned long *outs, /* array of 9 outputs */
+ unsigned long arg1,
+ unsigned long arg2,
+ unsigned long arg3,
+ unsigned long arg4,
+ unsigned long arg5,
+ unsigned long arg6,
+ unsigned long arg7,
+ unsigned long arg8,
+ unsigned long arg9)
+{
+ long ret;
+ int i, sleep_msecs;
+ unsigned long flags = 0;
+
+ if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT, opcode,
+ arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+
+ for (i = 0; i < 5; i++) {
+ /* serialize hCalls to work around firmware issue */
+ if (ehca_lock_hcalls)
+ spin_lock_irqsave(&hcall_lock, flags);
+
+ ret = plpar_hcall9(opcode, outs,
+ arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+
+ if (ehca_lock_hcalls)
+ spin_unlock_irqrestore(&hcall_lock, flags);
+
+ if (H_IS_LONG_BUSY(ret)) {
+ sleep_msecs = get_longbusy_msecs(ret);
+ msleep_interruptible(sleep_msecs);
+ continue;
+ }
+
+ if (ret < H_SUCCESS) {
+ ehca_gen_err("INPUT -- opcode=%lx " HCALL9_REGS_FORMAT,
+ opcode, arg1, arg2, arg3, arg4, arg5,
+ arg6, arg7, arg8, arg9);
+ ehca_gen_err("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
+ ret, outs[0], outs[1], outs[2], outs[3],
+ outs[4], outs[5], outs[6], outs[7],
+ outs[8]);
+ } else if (unlikely(ehca_debug_level >= 2))
+ ehca_gen_dbg("OUTPUT -- ret=%li " HCALL9_REGS_FORMAT,
+ ret, outs[0], outs[1], outs[2], outs[3],
+ outs[4], outs[5], outs[6], outs[7],
+ outs[8]);
+ return ret;
+ }
+
+ return H_BUSY;
+}
+
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_pfeq *pfeq,
+ const u32 neq_control,
+ const u32 number_of_entries,
+ struct ipz_eq_handle *eq_handle,
+ u32 *act_nr_of_entries,
+ u32 *act_pages,
+ u32 *eq_ist)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+ u64 allocate_controls;
+
+ /* resource type */
+ allocate_controls = 3ULL;
+
+ /* ISN is associated */
+ if (neq_control != 1)
+ allocate_controls = (1ULL << (63 - 7)) | allocate_controls;
+ else /* notification event queue */
+ allocate_controls = (1ULL << 63) | allocate_controls;
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ allocate_controls, /* r5 */
+ number_of_entries, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ eq_handle->handle = outs[0];
+ *act_nr_of_entries = (u32)outs[3];
+ *act_pages = (u32)outs[4];
+ *eq_ist = (u32)outs[5];
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resource - ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+ struct ipz_eq_handle eq_handle,
+ const u64 event_mask)
+{
+ return ehca_plpar_hcall_norets(H_RESET_EVENTS,
+ adapter_handle.handle, /* r4 */
+ eq_handle.handle, /* r5 */
+ event_mask, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ struct ehca_alloc_cq_parms *param)
+{
+ int rc;
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 2, /* r5 */
+ param->eq_handle.handle, /* r6 */
+ cq->token, /* r7 */
+ param->nr_cqe, /* r8 */
+ 0, 0, 0, 0);
+ cq->ipz_cq_handle.handle = outs[0];
+ param->act_nr_of_entries = (u32)outs[3];
+ param->act_pages = (u32)outs[4];
+
+ if (ret == H_SUCCESS) {
+ rc = hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]);
+ if (rc) {
+ ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
+ rc, outs[5]);
+
+ ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ cq->ipz_cq_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+ ret = H_NO_MEM;
+ }
+ }
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_alloc_qp_parms *parms, int is_user)
+{
+ int rc;
+ u64 ret;
+ u64 allocate_controls, max_r10_reg, r11, r12;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ allocate_controls =
+ EHCA_BMASK_SET(H_ALL_RES_QP_ENHANCED_OPS, parms->ext_type)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_STORAGE, parms->qp_storage)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_SQ_PAGE_SIZE,
+ parms->squeue.page_size)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_SMALL_RQ_PAGE_SIZE,
+ parms->rqueue.page_size)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
+ !!(parms->ll_comp_flags & LLQP_RECV_COMP))
+ | EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
+ !!(parms->ll_comp_flags & LLQP_SEND_COMP))
+ | EHCA_BMASK_SET(H_ALL_RES_QP_UD_AV_LKEY_CTRL,
+ parms->ud_av_l_key_ctl)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_RESOURCE_TYPE, 1);
+
+ max_r10_reg =
+ EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
+ parms->squeue.max_wr + 1)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
+ parms->rqueue.max_wr + 1)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
+ parms->squeue.max_sge)
+ | EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
+ parms->rqueue.max_sge);
+
+ r11 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QP_TOKEN, parms->srq_token);
+
+ if (parms->ext_type == EQPT_SRQ)
+ r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_LIMIT, parms->srq_limit);
+ else
+ r12 = EHCA_BMASK_SET(H_ALL_RES_QP_SRQ_QPN, parms->srq_qpn);
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ allocate_controls, /* r5 */
+ parms->send_cq_handle.handle,
+ parms->recv_cq_handle.handle,
+ parms->eq_handle.handle,
+ ((u64)parms->token << 32) | parms->pd.value,
+ max_r10_reg, r11, r12);
+
+ parms->qp_handle.handle = outs[0];
+ parms->real_qp_num = (u32)outs[1];
+ parms->squeue.act_nr_wqes =
+ (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
+ parms->rqueue.act_nr_wqes =
+ (u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
+ parms->squeue.act_nr_sges =
+ (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
+ parms->rqueue.act_nr_sges =
+ (u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
+ parms->squeue.queue_size =
+ (u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
+ parms->rqueue.queue_size =
+ (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
+
+ if (ret == H_SUCCESS) {
+ rc = hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]);
+ if (rc) {
+ ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx",
+ rc, outs[6]);
+
+ ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ parms->qp_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+ ret = H_NO_MEM;
+ }
+ }
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id,
+ struct hipz_query_port *query_port_response_block)
+{
+ u64 ret;
+ u64 r_cb = __pa(query_port_response_block);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("response block not page aligned");
+ return H_PARAMETER;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_QUERY_PORT,
+ adapter_handle.handle, /* r4 */
+ port_id, /* r5 */
+ r_cb, /* r6 */
+ 0, 0, 0, 0);
+
+ if (ehca_debug_level >= 2)
+ ehca_dmp(query_port_response_block, 64, "response_block");
+
+ return ret;
+}
+
+u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id, const u32 port_cap,
+ const u8 init_type, const int modify_mask)
+{
+ u64 port_attributes = port_cap;
+
+ if (modify_mask & IB_PORT_SHUTDOWN)
+ port_attributes |= EHCA_BMASK_SET(H_MP_SHUTDOWN, 1);
+ if (modify_mask & IB_PORT_INIT_TYPE)
+ port_attributes |= EHCA_BMASK_SET(H_MP_INIT_TYPE, init_type);
+ if (modify_mask & IB_PORT_RESET_QKEY_CNTR)
+ port_attributes |= EHCA_BMASK_SET(H_MP_RESET_QKEY_CTR, 1);
+
+ return ehca_plpar_hcall_norets(H_MODIFY_PORT,
+ adapter_handle.handle, /* r4 */
+ port_id, /* r5 */
+ port_attributes, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+ struct hipz_query_hca *query_hca_rblock)
+{
+ u64 r_cb = __pa(query_hca_rblock);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("response_block=%p not page aligned",
+ query_hca_rblock);
+ return H_PARAMETER;
+ }
+
+ return ehca_plpar_hcall_norets(H_QUERY_HCA,
+ adapter_handle.handle, /* r4 */
+ r_cb, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 resource_handle,
+ const u64 logical_address_of_page,
+ u64 count)
+{
+ return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
+ adapter_handle.handle, /* r4 */
+ (u64)queue_type | ((u64)pagesize) << 8,
+ /* r5 */
+ resource_handle, /* r6 */
+ logical_address_of_page, /* r7 */
+ count, /* r8 */
+ 0, 0);
+}
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_eq_handle eq_handle,
+ struct ehca_pfeq *pfeq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count)
+{
+ if (count != 1) {
+ ehca_gen_err("Ppage counter=%llx", count);
+ return H_PARAMETER;
+ }
+ return hipz_h_register_rpage(adapter_handle,
+ pagesize,
+ queue_type,
+ eq_handle.handle,
+ logical_address_of_page, count);
+}
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle adapter_handle,
+ u32 ist)
+{
+ u64 ret;
+ ret = ehca_plpar_hcall_norets(H_QUERY_INT_STATE,
+ adapter_handle.handle, /* r4 */
+ ist, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret != H_SUCCESS && ret != H_BUSY)
+ ehca_gen_err("Could not query interrupt state.");
+
+ return ret;
+}
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_cq_handle cq_handle,
+ struct ehca_pfcq *pfcq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa gal)
+{
+ if (count != 1) {
+ ehca_gen_err("Page counter=%llx", count);
+ return H_PARAMETER;
+ }
+
+ return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
+ cq_handle.handle, logical_address_of_page,
+ count);
+}
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa galpa)
+{
+ if (count > 1) {
+ ehca_gen_err("Page counter=%llx", count);
+ return H_PARAMETER;
+ }
+
+ return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
+ qp_handle.handle, logical_address_of_page,
+ count);
+}
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ void **log_addr_next_sq_wqe2processed,
+ void **log_addr_next_rq_wqe2processed,
+ int dis_and_get_function_code)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+ adapter_handle.handle, /* r4 */
+ dis_and_get_function_code, /* r5 */
+ qp_handle.handle, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ if (log_addr_next_sq_wqe2processed)
+ *log_addr_next_sq_wqe2processed = (void *)outs[0];
+ if (log_addr_next_rq_wqe2processed)
+ *log_addr_next_rq_wqe2processed = (void *)outs[1];
+
+ return ret;
+}
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u64 update_mask,
+ struct hcp_modify_qp_control_block *mqpcb,
+ struct h_galpa gal)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+ ret = ehca_plpar_hcall9(H_MODIFY_QP, outs,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ update_mask, /* r6 */
+ __pa(mqpcb), /* r7 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Insufficient resources ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ struct hcp_modify_qp_control_block *qqpcb,
+ struct h_galpa gal)
+{
+ return ehca_plpar_hcall_norets(H_QUERY_QP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ __pa(qqpcb), /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_qp *qp)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = hcp_galpas_dtor(&qp->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct qp->galpas");
+ return H_RESOURCE;
+ }
+ ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+ adapter_handle.handle, /* r4 */
+ /* function code */
+ 1, /* r5 */
+ qp->ipz_qp_handle.handle, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ if (ret == H_HARDWARE)
+ ehca_gen_err("HCA not operational. ret=%lli", ret);
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ qp->ipz_qp_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("Resource still in use. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port)
+{
+ return ehca_plpar_hcall_norets(H_DEFINE_AQP0,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ port, /* r6 */
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port, u32 * pma_qp_nr,
+ u32 * bma_qp_nr)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_DEFINE_AQP1, outs,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ port, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ *pma_qp_nr = (u32)outs[0];
+ *bma_qp_nr = (u32)outs[1];
+
+ if (ret == H_ALIAS_EXIST)
+ ehca_gen_err("AQP1 already exists. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id)
+{
+ u64 ret;
+
+ ret = ehca_plpar_hcall_norets(H_ATTACH_MCQP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ mcg_dlid, /* r6 */
+ interface_id, /* r7 */
+ subnet_prefix, /* r8 */
+ 0, 0);
+
+ if (ret == H_NOT_ENOUGH_RESOURCES)
+ ehca_gen_err("Not enough resources. ret=%lli", ret);
+
+ return ret;
+}
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id)
+{
+ return ehca_plpar_hcall_norets(H_DETACH_MCQP,
+ adapter_handle.handle, /* r4 */
+ qp_handle.handle, /* r5 */
+ mcg_dlid, /* r6 */
+ interface_id, /* r7 */
+ subnet_prefix, /* r8 */
+ 0, 0);
+}
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ u8 force_flag)
+{
+ u64 ret;
+
+ ret = hcp_galpas_dtor(&cq->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct cp->galpas");
+ return H_RESOURCE;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ cq->ipz_cq_handle.handle, /* r5 */
+ force_flag != 0 ? 1L : 0L, /* r6 */
+ 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("H_FREE_RESOURCE failed ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_eq *eq)
+{
+ u64 ret;
+
+ ret = hcp_galpas_dtor(&eq->galpas);
+ if (ret) {
+ ehca_gen_err("Could not destruct eq->galpas");
+ return H_RESOURCE;
+ }
+
+ ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ eq->ipz_eq_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+
+ if (ret == H_RESOURCE)
+ ehca_gen_err("Resource in use. ret=%lli ", ret);
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 5, /* r5 */
+ vaddr, /* r6 */
+ length, /* r7 */
+ (((u64)access_ctrl) << 32ULL), /* r8 */
+ pd.value, /* r9 */
+ 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count)
+{
+ u64 ret;
+
+ if (unlikely(ehca_debug_level >= 3)) {
+ if (count > 1) {
+ u64 *kpage;
+ int i;
+ kpage = __va(logical_address_of_page);
+ for (i = 0; i < count; i++)
+ ehca_gen_dbg("kpage[%d]=%p",
+ i, (void *)kpage[i]);
+ } else
+ ehca_gen_dbg("kpage=%p",
+ (void *)logical_address_of_page);
+ }
+
+ if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
+ ehca_gen_err("logical_address_of_page not on a 4k boundary "
+ "adapter_handle=%llx mr=%p mr_handle=%llx "
+ "pagesize=%x queue_type=%x "
+ "logical_address_of_page=%llx count=%llx",
+ adapter_handle.handle, mr,
+ mr->ipz_mr_handle.handle, pagesize, queue_type,
+ logical_address_of_page, count);
+ ret = H_PARAMETER;
+ } else
+ ret = hipz_h_register_rpage(adapter_handle, pagesize,
+ queue_type,
+ mr->ipz_mr_handle.handle,
+ logical_address_of_page, count);
+ return ret;
+}
+
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_QUERY_MR, outs,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0, 0, 0);
+ outparms->len = outs[0];
+ outparms->vaddr = outs[1];
+ outparms->acl = outs[4] >> 32;
+ outparms->lkey = (u32)(outs[5] >> 32);
+ outparms->rkey = (u32)(outs[5] & (0xffffffff));
+
+ return ret;
+}
+
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr)
+{
+ return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr_in,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ const u64 mr_addr_cb,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_REREGISTER_PMR, outs,
+ adapter_handle.handle, /* r4 */
+ mr->ipz_mr_handle.handle, /* r5 */
+ vaddr_in, /* r6 */
+ length, /* r7 */
+ /* r8 */
+ ((((u64)access_ctrl) << 32ULL) | pd.value),
+ mr_addr_cb, /* r9 */
+ 0, 0, 0);
+ outparms->vaddr = outs[1];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const struct ehca_mr *orig_mr,
+ const u64 vaddr_in,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_REGISTER_SMR, outs,
+ adapter_handle.handle, /* r4 */
+ orig_mr->ipz_mr_handle.handle, /* r5 */
+ vaddr_in, /* r6 */
+ (((u64)access_ctrl) << 32ULL), /* r7 */
+ pd.value, /* r8 */
+ 0, 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->lkey = (u32)outs[2];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ const struct ipz_pd pd,
+ struct ehca_mw_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+ adapter_handle.handle, /* r4 */
+ 6, /* r5 */
+ pd.value, /* r6 */
+ 0, 0, 0, 0, 0, 0);
+ outparms->handle.handle = outs[0];
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ struct ehca_mw_hipzout_parms *outparms)
+{
+ u64 ret;
+ unsigned long outs[PLPAR_HCALL9_BUFSIZE];
+
+ ret = ehca_plpar_hcall9(H_QUERY_MW, outs,
+ adapter_handle.handle, /* r4 */
+ mw->ipz_mw_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0, 0, 0);
+ outparms->rkey = (u32)outs[3];
+
+ return ret;
+}
+
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw)
+{
+ return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+ adapter_handle.handle, /* r4 */
+ mw->ipz_mw_handle.handle, /* r5 */
+ 0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+ const u64 ressource_handle,
+ void *rblock,
+ unsigned long *byte_count)
+{
+ u64 r_cb = __pa(rblock);
+
+ if (r_cb & (EHCA_PAGESIZE-1)) {
+ ehca_gen_err("rblock not page aligned.");
+ return H_PARAMETER;
+ }
+
+ return ehca_plpar_hcall_norets(H_ERROR_DATA,
+ adapter_handle.handle,
+ ressource_handle,
+ r_cb,
+ 0, 0, 0, 0);
+}
+
+u64 hipz_h_eoi(int irq)
+{
+ unsigned long xirr;
+
+ iosync();
+ xirr = (0xffULL << 24) | irq;
+
+ return plpar_hcall_norets(H_EOI, xirr);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware Infiniband Interface code for POWER
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_IF_H__
+#define __HCP_IF_H__
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "hipz_hw.h"
+
+/*
+ * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initialize
+ * resources, create the empty EQPT (ring).
+ */
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_pfeq *pfeq,
+ const u32 neq_control,
+ const u32 number_of_entries,
+ struct ipz_eq_handle *eq_handle,
+ u32 * act_nr_of_entries,
+ u32 * act_pages,
+ u32 * eq_ist);
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+ struct ipz_eq_handle eq_handle,
+ const u64 event_mask);
+/*
+ * hipz_h_allocate_resource_cq allocates CQ resources in HW and FW, initialize
+ * resources, create the empty CQPT (ring).
+ */
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ struct ehca_alloc_cq_parms *param);
+
+
+/*
+ * hipz_h_alloc_resource_qp allocates QP resources in HW and FW,
+ * initialize resources, create empty QPPTs (2 rings).
+ */
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_alloc_qp_parms *parms, int is_user);
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id,
+ struct hipz_query_port *query_port_response_block);
+
+u64 hipz_h_modify_port(const struct ipz_adapter_handle adapter_handle,
+ const u8 port_id, const u32 port_cap,
+ const u8 init_type, const int modify_mask);
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+ struct hipz_query_hca *query_hca_rblock);
+
+/*
+ * hipz_h_register_rpage internal function in hcp_if.h for all
+ * hcp_H_REGISTER_RPAGE calls.
+ */
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 resource_handle,
+ const u64 logical_address_of_page,
+ u64 count);
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_eq_handle eq_handle,
+ struct ehca_pfeq *pfeq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count);
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle
+ hcp_adapter_handle,
+ u32 ist);
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_cq_handle cq_handle,
+ struct ehca_pfcq *pfcq,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa gal);
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count,
+ const struct h_galpa galpa);
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ void **log_addr_next_sq_wqe_tb_processed,
+ void **log_addr_next_rq_wqe_tb_processed,
+ int dis_and_get_function_code);
+enum hcall_sigt {
+ HCALL_SIGT_NO_CQE = 0,
+ HCALL_SIGT_BY_WQE = 1,
+ HCALL_SIGT_EVERY = 2
+};
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ const u64 update_mask,
+ struct hcp_modify_qp_control_block *mqpcb,
+ struct h_galpa gal);
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct ehca_pfqp *pfqp,
+ struct hcp_modify_qp_control_block *qqpcb,
+ struct h_galpa gal);
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_qp *qp);
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port);
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u32 port, u32 * pma_qp_nr,
+ u32 * bma_qp_nr);
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+ const struct ipz_qp_handle qp_handle,
+ struct h_galpa gal,
+ u16 mcg_dlid,
+ u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_cq *cq,
+ u8 force_flag);
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+ struct ehca_eq *eq);
+
+/*
+ * hipz_h_alloc_resource_mr allocates MR resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_rpage_mr registers MR resource pages in HW and FW */
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u8 pagesize,
+ const u8 queue_type,
+ const u64 logical_address_of_page,
+ const u64 count);
+
+/* hipz_h_query_mr queries MR in HW and FW */
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mr frees MR resources in HW and FW */
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr);
+
+/* hipz_h_reregister_pmr reregisters MR in HW and FW */
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const u64 vaddr_in,
+ const u64 length,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ const u64 mr_addr_cb,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_smr register shared MR in HW and FW */
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mr *mr,
+ const struct ehca_mr *orig_mr,
+ const u64 vaddr_in,
+ const u32 access_ctrl,
+ const struct ipz_pd pd,
+ struct ehca_mr_hipzout_parms *outparms);
+
+/*
+ * hipz_h_alloc_resource_mw allocates MW resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ const struct ipz_pd pd,
+ struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_query_mw queries MW in HW and FW */
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw,
+ struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mw frees MW resources in HW and FW */
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+ const struct ehca_mw *mw);
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+ const u64 ressource_handle,
+ void *rblock,
+ unsigned long *byte_count);
+u64 hipz_h_eoi(int irq);
+
+#endif /* __HCP_IF_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * load store abstraction for ehca register access with tracing
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+u64 hcall_map_page(u64 physaddr)
+{
+ return (u64)ioremap(physaddr, EHCA_PAGESIZE);
+}
+
+int hcall_unmap_page(u64 mapaddr)
+{
+ iounmap((volatile void __iomem *) mapaddr);
+ return 0;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
+ u64 paddr_kernel, u64 paddr_user)
+{
+ if (!is_user) {
+ galpas->kernel.fw_handle = hcall_map_page(paddr_kernel);
+ if (!galpas->kernel.fw_handle)
+ return -ENOMEM;
+ } else
+ galpas->kernel.fw_handle = 0;
+
+ galpas->user.fw_handle = paddr_user;
+
+ return 0;
+}
+
+int hcp_galpas_dtor(struct h_galpas *galpas)
+{
+ if (galpas->kernel.fw_handle) {
+ int ret = hcall_unmap_page(galpas->kernel.fw_handle);
+ if (ret)
+ return ret;
+ }
+
+ galpas->user.fw_handle = galpas->kernel.fw_handle = 0;
+
+ return 0;
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * Firmware calls
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Waleri Fomin <fomin@de.ibm.com>
+ * Gerd Bayer <gerd.bayer@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_PHYP_H__
+#define __HCP_PHYP_H__
+
+
+/*
+ * eHCA page (mapped into memory)
+ * resource to access eHCA register pages in CPU address space
+*/
+struct h_galpa {
+ u64 fw_handle;
+ /* for pSeries this is a 64bit memory address where
+ I/O memory is mapped into CPU address space (kv) */
+};
+
+/*
+ * resource to access eHCA address space registers, all types
+ */
+struct h_galpas {
+ u32 pid; /*PID of userspace galpa checking */
+ struct h_galpa user; /* user space accessible resource,
+ set to 0 if unused */
+ struct h_galpa kernel; /* kernel space accessible resource,
+ set to 0 if unused */
+};
+
+static inline u64 hipz_galpa_load(struct h_galpa galpa, u32 offset)
+{
+ u64 addr = galpa.fw_handle + offset;
+ return *(volatile u64 __force *)addr;
+}
+
+static inline void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value)
+{
+ u64 addr = galpa.fw_handle + offset;
+ *(volatile u64 __force *)addr = value;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas, int is_user,
+ u64 paddr_kernel, u64 paddr_user);
+
+int hcp_galpas_dtor(struct h_galpas *galpas);
+
+u64 hcall_map_page(u64 physaddr);
+
+int hcall_unmap_page(u64 mapaddr);
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HW abstraction register functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_H__
+#define __HIPZ_FNS_H__
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+#include "hipz_fns_core.h"
+
+#define hipz_galpa_store_eq(gal, offset, value) \
+ hipz_galpa_store(gal, EQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_eq(gal, offset) \
+ hipz_galpa_load(gal, EQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qped(gal, offset, value) \
+ hipz_galpa_store(gal, QPEDMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_qped(gal, offset) \
+ hipz_galpa_load(gal, QPEDMM_OFFSET(offset))
+
+#define hipz_galpa_store_mrmw(gal, offset, value) \
+ hipz_galpa_store(gal, MRMWMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_mrmw(gal, offset) \
+ hipz_galpa_load(gal, MRMWMM_OFFSET(offset))
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * HW abstraction register functions
+ *
+ * Authors: Christoph Raisch <raisch@de.ibm.com>
+ * Heiko J Schick <schickhj@de.ibm.com>
+ * Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_CORE_H__
+#define __HIPZ_FNS_CORE_H__
+
+#include "hcp_phyp.h"
+#include "hipz_hw.h"
+
+#define hipz_galpa_store_cq(gal, offset, value) \
+ hipz_galpa_store(gal, CQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_cq(gal, offset) \
+ hipz_galpa_load(gal, CQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qp(gal, offset, value) \
+ hipz_galpa_store(gal, QPTEMM_OFFSET(offset), value)
+#define hipz_galpa_load_qp(gal, offset) \
+ hipz_galpa_load(gal, QPTEMM_OFFSET(offset))
+
+static inline void hipz_update_sqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+ /* ringing doorbell :-) */
+ hipz_galpa_store_qp(qp->galpas.kernel, qpx_sqa,
+ EHCA_BMASK_SET(QPX_SQADDER, nr_wqes));
+}
+
+static inline void hipz_update_rqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+ /* ringing doorbell :-) */
+ hipz_galpa_store_qp(qp->galpas.kernel, qpx_rqa,
+ EHCA_BMASK_SET(QPX_RQADDER, nr_wqes));
+}
+
+static inline void hipz_update_feca(struct ehca_cq *cq, u32 nr_cqes)
+{
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_feca,
+ EHCA_BMASK_SET(CQX_FECADDER, nr_cqes));
+}
+
+static inline void hipz_set_cqx_n0(struct ehca_cq *cq, u32 value)
+{
+ u64 cqx_n0_reg;
+
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_n0,
+ EHCA_BMASK_SET(CQX_N0_GENERATE_SOLICITED_COMP_EVENT,
+ value));
+ cqx_n0_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n0);
+}
+
+static inline void hipz_set_cqx_n1(struct ehca_cq *cq, u32 value)
+{
+ u64 cqx_n1_reg;
+
+ hipz_galpa_store_cq(cq->galpas.kernel, cqx_n1,
+ EHCA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, value));
+ cqx_n1_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n1);
+}
+
+#endif /* __HIPZ_FNC_CORE_H__ */
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * eHCA register definitions
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_HW_H__
+#define __HIPZ_HW_H__
+
+#include "ehca_tools.h"
+
+#define EHCA_MAX_MTU 4
+
+/* QP Table Entry Memory Map */
+struct hipz_qptemm {
+ u64 qpx_hcr;
+ u64 qpx_c;
+ u64 qpx_herr;
+ u64 qpx_aer;
+/* 0x20*/
+ u64 qpx_sqa;
+ u64 qpx_sqc;
+ u64 qpx_rqa;
+ u64 qpx_rqc;
+/* 0x40*/
+ u64 qpx_st;
+ u64 qpx_pmstate;
+ u64 qpx_pmfa;
+ u64 qpx_pkey;
+/* 0x60*/
+ u64 qpx_pkeya;
+ u64 qpx_pkeyb;
+ u64 qpx_pkeyc;
+ u64 qpx_pkeyd;
+/* 0x80*/
+ u64 qpx_qkey;
+ u64 qpx_dqp;
+ u64 qpx_dlidp;
+ u64 qpx_portp;
+/* 0xa0*/
+ u64 qpx_slidp;
+ u64 qpx_slidpp;
+ u64 qpx_dlida;
+ u64 qpx_porta;
+/* 0xc0*/
+ u64 qpx_slida;
+ u64 qpx_slidpa;
+ u64 qpx_slvl;
+ u64 qpx_ipd;
+/* 0xe0*/
+ u64 qpx_mtu;
+ u64 qpx_lato;
+ u64 qpx_rlimit;
+ u64 qpx_rnrlimit;
+/* 0x100*/
+ u64 qpx_t;
+ u64 qpx_sqhp;
+ u64 qpx_sqptp;
+ u64 qpx_nspsn;
+/* 0x120*/
+ u64 qpx_nspsnhwm;
+ u64 reserved1;
+ u64 qpx_sdsi;
+ u64 qpx_sdsbc;
+/* 0x140*/
+ u64 qpx_sqwsize;
+ u64 qpx_sqwts;
+ u64 qpx_lsn;
+ u64 qpx_nssn;
+/* 0x160 */
+ u64 qpx_mor;
+ u64 qpx_cor;
+ u64 qpx_sqsize;
+ u64 qpx_erc;
+/* 0x180*/
+ u64 qpx_rnrrc;
+ u64 qpx_ernrwt;
+ u64 qpx_rnrresp;
+ u64 qpx_lmsna;
+/* 0x1a0 */
+ u64 qpx_sqhpc;
+ u64 qpx_sqcptp;
+ u64 qpx_sigt;
+ u64 qpx_wqecnt;
+/* 0x1c0*/
+ u64 qpx_rqhp;
+ u64 qpx_rqptp;
+ u64 qpx_rqsize;
+ u64 qpx_nrr;
+/* 0x1e0*/
+ u64 qpx_rdmac;
+ u64 qpx_nrpsn;
+ u64 qpx_lapsn;
+ u64 qpx_lcr;
+/* 0x200*/
+ u64 qpx_rwc;
+ u64 qpx_rwva;
+ u64 qpx_rdsi;
+ u64 qpx_rdsbc;
+/* 0x220*/
+ u64 qpx_rqwsize;
+ u64 qpx_crmsn;
+ u64 qpx_rdd;
+ u64 qpx_larpsn;
+/* 0x240*/
+ u64 qpx_pd;
+ u64 qpx_scqn;
+ u64 qpx_rcqn;
+ u64 qpx_aeqn;
+/* 0x260*/
+ u64 qpx_aaelog;
+ u64 qpx_ram;
+ u64 qpx_rdmaqe0;
+ u64 qpx_rdmaqe1;
+/* 0x280*/
+ u64 qpx_rdmaqe2;
+ u64 qpx_rdmaqe3;
+ u64 qpx_nrpsnhwm;
+/* 0x298*/
+ u64 reserved[(0x400 - 0x298) / 8];
+/* 0x400 extended data */
+ u64 reserved_ext[(0x500 - 0x400) / 8];
+/* 0x500 */
+ u64 reserved2[(0x1000 - 0x500) / 8];
+/* 0x1000 */
+};
+
+#define QPX_SQADDER EHCA_BMASK_IBM(48, 63)
+#define QPX_RQADDER EHCA_BMASK_IBM(48, 63)
+#define QPX_AAELOG_RESET_SRQ_LIMIT EHCA_BMASK_IBM(3, 3)
+
+#define QPTEMM_OFFSET(x) offsetof(struct hipz_qptemm, x)
+
+/* MRMWPT Entry Memory Map */
+struct hipz_mrmwmm {
+ /* 0x00 */
+ u64 mrx_hcr;
+
+ u64 mrx_c;
+ u64 mrx_herr;
+ u64 mrx_aer;
+ /* 0x20 */
+ u64 mrx_pp;
+ u64 reserved1;
+ u64 reserved2;
+ u64 reserved3;
+ /* 0x40 */
+ u64 reserved4[(0x200 - 0x40) / 8];
+ /* 0x200 */
+ u64 mrx_ctl[64];
+
+};
+
+#define MRMWMM_OFFSET(x) offsetof(struct hipz_mrmwmm, x)
+
+struct hipz_qpedmm {
+ /* 0x00 */
+ u64 reserved0[(0x400) / 8];
+ /* 0x400 */
+ u64 qpedx_phh;
+ u64 qpedx_ppsgp;
+ /* 0x410 */
+ u64 qpedx_ppsgu;
+ u64 qpedx_ppdgp;
+ /* 0x420 */
+ u64 qpedx_ppdgu;
+ u64 qpedx_aph;
+ /* 0x430 */
+ u64 qpedx_apsgp;
+ u64 qpedx_apsgu;
+ /* 0x440 */
+ u64 qpedx_apdgp;
+ u64 qpedx_apdgu;
+ /* 0x450 */
+ u64 qpedx_apav;
+ u64 qpedx_apsav;
+ /* 0x460 */
+ u64 qpedx_hcr;
+ u64 reserved1[4];
+ /* 0x488 */
+ u64 qpedx_rrl0;
+ /* 0x490 */
+ u64 qpedx_rrrkey0;
+ u64 qpedx_rrva0;
+ /* 0x4a0 */
+ u64 reserved2;
+ u64 qpedx_rrl1;
+ /* 0x4b0 */
+ u64 qpedx_rrrkey1;
+ u64 qpedx_rrva1;
+ /* 0x4c0 */
+ u64 reserved3;
+ u64 qpedx_rrl2;
+ /* 0x4d0 */
+ u64 qpedx_rrrkey2;
+ u64 qpedx_rrva2;
+ /* 0x4e0 */
+ u64 reserved4;
+ u64 qpedx_rrl3;
+ /* 0x4f0 */
+ u64 qpedx_rrrkey3;
+ u64 qpedx_rrva3;
+};
+
+#define QPEDMM_OFFSET(x) offsetof(struct hipz_qpedmm, x)
+
+/* CQ Table Entry Memory Map */
+struct hipz_cqtemm {
+ u64 cqx_hcr;
+ u64 cqx_c;
+ u64 cqx_herr;
+ u64 cqx_aer;
+/* 0x20 */
+ u64 cqx_ptp;
+ u64 cqx_tp;
+ u64 cqx_fec;
+ u64 cqx_feca;
+/* 0x40 */
+ u64 cqx_ep;
+ u64 cqx_eq;
+/* 0x50 */
+ u64 reserved1;
+ u64 cqx_n0;
+/* 0x60 */
+ u64 cqx_n1;
+ u64 reserved2[(0x1000 - 0x60) / 8];
+/* 0x1000 */
+};
+
+#define CQX_FEC_CQE_CNT EHCA_BMASK_IBM(32, 63)
+#define CQX_FECADDER EHCA_BMASK_IBM(32, 63)
+#define CQX_N0_GENERATE_SOLICITED_COMP_EVENT EHCA_BMASK_IBM(0, 0)
+#define CQX_N1_GENERATE_COMP_EVENT EHCA_BMASK_IBM(0, 0)
+
+#define CQTEMM_OFFSET(x) offsetof(struct hipz_cqtemm, x)
+
+/* EQ Table Entry Memory Map */
+struct hipz_eqtemm {
+ u64 eqx_hcr;
+ u64 eqx_c;
+
+ u64 eqx_herr;
+ u64 eqx_aer;
+/* 0x20 */
+ u64 eqx_ptp;
+ u64 eqx_tp;
+ u64 eqx_ssba;
+ u64 eqx_psba;
+
+/* 0x40 */
+ u64 eqx_cec;
+ u64 eqx_meql;
+ u64 eqx_xisbi;
+ u64 eqx_xisc;
+/* 0x60 */
+ u64 eqx_it;
+
+};
+
+#define EQTEMM_OFFSET(x) offsetof(struct hipz_eqtemm, x)
+
+/* access control defines for MR/MW */
+#define HIPZ_ACCESSCTRL_L_WRITE 0x00800000
+#define HIPZ_ACCESSCTRL_R_WRITE 0x00400000
+#define HIPZ_ACCESSCTRL_R_READ 0x00200000
+#define HIPZ_ACCESSCTRL_R_ATOMIC 0x00100000
+#define HIPZ_ACCESSCTRL_MW_BIND 0x00080000
+
+/* query hca response block */
+struct hipz_query_hca {
+ u32 cur_reliable_dg;
+ u32 cur_qp;
+ u32 cur_cq;
+ u32 cur_eq;
+ u32 cur_mr;
+ u32 cur_mw;
+ u32 cur_ee_context;
+ u32 cur_mcast_grp;
+ u32 cur_qp_attached_mcast_grp;
+ u32 reserved1;
+ u32 cur_ipv6_qp;
+ u32 cur_eth_qp;
+ u32 cur_hp_mr;
+ u32 reserved2[3];
+ u32 max_rd_domain;
+ u32 max_qp;
+ u32 max_cq;
+ u32 max_eq;
+ u32 max_mr;
+ u32 max_hp_mr;
+ u32 max_mw;
+ u32 max_mrwpte;
+ u32 max_special_mrwpte;
+ u32 max_rd_ee_context;
+ u32 max_mcast_grp;
+ u32 max_total_mcast_qp_attach;
+ u32 max_mcast_qp_attach;
+ u32 max_raw_ipv6_qp;
+ u32 max_raw_ethy_qp;
+ u32 internal_clock_frequency;
+ u32 max_pd;
+ u32 max_ah;
+ u32 max_cqe;
+ u32 max_wqes_wq;
+ u32 max_partitions;
+ u32 max_rr_ee_context;
+ u32 max_rr_qp;
+ u32 max_rr_hca;
+ u32 max_act_wqs_ee_context;
+ u32 max_act_wqs_qp;
+ u32 max_sge;
+ u32 max_sge_rd;
+ u32 memory_page_size_supported;
+ u64 max_mr_size;
+ u32 local_ca_ack_delay;
+ u32 num_ports;
+ u32 vendor_id;
+ u32 vendor_part_id;
+ u32 hw_ver;
+ u64 node_guid;
+ u64 hca_cap_indicators;
+ u32 data_counter_register_size;
+ u32 max_shared_rq;
+ u32 max_isns_eq;
+ u32 max_neq;
+} __attribute__ ((packed));
+
+#define HCA_CAP_AH_PORT_NR_CHECK EHCA_BMASK_IBM( 0, 0)
+#define HCA_CAP_ATOMIC EHCA_BMASK_IBM( 1, 1)
+#define HCA_CAP_AUTO_PATH_MIG EHCA_BMASK_IBM( 2, 2)
+#define HCA_CAP_BAD_P_KEY_CTR EHCA_BMASK_IBM( 3, 3)
+#define HCA_CAP_SQD_RTS_PORT_CHANGE EHCA_BMASK_IBM( 4, 4)
+#define HCA_CAP_CUR_QP_STATE_MOD EHCA_BMASK_IBM( 5, 5)
+#define HCA_CAP_INIT_TYPE EHCA_BMASK_IBM( 6, 6)
+#define HCA_CAP_PORT_ACTIVE_EVENT EHCA_BMASK_IBM( 7, 7)
+#define HCA_CAP_Q_KEY_VIOL_CTR EHCA_BMASK_IBM( 8, 8)
+#define HCA_CAP_WQE_RESIZE EHCA_BMASK_IBM( 9, 9)
+#define HCA_CAP_RAW_PACKET_MCAST EHCA_BMASK_IBM(10, 10)
+#define HCA_CAP_SHUTDOWN_PORT EHCA_BMASK_IBM(11, 11)
+#define HCA_CAP_RC_LL_QP EHCA_BMASK_IBM(12, 12)
+#define HCA_CAP_SRQ EHCA_BMASK_IBM(13, 13)
+#define HCA_CAP_UD_LL_QP EHCA_BMASK_IBM(16, 16)
+#define HCA_CAP_RESIZE_MR EHCA_BMASK_IBM(17, 17)
+#define HCA_CAP_MINI_QP EHCA_BMASK_IBM(18, 18)
+#define HCA_CAP_H_ALLOC_RES_SYNC EHCA_BMASK_IBM(19, 19)
+
+/* query port response block */
+struct hipz_query_port {
+ u32 state;
+ u32 bad_pkey_cntr;
+ u32 lmc;
+ u32 lid;
+ u32 subnet_timeout;
+ u32 qkey_viol_cntr;
+ u32 sm_sl;
+ u32 sm_lid;
+ u32 capability_mask;
+ u32 init_type_reply;
+ u32 pkey_tbl_len;
+ u32 gid_tbl_len;
+ u64 gid_prefix;
+ u32 port_nr;
+ u16 pkey_entries[16];
+ u8 reserved1[32];
+ u32 trent_size;
+ u32 trbuf_size;
+ u64 max_msg_sz;
+ u32 max_mtu;
+ u32 vl_cap;
+ u32 phys_pstate;
+ u32 phys_state;
+ u32 phys_speed;
+ u32 phys_width;
+ u8 reserved2[1884];
+ u64 guid_entries[255];
+} __attribute__ ((packed));
+
+#endif
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * internal queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/slab.h>
+
+#include "ehca_tools.h"
+#include "ipz_pt_fn.h"
+#include "ehca_classes.h"
+
+#define PAGES_PER_KPAGE (PAGE_SIZE >> EHCA_PAGESHIFT)
+
+struct kmem_cache *small_qp_cache;
+
+void *ipz_qpageit_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ queue->current_q_offset += queue->pagesize;
+ if (queue->current_q_offset > queue->queue_length) {
+ queue->current_q_offset -= queue->pagesize;
+ ret = NULL;
+ }
+ if (((u64)ret) % queue->pagesize) {
+ ehca_gen_err("ERROR!! not at PAGE-Boundary");
+ return NULL;
+ }
+ return ret;
+}
+
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u64 last_entry_in_q = queue->queue_length - queue->qe_size;
+
+ queue->current_q_offset += queue->qe_size;
+ if (queue->current_q_offset > last_entry_in_q) {
+ queue->current_q_offset = 0;
+ queue->toggle_state = (~queue->toggle_state) & 1;
+ }
+
+ return ret;
+}
+
+int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset)
+{
+ int i;
+ for (i = 0; i < queue->queue_length / queue->pagesize; i++) {
+ u64 page = __pa(queue->queue_pages[i]);
+ if (addr >= page && addr < page + queue->pagesize) {
+ *q_offset = addr - page + i * queue->pagesize;
+ return 0;
+ }
+ }
+ return -EINVAL;
+}
+
+#if PAGE_SHIFT < EHCA_PAGESHIFT
+#error Kernel pages must be at least as large than eHCA pages (4K) !
+#endif
+
+/*
+ * allocate pages for queue:
+ * outer loop allocates whole kernel pages (page aligned) and
+ * inner loop divides a kernel page into smaller hca queue pages
+ */
+static int alloc_queue_pages(struct ipz_queue *queue, const u32 nr_of_pages)
+{
+ int k, f = 0;
+ u8 *kpage;
+
+ while (f < nr_of_pages) {
+ kpage = (u8 *)get_zeroed_page(GFP_KERNEL);
+ if (!kpage)
+ goto out;
+
+ for (k = 0; k < PAGES_PER_KPAGE && f < nr_of_pages; k++) {
+ queue->queue_pages[f] = (struct ipz_page *)kpage;
+ kpage += EHCA_PAGESIZE;
+ f++;
+ }
+ }
+ return 1;
+
+out:
+ for (f = 0; f < nr_of_pages && queue->queue_pages[f];
+ f += PAGES_PER_KPAGE)
+ free_page((unsigned long)(queue->queue_pages)[f]);
+ return 0;
+}
+
+static int alloc_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page;
+ unsigned long bit;
+
+ mutex_lock(&pd->lock);
+
+ if (!list_empty(&pd->free[order]))
+ page = list_entry(pd->free[order].next,
+ struct ipz_small_queue_page, list);
+ else {
+ page = kmem_cache_zalloc(small_qp_cache, GFP_KERNEL);
+ if (!page)
+ goto out;
+
+ page->page = get_zeroed_page(GFP_KERNEL);
+ if (!page->page) {
+ kmem_cache_free(small_qp_cache, page);
+ goto out;
+ }
+
+ list_add(&page->list, &pd->free[order]);
+ }
+
+ bit = find_first_zero_bit(page->bitmap, IPZ_SPAGE_PER_KPAGE >> order);
+ __set_bit(bit, page->bitmap);
+ page->fill++;
+
+ if (page->fill == IPZ_SPAGE_PER_KPAGE >> order)
+ list_move(&page->list, &pd->full[order]);
+
+ mutex_unlock(&pd->lock);
+
+ queue->queue_pages[0] = (void *)(page->page | (bit << (order + 9)));
+ queue->small_page = page;
+ queue->offset = bit << (order + 9);
+ return 1;
+
+out:
+ ehca_err(pd->ib_pd.device, "failed to allocate small queue page");
+ mutex_unlock(&pd->lock);
+ return 0;
+}
+
+static void free_small_queue_page(struct ipz_queue *queue, struct ehca_pd *pd)
+{
+ int order = ilog2(queue->pagesize) - 9;
+ struct ipz_small_queue_page *page = queue->small_page;
+ unsigned long bit;
+ int free_page = 0;
+
+ bit = ((unsigned long)queue->queue_pages[0] & ~PAGE_MASK)
+ >> (order + 9);
+
+ mutex_lock(&pd->lock);
+
+ __clear_bit(bit, page->bitmap);
+ page->fill--;
+
+ if (page->fill == 0) {
+ list_del(&page->list);
+ free_page = 1;
+ }
+
+ if (page->fill == (IPZ_SPAGE_PER_KPAGE >> order) - 1)
+ /* the page was full until we freed the chunk */
+ list_move_tail(&page->list, &pd->free[order]);
+
+ mutex_unlock(&pd->lock);
+
+ if (free_page) {
+ free_page(page->page);
+ kmem_cache_free(small_qp_cache, page);
+ }
+}
+
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small)
+{
+ if (pagesize > PAGE_SIZE) {
+ ehca_gen_err("FATAL ERROR: pagesize=%x "
+ "is greater than kernel page size", pagesize);
+ return 0;
+ }
+
+ /* init queue fields */
+ queue->queue_length = nr_of_pages * pagesize;
+ queue->pagesize = pagesize;
+ queue->qe_size = qe_size;
+ queue->act_nr_of_sg = nr_of_sg;
+ queue->current_q_offset = 0;
+ queue->toggle_state = 1;
+ queue->small_page = NULL;
+
+ /* allocate queue page pointers */
+ queue->queue_pages = kzalloc(nr_of_pages * sizeof(void *),
+ GFP_KERNEL | __GFP_NOWARN);
+ if (!queue->queue_pages) {
+ queue->queue_pages = vzalloc(nr_of_pages * sizeof(void *));
+ if (!queue->queue_pages) {
+ ehca_gen_err("Couldn't allocate queue page list");
+ return 0;
+ }
+ }
+
+ /* allocate actual queue pages */
+ if (is_small) {
+ if (!alloc_small_queue_page(queue, pd))
+ goto ipz_queue_ctor_exit0;
+ } else
+ if (!alloc_queue_pages(queue, nr_of_pages))
+ goto ipz_queue_ctor_exit0;
+
+ return 1;
+
+ipz_queue_ctor_exit0:
+ ehca_gen_err("Couldn't alloc pages queue=%p "
+ "nr_of_pages=%x", queue, nr_of_pages);
+ kvfree(queue->queue_pages);
+
+ return 0;
+}
+
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue)
+{
+ int i, nr_pages;
+
+ if (!queue || !queue->queue_pages) {
+ ehca_gen_dbg("queue or queue_pages is NULL");
+ return 0;
+ }
+
+ if (queue->small_page)
+ free_small_queue_page(queue, pd);
+ else {
+ nr_pages = queue->queue_length / queue->pagesize;
+ for (i = 0; i < nr_pages; i += PAGES_PER_KPAGE)
+ free_page((unsigned long)queue->queue_pages[i]);
+ }
+
+ kvfree(queue->queue_pages);
+
+ return 1;
+}
+
+int ehca_init_small_qp_cache(void)
+{
+ small_qp_cache = kmem_cache_create("ehca_cache_small_qp",
+ sizeof(struct ipz_small_queue_page),
+ 0, SLAB_HWCACHE_ALIGN, NULL);
+ if (!small_qp_cache)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void ehca_cleanup_small_qp_cache(void)
+{
+ kmem_cache_destroy(small_qp_cache);
+}
--- /dev/null
+/*
+ * IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ * internal queue handling
+ *
+ * Authors: Waleri Fomin <fomin@de.ibm.com>
+ * Reinhard Ernst <rernst@de.ibm.com>
+ * Christoph Raisch <raisch@de.ibm.com>
+ *
+ * Copyright (c) 2005 IBM Corporation
+ *
+ * All rights reserved.
+ *
+ * This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ * BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __IPZ_PT_FN_H__
+#define __IPZ_PT_FN_H__
+
+#define EHCA_PAGESHIFT 12
+#define EHCA_PAGESIZE 4096UL
+#define EHCA_PAGEMASK (~(EHCA_PAGESIZE-1))
+#define EHCA_PT_ENTRIES 512UL
+
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+
+struct ehca_pd;
+struct ipz_small_queue_page;
+
+extern struct kmem_cache *small_qp_cache;
+
+/* struct generic ehca page */
+struct ipz_page {
+ u8 entries[EHCA_PAGESIZE];
+};
+
+#define IPZ_SPAGE_PER_KPAGE (PAGE_SIZE / 512)
+
+struct ipz_small_queue_page {
+ unsigned long page;
+ unsigned long bitmap[IPZ_SPAGE_PER_KPAGE / BITS_PER_LONG];
+ int fill;
+ void *mapped_addr;
+ u32 mmap_count;
+ struct list_head list;
+};
+
+/* struct generic queue in linux kernel virtual memory (kv) */
+struct ipz_queue {
+ u64 current_q_offset; /* current queue entry */
+
+ struct ipz_page **queue_pages; /* array of pages belonging to queue */
+ u32 qe_size; /* queue entry size */
+ u32 act_nr_of_sg;
+ u32 queue_length; /* queue length allocated in bytes */
+ u32 pagesize;
+ u32 toggle_state; /* toggle flag - per page */
+ u32 offset; /* save offset within page for small_qp */
+ struct ipz_small_queue_page *small_page;
+};
+
+/*
+ * return current Queue Entry for a certain q_offset
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_calc(struct ipz_queue *queue, u64 q_offset)
+{
+ struct ipz_page *current_page;
+ if (q_offset >= queue->queue_length)
+ return NULL;
+ current_page = (queue->queue_pages)[q_offset >> EHCA_PAGESHIFT];
+ return ¤t_page->entries[q_offset & (EHCA_PAGESIZE - 1)];
+}
+
+/*
+ * return current Queue Entry
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_get(struct ipz_queue *queue)
+{
+ return ipz_qeit_calc(queue, queue->current_q_offset);
+}
+
+/*
+ * return current Queue Page , increment Queue Page iterator from
+ * page to page in struct ipz_queue, last increment will return 0! and
+ * NOT wrap
+ * returns address (kv) of Queue Page
+ * warning don't use in parallel with ipz_QE_get_inc()
+ */
+void *ipz_qpageit_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ */
+static inline void *ipz_qeit_get_inc(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ queue->current_q_offset += queue->qe_size;
+ if (queue->current_q_offset >= queue->queue_length) {
+ queue->current_q_offset = 0;
+ /* toggle the valid flag */
+ queue->toggle_state = (~queue->toggle_state) & 1;
+ }
+
+ return ret;
+}
+
+/*
+ * return a bool indicating whether current Queue Entry is valid
+ */
+static inline int ipz_qeit_is_valid(struct ipz_queue *queue)
+{
+ struct ehca_cqe *cqe = ipz_qeit_get(queue);
+ return ((cqe->cqe_flags >> 7) == (queue->toggle_state & 1));
+}
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ */
+static inline void *ipz_qeit_get_inc_valid(struct ipz_queue *queue)
+{
+ return ipz_qeit_is_valid(queue) ? ipz_qeit_get_inc(queue) : NULL;
+}
+
+/*
+ * returns and resets Queue Entry iterator
+ * returns address (kv) of first Queue Entry
+ */
+static inline void *ipz_qeit_reset(struct ipz_queue *queue)
+{
+ queue->current_q_offset = 0;
+ return ipz_qeit_get(queue);
+}
+
+/*
+ * return the q_offset corresponding to an absolute address
+ */
+int ipz_queue_abs_to_offset(struct ipz_queue *queue, u64 addr, u64 *q_offset);
+
+/*
+ * return the next queue offset. don't modify the queue.
+ */
+static inline u64 ipz_queue_advance_offset(struct ipz_queue *queue, u64 offset)
+{
+ offset += queue->qe_size;
+ if (offset >= queue->queue_length) offset = 0;
+ return offset;
+}
+
+/* struct generic page table */
+struct ipz_pt {
+ u64 entries[EHCA_PT_ENTRIES];
+};
+
+/* struct page table for a queue, only to be used in pf */
+struct ipz_qpt {
+ /* queue page tables (kv), use u64 because we know the element length */
+ u64 *qpts;
+ u32 n_qpts;
+ u32 n_ptes; /* number of page table entries */
+ u64 *current_pte_addr;
+};
+
+/*
+ * constructor for a ipz_queue_t, placement new for ipz_queue_t,
+ * new for all dependent datastructors
+ * all QP Tables are the same
+ * flow:
+ * allocate+pin queue
+ * see ipz_qpt_ctor()
+ * returns true if ok, false if out of memory
+ */
+int ipz_queue_ctor(struct ehca_pd *pd, struct ipz_queue *queue,
+ const u32 nr_of_pages, const u32 pagesize,
+ const u32 qe_size, const u32 nr_of_sg,
+ int is_small);
+
+/*
+ * destructor for a ipz_queue_t
+ * -# free queue
+ * see ipz_queue_ctor()
+ * returns true if ok, false if queue was NULL-ptr of free failed
+ */
+int ipz_queue_dtor(struct ehca_pd *pd, struct ipz_queue *queue);
+
+/*
+ * constructor for a ipz_qpt_t,
+ * placement new for struct ipz_queue, new for all dependent datastructors
+ * all QP Tables are the same,
+ * flow:
+ * -# allocate+pin queue
+ * -# initialise ptcb
+ * -# allocate+pin PTs
+ * -# link PTs to a ring, according to HCA Arch, set bit62 id needed
+ * -# the ring must have room for exactly nr_of_PTEs
+ * see ipz_qpt_ctor()
+ */
+void ipz_qpt_ctor(struct ipz_qpt *qpt,
+ const u32 nr_of_qes,
+ const u32 pagesize,
+ const u32 qe_size,
+ const u8 lowbyte, const u8 toggle,
+ u32 * act_nr_of_QEs, u32 * act_nr_of_pages);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ * fix EQ page problems
+ */
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Event Queue Entry, increment Queue Entry iterator
+ * by one step in struct ipz_queue if valid, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_queue_QPageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ */
+static inline void *ipz_eqit_eq_get_inc_valid(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u32 qe = *(u8 *)ret;
+ if ((qe >> 7) != (queue->toggle_state & 1))
+ return NULL;
+ ipz_qeit_eq_get_inc(queue); /* this is a good one */
+ return ret;
+}
+
+static inline void *ipz_eqit_eq_peek_valid(struct ipz_queue *queue)
+{
+ void *ret = ipz_qeit_get(queue);
+ u32 qe = *(u8 *)ret;
+ if ((qe >> 7) != (queue->toggle_state & 1))
+ return NULL;
+ return ret;
+}
+
+/* returns address (GX) of first queue entry */
+static inline u64 ipz_qpt_get_firstpage(struct ipz_qpt *qpt)
+{
+ return be64_to_cpu(qpt->qpts[0]);
+}
+
+/* returns address (kv) of first page of queue page table */
+static inline void *ipz_qpt_get_qpt(struct ipz_qpt *qpt)
+{
+ return qpt->qpts;
+}
+
+#endif /* __IPZ_PT_FN_H__ */
projects / GitHub / exynos8895 / android_kernel_samsung_universal8895.git / commitdiff