Commit | Line | Data |
---|---|---|
225c7b1f RD |
1 | /* |
2 | * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. | |
3 | * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. | |
4 | * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved. | |
5 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. | |
6 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. | |
7 | * | |
8 | * This software is available to you under a choice of one of two | |
9 | * licenses. You may choose to be licensed under the terms of the GNU | |
10 | * General Public License (GPL) Version 2, available from the file | |
11 | * COPYING in the main directory of this source tree, or the | |
12 | * OpenIB.org BSD license below: | |
13 | * | |
14 | * Redistribution and use in source and binary forms, with or | |
15 | * without modification, are permitted provided that the following | |
16 | * conditions are met: | |
17 | * | |
18 | * - Redistributions of source code must retain the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer. | |
21 | * | |
22 | * - Redistributions in binary form must reproduce the above | |
23 | * copyright notice, this list of conditions and the following | |
24 | * disclaimer in the documentation and/or other materials | |
25 | * provided with the distribution. | |
26 | * | |
27 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
28 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
29 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
30 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
31 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
32 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
33 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
34 | * SOFTWARE. | |
35 | */ | |
36 | ||
37 | #include <linux/init.h> | |
38 | #include <linux/hardirq.h> | |
39 | ||
40 | #include <linux/mlx4/cmd.h> | |
41 | ||
42 | #include "mlx4.h" | |
43 | #include "icm.h" | |
44 | ||
45 | struct mlx4_cq_context { | |
46 | __be32 flags; | |
47 | u16 reserved1[3]; | |
48 | __be16 page_offset; | |
49 | __be32 logsize_usrpage; | |
50 | u8 reserved2; | |
51 | u8 cq_period; | |
52 | u8 reserved3; | |
53 | u8 cq_max_count; | |
54 | u8 reserved4[3]; | |
55 | u8 comp_eqn; | |
56 | u8 log_page_size; | |
57 | u8 reserved5[2]; | |
58 | u8 mtt_base_addr_h; | |
59 | __be32 mtt_base_addr_l; | |
60 | __be32 last_notified_index; | |
61 | __be32 solicit_producer_index; | |
62 | __be32 consumer_index; | |
63 | __be32 producer_index; | |
64 | u8 reserved6[2]; | |
65 | __be64 db_rec_addr; | |
66 | }; | |
67 | ||
68 | #define MLX4_CQ_STATUS_OK ( 0 << 28) | |
69 | #define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28) | |
70 | #define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28) | |
71 | #define MLX4_CQ_FLAG_CC ( 1 << 18) | |
72 | #define MLX4_CQ_FLAG_OI ( 1 << 17) | |
73 | #define MLX4_CQ_STATE_ARMED ( 9 << 8) | |
74 | #define MLX4_CQ_STATE_ARMED_SOL ( 6 << 8) | |
75 | #define MLX4_EQ_STATE_FIRED (10 << 8) | |
76 | ||
77 | void mlx4_cq_completion(struct mlx4_dev *dev, u32 cqn) | |
78 | { | |
79 | struct mlx4_cq *cq; | |
80 | ||
81 | cq = radix_tree_lookup(&mlx4_priv(dev)->cq_table.tree, | |
82 | cqn & (dev->caps.num_cqs - 1)); | |
83 | if (!cq) { | |
84 | mlx4_warn(dev, "Completion event for bogus CQ %08x\n", cqn); | |
85 | return; | |
86 | } | |
87 | ||
88 | ++cq->arm_sn; | |
89 | ||
90 | cq->comp(cq); | |
91 | } | |
92 | ||
93 | void mlx4_cq_event(struct mlx4_dev *dev, u32 cqn, int event_type) | |
94 | { | |
95 | struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; | |
96 | struct mlx4_cq *cq; | |
97 | ||
98 | spin_lock(&cq_table->lock); | |
99 | ||
100 | cq = radix_tree_lookup(&cq_table->tree, cqn & (dev->caps.num_cqs - 1)); | |
101 | if (cq) | |
102 | atomic_inc(&cq->refcount); | |
103 | ||
104 | spin_unlock(&cq_table->lock); | |
105 | ||
106 | if (!cq) { | |
107 | mlx4_warn(dev, "Async event for bogus CQ %08x\n", cqn); | |
108 | return; | |
109 | } | |
110 | ||
111 | cq->event(cq, event_type); | |
112 | ||
113 | if (atomic_dec_and_test(&cq->refcount)) | |
114 | complete(&cq->free); | |
115 | } | |
116 | ||
117 | static int mlx4_SW2HW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, | |
118 | int cq_num) | |
119 | { | |
120 | return mlx4_cmd(dev, mailbox->dma, cq_num, 0, MLX4_CMD_SW2HW_CQ, | |
121 | MLX4_CMD_TIME_CLASS_A); | |
122 | } | |
123 | ||
124 | static int mlx4_HW2SW_CQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, | |
125 | int cq_num) | |
126 | { | |
127 | return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, cq_num, | |
128 | mailbox ? 0 : 1, MLX4_CMD_HW2SW_CQ, | |
129 | MLX4_CMD_TIME_CLASS_A); | |
130 | } | |
131 | ||
132 | int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt, | |
133 | struct mlx4_uar *uar, u64 db_rec, struct mlx4_cq *cq) | |
134 | { | |
135 | struct mlx4_priv *priv = mlx4_priv(dev); | |
136 | struct mlx4_cq_table *cq_table = &priv->cq_table; | |
137 | struct mlx4_cmd_mailbox *mailbox; | |
138 | struct mlx4_cq_context *cq_context; | |
139 | u64 mtt_addr; | |
140 | int err; | |
141 | ||
142 | cq->cqn = mlx4_bitmap_alloc(&cq_table->bitmap); | |
143 | if (cq->cqn == -1) | |
144 | return -ENOMEM; | |
145 | ||
146 | err = mlx4_table_get(dev, &cq_table->table, cq->cqn); | |
147 | if (err) | |
148 | goto err_out; | |
149 | ||
150 | err = mlx4_table_get(dev, &cq_table->cmpt_table, cq->cqn); | |
151 | if (err) | |
152 | goto err_put; | |
153 | ||
154 | spin_lock_irq(&cq_table->lock); | |
155 | err = radix_tree_insert(&cq_table->tree, cq->cqn, cq); | |
156 | spin_unlock_irq(&cq_table->lock); | |
157 | if (err) | |
158 | goto err_cmpt_put; | |
159 | ||
160 | mailbox = mlx4_alloc_cmd_mailbox(dev); | |
161 | if (IS_ERR(mailbox)) { | |
162 | err = PTR_ERR(mailbox); | |
163 | goto err_radix; | |
164 | } | |
165 | ||
166 | cq_context = mailbox->buf; | |
167 | memset(cq_context, 0, sizeof *cq_context); | |
168 | ||
169 | cq_context->logsize_usrpage = cpu_to_be32((ilog2(nent) << 24) | uar->index); | |
170 | cq_context->comp_eqn = priv->eq_table.eq[MLX4_EQ_COMP].eqn; | |
171 | cq_context->log_page_size = mtt->page_shift - MLX4_ICM_PAGE_SHIFT; | |
172 | ||
173 | mtt_addr = mlx4_mtt_addr(dev, mtt); | |
174 | cq_context->mtt_base_addr_h = mtt_addr >> 32; | |
175 | cq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff); | |
176 | cq_context->db_rec_addr = cpu_to_be64(db_rec); | |
177 | ||
178 | err = mlx4_SW2HW_CQ(dev, mailbox, cq->cqn); | |
179 | mlx4_free_cmd_mailbox(dev, mailbox); | |
180 | if (err) | |
181 | goto err_radix; | |
182 | ||
183 | cq->cons_index = 0; | |
184 | cq->arm_sn = 1; | |
185 | cq->uar = uar; | |
186 | atomic_set(&cq->refcount, 1); | |
187 | init_completion(&cq->free); | |
188 | ||
189 | return 0; | |
190 | ||
191 | err_radix: | |
192 | spin_lock_irq(&cq_table->lock); | |
193 | radix_tree_delete(&cq_table->tree, cq->cqn); | |
194 | spin_unlock_irq(&cq_table->lock); | |
195 | ||
196 | err_cmpt_put: | |
197 | mlx4_table_put(dev, &cq_table->cmpt_table, cq->cqn); | |
198 | ||
199 | err_put: | |
200 | mlx4_table_put(dev, &cq_table->table, cq->cqn); | |
201 | ||
202 | err_out: | |
203 | mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); | |
204 | ||
205 | return err; | |
206 | } | |
207 | EXPORT_SYMBOL_GPL(mlx4_cq_alloc); | |
208 | ||
209 | void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq) | |
210 | { | |
211 | struct mlx4_priv *priv = mlx4_priv(dev); | |
212 | struct mlx4_cq_table *cq_table = &priv->cq_table; | |
213 | int err; | |
214 | ||
215 | err = mlx4_HW2SW_CQ(dev, NULL, cq->cqn); | |
216 | if (err) | |
217 | mlx4_warn(dev, "HW2SW_CQ failed (%d) for CQN %06x\n", err, cq->cqn); | |
218 | ||
219 | synchronize_irq(priv->eq_table.eq[MLX4_EQ_COMP].irq); | |
220 | ||
221 | spin_lock_irq(&cq_table->lock); | |
222 | radix_tree_delete(&cq_table->tree, cq->cqn); | |
223 | spin_unlock_irq(&cq_table->lock); | |
224 | ||
225 | if (atomic_dec_and_test(&cq->refcount)) | |
226 | complete(&cq->free); | |
227 | wait_for_completion(&cq->free); | |
228 | ||
229 | mlx4_table_put(dev, &cq_table->table, cq->cqn); | |
230 | mlx4_bitmap_free(&cq_table->bitmap, cq->cqn); | |
231 | } | |
232 | EXPORT_SYMBOL_GPL(mlx4_cq_free); | |
233 | ||
234 | int __devinit mlx4_init_cq_table(struct mlx4_dev *dev) | |
235 | { | |
236 | struct mlx4_cq_table *cq_table = &mlx4_priv(dev)->cq_table; | |
237 | int err; | |
238 | ||
239 | spin_lock_init(&cq_table->lock); | |
240 | INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC); | |
241 | ||
242 | err = mlx4_bitmap_init(&cq_table->bitmap, dev->caps.num_cqs, | |
243 | dev->caps.num_cqs - 1, dev->caps.reserved_cqs); | |
244 | if (err) | |
245 | return err; | |
246 | ||
247 | return 0; | |
248 | } | |
249 | ||
250 | void mlx4_cleanup_cq_table(struct mlx4_dev *dev) | |
251 | { | |
252 | /* Nothing to do to clean up radix_tree */ | |
253 | mlx4_bitmap_cleanup(&mlx4_priv(dev)->cq_table.bitmap); | |
254 | } |