Commit | Line | Data |
---|---|---|
fce8a7bb JM |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * BSD LICENSE | |
14 | * | |
15 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
16 | * | |
17 | * Redistribution and use in source and binary forms, with or without | |
18 | * modification, are permitted provided that the following conditions | |
19 | * are met: | |
20 | * | |
21 | * * Redistributions of source code must retain the above copyright | |
22 | * notice, this list of conditions and the following disclaimer. | |
23 | * * Redistributions in binary form must reproduce the above copy | |
24 | * notice, this list of conditions and the following disclaimer in | |
25 | * the documentation and/or other materials provided with the | |
26 | * distribution. | |
27 | * * Neither the name of Intel Corporation nor the names of its | |
28 | * contributors may be used to endorse or promote products derived | |
29 | * from this software without specific prior written permission. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
32 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
33 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
34 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
35 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
36 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
37 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
38 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
39 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
40 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
41 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
42 | * | |
43 | * Intel PCIe NTB Linux driver | |
44 | * | |
45 | * Contact Information: | |
46 | * Jon Mason <jon.mason@intel.com> | |
47 | */ | |
48 | #include <linux/debugfs.h> | |
49 | #include <linux/init.h> | |
50 | #include <linux/interrupt.h> | |
51 | #include <linux/module.h> | |
52 | #include <linux/pci.h> | |
53 | #include <linux/slab.h> | |
54 | #include "ntb_hw.h" | |
55 | #include "ntb_regs.h" | |
56 | ||
57 | #define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver" | |
50228c55 | 58 | #define NTB_VER "0.25" |
fce8a7bb JM |
59 | |
60 | MODULE_DESCRIPTION(NTB_NAME); | |
61 | MODULE_VERSION(NTB_VER); | |
62 | MODULE_LICENSE("Dual BSD/GPL"); | |
63 | MODULE_AUTHOR("Intel Corporation"); | |
64 | ||
65 | enum { | |
66 | NTB_CONN_CLASSIC = 0, | |
67 | NTB_CONN_B2B, | |
68 | NTB_CONN_RP, | |
69 | }; | |
70 | ||
71 | enum { | |
72 | NTB_DEV_USD = 0, | |
73 | NTB_DEV_DSD, | |
74 | }; | |
75 | ||
76 | enum { | |
77 | SNB_HW = 0, | |
78 | BWD_HW, | |
79 | }; | |
80 | ||
81 | /* Translate memory window 0,1 to BAR 2,4 */ | |
82 | #define MW_TO_BAR(mw) (mw * 2 + 2) | |
83 | ||
84 | static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = { | |
85 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)}, | |
86 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, | |
87 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_CLASSIC_JSF)}, | |
88 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_RP_JSF)}, | |
89 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_RP_SNB)}, | |
90 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, | |
91 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_CLASSIC_SNB)}, | |
92 | {0} | |
93 | }; | |
94 | MODULE_DEVICE_TABLE(pci, ntb_pci_tbl); | |
95 | ||
96 | /** | |
97 | * ntb_register_event_callback() - register event callback | |
98 | * @ndev: pointer to ntb_device instance | |
99 | * @func: callback function to register | |
100 | * | |
101 | * This function registers a callback for any HW driver events such as link | |
102 | * up/down, power management notices and etc. | |
103 | * | |
104 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
105 | */ | |
106 | int ntb_register_event_callback(struct ntb_device *ndev, | |
74465645 | 107 | void (*func)(void *handle, enum ntb_hw_event event)) |
fce8a7bb JM |
108 | { |
109 | if (ndev->event_cb) | |
110 | return -EINVAL; | |
111 | ||
112 | ndev->event_cb = func; | |
113 | ||
114 | return 0; | |
115 | } | |
116 | ||
117 | /** | |
118 | * ntb_unregister_event_callback() - unregisters the event callback | |
119 | * @ndev: pointer to ntb_device instance | |
120 | * | |
121 | * This function unregisters the existing callback from transport | |
122 | */ | |
123 | void ntb_unregister_event_callback(struct ntb_device *ndev) | |
124 | { | |
125 | ndev->event_cb = NULL; | |
126 | } | |
127 | ||
128 | /** | |
129 | * ntb_register_db_callback() - register a callback for doorbell interrupt | |
130 | * @ndev: pointer to ntb_device instance | |
131 | * @idx: doorbell index to register callback, zero based | |
132 | * @func: callback function to register | |
133 | * | |
134 | * This function registers a callback function for the doorbell interrupt | |
135 | * on the primary side. The function will unmask the doorbell as well to | |
136 | * allow interrupt. | |
137 | * | |
138 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
139 | */ | |
140 | int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, | |
141 | void *data, void (*func)(void *data, int db_num)) | |
142 | { | |
143 | unsigned long mask; | |
144 | ||
145 | if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) { | |
146 | dev_warn(&ndev->pdev->dev, "Invalid Index.\n"); | |
147 | return -EINVAL; | |
148 | } | |
149 | ||
150 | ndev->db_cb[idx].callback = func; | |
151 | ndev->db_cb[idx].data = data; | |
152 | ||
153 | /* unmask interrupt */ | |
154 | mask = readw(ndev->reg_ofs.pdb_mask); | |
155 | clear_bit(idx * ndev->bits_per_vector, &mask); | |
156 | writew(mask, ndev->reg_ofs.pdb_mask); | |
157 | ||
158 | return 0; | |
159 | } | |
160 | ||
161 | /** | |
162 | * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt | |
163 | * @ndev: pointer to ntb_device instance | |
164 | * @idx: doorbell index to register callback, zero based | |
165 | * | |
166 | * This function unregisters a callback function for the doorbell interrupt | |
167 | * on the primary side. The function will also mask the said doorbell. | |
168 | */ | |
169 | void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) | |
170 | { | |
171 | unsigned long mask; | |
172 | ||
173 | if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) | |
174 | return; | |
175 | ||
176 | mask = readw(ndev->reg_ofs.pdb_mask); | |
177 | set_bit(idx * ndev->bits_per_vector, &mask); | |
178 | writew(mask, ndev->reg_ofs.pdb_mask); | |
179 | ||
180 | ndev->db_cb[idx].callback = NULL; | |
181 | } | |
182 | ||
183 | /** | |
184 | * ntb_find_transport() - find the transport pointer | |
185 | * @transport: pointer to pci device | |
186 | * | |
187 | * Given the pci device pointer, return the transport pointer passed in when | |
188 | * the transport attached when it was inited. | |
189 | * | |
190 | * RETURNS: pointer to transport. | |
191 | */ | |
192 | void *ntb_find_transport(struct pci_dev *pdev) | |
193 | { | |
194 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
195 | return ndev->ntb_transport; | |
196 | } | |
197 | ||
198 | /** | |
199 | * ntb_register_transport() - Register NTB transport with NTB HW driver | |
200 | * @transport: transport identifier | |
201 | * | |
202 | * This function allows a transport to reserve the hardware driver for | |
203 | * NTB usage. | |
204 | * | |
205 | * RETURNS: pointer to ntb_device, NULL on error. | |
206 | */ | |
207 | struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport) | |
208 | { | |
209 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
210 | ||
211 | if (ndev->ntb_transport) | |
212 | return NULL; | |
213 | ||
214 | ndev->ntb_transport = transport; | |
215 | return ndev; | |
216 | } | |
217 | ||
218 | /** | |
219 | * ntb_unregister_transport() - Unregister the transport with the NTB HW driver | |
220 | * @ndev - ntb_device of the transport to be freed | |
221 | * | |
222 | * This function unregisters the transport from the HW driver and performs any | |
223 | * necessary cleanups. | |
224 | */ | |
225 | void ntb_unregister_transport(struct ntb_device *ndev) | |
226 | { | |
227 | int i; | |
228 | ||
229 | if (!ndev->ntb_transport) | |
230 | return; | |
231 | ||
232 | for (i = 0; i < ndev->max_cbs; i++) | |
233 | ntb_unregister_db_callback(ndev, i); | |
234 | ||
235 | ntb_unregister_event_callback(ndev); | |
236 | ndev->ntb_transport = NULL; | |
237 | } | |
238 | ||
fce8a7bb JM |
239 | /** |
240 | * ntb_write_local_spad() - write to the secondary scratchpad register | |
241 | * @ndev: pointer to ntb_device instance | |
242 | * @idx: index to the scratchpad register, 0 based | |
243 | * @val: the data value to put into the register | |
244 | * | |
245 | * This function allows writing of a 32bit value to the indexed scratchpad | |
246 | * register. This writes over the data mirrored to the local scratchpad register | |
247 | * by the remote system. | |
248 | * | |
249 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
250 | */ | |
251 | int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
252 | { | |
253 | if (idx >= ndev->limits.max_spads) | |
254 | return -EINVAL; | |
255 | ||
256 | dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", | |
257 | val, idx); | |
258 | writel(val, ndev->reg_ofs.spad_read + idx * 4); | |
259 | ||
260 | return 0; | |
261 | } | |
262 | ||
263 | /** | |
264 | * ntb_read_local_spad() - read from the primary scratchpad register | |
265 | * @ndev: pointer to ntb_device instance | |
266 | * @idx: index to scratchpad register, 0 based | |
267 | * @val: pointer to 32bit integer for storing the register value | |
268 | * | |
269 | * This function allows reading of the 32bit scratchpad register on | |
270 | * the primary (internal) side. This allows the local system to read data | |
271 | * written and mirrored to the scratchpad register by the remote system. | |
272 | * | |
273 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
274 | */ | |
275 | int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
276 | { | |
277 | if (idx >= ndev->limits.max_spads) | |
278 | return -EINVAL; | |
279 | ||
280 | *val = readl(ndev->reg_ofs.spad_write + idx * 4); | |
281 | dev_dbg(&ndev->pdev->dev, | |
282 | "Reading %x from local scratch pad index %d\n", *val, idx); | |
283 | ||
284 | return 0; | |
285 | } | |
286 | ||
287 | /** | |
288 | * ntb_write_remote_spad() - write to the secondary scratchpad register | |
289 | * @ndev: pointer to ntb_device instance | |
290 | * @idx: index to the scratchpad register, 0 based | |
291 | * @val: the data value to put into the register | |
292 | * | |
293 | * This function allows writing of a 32bit value to the indexed scratchpad | |
294 | * register. The register resides on the secondary (external) side. This allows | |
295 | * the local system to write data to be mirrored to the remote systems | |
296 | * scratchpad register. | |
297 | * | |
298 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
299 | */ | |
300 | int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
301 | { | |
302 | if (idx >= ndev->limits.max_spads) | |
303 | return -EINVAL; | |
304 | ||
305 | dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", | |
306 | val, idx); | |
307 | writel(val, ndev->reg_ofs.spad_write + idx * 4); | |
308 | ||
309 | return 0; | |
310 | } | |
311 | ||
312 | /** | |
313 | * ntb_read_remote_spad() - read from the primary scratchpad register | |
314 | * @ndev: pointer to ntb_device instance | |
315 | * @idx: index to scratchpad register, 0 based | |
316 | * @val: pointer to 32bit integer for storing the register value | |
317 | * | |
318 | * This function allows reading of the 32bit scratchpad register on | |
319 | * the primary (internal) side. This alloows the local system to read the data | |
320 | * it wrote to be mirrored on the remote system. | |
321 | * | |
322 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
323 | */ | |
324 | int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
325 | { | |
326 | if (idx >= ndev->limits.max_spads) | |
327 | return -EINVAL; | |
328 | ||
329 | *val = readl(ndev->reg_ofs.spad_read + idx * 4); | |
330 | dev_dbg(&ndev->pdev->dev, | |
331 | "Reading %x from remote scratch pad index %d\n", *val, idx); | |
332 | ||
333 | return 0; | |
334 | } | |
335 | ||
336 | /** | |
337 | * ntb_get_mw_vbase() - get virtual addr for the NTB memory window | |
338 | * @ndev: pointer to ntb_device instance | |
339 | * @mw: memory window number | |
340 | * | |
341 | * This function provides the base virtual address of the memory window | |
342 | * specified. | |
343 | * | |
344 | * RETURNS: pointer to virtual address, or NULL on error. | |
345 | */ | |
74465645 | 346 | void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw) |
fce8a7bb | 347 | { |
ad3e2751 | 348 | if (mw >= NTB_NUM_MW) |
fce8a7bb JM |
349 | return NULL; |
350 | ||
351 | return ndev->mw[mw].vbase; | |
352 | } | |
353 | ||
354 | /** | |
355 | * ntb_get_mw_size() - return size of NTB memory window | |
356 | * @ndev: pointer to ntb_device instance | |
357 | * @mw: memory window number | |
358 | * | |
359 | * This function provides the physical size of the memory window specified | |
360 | * | |
361 | * RETURNS: the size of the memory window or zero on error | |
362 | */ | |
363 | resource_size_t ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw) | |
364 | { | |
ad3e2751 | 365 | if (mw >= NTB_NUM_MW) |
fce8a7bb JM |
366 | return 0; |
367 | ||
368 | return ndev->mw[mw].bar_sz; | |
369 | } | |
370 | ||
371 | /** | |
372 | * ntb_set_mw_addr - set the memory window address | |
373 | * @ndev: pointer to ntb_device instance | |
374 | * @mw: memory window number | |
375 | * @addr: base address for data | |
376 | * | |
377 | * This function sets the base physical address of the memory window. This | |
378 | * memory address is where data from the remote system will be transfered into | |
379 | * or out of depending on how the transport is configured. | |
380 | */ | |
381 | void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr) | |
382 | { | |
ad3e2751 | 383 | if (mw >= NTB_NUM_MW) |
fce8a7bb JM |
384 | return; |
385 | ||
386 | dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, | |
387 | MW_TO_BAR(mw)); | |
388 | ||
389 | ndev->mw[mw].phys_addr = addr; | |
390 | ||
391 | switch (MW_TO_BAR(mw)) { | |
392 | case NTB_BAR_23: | |
393 | writeq(addr, ndev->reg_ofs.sbar2_xlat); | |
394 | break; | |
395 | case NTB_BAR_45: | |
396 | writeq(addr, ndev->reg_ofs.sbar4_xlat); | |
397 | break; | |
398 | } | |
399 | } | |
400 | ||
401 | /** | |
402 | * ntb_ring_sdb() - Set the doorbell on the secondary/external side | |
403 | * @ndev: pointer to ntb_device instance | |
404 | * @db: doorbell to ring | |
405 | * | |
406 | * This function allows triggering of a doorbell on the secondary/external | |
407 | * side that will initiate an interrupt on the remote host | |
408 | * | |
409 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
410 | */ | |
411 | void ntb_ring_sdb(struct ntb_device *ndev, unsigned int db) | |
412 | { | |
413 | dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db); | |
414 | ||
415 | if (ndev->hw_type == BWD_HW) | |
416 | writeq((u64) 1 << db, ndev->reg_ofs.sdb); | |
417 | else | |
418 | writew(((1 << ndev->bits_per_vector) - 1) << | |
419 | (db * ndev->bits_per_vector), ndev->reg_ofs.sdb); | |
420 | } | |
421 | ||
422 | static void ntb_link_event(struct ntb_device *ndev, int link_state) | |
423 | { | |
424 | unsigned int event; | |
425 | ||
426 | if (ndev->link_status == link_state) | |
427 | return; | |
428 | ||
429 | if (link_state == NTB_LINK_UP) { | |
430 | u16 status; | |
431 | ||
432 | dev_info(&ndev->pdev->dev, "Link Up\n"); | |
433 | ndev->link_status = NTB_LINK_UP; | |
434 | event = NTB_EVENT_HW_LINK_UP; | |
435 | ||
436 | if (ndev->hw_type == BWD_HW) | |
437 | status = readw(ndev->reg_ofs.lnk_stat); | |
438 | else { | |
439 | int rc = pci_read_config_word(ndev->pdev, | |
440 | SNB_LINK_STATUS_OFFSET, | |
441 | &status); | |
442 | if (rc) | |
443 | return; | |
444 | } | |
445 | dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", | |
446 | (status & NTB_LINK_WIDTH_MASK) >> 4, | |
447 | (status & NTB_LINK_SPEED_MASK)); | |
448 | } else { | |
449 | dev_info(&ndev->pdev->dev, "Link Down\n"); | |
450 | ndev->link_status = NTB_LINK_DOWN; | |
451 | event = NTB_EVENT_HW_LINK_DOWN; | |
452 | } | |
453 | ||
454 | /* notify the upper layer if we have an event change */ | |
455 | if (ndev->event_cb) | |
456 | ndev->event_cb(ndev->ntb_transport, event); | |
457 | } | |
458 | ||
459 | static int ntb_link_status(struct ntb_device *ndev) | |
460 | { | |
461 | int link_state; | |
462 | ||
463 | if (ndev->hw_type == BWD_HW) { | |
464 | u32 ntb_cntl; | |
465 | ||
466 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
467 | if (ntb_cntl & BWD_CNTL_LINK_DOWN) | |
468 | link_state = NTB_LINK_DOWN; | |
469 | else | |
470 | link_state = NTB_LINK_UP; | |
471 | } else { | |
472 | u16 status; | |
473 | int rc; | |
474 | ||
475 | rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, | |
476 | &status); | |
477 | if (rc) | |
478 | return rc; | |
479 | ||
480 | if (status & NTB_LINK_STATUS_ACTIVE) | |
481 | link_state = NTB_LINK_UP; | |
482 | else | |
483 | link_state = NTB_LINK_DOWN; | |
484 | } | |
485 | ||
486 | ntb_link_event(ndev, link_state); | |
487 | ||
488 | return 0; | |
489 | } | |
490 | ||
491 | /* BWD doesn't have link status interrupt, poll on that platform */ | |
492 | static void bwd_link_poll(struct work_struct *work) | |
493 | { | |
494 | struct ntb_device *ndev = container_of(work, struct ntb_device, | |
495 | hb_timer.work); | |
496 | unsigned long ts = jiffies; | |
497 | ||
498 | /* If we haven't gotten an interrupt in a while, check the BWD link | |
499 | * status bit | |
500 | */ | |
501 | if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { | |
502 | int rc = ntb_link_status(ndev); | |
503 | if (rc) | |
504 | dev_err(&ndev->pdev->dev, | |
505 | "Error determining link status\n"); | |
506 | } | |
507 | ||
508 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
509 | } | |
510 | ||
511 | static int ntb_xeon_setup(struct ntb_device *ndev) | |
512 | { | |
513 | int rc; | |
514 | u8 val; | |
515 | ||
516 | ndev->hw_type = SNB_HW; | |
517 | ||
518 | rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val); | |
519 | if (rc) | |
520 | return rc; | |
521 | ||
522 | switch (val & SNB_PPD_CONN_TYPE) { | |
523 | case NTB_CONN_B2B: | |
524 | ndev->conn_type = NTB_CONN_B2B; | |
525 | break; | |
526 | case NTB_CONN_CLASSIC: | |
527 | case NTB_CONN_RP: | |
528 | default: | |
529 | dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n"); | |
530 | return -EINVAL; | |
531 | } | |
532 | ||
533 | if (val & SNB_PPD_DEV_TYPE) | |
534 | ndev->dev_type = NTB_DEV_DSD; | |
535 | else | |
536 | ndev->dev_type = NTB_DEV_USD; | |
537 | ||
538 | ndev->reg_ofs.pdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; | |
539 | ndev->reg_ofs.pdb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; | |
540 | ndev->reg_ofs.sbar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; | |
541 | ndev->reg_ofs.sbar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; | |
542 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; | |
543 | ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_LINK_STATUS_OFFSET; | |
544 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; | |
545 | ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; | |
546 | ||
547 | if (ndev->conn_type == NTB_CONN_B2B) { | |
548 | ndev->reg_ofs.sdb = ndev->reg_base + SNB_B2B_DOORBELL_OFFSET; | |
549 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_B2B_SPAD_OFFSET; | |
576db18c | 550 | ndev->limits.max_spads = SNB_MAX_B2B_SPADS; |
fce8a7bb JM |
551 | } else { |
552 | ndev->reg_ofs.sdb = ndev->reg_base + SNB_SDOORBELL_OFFSET; | |
553 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET; | |
554 | ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS; | |
555 | } | |
556 | ||
557 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; | |
558 | ndev->limits.msix_cnt = SNB_MSIX_CNT; | |
559 | ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; | |
560 | ||
561 | return 0; | |
562 | } | |
563 | ||
564 | static int ntb_bwd_setup(struct ntb_device *ndev) | |
565 | { | |
566 | int rc; | |
567 | u32 val; | |
568 | ||
569 | ndev->hw_type = BWD_HW; | |
570 | ||
571 | rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); | |
572 | if (rc) | |
573 | return rc; | |
574 | ||
575 | switch ((val & BWD_PPD_CONN_TYPE) >> 8) { | |
576 | case NTB_CONN_B2B: | |
577 | ndev->conn_type = NTB_CONN_B2B; | |
578 | break; | |
579 | case NTB_CONN_RP: | |
580 | default: | |
581 | dev_err(&ndev->pdev->dev, "Only B2B supported at this time\n"); | |
582 | return -EINVAL; | |
583 | } | |
584 | ||
585 | if (val & BWD_PPD_DEV_TYPE) | |
586 | ndev->dev_type = NTB_DEV_DSD; | |
587 | else | |
588 | ndev->dev_type = NTB_DEV_USD; | |
589 | ||
590 | /* Initiate PCI-E link training */ | |
591 | rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET, | |
592 | val | BWD_PPD_INIT_LINK); | |
593 | if (rc) | |
594 | return rc; | |
595 | ||
596 | ndev->reg_ofs.pdb = ndev->reg_base + BWD_PDOORBELL_OFFSET; | |
597 | ndev->reg_ofs.pdb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; | |
598 | ndev->reg_ofs.sbar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; | |
599 | ndev->reg_ofs.sbar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; | |
600 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; | |
601 | ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; | |
602 | ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; | |
603 | ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; | |
604 | ||
605 | if (ndev->conn_type == NTB_CONN_B2B) { | |
606 | ndev->reg_ofs.sdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; | |
607 | ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; | |
608 | ndev->limits.max_spads = BWD_MAX_SPADS; | |
609 | } else { | |
610 | ndev->reg_ofs.sdb = ndev->reg_base + BWD_PDOORBELL_OFFSET; | |
611 | ndev->reg_ofs.spad_write = ndev->reg_base + BWD_SPAD_OFFSET; | |
612 | ndev->limits.max_spads = BWD_MAX_COMPAT_SPADS; | |
613 | } | |
614 | ||
615 | ndev->limits.max_db_bits = BWD_MAX_DB_BITS; | |
616 | ndev->limits.msix_cnt = BWD_MSIX_CNT; | |
617 | ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; | |
618 | ||
619 | /* Since bwd doesn't have a link interrupt, setup a poll timer */ | |
620 | INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); | |
621 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
622 | ||
623 | return 0; | |
624 | } | |
625 | ||
78a61ab7 | 626 | static int ntb_device_setup(struct ntb_device *ndev) |
fce8a7bb JM |
627 | { |
628 | int rc; | |
629 | ||
630 | switch (ndev->pdev->device) { | |
631 | case PCI_DEVICE_ID_INTEL_NTB_2ND_SNB: | |
632 | case PCI_DEVICE_ID_INTEL_NTB_RP_JSF: | |
633 | case PCI_DEVICE_ID_INTEL_NTB_RP_SNB: | |
634 | case PCI_DEVICE_ID_INTEL_NTB_CLASSIC_JSF: | |
635 | case PCI_DEVICE_ID_INTEL_NTB_CLASSIC_SNB: | |
636 | case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: | |
637 | case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: | |
638 | rc = ntb_xeon_setup(ndev); | |
639 | break; | |
640 | case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: | |
641 | rc = ntb_bwd_setup(ndev); | |
642 | break; | |
643 | default: | |
644 | rc = -ENODEV; | |
645 | } | |
646 | ||
21720562 JM |
647 | if (rc) |
648 | return rc; | |
649 | ||
fce8a7bb JM |
650 | /* Enable Bus Master and Memory Space on the secondary side */ |
651 | writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, ndev->reg_ofs.spci_cmd); | |
652 | ||
21720562 | 653 | return 0; |
fce8a7bb JM |
654 | } |
655 | ||
656 | static void ntb_device_free(struct ntb_device *ndev) | |
657 | { | |
658 | if (ndev->hw_type == BWD_HW) | |
659 | cancel_delayed_work_sync(&ndev->hb_timer); | |
660 | } | |
661 | ||
662 | static irqreturn_t bwd_callback_msix_irq(int irq, void *data) | |
663 | { | |
664 | struct ntb_db_cb *db_cb = data; | |
665 | struct ntb_device *ndev = db_cb->ndev; | |
666 | ||
667 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
668 | db_cb->db_num); | |
669 | ||
670 | if (db_cb->callback) | |
671 | db_cb->callback(db_cb->data, db_cb->db_num); | |
672 | ||
673 | /* No need to check for the specific HB irq, any interrupt means | |
674 | * we're connected. | |
675 | */ | |
676 | ndev->last_ts = jiffies; | |
677 | ||
678 | writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.pdb); | |
679 | ||
680 | return IRQ_HANDLED; | |
681 | } | |
682 | ||
683 | static irqreturn_t xeon_callback_msix_irq(int irq, void *data) | |
684 | { | |
685 | struct ntb_db_cb *db_cb = data; | |
686 | struct ntb_device *ndev = db_cb->ndev; | |
687 | ||
688 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
689 | db_cb->db_num); | |
690 | ||
691 | if (db_cb->callback) | |
692 | db_cb->callback(db_cb->data, db_cb->db_num); | |
693 | ||
694 | /* On Sandybridge, there are 16 bits in the interrupt register | |
695 | * but only 4 vectors. So, 5 bits are assigned to the first 3 | |
696 | * vectors, with the 4th having a single bit for link | |
697 | * interrupts. | |
698 | */ | |
699 | writew(((1 << ndev->bits_per_vector) - 1) << | |
700 | (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.pdb); | |
701 | ||
702 | return IRQ_HANDLED; | |
703 | } | |
704 | ||
705 | /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */ | |
706 | static irqreturn_t xeon_event_msix_irq(int irq, void *dev) | |
707 | { | |
708 | struct ntb_device *ndev = dev; | |
709 | int rc; | |
710 | ||
711 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq); | |
712 | ||
713 | rc = ntb_link_status(ndev); | |
714 | if (rc) | |
715 | dev_err(&ndev->pdev->dev, "Error determining link status\n"); | |
716 | ||
717 | /* bit 15 is always the link bit */ | |
718 | writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.pdb); | |
719 | ||
720 | return IRQ_HANDLED; | |
721 | } | |
722 | ||
723 | static irqreturn_t ntb_interrupt(int irq, void *dev) | |
724 | { | |
725 | struct ntb_device *ndev = dev; | |
726 | unsigned int i = 0; | |
727 | ||
728 | if (ndev->hw_type == BWD_HW) { | |
729 | u64 pdb = readq(ndev->reg_ofs.pdb); | |
730 | ||
731 | dev_dbg(&ndev->pdev->dev, "irq %d - pdb = %Lx\n", irq, pdb); | |
732 | ||
733 | while (pdb) { | |
734 | i = __ffs(pdb); | |
735 | pdb &= pdb - 1; | |
736 | bwd_callback_msix_irq(irq, &ndev->db_cb[i]); | |
737 | } | |
738 | } else { | |
739 | u16 pdb = readw(ndev->reg_ofs.pdb); | |
740 | ||
741 | dev_dbg(&ndev->pdev->dev, "irq %d - pdb = %x sdb %x\n", irq, | |
742 | pdb, readw(ndev->reg_ofs.sdb)); | |
743 | ||
744 | if (pdb & SNB_DB_HW_LINK) { | |
745 | xeon_event_msix_irq(irq, dev); | |
746 | pdb &= ~SNB_DB_HW_LINK; | |
747 | } | |
748 | ||
749 | while (pdb) { | |
750 | i = __ffs(pdb); | |
751 | pdb &= pdb - 1; | |
752 | xeon_callback_msix_irq(irq, &ndev->db_cb[i]); | |
753 | } | |
754 | } | |
755 | ||
756 | return IRQ_HANDLED; | |
757 | } | |
758 | ||
759 | static int ntb_setup_msix(struct ntb_device *ndev) | |
760 | { | |
761 | struct pci_dev *pdev = ndev->pdev; | |
762 | struct msix_entry *msix; | |
763 | int msix_entries; | |
764 | int rc, i, pos; | |
765 | u16 val; | |
766 | ||
767 | pos = pci_find_capability(pdev, PCI_CAP_ID_MSIX); | |
768 | if (!pos) { | |
769 | rc = -EIO; | |
770 | goto err; | |
771 | } | |
772 | ||
773 | rc = pci_read_config_word(pdev, pos + PCI_MSIX_FLAGS, &val); | |
774 | if (rc) | |
775 | goto err; | |
776 | ||
777 | msix_entries = msix_table_size(val); | |
778 | if (msix_entries > ndev->limits.msix_cnt) { | |
779 | rc = -EINVAL; | |
780 | goto err; | |
781 | } | |
782 | ||
783 | ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, | |
784 | GFP_KERNEL); | |
785 | if (!ndev->msix_entries) { | |
786 | rc = -ENOMEM; | |
787 | goto err; | |
788 | } | |
789 | ||
790 | for (i = 0; i < msix_entries; i++) | |
791 | ndev->msix_entries[i].entry = i; | |
792 | ||
793 | rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries); | |
794 | if (rc < 0) | |
795 | goto err1; | |
796 | if (rc > 0) { | |
797 | /* On SNB, the link interrupt is always tied to 4th vector. If | |
798 | * we can't get all 4, then we can't use MSI-X. | |
799 | */ | |
800 | if (ndev->hw_type != BWD_HW) { | |
801 | rc = -EIO; | |
802 | goto err1; | |
803 | } | |
804 | ||
805 | dev_warn(&pdev->dev, | |
806 | "Only %d MSI-X vectors. Limiting the number of queues to that number.\n", | |
807 | rc); | |
808 | msix_entries = rc; | |
809 | } | |
810 | ||
811 | for (i = 0; i < msix_entries; i++) { | |
812 | msix = &ndev->msix_entries[i]; | |
813 | WARN_ON(!msix->vector); | |
814 | ||
815 | /* Use the last MSI-X vector for Link status */ | |
816 | if (ndev->hw_type == BWD_HW) { | |
817 | rc = request_irq(msix->vector, bwd_callback_msix_irq, 0, | |
818 | "ntb-callback-msix", &ndev->db_cb[i]); | |
819 | if (rc) | |
820 | goto err2; | |
821 | } else { | |
822 | if (i == msix_entries - 1) { | |
823 | rc = request_irq(msix->vector, | |
824 | xeon_event_msix_irq, 0, | |
825 | "ntb-event-msix", ndev); | |
826 | if (rc) | |
827 | goto err2; | |
828 | } else { | |
829 | rc = request_irq(msix->vector, | |
830 | xeon_callback_msix_irq, 0, | |
831 | "ntb-callback-msix", | |
832 | &ndev->db_cb[i]); | |
833 | if (rc) | |
834 | goto err2; | |
835 | } | |
836 | } | |
837 | } | |
838 | ||
839 | ndev->num_msix = msix_entries; | |
840 | if (ndev->hw_type == BWD_HW) | |
841 | ndev->max_cbs = msix_entries; | |
842 | else | |
843 | ndev->max_cbs = msix_entries - 1; | |
844 | ||
845 | return 0; | |
846 | ||
847 | err2: | |
848 | while (--i >= 0) { | |
849 | msix = &ndev->msix_entries[i]; | |
850 | if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1) | |
851 | free_irq(msix->vector, ndev); | |
852 | else | |
853 | free_irq(msix->vector, &ndev->db_cb[i]); | |
854 | } | |
855 | pci_disable_msix(pdev); | |
856 | err1: | |
857 | kfree(ndev->msix_entries); | |
858 | dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n"); | |
859 | err: | |
860 | ndev->num_msix = 0; | |
861 | return rc; | |
862 | } | |
863 | ||
864 | static int ntb_setup_msi(struct ntb_device *ndev) | |
865 | { | |
866 | struct pci_dev *pdev = ndev->pdev; | |
867 | int rc; | |
868 | ||
869 | rc = pci_enable_msi(pdev); | |
870 | if (rc) | |
871 | return rc; | |
872 | ||
873 | rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); | |
874 | if (rc) { | |
875 | pci_disable_msi(pdev); | |
876 | dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); | |
877 | return rc; | |
878 | } | |
879 | ||
880 | return 0; | |
881 | } | |
882 | ||
883 | static int ntb_setup_intx(struct ntb_device *ndev) | |
884 | { | |
885 | struct pci_dev *pdev = ndev->pdev; | |
886 | int rc; | |
887 | ||
888 | pci_msi_off(pdev); | |
889 | ||
890 | /* Verify intx is enabled */ | |
891 | pci_intx(pdev, 1); | |
892 | ||
893 | rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx", | |
894 | ndev); | |
895 | if (rc) | |
896 | return rc; | |
897 | ||
898 | return 0; | |
899 | } | |
900 | ||
78a61ab7 | 901 | static int ntb_setup_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
902 | { |
903 | int rc; | |
904 | ||
905 | /* On BWD, disable all interrupts. On SNB, disable all but Link | |
906 | * Interrupt. The rest will be unmasked as callbacks are registered. | |
907 | */ | |
908 | if (ndev->hw_type == BWD_HW) | |
909 | writeq(~0, ndev->reg_ofs.pdb_mask); | |
910 | else | |
911 | writew(~(1 << ndev->limits.max_db_bits), | |
912 | ndev->reg_ofs.pdb_mask); | |
913 | ||
914 | rc = ntb_setup_msix(ndev); | |
915 | if (!rc) | |
916 | goto done; | |
917 | ||
918 | ndev->bits_per_vector = 1; | |
919 | ndev->max_cbs = ndev->limits.max_db_bits; | |
920 | ||
921 | rc = ntb_setup_msi(ndev); | |
922 | if (!rc) | |
923 | goto done; | |
924 | ||
925 | rc = ntb_setup_intx(ndev); | |
926 | if (rc) { | |
927 | dev_err(&ndev->pdev->dev, "no usable interrupts\n"); | |
928 | return rc; | |
929 | } | |
930 | ||
931 | done: | |
932 | return 0; | |
933 | } | |
934 | ||
78a61ab7 | 935 | static void ntb_free_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
936 | { |
937 | struct pci_dev *pdev = ndev->pdev; | |
938 | ||
939 | /* mask interrupts */ | |
940 | if (ndev->hw_type == BWD_HW) | |
941 | writeq(~0, ndev->reg_ofs.pdb_mask); | |
942 | else | |
943 | writew(~0, ndev->reg_ofs.pdb_mask); | |
944 | ||
945 | if (ndev->num_msix) { | |
946 | struct msix_entry *msix; | |
947 | u32 i; | |
948 | ||
949 | for (i = 0; i < ndev->num_msix; i++) { | |
950 | msix = &ndev->msix_entries[i]; | |
951 | if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1) | |
952 | free_irq(msix->vector, ndev); | |
953 | else | |
954 | free_irq(msix->vector, &ndev->db_cb[i]); | |
955 | } | |
956 | pci_disable_msix(pdev); | |
957 | } else { | |
958 | free_irq(pdev->irq, ndev); | |
959 | ||
960 | if (pci_dev_msi_enabled(pdev)) | |
961 | pci_disable_msi(pdev); | |
962 | } | |
963 | } | |
964 | ||
78a61ab7 | 965 | static int ntb_create_callbacks(struct ntb_device *ndev) |
fce8a7bb JM |
966 | { |
967 | int i; | |
968 | ||
969 | /* Checken-egg issue. We won't know how many callbacks are necessary | |
970 | * until we see how many MSI-X vectors we get, but these pointers need | |
971 | * to be passed into the MSI-X register fucntion. So, we allocate the | |
972 | * max, knowing that they might not all be used, to work around this. | |
973 | */ | |
974 | ndev->db_cb = kcalloc(ndev->limits.max_db_bits, | |
975 | sizeof(struct ntb_db_cb), | |
976 | GFP_KERNEL); | |
977 | if (!ndev->db_cb) | |
978 | return -ENOMEM; | |
979 | ||
980 | for (i = 0; i < ndev->limits.max_db_bits; i++) { | |
981 | ndev->db_cb[i].db_num = i; | |
982 | ndev->db_cb[i].ndev = ndev; | |
983 | } | |
984 | ||
985 | return 0; | |
986 | } | |
987 | ||
988 | static void ntb_free_callbacks(struct ntb_device *ndev) | |
989 | { | |
990 | int i; | |
991 | ||
992 | for (i = 0; i < ndev->limits.max_db_bits; i++) | |
993 | ntb_unregister_db_callback(ndev, i); | |
994 | ||
995 | kfree(ndev->db_cb); | |
996 | } | |
997 | ||
78a61ab7 | 998 | static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
fce8a7bb JM |
999 | { |
1000 | struct ntb_device *ndev; | |
1001 | int rc, i; | |
1002 | ||
1003 | ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); | |
1004 | if (!ndev) | |
1005 | return -ENOMEM; | |
1006 | ||
1007 | ndev->pdev = pdev; | |
1008 | ndev->link_status = NTB_LINK_DOWN; | |
1009 | pci_set_drvdata(pdev, ndev); | |
1010 | ||
1011 | rc = pci_enable_device(pdev); | |
1012 | if (rc) | |
1013 | goto err; | |
1014 | ||
1015 | pci_set_master(ndev->pdev); | |
1016 | ||
1017 | rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME); | |
1018 | if (rc) | |
1019 | goto err1; | |
1020 | ||
1021 | ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); | |
1022 | if (!ndev->reg_base) { | |
1023 | dev_warn(&pdev->dev, "Cannot remap BAR 0\n"); | |
1024 | rc = -EIO; | |
1025 | goto err2; | |
1026 | } | |
1027 | ||
1028 | for (i = 0; i < NTB_NUM_MW; i++) { | |
1029 | ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i)); | |
1030 | ndev->mw[i].vbase = | |
1031 | ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)), | |
1032 | ndev->mw[i].bar_sz); | |
113fc505 JM |
1033 | dev_info(&pdev->dev, "MW %d size %llu\n", i, |
1034 | pci_resource_len(pdev, MW_TO_BAR(i))); | |
fce8a7bb JM |
1035 | if (!ndev->mw[i].vbase) { |
1036 | dev_warn(&pdev->dev, "Cannot remap BAR %d\n", | |
1037 | MW_TO_BAR(i)); | |
1038 | rc = -EIO; | |
1039 | goto err3; | |
1040 | } | |
1041 | } | |
1042 | ||
1043 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1044 | if (rc) { | |
1045 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1046 | if (rc) | |
1047 | goto err3; | |
1048 | ||
1049 | dev_warn(&pdev->dev, "Cannot DMA highmem\n"); | |
1050 | } | |
1051 | ||
1052 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1053 | if (rc) { | |
1054 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1055 | if (rc) | |
1056 | goto err3; | |
1057 | ||
1058 | dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); | |
1059 | } | |
1060 | ||
1061 | rc = ntb_device_setup(ndev); | |
1062 | if (rc) | |
1063 | goto err3; | |
1064 | ||
1065 | rc = ntb_create_callbacks(ndev); | |
1066 | if (rc) | |
1067 | goto err4; | |
1068 | ||
1069 | rc = ntb_setup_interrupts(ndev); | |
1070 | if (rc) | |
1071 | goto err5; | |
1072 | ||
1073 | /* The scratchpad registers keep the values between rmmod/insmod, | |
1074 | * blast them now | |
1075 | */ | |
1076 | for (i = 0; i < ndev->limits.max_spads; i++) { | |
1077 | ntb_write_local_spad(ndev, i, 0); | |
1078 | ntb_write_remote_spad(ndev, i, 0); | |
1079 | } | |
1080 | ||
1081 | rc = ntb_transport_init(pdev); | |
1082 | if (rc) | |
1083 | goto err6; | |
1084 | ||
1085 | /* Let's bring the NTB link up */ | |
1086 | writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP, | |
1087 | ndev->reg_ofs.lnk_cntl); | |
1088 | ||
1089 | return 0; | |
1090 | ||
1091 | err6: | |
1092 | ntb_free_interrupts(ndev); | |
1093 | err5: | |
1094 | ntb_free_callbacks(ndev); | |
1095 | err4: | |
1096 | ntb_device_free(ndev); | |
1097 | err3: | |
1098 | for (i--; i >= 0; i--) | |
1099 | iounmap(ndev->mw[i].vbase); | |
1100 | iounmap(ndev->reg_base); | |
1101 | err2: | |
1102 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
1103 | err1: | |
1104 | pci_disable_device(pdev); | |
1105 | err: | |
1106 | kfree(ndev); | |
1107 | ||
1108 | dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME); | |
1109 | return rc; | |
1110 | } | |
1111 | ||
78a61ab7 | 1112 | static void ntb_pci_remove(struct pci_dev *pdev) |
fce8a7bb JM |
1113 | { |
1114 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
1115 | int i; | |
1116 | u32 ntb_cntl; | |
1117 | ||
1118 | /* Bring NTB link down */ | |
1119 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
1120 | ntb_cntl |= NTB_LINK_DISABLE; | |
1121 | writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); | |
1122 | ||
1123 | ntb_transport_free(ndev->ntb_transport); | |
1124 | ||
1125 | ntb_free_interrupts(ndev); | |
1126 | ntb_free_callbacks(ndev); | |
1127 | ntb_device_free(ndev); | |
1128 | ||
1129 | for (i = 0; i < NTB_NUM_MW; i++) | |
1130 | iounmap(ndev->mw[i].vbase); | |
1131 | ||
1132 | iounmap(ndev->reg_base); | |
1133 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
1134 | pci_disable_device(pdev); | |
1135 | kfree(ndev); | |
1136 | } | |
1137 | ||
1138 | static struct pci_driver ntb_pci_driver = { | |
1139 | .name = KBUILD_MODNAME, | |
1140 | .id_table = ntb_pci_tbl, | |
1141 | .probe = ntb_pci_probe, | |
78a61ab7 | 1142 | .remove = ntb_pci_remove, |
fce8a7bb JM |
1143 | }; |
1144 | module_pci_driver(ntb_pci_driver); |