2 * iSCSI Initiator over iSER Data-Path
4 * Copyright (C) 2004 Dmitry Yusupov
5 * Copyright (C) 2004 Alex Aizman
6 * Copyright (C) 2005 Mike Christie
7 * Copyright (c) 2005, 2006 Voltaire, Inc. All rights reserved.
8 * maintained by openib-general@openib.org
10 * This software is available to you under a choice of one of two
11 * licenses. You may choose to be licensed under the terms of the GNU
12 * General Public License (GPL) Version 2, available from the file
13 * COPYING in the main directory of this source tree, or the
14 * OpenIB.org BSD license below:
16 * Redistribution and use in source and binary forms, with or
17 * without modification, are permitted provided that the following
20 * - Redistributions of source code must retain the above
21 * copyright notice, this list of conditions and the following
24 * - Redistributions in binary form must reproduce the above
25 * copyright notice, this list of conditions and the following
26 * disclaimer in the documentation and/or other materials
27 * provided with the distribution.
29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
47 #include <linux/types.h>
48 #include <linux/list.h>
49 #include <linux/hardirq.h>
50 #include <linux/kfifo.h>
51 #include <linux/blkdev.h>
52 #include <linux/init.h>
53 #include <linux/ioctl.h>
54 #include <linux/cdev.h>
56 #include <linux/net.h>
57 #include <linux/scatterlist.h>
58 #include <linux/delay.h>
62 #include <asm/uaccess.h>
64 #include <scsi/scsi_cmnd.h>
65 #include <scsi/scsi_device.h>
66 #include <scsi/scsi_eh.h>
67 #include <scsi/scsi_tcq.h>
68 #include <scsi/scsi_host.h>
69 #include <scsi/scsi.h>
70 #include <scsi/scsi_transport_iscsi.h>
72 #include "iscsi_iser.h"
74 static struct scsi_host_template iscsi_iser_sht
;
75 static struct iscsi_transport iscsi_iser_transport
;
76 static struct scsi_transport_template
*iscsi_iser_scsi_transport
;
78 static unsigned int iscsi_max_lun
= 512;
79 module_param_named(max_lun
, iscsi_max_lun
, uint
, S_IRUGO
);
81 int iser_debug_level
= 0;
83 MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover "
84 "v" DRV_VER
" (" DRV_DATE
")");
85 MODULE_LICENSE("Dual BSD/GPL");
86 MODULE_AUTHOR("Alex Nezhinsky, Dan Bar Dov, Or Gerlitz");
88 module_param_named(debug_level
, iser_debug_level
, int, 0644);
89 MODULE_PARM_DESC(debug_level
, "Enable debug tracing if > 0 (default:disabled)");
91 struct iser_global ig
;
94 iscsi_iser_recv(struct iscsi_conn
*conn
,
95 struct iscsi_hdr
*hdr
, char *rx_data
, int rx_data_len
)
101 /* verify PDU length */
102 datalen
= ntoh24(hdr
->dlength
);
103 if (datalen
!= rx_data_len
) {
104 printk(KERN_ERR
"iscsi_iser: datalen %d (hdr) != %d (IB) \n",
105 datalen
, rx_data_len
);
106 rc
= ISCSI_ERR_DATALEN
;
111 ahslen
= hdr
->hlength
* 4;
113 rc
= iscsi_complete_pdu(conn
, hdr
, rx_data
, rx_data_len
);
114 if (rc
&& rc
!= ISCSI_ERR_NO_SCSI_CMD
)
119 iscsi_conn_failure(conn
, rc
);
122 static int iscsi_iser_pdu_alloc(struct iscsi_task
*task
, uint8_t opcode
)
124 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
126 task
->hdr
= (struct iscsi_hdr
*)&iser_task
->desc
.iscsi_header
;
127 task
->hdr_max
= sizeof(iser_task
->desc
.iscsi_header
);
132 * iscsi_iser_task_init - Initialize task
135 * Initialize the task for the scsi command or mgmt command.
138 iscsi_iser_task_init(struct iscsi_task
*task
)
140 struct iscsi_iser_conn
*iser_conn
= task
->conn
->dd_data
;
141 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
145 iser_task
->desc
.data
= task
->data
;
149 iser_task
->command_sent
= 0;
150 iser_task
->iser_conn
= iser_conn
;
151 iser_task_rdma_init(iser_task
);
156 * iscsi_iser_mtask_xmit - xmit management(immediate) task
157 * @conn: iscsi connection
158 * @task: task management task
161 * The function can return -EAGAIN in which case caller must
162 * call it again later, or recover. '0' return code means successful
167 iscsi_iser_mtask_xmit(struct iscsi_conn
*conn
, struct iscsi_task
*task
)
171 iser_dbg("task deq [cid %d itt 0x%x]\n", conn
->id
, task
->itt
);
173 error
= iser_send_control(conn
, task
);
175 /* since iser xmits control with zero copy, tasks can not be recycled
176 * right after sending them.
177 * The recycling scheme is based on whether a response is expected
178 * - if yes, the task is recycled at iscsi_complete_pdu
179 * - if no, the task is recycled at iser_snd_completion
181 if (error
&& error
!= -ENOBUFS
)
182 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
188 iscsi_iser_task_xmit_unsol_data(struct iscsi_conn
*conn
,
189 struct iscsi_task
*task
)
191 struct iscsi_r2t_info
*r2t
= &task
->unsol_r2t
;
192 struct iscsi_data hdr
;
195 /* Send data-out PDUs while there's still unsolicited data to send */
196 while (iscsi_task_has_unsol_data(task
)) {
197 iscsi_prep_data_out_pdu(task
, r2t
, &hdr
);
198 iser_dbg("Sending data-out: itt 0x%x, data count %d\n",
199 hdr
.itt
, r2t
->data_count
);
201 /* the buffer description has been passed with the command */
202 /* Send the command */
203 error
= iser_send_data_out(conn
, task
, &hdr
);
206 goto iscsi_iser_task_xmit_unsol_data_exit
;
208 r2t
->sent
+= r2t
->data_count
;
209 iser_dbg("Need to send %d more as data-out PDUs\n",
210 r2t
->data_length
- r2t
->sent
);
213 iscsi_iser_task_xmit_unsol_data_exit
:
218 iscsi_iser_task_xmit(struct iscsi_task
*task
)
220 struct iscsi_conn
*conn
= task
->conn
;
221 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
225 return iscsi_iser_mtask_xmit(conn
, task
);
227 if (task
->sc
->sc_data_direction
== DMA_TO_DEVICE
) {
228 BUG_ON(scsi_bufflen(task
->sc
) == 0);
230 iser_dbg("cmd [itt %x total %d imm %d unsol_data %d\n",
231 task
->itt
, scsi_bufflen(task
->sc
),
232 task
->imm_count
, task
->unsol_r2t
.data_length
);
235 iser_dbg("task deq [cid %d itt 0x%x]\n",
236 conn
->id
, task
->itt
);
238 /* Send the cmd PDU */
239 if (!iser_task
->command_sent
) {
240 error
= iser_send_command(conn
, task
);
242 goto iscsi_iser_task_xmit_exit
;
243 iser_task
->command_sent
= 1;
246 /* Send unsolicited data-out PDU(s) if necessary */
247 if (iscsi_task_has_unsol_data(task
))
248 error
= iscsi_iser_task_xmit_unsol_data(conn
, task
);
250 iscsi_iser_task_xmit_exit
:
251 if (error
&& error
!= -ENOBUFS
)
252 iscsi_conn_failure(conn
, ISCSI_ERR_CONN_FAILED
);
256 static void iscsi_iser_cleanup_task(struct iscsi_task
*task
)
258 struct iscsi_iser_task
*iser_task
= task
->dd_data
;
261 * mgmt tasks do not need special cleanup and we do not
262 * allocate anything in the init task callout
264 if (!task
->sc
|| task
->state
== ISCSI_TASK_PENDING
)
267 if (iser_task
->status
== ISER_TASK_STATUS_STARTED
) {
268 iser_task
->status
= ISER_TASK_STATUS_COMPLETED
;
269 iser_task_rdma_finalize(iser_task
);
273 static struct iscsi_cls_conn
*
274 iscsi_iser_conn_create(struct iscsi_cls_session
*cls_session
, uint32_t conn_idx
)
276 struct iscsi_conn
*conn
;
277 struct iscsi_cls_conn
*cls_conn
;
278 struct iscsi_iser_conn
*iser_conn
;
280 cls_conn
= iscsi_conn_setup(cls_session
, sizeof(*iser_conn
), conn_idx
);
283 conn
= cls_conn
->dd_data
;
286 * due to issues with the login code re iser sematics
287 * this not set in iscsi_conn_setup - FIXME
289 conn
->max_recv_dlength
= 128;
291 iser_conn
= conn
->dd_data
;
292 conn
->dd_data
= iser_conn
;
293 iser_conn
->iscsi_conn
= conn
;
299 iscsi_iser_conn_destroy(struct iscsi_cls_conn
*cls_conn
)
301 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
302 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
303 struct iser_conn
*ib_conn
= iser_conn
->ib_conn
;
305 iscsi_conn_teardown(cls_conn
);
307 * Userspace will normally call the stop callback and
308 * already have freed the ib_conn, but if it goofed up then
312 ib_conn
->iser_conn
= NULL
;
313 iser_conn_put(ib_conn
);
318 iscsi_iser_conn_bind(struct iscsi_cls_session
*cls_session
,
319 struct iscsi_cls_conn
*cls_conn
, uint64_t transport_eph
,
322 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
323 struct iscsi_iser_conn
*iser_conn
;
324 struct iser_conn
*ib_conn
;
325 struct iscsi_endpoint
*ep
;
328 error
= iscsi_conn_bind(cls_session
, cls_conn
, is_leading
);
332 /* the transport ep handle comes from user space so it must be
333 * verified against the global ib connections list */
334 ep
= iscsi_lookup_endpoint(transport_eph
);
336 iser_err("can't bind eph %llx\n",
337 (unsigned long long)transport_eph
);
340 ib_conn
= ep
->dd_data
;
342 /* binds the iSER connection retrieved from the previously
343 * connected ep_handle to the iSCSI layer connection. exchanges
344 * connection pointers */
345 iser_err("binding iscsi conn %p to iser_conn %p\n",conn
,ib_conn
);
346 iser_conn
= conn
->dd_data
;
347 ib_conn
->iser_conn
= iser_conn
;
348 iser_conn
->ib_conn
= ib_conn
;
349 iser_conn_get(ib_conn
);
354 iscsi_iser_conn_stop(struct iscsi_cls_conn
*cls_conn
, int flag
)
356 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
357 struct iscsi_iser_conn
*iser_conn
= conn
->dd_data
;
358 struct iser_conn
*ib_conn
= iser_conn
->ib_conn
;
361 * Userspace may have goofed up and not bound the connection or
362 * might have only partially setup the connection.
365 iscsi_conn_stop(cls_conn
, flag
);
367 * There is no unbind event so the stop callback
368 * must release the ref from the bind.
370 iser_conn_put(ib_conn
);
372 iser_conn
->ib_conn
= NULL
;
376 iscsi_iser_conn_start(struct iscsi_cls_conn
*cls_conn
)
378 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
381 err
= iser_conn_set_full_featured_mode(conn
);
385 return iscsi_conn_start(cls_conn
);
388 static void iscsi_iser_session_destroy(struct iscsi_cls_session
*cls_session
)
390 struct Scsi_Host
*shost
= iscsi_session_to_shost(cls_session
);
392 iscsi_session_teardown(cls_session
);
393 iscsi_host_remove(shost
);
394 iscsi_host_free(shost
);
397 static struct iscsi_cls_session
*
398 iscsi_iser_session_create(struct iscsi_endpoint
*ep
,
399 uint16_t cmds_max
, uint16_t qdepth
,
400 uint32_t initial_cmdsn
)
402 struct iscsi_cls_session
*cls_session
;
403 struct iscsi_session
*session
;
404 struct Scsi_Host
*shost
;
405 struct iser_conn
*ib_conn
;
407 shost
= iscsi_host_alloc(&iscsi_iser_sht
, 0, 1);
410 shost
->transportt
= iscsi_iser_scsi_transport
;
411 shost
->max_lun
= iscsi_max_lun
;
413 shost
->max_channel
= 0;
414 shost
->max_cmd_len
= 16;
417 * older userspace tools (before 2.0-870) did not pass us
418 * the leading conn's ep so this will be NULL;
421 ib_conn
= ep
->dd_data
;
423 if (iscsi_host_add(shost
,
424 ep
? ib_conn
->device
->ib_device
->dma_device
: NULL
))
428 * we do not support setting can_queue cmd_per_lun from userspace yet
429 * because we preallocate so many resources
431 cls_session
= iscsi_session_setup(&iscsi_iser_transport
, shost
,
432 ISCSI_DEF_XMIT_CMDS_MAX
,
433 sizeof(struct iscsi_iser_task
),
437 session
= cls_session
->dd_data
;
439 shost
->can_queue
= session
->scsi_cmds_max
;
443 iscsi_host_remove(shost
);
445 iscsi_host_free(shost
);
450 iscsi_iser_set_param(struct iscsi_cls_conn
*cls_conn
,
451 enum iscsi_param param
, char *buf
, int buflen
)
456 case ISCSI_PARAM_MAX_RECV_DLENGTH
:
459 case ISCSI_PARAM_HDRDGST_EN
:
460 sscanf(buf
, "%d", &value
);
462 printk(KERN_ERR
"DataDigest wasn't negotiated to None");
466 case ISCSI_PARAM_DATADGST_EN
:
467 sscanf(buf
, "%d", &value
);
469 printk(KERN_ERR
"DataDigest wasn't negotiated to None");
473 case ISCSI_PARAM_IFMARKER_EN
:
474 sscanf(buf
, "%d", &value
);
476 printk(KERN_ERR
"IFMarker wasn't negotiated to No");
480 case ISCSI_PARAM_OFMARKER_EN
:
481 sscanf(buf
, "%d", &value
);
483 printk(KERN_ERR
"OFMarker wasn't negotiated to No");
488 return iscsi_set_param(cls_conn
, param
, buf
, buflen
);
495 iscsi_iser_conn_get_stats(struct iscsi_cls_conn
*cls_conn
, struct iscsi_stats
*stats
)
497 struct iscsi_conn
*conn
= cls_conn
->dd_data
;
499 stats
->txdata_octets
= conn
->txdata_octets
;
500 stats
->rxdata_octets
= conn
->rxdata_octets
;
501 stats
->scsicmd_pdus
= conn
->scsicmd_pdus_cnt
;
502 stats
->dataout_pdus
= conn
->dataout_pdus_cnt
;
503 stats
->scsirsp_pdus
= conn
->scsirsp_pdus_cnt
;
504 stats
->datain_pdus
= conn
->datain_pdus_cnt
; /* always 0 */
505 stats
->r2t_pdus
= conn
->r2t_pdus_cnt
; /* always 0 */
506 stats
->tmfcmd_pdus
= conn
->tmfcmd_pdus_cnt
;
507 stats
->tmfrsp_pdus
= conn
->tmfrsp_pdus_cnt
;
508 stats
->custom_length
= 4;
509 strcpy(stats
->custom
[0].desc
, "qp_tx_queue_full");
510 stats
->custom
[0].value
= 0; /* TB iser_conn->qp_tx_queue_full; */
511 strcpy(stats
->custom
[1].desc
, "fmr_map_not_avail");
512 stats
->custom
[1].value
= 0; /* TB iser_conn->fmr_map_not_avail */;
513 strcpy(stats
->custom
[2].desc
, "eh_abort_cnt");
514 stats
->custom
[2].value
= conn
->eh_abort_cnt
;
515 strcpy(stats
->custom
[3].desc
, "fmr_unalign_cnt");
516 stats
->custom
[3].value
= conn
->fmr_unalign_cnt
;
519 static struct iscsi_endpoint
*
520 iscsi_iser_ep_connect(struct sockaddr
*dst_addr
, int non_blocking
)
523 struct iser_conn
*ib_conn
;
524 struct iscsi_endpoint
*ep
;
526 ep
= iscsi_create_endpoint(sizeof(*ib_conn
));
528 return ERR_PTR(-ENOMEM
);
530 ib_conn
= ep
->dd_data
;
532 iser_conn_init(ib_conn
);
534 err
= iser_connect(ib_conn
, NULL
, (struct sockaddr_in
*)dst_addr
,
537 iscsi_destroy_endpoint(ep
);
544 iscsi_iser_ep_poll(struct iscsi_endpoint
*ep
, int timeout_ms
)
546 struct iser_conn
*ib_conn
;
549 ib_conn
= ep
->dd_data
;
550 rc
= wait_event_interruptible_timeout(ib_conn
->wait
,
551 ib_conn
->state
== ISER_CONN_UP
,
552 msecs_to_jiffies(timeout_ms
));
554 /* if conn establishment failed, return error code to iscsi */
556 (ib_conn
->state
== ISER_CONN_TERMINATING
||
557 ib_conn
->state
== ISER_CONN_DOWN
))
560 iser_err("ib conn %p rc = %d\n", ib_conn
, rc
);
563 return 1; /* success, this is the equivalent of POLLOUT */
565 return 0; /* timeout */
567 return rc
; /* signal */
571 iscsi_iser_ep_disconnect(struct iscsi_endpoint
*ep
)
573 struct iser_conn
*ib_conn
;
575 ib_conn
= ep
->dd_data
;
576 if (ib_conn
->iser_conn
)
578 * Must suspend xmit path if the ep is bound to the
579 * iscsi_conn, so we know we are not accessing the ib_conn
582 * This may not be bound if the ep poll failed.
584 iscsi_suspend_tx(ib_conn
->iser_conn
->iscsi_conn
);
587 iser_err("ib conn %p state %d\n",ib_conn
, ib_conn
->state
);
588 iser_conn_terminate(ib_conn
);
591 static struct scsi_host_template iscsi_iser_sht
= {
592 .module
= THIS_MODULE
,
593 .name
= "iSCSI Initiator over iSER, v." DRV_VER
,
594 .queuecommand
= iscsi_queuecommand
,
595 .change_queue_depth
= iscsi_change_queue_depth
,
596 .sg_tablesize
= ISCSI_ISER_SG_TABLESIZE
,
598 .cmd_per_lun
= ISER_DEF_CMD_PER_LUN
,
599 .eh_abort_handler
= iscsi_eh_abort
,
600 .eh_device_reset_handler
= iscsi_eh_device_reset
,
601 .eh_target_reset_handler
= iscsi_eh_target_reset
,
602 .target_alloc
= iscsi_target_alloc
,
603 .use_clustering
= DISABLE_CLUSTERING
,
604 .proc_name
= "iscsi_iser",
608 static struct iscsi_transport iscsi_iser_transport
= {
609 .owner
= THIS_MODULE
,
611 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
,
612 .param_mask
= ISCSI_MAX_RECV_DLENGTH
|
613 ISCSI_MAX_XMIT_DLENGTH
|
616 ISCSI_INITIAL_R2T_EN
|
621 ISCSI_PDU_INORDER_EN
|
622 ISCSI_DATASEQ_INORDER_EN
|
624 ISCSI_PERSISTENT_PORT
|
625 ISCSI_PERSISTENT_ADDRESS
|
626 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
627 ISCSI_USERNAME
| ISCSI_PASSWORD
|
628 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
629 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
630 ISCSI_PING_TMO
| ISCSI_RECV_TMO
|
631 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
632 .host_param_mask
= ISCSI_HOST_HWADDRESS
|
633 ISCSI_HOST_NETDEV_NAME
|
634 ISCSI_HOST_INITIATOR_NAME
,
635 /* session management */
636 .create_session
= iscsi_iser_session_create
,
637 .destroy_session
= iscsi_iser_session_destroy
,
638 /* connection management */
639 .create_conn
= iscsi_iser_conn_create
,
640 .bind_conn
= iscsi_iser_conn_bind
,
641 .destroy_conn
= iscsi_iser_conn_destroy
,
642 .set_param
= iscsi_iser_set_param
,
643 .get_conn_param
= iscsi_conn_get_param
,
644 .get_session_param
= iscsi_session_get_param
,
645 .start_conn
= iscsi_iser_conn_start
,
646 .stop_conn
= iscsi_iser_conn_stop
,
647 /* iscsi host params */
648 .get_host_param
= iscsi_host_get_param
,
649 .set_host_param
= iscsi_host_set_param
,
651 .send_pdu
= iscsi_conn_send_pdu
,
652 .get_stats
= iscsi_iser_conn_get_stats
,
653 .init_task
= iscsi_iser_task_init
,
654 .xmit_task
= iscsi_iser_task_xmit
,
655 .cleanup_task
= iscsi_iser_cleanup_task
,
656 .alloc_pdu
= iscsi_iser_pdu_alloc
,
658 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
660 .ep_connect
= iscsi_iser_ep_connect
,
661 .ep_poll
= iscsi_iser_ep_poll
,
662 .ep_disconnect
= iscsi_iser_ep_disconnect
665 static int __init
iser_init(void)
669 iser_dbg("Starting iSER datamover...\n");
671 if (iscsi_max_lun
< 1) {
672 printk(KERN_ERR
"Invalid max_lun value of %u\n", iscsi_max_lun
);
676 memset(&ig
, 0, sizeof(struct iser_global
));
678 ig
.desc_cache
= kmem_cache_create("iser_descriptors",
679 sizeof (struct iser_desc
),
680 0, SLAB_HWCACHE_ALIGN
,
682 if (ig
.desc_cache
== NULL
)
685 /* device init is called only after the first addr resolution */
686 mutex_init(&ig
.device_list_mutex
);
687 INIT_LIST_HEAD(&ig
.device_list
);
688 mutex_init(&ig
.connlist_mutex
);
689 INIT_LIST_HEAD(&ig
.connlist
);
691 iscsi_iser_scsi_transport
= iscsi_register_transport(
692 &iscsi_iser_transport
);
693 if (!iscsi_iser_scsi_transport
) {
694 iser_err("iscsi_register_transport failed\n");
696 goto register_transport_failure
;
701 register_transport_failure
:
702 kmem_cache_destroy(ig
.desc_cache
);
707 static void __exit
iser_exit(void)
709 iser_dbg("Removing iSER datamover...\n");
710 iscsi_unregister_transport(&iscsi_iser_transport
);
711 kmem_cache_destroy(ig
.desc_cache
);
714 module_init(iser_init
);
715 module_exit(iser_exit
);