projects / GitHub / mt8127 / android_kernel_alcatel_ttab.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iscsi-target: Refactor RX PDU logic + export request PDU handling
[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / target / iscsi / iscsi_target.c
CommitLineData
e48354ce
NB		 1/*******************************************************************************
2 * This file contains main functions related to the iSCSI Target Core Driver.
3 *
4 * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
5 *
6 * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
7 *
8 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 ******************************************************************************/
20
21#include <linux/string.h>
22#include <linux/kthread.h>
23#include <linux/crypto.h>
24#include <linux/completion.h>
827509e3 25#include <linux/module.h>
40401530 26#include <linux/idr.h>
e48354ce
NB		 27#include <asm/unaligned.h>
28#include <scsi/scsi_device.h>
29#include <scsi/iscsi_proto.h>
d28b1169 30#include <scsi/scsi_tcq.h>
e48354ce 31#include <target/target_core_base.h>
c4795fb2 32#include <target/target_core_fabric.h>
d28b1169 33#include <target/target_core_configfs.h>
e48354ce
NB		 34
35#include "iscsi_target_core.h"
36#include "iscsi_target_parameters.h"
37#include "iscsi_target_seq_pdu_list.h"
38#include "iscsi_target_tq.h"
39#include "iscsi_target_configfs.h"
40#include "iscsi_target_datain_values.h"
41#include "iscsi_target_erl0.h"
42#include "iscsi_target_erl1.h"
43#include "iscsi_target_erl2.h"
44#include "iscsi_target_login.h"
45#include "iscsi_target_tmr.h"
46#include "iscsi_target_tpg.h"
47#include "iscsi_target_util.h"
48#include "iscsi_target.h"
49#include "iscsi_target_device.h"
50#include "iscsi_target_stat.h"
51
baa4d64b
NB		 52#include <target/iscsi/iscsi_transport.h>
53
e48354ce
NB		 54static LIST_HEAD(g_tiqn_list);
55static LIST_HEAD(g_np_list);
56static DEFINE_SPINLOCK(tiqn_lock);
57static DEFINE_SPINLOCK(np_lock);
58
59static struct idr tiqn_idr;
60struct idr sess_idr;
61struct mutex auth_id_lock;
62spinlock_t sess_idr_lock;
63
64struct iscsit_global *iscsit_global;
65
66struct kmem_cache *lio_cmd_cache;
67struct kmem_cache *lio_qr_cache;
68struct kmem_cache *lio_dr_cache;
69struct kmem_cache *lio_ooo_cache;
70struct kmem_cache *lio_r2t_cache;
71
72static int iscsit_handle_immediate_data(struct iscsi_cmd *,
73 unsigned char *buf, u32);
74static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
75
76struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
77{
78 struct iscsi_tiqn *tiqn = NULL;
79
80 spin_lock(&tiqn_lock);
81 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
82 if (!strcmp(tiqn->tiqn, buf)) {
83
84 spin_lock(&tiqn->tiqn_state_lock);
85 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
86 tiqn->tiqn_access_count++;
87 spin_unlock(&tiqn->tiqn_state_lock);
88 spin_unlock(&tiqn_lock);
89 return tiqn;
90 }
91 spin_unlock(&tiqn->tiqn_state_lock);
92 }
93 }
94 spin_unlock(&tiqn_lock);
95
96 return NULL;
97}
98
99static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
100{
101 spin_lock(&tiqn->tiqn_state_lock);
102 if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
103 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
104 spin_unlock(&tiqn->tiqn_state_lock);
105 return 0;
106 }
107 spin_unlock(&tiqn->tiqn_state_lock);
108
109 return -1;
110}
111
112void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
113{
114 spin_lock(&tiqn->tiqn_state_lock);
115 tiqn->tiqn_access_count--;
116 spin_unlock(&tiqn->tiqn_state_lock);
117}
118
119/*
120 * Note that IQN formatting is expected to be done in userspace, and
121 * no explict IQN format checks are done here.
122 */
123struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
124{
125 struct iscsi_tiqn *tiqn = NULL;
126 int ret;
127
8f50c7f5 128 if (strlen(buf) >= ISCSI_IQN_LEN) {
e48354ce
NB		 129 pr_err("Target IQN exceeds %d bytes\n",
130 ISCSI_IQN_LEN);
131 return ERR_PTR(-EINVAL);
132 }
133
134 tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
135 if (!tiqn) {
136 pr_err("Unable to allocate struct iscsi_tiqn\n");
137 return ERR_PTR(-ENOMEM);
138 }
139
140 sprintf(tiqn->tiqn, "%s", buf);
141 INIT_LIST_HEAD(&tiqn->tiqn_list);
142 INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
143 spin_lock_init(&tiqn->tiqn_state_lock);
144 spin_lock_init(&tiqn->tiqn_tpg_lock);
145 spin_lock_init(&tiqn->sess_err_stats.lock);
146 spin_lock_init(&tiqn->login_stats.lock);
147 spin_lock_init(&tiqn->logout_stats.lock);
148
e48354ce
NB		 149 tiqn->tiqn_state = TIQN_STATE_ACTIVE;
150
c9365bd0 151 idr_preload(GFP_KERNEL);
e48354ce 152 spin_lock(&tiqn_lock);
c9365bd0
TH		 153
154 ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
e48354ce 155 if (ret < 0) {
c9365bd0 156 pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
e48354ce 157 spin_unlock(&tiqn_lock);
c9365bd0 158 idr_preload_end();
e48354ce
NB		 159 kfree(tiqn);
160 return ERR_PTR(ret);
161 }
c9365bd0 162 tiqn->tiqn_index = ret;
e48354ce 163 list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
c9365bd0 164
e48354ce 165 spin_unlock(&tiqn_lock);
c9365bd0 166 idr_preload_end();
e48354ce
NB		 167
168 pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
169
170 return tiqn;
171
172}
173
174static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
175{
176 /*
177 * Wait for accesses to said struct iscsi_tiqn to end.
178 */
179 spin_lock(&tiqn->tiqn_state_lock);
180 while (tiqn->tiqn_access_count != 0) {
181 spin_unlock(&tiqn->tiqn_state_lock);
182 msleep(10);
183 spin_lock(&tiqn->tiqn_state_lock);
184 }
185 spin_unlock(&tiqn->tiqn_state_lock);
186}
187
188void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
189{
190 /*
191 * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
192 * while holding tiqn->tiqn_state_lock. This means that all subsequent
193 * attempts to access this struct iscsi_tiqn will fail from both transport
194 * fabric and control code paths.
195 */
196 if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
197 pr_err("iscsit_set_tiqn_shutdown() failed\n");
198 return;
199 }
200
201 iscsit_wait_for_tiqn(tiqn);
202
203 spin_lock(&tiqn_lock);
204 list_del(&tiqn->tiqn_list);
205 idr_remove(&tiqn_idr, tiqn->tiqn_index);
206 spin_unlock(&tiqn_lock);
207
208 pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
209 tiqn->tiqn);
210 kfree(tiqn);
211}
212
213int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
214{
215 int ret;
216 /*
217 * Determine if the network portal is accepting storage traffic.
218 */
219 spin_lock_bh(&np->np_thread_lock);
220 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
221 spin_unlock_bh(&np->np_thread_lock);
222 return -1;
223 }
224 if (np->np_login_tpg) {
225 pr_err("np->np_login_tpg() is not NULL!\n");
226 spin_unlock_bh(&np->np_thread_lock);
227 return -1;
228 }
229 spin_unlock_bh(&np->np_thread_lock);
230 /*
231 * Determine if the portal group is accepting storage traffic.
232 */
233 spin_lock_bh(&tpg->tpg_state_lock);
234 if (tpg->tpg_state != TPG_STATE_ACTIVE) {
235 spin_unlock_bh(&tpg->tpg_state_lock);
236 return -1;
237 }
238 spin_unlock_bh(&tpg->tpg_state_lock);
239
240 /*
241 * Here we serialize access across the TIQN+TPG Tuple.
242 */
243 ret = mutex_lock_interruptible(&tpg->np_login_lock);
244 if ((ret != 0) || signal_pending(current))
245 return -1;
246
247 spin_lock_bh(&np->np_thread_lock);
248 np->np_login_tpg = tpg;
249 spin_unlock_bh(&np->np_thread_lock);
250
251 return 0;
252}
253
254int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
255{
256 struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
257
258 spin_lock_bh(&np->np_thread_lock);
259 np->np_login_tpg = NULL;
260 spin_unlock_bh(&np->np_thread_lock);
261
262 mutex_unlock(&tpg->np_login_lock);
263
264 if (tiqn)
265 iscsit_put_tiqn_for_login(tiqn);
266
267 return 0;
268}
269
05b96892 270bool iscsit_check_np_match(
e48354ce 271 struct __kernel_sockaddr_storage *sockaddr,
05b96892 272 struct iscsi_np *np,
e48354ce
NB		 273 int network_transport)
274{
275 struct sockaddr_in *sock_in, *sock_in_e;
276 struct sockaddr_in6 *sock_in6, *sock_in6_e;
05b96892 277 bool ip_match = false;
e48354ce
NB		 278 u16 port;
279
05b96892
NB		 280 if (sockaddr->ss_family == AF_INET6) {
281 sock_in6 = (struct sockaddr_in6 *)sockaddr;
282 sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
283
284 if (!memcmp(&sock_in6->sin6_addr.in6_u,
285 &sock_in6_e->sin6_addr.in6_u,
286 sizeof(struct in6_addr)))
287 ip_match = true;
288
289 port = ntohs(sock_in6->sin6_port);
290 } else {
291 sock_in = (struct sockaddr_in *)sockaddr;
292 sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
293
294 if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
295 ip_match = true;
296
297 port = ntohs(sock_in->sin_port);
298 }
299
300 if ((ip_match == true) && (np->np_port == port) &&
301 (np->np_network_transport == network_transport))
302 return true;
303
304 return false;
305}
306
307static struct iscsi_np *iscsit_get_np(
308 struct __kernel_sockaddr_storage *sockaddr,
309 int network_transport)
310{
311 struct iscsi_np *np;
312 bool match;
313
e48354ce
NB		 314 spin_lock_bh(&np_lock);
315 list_for_each_entry(np, &g_np_list, np_list) {
316 spin_lock(&np->np_thread_lock);
317 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
318 spin_unlock(&np->np_thread_lock);
319 continue;
320 }
321
05b96892
NB		 322 match = iscsit_check_np_match(sockaddr, np, network_transport);
323 if (match == true) {
e48354ce
NB		 324 /*
325 * Increment the np_exports reference count now to
326 * prevent iscsit_del_np() below from being called
327 * while iscsi_tpg_add_network_portal() is called.
328 */
329 np->np_exports++;
330 spin_unlock(&np->np_thread_lock);
331 spin_unlock_bh(&np_lock);
332 return np;
333 }
334 spin_unlock(&np->np_thread_lock);
335 }
336 spin_unlock_bh(&np_lock);
337
338 return NULL;
339}
340
341struct iscsi_np *iscsit_add_np(
342 struct __kernel_sockaddr_storage *sockaddr,
343 char *ip_str,
344 int network_transport)
345{
346 struct sockaddr_in *sock_in;
347 struct sockaddr_in6 *sock_in6;
348 struct iscsi_np *np;
349 int ret;
350 /*
351 * Locate the existing struct iscsi_np if already active..
352 */
353 np = iscsit_get_np(sockaddr, network_transport);
354 if (np)
355 return np;
356
357 np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
358 if (!np) {
359 pr_err("Unable to allocate memory for struct iscsi_np\n");
360 return ERR_PTR(-ENOMEM);
361 }
362
363 np->np_flags |= NPF_IP_NETWORK;
364 if (sockaddr->ss_family == AF_INET6) {
365 sock_in6 = (struct sockaddr_in6 *)sockaddr;
366 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str);
367 np->np_port = ntohs(sock_in6->sin6_port);
368 } else {
369 sock_in = (struct sockaddr_in *)sockaddr;
370 sprintf(np->np_ip, "%s", ip_str);
371 np->np_port = ntohs(sock_in->sin_port);
372 }
373
374 np->np_network_transport = network_transport;
375 spin_lock_init(&np->np_thread_lock);
376 init_completion(&np->np_restart_comp);
377 INIT_LIST_HEAD(&np->np_list);
378
379 ret = iscsi_target_setup_login_socket(np, sockaddr);
380 if (ret != 0) {
381 kfree(np);
382 return ERR_PTR(ret);
383 }
384
385 np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
386 if (IS_ERR(np->np_thread)) {
387 pr_err("Unable to create kthread: iscsi_np\n");
388 ret = PTR_ERR(np->np_thread);
389 kfree(np);
390 return ERR_PTR(ret);
391 }
392 /*
393 * Increment the np_exports reference count now to prevent
394 * iscsit_del_np() below from being run while a new call to
395 * iscsi_tpg_add_network_portal() for a matching iscsi_np is
396 * active. We don't need to hold np->np_thread_lock at this
397 * point because iscsi_np has not been added to g_np_list yet.
398 */
399 np->np_exports = 1;
400
401 spin_lock_bh(&np_lock);
402 list_add_tail(&np->np_list, &g_np_list);
403 spin_unlock_bh(&np_lock);
404
405 pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
baa4d64b 406 np->np_ip, np->np_port, np->np_transport->name);
e48354ce
NB		 407
408 return np;
409}
410
411int iscsit_reset_np_thread(
412 struct iscsi_np *np,
413 struct iscsi_tpg_np *tpg_np,
414 struct iscsi_portal_group *tpg)
415{
416 spin_lock_bh(&np->np_thread_lock);
417 if (tpg && tpg_np) {
418 /*
419 * The reset operation need only be performed when the
420 * passed struct iscsi_portal_group has a login in progress
421 * to one of the network portals.
422 */
423 if (tpg_np->tpg_np->np_login_tpg != tpg) {
424 spin_unlock_bh(&np->np_thread_lock);
425 return 0;
426 }
427 }
428 if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
429 spin_unlock_bh(&np->np_thread_lock);
430 return 0;
431 }
432 np->np_thread_state = ISCSI_NP_THREAD_RESET;
433
434 if (np->np_thread) {
435 spin_unlock_bh(&np->np_thread_lock);
436 send_sig(SIGINT, np->np_thread, 1);
437 wait_for_completion(&np->np_restart_comp);
438 spin_lock_bh(&np->np_thread_lock);
439 }
440 spin_unlock_bh(&np->np_thread_lock);
441
442 return 0;
443}
444
baa4d64b 445static void iscsit_free_np(struct iscsi_np *np)
e48354ce 446{
bf6932f4
AV		 447 if (np->np_socket)
448 sock_release(np->np_socket);
e48354ce
NB		 449}
450
451int iscsit_del_np(struct iscsi_np *np)
452{
453 spin_lock_bh(&np->np_thread_lock);
454 np->np_exports--;
455 if (np->np_exports) {
456 spin_unlock_bh(&np->np_thread_lock);
457 return 0;
458 }
459 np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
460 spin_unlock_bh(&np->np_thread_lock);
461
462 if (np->np_thread) {
463 /*
464 * We need to send the signal to wakeup Linux/Net
465 * which may be sleeping in sock_accept()..
466 */
467 send_sig(SIGINT, np->np_thread, 1);
468 kthread_stop(np->np_thread);
469 }
baa4d64b
NB		 470
471 np->np_transport->iscsit_free_np(np);
e48354ce
NB		 472
473 spin_lock_bh(&np_lock);
474 list_del(&np->np_list);
475 spin_unlock_bh(&np_lock);
476
477 pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
baa4d64b 478 np->np_ip, np->np_port, np->np_transport->name);
e48354ce 479
baa4d64b 480 iscsit_put_transport(np->np_transport);
e48354ce
NB		 481 kfree(np);
482 return 0;
483}
484
baa4d64b
NB		 485static struct iscsit_transport iscsi_target_transport = {
486 .name = "iSCSI/TCP",
487 .transport_type = ISCSI_TCP,
488 .owner = NULL,
489 .iscsit_setup_np = iscsit_setup_np,
490 .iscsit_accept_np = iscsit_accept_np,
491 .iscsit_free_np = iscsit_free_np,
cdb72665 492 .iscsit_alloc_cmd = iscsit_alloc_cmd,
baa4d64b
NB		 493 .iscsit_get_login_rx = iscsit_get_login_rx,
494 .iscsit_put_login_tx = iscsit_put_login_tx,
3e1c81a9 495 .iscsit_get_dataout = iscsit_build_r2ts_for_cmd,
baa4d64b
NB		 496};
497
e48354ce
NB		 498static int __init iscsi_target_init_module(void)
499{
500 int ret = 0;
501
502 pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
503
504 iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
505 if (!iscsit_global) {
506 pr_err("Unable to allocate memory for iscsit_global\n");
507 return -1;
508 }
509 mutex_init(&auth_id_lock);
510 spin_lock_init(&sess_idr_lock);
511 idr_init(&tiqn_idr);
512 idr_init(&sess_idr);
513
514 ret = iscsi_target_register_configfs();
515 if (ret < 0)
516 goto out;
517
518 ret = iscsi_thread_set_init();
519 if (ret < 0)
520 goto configfs_out;
521
522 if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
523 TARGET_THREAD_SET_COUNT) {
524 pr_err("iscsi_allocate_thread_sets() returned"
525 " unexpected value!\n");
526 goto ts_out1;
527 }
528
529 lio_cmd_cache = kmem_cache_create("lio_cmd_cache",
530 sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd),
531 0, NULL);
532 if (!lio_cmd_cache) {
533 pr_err("Unable to kmem_cache_create() for"
534 " lio_cmd_cache\n");
535 goto ts_out2;
536 }
537
538 lio_qr_cache = kmem_cache_create("lio_qr_cache",
539 sizeof(struct iscsi_queue_req),
540 __alignof__(struct iscsi_queue_req), 0, NULL);
541 if (!lio_qr_cache) {
542 pr_err("nable to kmem_cache_create() for"
543 " lio_qr_cache\n");
544 goto cmd_out;
545 }
546
547 lio_dr_cache = kmem_cache_create("lio_dr_cache",
548 sizeof(struct iscsi_datain_req),
549 __alignof__(struct iscsi_datain_req), 0, NULL);
550 if (!lio_dr_cache) {
551 pr_err("Unable to kmem_cache_create() for"
552 " lio_dr_cache\n");
553 goto qr_out;
554 }
555
556 lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
557 sizeof(struct iscsi_ooo_cmdsn),
558 __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
559 if (!lio_ooo_cache) {
560 pr_err("Unable to kmem_cache_create() for"
561 " lio_ooo_cache\n");
562 goto dr_out;
563 }
564
565 lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
566 sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
567 0, NULL);
568 if (!lio_r2t_cache) {
569 pr_err("Unable to kmem_cache_create() for"
570 " lio_r2t_cache\n");
571 goto ooo_out;
572 }
573
baa4d64b
NB		 574 iscsit_register_transport(&iscsi_target_transport);
575
e48354ce
NB		 576 if (iscsit_load_discovery_tpg() < 0)
577 goto r2t_out;
578
579 return ret;
580r2t_out:
581 kmem_cache_destroy(lio_r2t_cache);
582ooo_out:
583 kmem_cache_destroy(lio_ooo_cache);
584dr_out:
585 kmem_cache_destroy(lio_dr_cache);
586qr_out:
587 kmem_cache_destroy(lio_qr_cache);
588cmd_out:
589 kmem_cache_destroy(lio_cmd_cache);
590ts_out2:
591 iscsi_deallocate_thread_sets();
592ts_out1:
593 iscsi_thread_set_free();
594configfs_out:
595 iscsi_target_deregister_configfs();
596out:
597 kfree(iscsit_global);
598 return -ENOMEM;
599}
600
601static void __exit iscsi_target_cleanup_module(void)
602{
603 iscsi_deallocate_thread_sets();
604 iscsi_thread_set_free();
605 iscsit_release_discovery_tpg();
baa4d64b 606 iscsit_unregister_transport(&iscsi_target_transport);
e48354ce
NB		 607 kmem_cache_destroy(lio_cmd_cache);
608 kmem_cache_destroy(lio_qr_cache);
609 kmem_cache_destroy(lio_dr_cache);
610 kmem_cache_destroy(lio_ooo_cache);
611 kmem_cache_destroy(lio_r2t_cache);
612
613 iscsi_target_deregister_configfs();
614
615 kfree(iscsit_global);
616}
617
8b1e1244 618static int iscsit_add_reject(
e48354ce
NB		 619 u8 reason,
620 int fail_conn,
621 unsigned char *buf,
622 struct iscsi_conn *conn)
623{
624 struct iscsi_cmd *cmd;
625 struct iscsi_reject *hdr;
626 int ret;
627
628 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
629 if (!cmd)
630 return -1;
631
632 cmd->iscsi_opcode = ISCSI_OP_REJECT;
633 if (fail_conn)
634 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
635
636 hdr = (struct iscsi_reject *) cmd->pdu;
637 hdr->reason = reason;
638
1c3d5794 639 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
e48354ce
NB		 640 if (!cmd->buf_ptr) {
641 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
642 iscsit_release_cmd(cmd);
643 return -1;
644 }
e48354ce
NB		 645
646 spin_lock_bh(&conn->cmd_lock);
2fbb471e 647 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 648 spin_unlock_bh(&conn->cmd_lock);
649
650 cmd->i_state = ISTATE_SEND_REJECT;
651 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
652
653 ret = wait_for_completion_interruptible(&cmd->reject_comp);
3e1c81a9
NB		 654 /*
655 * Perform the kref_put now if se_cmd has been setup by
656 * iscsit_setup_scsi_cmd()
657 */
658 if (cmd->se_cmd.se_tfo != NULL) {
659 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
660 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
661 }
e48354ce
NB		 662 if (ret != 0)
663 return -1;
664
665 return (!fail_conn) ? 0 : -1;
666}
667
668int iscsit_add_reject_from_cmd(
669 u8 reason,
670 int fail_conn,
671 int add_to_conn,
672 unsigned char *buf,
673 struct iscsi_cmd *cmd)
674{
675 struct iscsi_conn *conn;
676 struct iscsi_reject *hdr;
677 int ret;
678
679 if (!cmd->conn) {
680 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
681 cmd->init_task_tag);
682 return -1;
683 }
684 conn = cmd->conn;
685
686 cmd->iscsi_opcode = ISCSI_OP_REJECT;
687 if (fail_conn)
688 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
689
690 hdr = (struct iscsi_reject *) cmd->pdu;
691 hdr->reason = reason;
692
1c3d5794 693 cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
e48354ce
NB		 694 if (!cmd->buf_ptr) {
695 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
696 iscsit_release_cmd(cmd);
697 return -1;
698 }
e48354ce
NB		 699
700 if (add_to_conn) {
701 spin_lock_bh(&conn->cmd_lock);
2fbb471e 702 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 703 spin_unlock_bh(&conn->cmd_lock);
704 }
705
706 cmd->i_state = ISTATE_SEND_REJECT;
707 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
708
709 ret = wait_for_completion_interruptible(&cmd->reject_comp);
3e1c81a9
NB		 710 /*
711 * Perform the kref_put now if se_cmd has already been setup by
712 * scsit_setup_scsi_cmd()
713 */
714 if (cmd->se_cmd.se_tfo != NULL) {
715 pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
716 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
717 }
e48354ce
NB		 718 if (ret != 0)
719 return -1;
720
721 return (!fail_conn) ? 0 : -1;
722}
3e1c81a9 723EXPORT_SYMBOL(iscsit_add_reject_from_cmd);
e48354ce
NB		 724
725/*
726 * Map some portion of the allocated scatterlist to an iovec, suitable for
bfb79eac 727 * kernel sockets to copy data in/out.
e48354ce
NB		 728 */
729static int iscsit_map_iovec(
730 struct iscsi_cmd *cmd,
731 struct kvec *iov,
732 u32 data_offset,
733 u32 data_length)
734{
735 u32 i = 0;
736 struct scatterlist *sg;
737 unsigned int page_off;
738
739 /*
bfb79eac 740 * We know each entry in t_data_sg contains a page.
e48354ce 741 */
bfb79eac 742 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
e48354ce
NB		 743 page_off = (data_offset % PAGE_SIZE);
744
745 cmd->first_data_sg = sg;
746 cmd->first_data_sg_off = page_off;
747
748 while (data_length) {
749 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
750
751 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
752 iov[i].iov_len = cur_len;
753
754 data_length -= cur_len;
755 page_off = 0;
756 sg = sg_next(sg);
757 i++;
758 }
759
760 cmd->kmapped_nents = i;
761
762 return i;
763}
764
765static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
766{
767 u32 i;
768 struct scatterlist *sg;
769
770 sg = cmd->first_data_sg;
771
772 for (i = 0; i < cmd->kmapped_nents; i++)
773 kunmap(sg_page(&sg[i]));
774}
775
776static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
777{
778 struct iscsi_cmd *cmd;
779
780 conn->exp_statsn = exp_statsn;
781
3e1c81a9
NB		 782 if (conn->sess->sess_ops->RDMAExtensions)
783 return;
784
e48354ce 785 spin_lock_bh(&conn->cmd_lock);
2fbb471e 786 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
e48354ce
NB		 787 spin_lock(&cmd->istate_lock);
788 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
64c13330 789 iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
e48354ce
NB		 790 cmd->i_state = ISTATE_REMOVE;
791 spin_unlock(&cmd->istate_lock);
792 iscsit_add_cmd_to_immediate_queue(cmd, conn,
793 cmd->i_state);
794 continue;
795 }
796 spin_unlock(&cmd->istate_lock);
797 }
798 spin_unlock_bh(&conn->cmd_lock);
799}
800
801static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
802{
f80e8ed3 803 u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
e48354ce 804
c0427f15 805 iov_count += ISCSI_IOV_DATA_BUFFER;
e48354ce
NB		 806
807 cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
808 if (!cmd->iov_data) {
809 pr_err("Unable to allocate cmd->iov_data\n");
810 return -ENOMEM;
811 }
812
813 cmd->orig_iov_data_count = iov_count;
814 return 0;
815}
816
3e1c81a9
NB		 817int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
818 unsigned char *buf)
e48354ce 819{
3e1c81a9 820 int data_direction, payload_length;
e48354ce 821 struct iscsi_scsi_req *hdr;
d28b1169
AG		 822 int iscsi_task_attr;
823 int sam_task_attr;
e48354ce
NB		 824
825 spin_lock_bh(&conn->sess->session_stats_lock);
826 conn->sess->cmd_pdus++;
827 if (conn->sess->se_sess->se_node_acl) {
828 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
829 conn->sess->se_sess->se_node_acl->num_cmds++;
830 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
831 }
832 spin_unlock_bh(&conn->sess->session_stats_lock);
833
834 hdr = (struct iscsi_scsi_req *) buf;
835 payload_length = ntoh24(hdr->dlength);
e48354ce
NB		 836
837 /* FIXME; Add checks for AdditionalHeaderSegment */
838
839 if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
840 !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
841 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
842 " not set. Bad iSCSI Initiator.\n");
3e1c81a9
NB		 843 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
844 1, 1, buf, cmd);
e48354ce
NB		 845 }
846
847 if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
848 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
849 /*
850 * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2)
851 * that adds support for RESERVE/RELEASE. There is a bug
852 * add with this new functionality that sets R/W bits when
853 * neither CDB carries any READ or WRITE datapayloads.
854 */
855 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) {
856 hdr->flags &= ~ISCSI_FLAG_CMD_READ;
857 hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
858 goto done;
859 }
860
861 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
862 " set when Expected Data Transfer Length is 0 for"
863 " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]);
3e1c81a9
NB		 864 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
865 1, 1, buf, cmd);
e48354ce
NB		 866 }
867done:
868
869 if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
870 !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
871 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
872 " MUST be set if Expected Data Transfer Length is not 0."
873 " Bad iSCSI Initiator\n");
3e1c81a9
NB		 874 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
875 1, 1, buf, cmd);
e48354ce
NB		 876 }
877
878 if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
879 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
880 pr_err("Bidirectional operations not supported!\n");
3e1c81a9
NB		 881 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
882 1, 1, buf, cmd);
e48354ce
NB		 883 }
884
885 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
886 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
887 " Scsi Command PDU.\n");
3e1c81a9
NB		 888 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
889 1, 1, buf, cmd);
e48354ce
NB		 890 }
891
892 if (payload_length && !conn->sess->sess_ops->ImmediateData) {
893 pr_err("ImmediateData=No but DataSegmentLength=%u,"
894 " protocol error.\n", payload_length);
3e1c81a9
NB		 895 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
896 1, 1, buf, cmd);
e48354ce
NB		 897 }
898
50e5c87d 899 if ((be32_to_cpu(hdr->data_length )== payload_length) &&
e48354ce
NB		 900 (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
901 pr_err("Expected Data Transfer Length and Length of"
902 " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
903 " bit is not set protocol error\n");
3e1c81a9
NB		 904 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
905 1, 1, buf, cmd);
e48354ce
NB		 906 }
907
50e5c87d 908 if (payload_length > be32_to_cpu(hdr->data_length)) {
e48354ce
NB		 909 pr_err("DataSegmentLength: %u is greater than"
910 " EDTL: %u, protocol error.\n", payload_length,
911 hdr->data_length);
3e1c81a9
NB		 912 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
913 1, 1, buf, cmd);
e48354ce
NB		 914 }
915
21f5aa7e 916 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
e48354ce 917 pr_err("DataSegmentLength: %u is greater than"
21f5aa7e
NB		 918 " MaxXmitDataSegmentLength: %u, protocol error.\n",
919 payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
3e1c81a9
NB		 920 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
921 1, 1, buf, cmd);
e48354ce
NB		 922 }
923
924 if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
925 pr_err("DataSegmentLength: %u is greater than"
926 " FirstBurstLength: %u, protocol error.\n",
927 payload_length, conn->sess->sess_ops->FirstBurstLength);
3e1c81a9
NB		 928 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
929 1, 1, buf, cmd);
e48354ce
NB		 930 }
931
932 data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
933 (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
934 DMA_NONE;
935
d28b1169 936 cmd->data_direction = data_direction;
d28b1169
AG		 937 iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
938 /*
939 * Figure out the SAM Task Attribute for the incoming SCSI CDB
940 */
941 if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
942 (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
943 sam_task_attr = MSG_SIMPLE_TAG;
944 else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
945 sam_task_attr = MSG_ORDERED_TAG;
946 else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
947 sam_task_attr = MSG_HEAD_TAG;
948 else if (iscsi_task_attr == ISCSI_ATTR_ACA)
949 sam_task_attr = MSG_ACA_TAG;
950 else {
951 pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
952 " MSG_SIMPLE_TAG\n", iscsi_task_attr);
953 sam_task_attr = MSG_SIMPLE_TAG;
954 }
955
e48354ce
NB		 956 cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD;
957 cmd->i_state = ISTATE_NEW_CMD;
958 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
959 cmd->immediate_data = (payload_length) ? 1 : 0;
960 cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
961 (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
962 if (cmd->unsolicited_data)
963 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
964
965 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
966 if (hdr->flags & ISCSI_FLAG_CMD_READ) {
967 spin_lock_bh(&conn->sess->ttt_lock);
968 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
969 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
970 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
971 spin_unlock_bh(&conn->sess->ttt_lock);
972 } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE)
973 cmd->targ_xfer_tag = 0xFFFFFFFF;
50e5c87d
CH		 974 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
975 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
e48354ce
NB		 976 cmd->first_burst_len = payload_length;
977
3e1c81a9
NB		 978 if (!conn->sess->sess_ops->RDMAExtensions &&
979 cmd->data_direction == DMA_FROM_DEVICE) {
e48354ce
NB		 980 struct iscsi_datain_req *dr;
981
982 dr = iscsit_allocate_datain_req();
983 if (!dr)
984 return iscsit_add_reject_from_cmd(
985 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
986 1, 1, buf, cmd);
987
988 iscsit_attach_datain_req(cmd, dr);
989 }
990
065ca1e4
AG		 991 /*
992 * Initialize struct se_cmd descriptor from target_core_mod infrastructure
993 */
994 transport_init_se_cmd(&cmd->se_cmd, &lio_target_fabric_configfs->tf_ops,
50e5c87d
CH		 995 conn->sess->se_sess, be32_to_cpu(hdr->data_length),
996 cmd->data_direction, sam_task_attr,
997 cmd->sense_buffer + 2);
065ca1e4
AG		 998
999 pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
1000 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
3e1c81a9
NB		 1001 hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
1002 conn->cid);
1003
1004 target_get_sess_cmd(conn->sess->se_sess, &cmd->se_cmd, true);
065ca1e4 1005
de103c93
CH		 1006 cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd,
1007 scsilun_to_int(&hdr->lun));
1008 if (cmd->sense_reason)
1009 goto attach_cmd;
1010
1011 cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb);
1012 if (cmd->sense_reason) {
1013 if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
1014 return iscsit_add_reject_from_cmd(
1015 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1016 1, 1, buf, cmd);
e48354ce 1017 }
de103c93 1018
e48354ce
NB		 1019 goto attach_cmd;
1020 }
a12f41f8 1021
de103c93 1022 if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
e48354ce 1023 return iscsit_add_reject_from_cmd(
de103c93
CH		 1024 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1025 1, 1, buf, cmd);
e48354ce
NB		 1026 }
1027
1028attach_cmd:
1029 spin_lock_bh(&conn->cmd_lock);
2fbb471e 1030 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 1031 spin_unlock_bh(&conn->cmd_lock);
1032 /*
1033 * Check if we need to delay processing because of ALUA
1034 * Active/NonOptimized primary access state..
1035 */
1036 core_alua_check_nonop_delay(&cmd->se_cmd);
bfb79eac 1037
3e1c81a9
NB		 1038 return 0;
1039}
1040EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
de103c93 1041
3e1c81a9
NB		 1042void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd)
1043{
1044 iscsit_set_dataout_sequence_values(cmd);
1045
1046 spin_lock_bh(&cmd->dataout_timeout_lock);
1047 iscsit_start_dataout_timer(cmd, cmd->conn);
1048 spin_unlock_bh(&cmd->dataout_timeout_lock);
1049}
1050EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout);
1051
1052int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1053 struct iscsi_scsi_req *hdr)
1054{
1055 int cmdsn_ret = 0;
e48354ce
NB		 1056 /*
1057 * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1058 * the Immediate Bit is not set, and no Immediate
1059 * Data is attached.
1060 *
1061 * A PDU/CmdSN carrying Immediate Data can only
1062 * be processed after the DataCRC has passed.
1063 * If the DataCRC fails, the CmdSN MUST NOT
1064 * be acknowledged. (See below)
1065 */
1066 if (!cmd->immediate_data) {
1067 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
3e1c81a9
NB		 1068 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1069 if (!cmd->sense_reason)
1070 return 0;
1071
1072 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
7e32da55 1073 return 0;
3e1c81a9 1074 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
e48354ce
NB		 1075 return iscsit_add_reject_from_cmd(
1076 ISCSI_REASON_PROTOCOL_ERROR,
3e1c81a9
NB		 1077 1, 0, (unsigned char *)hdr, cmd);
1078 }
e48354ce
NB		 1079 }
1080
50e5c87d 1081 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
e48354ce
NB		 1082
1083 /*
1084 * If no Immediate Data is attached, it's OK to return now.
1085 */
1086 if (!cmd->immediate_data) {
3e1c81a9
NB		 1087 if (!cmd->sense_reason && cmd->unsolicited_data)
1088 iscsit_set_unsoliticed_dataout(cmd);
1089 if (!cmd->sense_reason)
1090 return 0;
e48354ce 1091
3e1c81a9 1092 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
e48354ce
NB		 1093 return 0;
1094 }
1095
1096 /*
3e1c81a9
NB		 1097 * Early CHECK_CONDITIONs with ImmediateData never make it to command
1098 * execution. These exceptions are processed in CmdSN order using
1099 * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
e48354ce 1100 */
de103c93 1101 if (cmd->sense_reason) {
3e1c81a9
NB		 1102 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1103 return 1;
e48354ce
NB		 1104 }
1105 /*
1106 * Call directly into transport_generic_new_cmd() to perform
1107 * the backend memory allocation.
1108 */
de103c93
CH		 1109 cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
1110 if (cmd->sense_reason) {
3e1c81a9
NB		 1111 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd);
1112 return 1;
e48354ce
NB		 1113 }
1114
3e1c81a9
NB		 1115 return 0;
1116}
1117EXPORT_SYMBOL(iscsit_process_scsi_cmd);
1118
1119static int
1120iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
1121 bool dump_payload)
1122{
1123 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1124 /*
1125 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
1126 */
1127 if (dump_payload == true)
1128 goto after_immediate_data;
1129
1130 immed_ret = iscsit_handle_immediate_data(cmd, hdr,
1131 cmd->first_burst_len);
e48354ce
NB		 1132after_immediate_data:
1133 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1134 /*
1135 * A PDU/CmdSN carrying Immediate Data passed
1136 * DataCRC, check against ExpCmdSN/MaxCmdSN if
1137 * Immediate Bit is not set.
1138 */
3e1c81a9
NB		 1139 cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, hdr->cmdsn);
1140
de103c93 1141 if (cmd->sense_reason) {
3e1c81a9
NB		 1142 if (iscsit_dump_data_payload(cmd->conn,
1143 cmd->first_burst_len, 1) < 0)
e48354ce 1144 return -1;
3e1c81a9
NB		 1145 } else if (cmd->unsolicited_data)
1146 iscsit_set_unsoliticed_dataout(cmd);
e48354ce
NB		 1147
1148 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1149 return iscsit_add_reject_from_cmd(
1150 ISCSI_REASON_PROTOCOL_ERROR,
3e1c81a9 1151 1, 0, (unsigned char *)hdr, cmd);
e48354ce
NB		 1152
1153 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1154 /*
1155 * Immediate Data failed DataCRC and ERL>=1,
1156 * silently drop this PDU and let the initiator
1157 * plug the CmdSN gap.
1158 *
1159 * FIXME: Send Unsolicited NOPIN with reserved
1160 * TTT here to help the initiator figure out
1161 * the missing CmdSN, although they should be
1162 * intelligent enough to determine the missing
1163 * CmdSN and issue a retry to plug the sequence.
1164 */
1165 cmd->i_state = ISTATE_REMOVE;
3e1c81a9 1166 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
e48354ce
NB		 1167 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1168 return -1;
1169
1170 return 0;
1171}
1172
3e1c81a9
NB		 1173static int
1174iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1175 unsigned char *buf)
1176{
1177 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1178 int rc, immed_data;
1179 bool dump_payload = false;
1180
1181 rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1182 if (rc < 0)
1183 return rc;
1184 /*
1185 * Allocation iovecs needed for struct socket operations for
1186 * traditional iSCSI block I/O.
1187 */
1188 if (iscsit_allocate_iovecs(cmd) < 0) {
1189 return iscsit_add_reject_from_cmd(
1190 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1191 1, 0, buf, cmd);
1192 }
1193 immed_data = cmd->immediate_data;
1194
1195 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1196 if (rc < 0)
1197 return rc;
1198 else if (rc > 0)
1199 dump_payload = true;
1200
1201 if (!immed_data)
1202 return 0;
1203
1204 return iscsit_get_immediate_data(cmd, hdr, dump_payload);
1205}
1206
e48354ce
NB		 1207static u32 iscsit_do_crypto_hash_sg(
1208 struct hash_desc *hash,
1209 struct iscsi_cmd *cmd,
1210 u32 data_offset,
1211 u32 data_length,
1212 u32 padding,
1213 u8 *pad_bytes)
1214{
1215 u32 data_crc;
1216 u32 i;
1217 struct scatterlist *sg;
1218 unsigned int page_off;
1219
1220 crypto_hash_init(hash);
1221
1222 sg = cmd->first_data_sg;
1223 page_off = cmd->first_data_sg_off;
1224
1225 i = 0;
1226 while (data_length) {
1227 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off));
1228
1229 crypto_hash_update(hash, &sg[i], cur_len);
1230
1231 data_length -= cur_len;
1232 page_off = 0;
1233 i++;
1234 }
1235
1236 if (padding) {
1237 struct scatterlist pad_sg;
1238
1239 sg_init_one(&pad_sg, pad_bytes, padding);
1240 crypto_hash_update(hash, &pad_sg, padding);
1241 }
1242 crypto_hash_final(hash, (u8 *) &data_crc);
1243
1244 return data_crc;
1245}
1246
1247static void iscsit_do_crypto_hash_buf(
1248 struct hash_desc *hash,
1249 unsigned char *buf,
1250 u32 payload_length,
1251 u32 padding,
1252 u8 *pad_bytes,
1253 u8 *data_crc)
1254{
1255 struct scatterlist sg;
1256
1257 crypto_hash_init(hash);
1258
8359cf43 1259 sg_init_one(&sg, buf, payload_length);
e48354ce
NB		 1260 crypto_hash_update(hash, &sg, payload_length);
1261
1262 if (padding) {
1263 sg_init_one(&sg, pad_bytes, padding);
1264 crypto_hash_update(hash, &sg, padding);
1265 }
1266 crypto_hash_final(hash, data_crc);
1267}
1268
3e1c81a9
NB		 1269int
1270iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf,
1271 struct iscsi_cmd **out_cmd)
e48354ce 1272{
3e1c81a9 1273 struct iscsi_data *hdr = (struct iscsi_data *)buf;
e48354ce
NB		 1274 struct iscsi_cmd *cmd = NULL;
1275 struct se_cmd *se_cmd;
e48354ce 1276 unsigned long flags;
3e1c81a9
NB		 1277 u32 payload_length = ntoh24(hdr->dlength);
1278 int rc;
e48354ce
NB		 1279
1280 if (!payload_length) {
1281 pr_err("DataOUT payload is ZERO, protocol error.\n");
1282 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1283 buf, conn);
1284 }
1285
1286 /* iSCSI write */
1287 spin_lock_bh(&conn->sess->session_stats_lock);
1288 conn->sess->rx_data_octets += payload_length;
1289 if (conn->sess->se_sess->se_node_acl) {
1290 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
1291 conn->sess->se_sess->se_node_acl->write_bytes += payload_length;
1292 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
1293 }
1294 spin_unlock_bh(&conn->sess->session_stats_lock);
1295
21f5aa7e 1296 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
e48354ce 1297 pr_err("DataSegmentLength: %u is greater than"
21f5aa7e
NB		 1298 " MaxXmitDataSegmentLength: %u\n", payload_length,
1299 conn->conn_ops->MaxXmitDataSegmentLength);
e48354ce
NB		 1300 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1301 buf, conn);
1302 }
1303
1304 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
1305 payload_length);
1306 if (!cmd)
1307 return 0;
1308
1309 pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1310 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
3e1c81a9 1311 hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
e48354ce
NB		 1312 payload_length, conn->cid);
1313
1314 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1315 pr_err("Command ITT: 0x%08x received DataOUT after"
1316 " last DataOUT received, dumping payload\n",
1317 cmd->init_task_tag);
1318 return iscsit_dump_data_payload(conn, payload_length, 1);
1319 }
1320
1321 if (cmd->data_direction != DMA_TO_DEVICE) {
1322 pr_err("Command ITT: 0x%08x received DataOUT for a"
1323 " NON-WRITE command.\n", cmd->init_task_tag);
1324 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
1325 1, 0, buf, cmd);
1326 }
1327 se_cmd = &cmd->se_cmd;
1328 iscsit_mod_dataout_timer(cmd);
1329
50e5c87d 1330 if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
e48354ce
NB		 1331 pr_err("DataOut Offset: %u, Length %u greater than"
1332 " iSCSI Command EDTL %u, protocol error.\n",
ebf1d95c 1333 hdr->offset, payload_length, cmd->se_cmd.data_length);
e48354ce
NB		 1334 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
1335 1, 0, buf, cmd);
1336 }
1337
1338 if (cmd->unsolicited_data) {
1339 int dump_unsolicited_data = 0;
1340
1341 if (conn->sess->sess_ops->InitialR2T) {
1342 pr_err("Received unexpected unsolicited data"
1343 " while InitialR2T=Yes, protocol error.\n");
1344 transport_send_check_condition_and_sense(&cmd->se_cmd,
1345 TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1346 return -1;
1347 }
1348 /*
1349 * Special case for dealing with Unsolicited DataOUT
1350 * and Unsupported SAM WRITE Opcodes and SE resource allocation
1351 * failures;
1352 */
1353
1354 /* Something's amiss if we're not in WRITE_PENDING state... */
1355 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1356 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1357 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1358
1359 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
de103c93 1360 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
e48354ce
NB		 1361 dump_unsolicited_data = 1;
1362 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1363
1364 if (dump_unsolicited_data) {
1365 /*
1366 * Check if a delayed TASK_ABORTED status needs to
1367 * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1368 * received with the unsolicitied data out.
1369 */
1370 if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1371 iscsit_stop_dataout_timer(cmd);
1372
1373 transport_check_aborted_status(se_cmd,
1374 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1375 return iscsit_dump_data_payload(conn, payload_length, 1);
1376 }
1377 } else {
1378 /*
1379 * For the normal solicited data path:
1380 *
1381 * Check for a delayed TASK_ABORTED status and dump any
1382 * incoming data out payload if one exists. Also, when the
1383 * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1384 * data out sequence, we decrement outstanding_r2ts. Once
1385 * outstanding_r2ts reaches zero, go ahead and send the delayed
1386 * TASK_ABORTED status.
1387 */
7d680f3b 1388 if (se_cmd->transport_state & CMD_T_ABORTED) {
e48354ce
NB		 1389 if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1390 if (--cmd->outstanding_r2ts < 1) {
1391 iscsit_stop_dataout_timer(cmd);
1392 transport_check_aborted_status(
1393 se_cmd, 1);
1394 }
1395
1396 return iscsit_dump_data_payload(conn, payload_length, 1);
1397 }
1398 }
1399 /*
1400 * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1401 * within-command recovery checks before receiving the payload.
1402 */
3e1c81a9
NB		 1403 rc = iscsit_check_pre_dataout(cmd, buf);
1404 if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
e48354ce 1405 return 0;
3e1c81a9 1406 else if (rc == DATAOUT_CANNOT_RECOVER)
e48354ce
NB		 1407 return -1;
1408
3e1c81a9
NB		 1409 *out_cmd = cmd;
1410 return 0;
1411}
1412EXPORT_SYMBOL(iscsit_check_dataout_hdr);
1413
1414static int
1415iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1416 struct iscsi_data *hdr)
1417{
1418 struct kvec *iov;
1419 u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
1420 u32 payload_length = ntoh24(hdr->dlength);
1421 int iov_ret, data_crc_failed = 0;
1422
e48354ce
NB		 1423 rx_size += payload_length;
1424 iov = &cmd->iov_data[0];
1425
50e5c87d
CH		 1426 iov_ret = iscsit_map_iovec(cmd, iov, be32_to_cpu(hdr->offset),
1427 payload_length);
e48354ce
NB		 1428 if (iov_ret < 0)
1429 return -1;
1430
1431 iov_count += iov_ret;
1432
1433 padding = ((-payload_length) & 3);
1434 if (padding != 0) {
1435 iov[iov_count].iov_base = cmd->pad_bytes;
1436 iov[iov_count++].iov_len = padding;
1437 rx_size += padding;
1438 pr_debug("Receiving %u padding bytes.\n", padding);
1439 }
1440
1441 if (conn->conn_ops->DataDigest) {
1442 iov[iov_count].iov_base = &checksum;
1443 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1444 rx_size += ISCSI_CRC_LEN;
1445 }
1446
1447 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1448
1449 iscsit_unmap_iovec(cmd);
1450
1451 if (rx_got != rx_size)
1452 return -1;
1453
1454 if (conn->conn_ops->DataDigest) {
1455 u32 data_crc;
1456
1457 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
50e5c87d
CH		 1458 be32_to_cpu(hdr->offset),
1459 payload_length, padding,
e48354ce
NB		 1460 cmd->pad_bytes);
1461
1462 if (checksum != data_crc) {
1463 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1464 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1465 " does not match computed 0x%08x\n",
1466 hdr->itt, hdr->offset, payload_length,
1467 hdr->datasn, checksum, data_crc);
1468 data_crc_failed = 1;
1469 } else {
1470 pr_debug("Got CRC32C DataDigest 0x%08x for"
1471 " %u bytes of Data Out\n", checksum,
1472 payload_length);
1473 }
1474 }
3e1c81a9
NB		 1475
1476 return data_crc_failed;
1477}
1478
1479int
1480iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
1481 bool data_crc_failed)
1482{
1483 struct iscsi_conn *conn = cmd->conn;
1484 int rc, ooo_cmdsn;
e48354ce
NB		 1485 /*
1486 * Increment post receive data and CRC values or perform
1487 * within-command recovery.
1488 */
3e1c81a9
NB		 1489 rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
1490 if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
e48354ce 1491 return 0;
3e1c81a9 1492 else if (rc == DATAOUT_SEND_R2T) {
e48354ce 1493 iscsit_set_dataout_sequence_values(cmd);
3e1c81a9
NB		 1494 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
1495 } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
e48354ce
NB		 1496 /*
1497 * Handle extra special case for out of order
1498 * Unsolicited Data Out.
1499 */
1500 spin_lock_bh(&cmd->istate_lock);
1501 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1502 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1503 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1504 spin_unlock_bh(&cmd->istate_lock);
1505
1506 iscsit_stop_dataout_timer(cmd);
67441b68
CH		 1507 if (ooo_cmdsn)
1508 return 0;
1509 target_execute_cmd(&cmd->se_cmd);
1510 return 0;
e48354ce
NB		 1511 } else /* DATAOUT_CANNOT_RECOVER */
1512 return -1;
1513
1514 return 0;
1515}
3e1c81a9 1516EXPORT_SYMBOL(iscsit_check_dataout_payload);
e48354ce 1517
3e1c81a9
NB		 1518static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1519{
1520 struct iscsi_cmd *cmd;
1521 struct iscsi_data *hdr = (struct iscsi_data *)buf;
1522 int rc;
1523 bool data_crc_failed = false;
1524
1525 rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1526 if (rc < 0)
1527 return rc;
1528 else if (!cmd)
1529 return 0;
1530
1531 rc = iscsit_get_dataout(conn, cmd, hdr);
1532 if (rc < 0)
1533 return rc;
1534 else if (rc > 0)
1535 data_crc_failed = true;
1536
1537 return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
1538}
1539
1540int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1541 unsigned char *buf)
e48354ce
NB		 1542{
1543 unsigned char *ping_data = NULL;
1544 int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
1545 u32 checksum, data_crc, padding = 0, payload_length;
3e1c81a9 1546 struct iscsi_cmd *cmd_p = NULL;
e48354ce
NB		 1547 struct kvec *iov = NULL;
1548 struct iscsi_nopout *hdr;
1549
1550 hdr = (struct iscsi_nopout *) buf;
1551 payload_length = ntoh24(hdr->dlength);
e48354ce 1552
66c7db68 1553 if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
e48354ce
NB		 1554 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1555 " not set, protocol error.\n");
1556 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1557 buf, conn);
1558 }
1559
21f5aa7e 1560 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
e48354ce 1561 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
21f5aa7e 1562 " greater than MaxXmitDataSegmentLength: %u, protocol"
e48354ce 1563 " error.\n", payload_length,
21f5aa7e 1564 conn->conn_ops->MaxXmitDataSegmentLength);
e48354ce
NB		 1565 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1566 buf, conn);
1567 }
1568
3e1c81a9 1569 pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
e48354ce 1570 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
66c7db68 1571 hdr->itt == RESERVED_ITT ? "Response" : "Request",
e48354ce
NB		 1572 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1573 payload_length);
1574 /*
1575 * This is not a response to a Unsolicited NopIN, which means
1576 * it can either be a NOPOUT ping request (with a valid ITT),
1577 * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1578 * Either way, make sure we allocate an struct iscsi_cmd, as both
1579 * can contain ping data.
1580 */
50e5c87d 1581 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
e48354ce
NB		 1582 if (!cmd)
1583 return iscsit_add_reject(
1584 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1585 1, buf, conn);
1586
1587 cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT;
1588 cmd->i_state = ISTATE_SEND_NOPIN;
1589 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1590 1 : 0);
1591 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1592 cmd->targ_xfer_tag = 0xFFFFFFFF;
50e5c87d
CH		 1593 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
1594 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
e48354ce
NB		 1595 cmd->data_direction = DMA_NONE;
1596 }
1597
50e5c87d 1598 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
e48354ce
NB		 1599 rx_size = payload_length;
1600 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1601 if (!ping_data) {
1602 pr_err("Unable to allocate memory for"
1603 " NOPOUT ping data.\n");
1604 ret = -1;
1605 goto out;
1606 }
1607
1608 iov = &cmd->iov_misc[0];
1609 iov[niov].iov_base = ping_data;
1610 iov[niov++].iov_len = payload_length;
1611
1612 padding = ((-payload_length) & 3);
1613 if (padding != 0) {
1614 pr_debug("Receiving %u additional bytes"
1615 " for padding.\n", padding);
1616 iov[niov].iov_base = &cmd->pad_bytes;
1617 iov[niov++].iov_len = padding;
1618 rx_size += padding;
1619 }
1620 if (conn->conn_ops->DataDigest) {
1621 iov[niov].iov_base = &checksum;
1622 iov[niov++].iov_len = ISCSI_CRC_LEN;
1623 rx_size += ISCSI_CRC_LEN;
1624 }
1625
1626 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1627 if (rx_got != rx_size) {
1628 ret = -1;
1629 goto out;
1630 }
1631
1632 if (conn->conn_ops->DataDigest) {
1633 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1634 ping_data, payload_length,
1635 padding, cmd->pad_bytes,
1636 (u8 *)&data_crc);
1637
1638 if (checksum != data_crc) {
1639 pr_err("Ping data CRC32C DataDigest"
1640 " 0x%08x does not match computed 0x%08x\n",
1641 checksum, data_crc);
1642 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1643 pr_err("Unable to recover from"
1644 " NOPOUT Ping DataCRC failure while in"
1645 " ERL=0.\n");
1646 ret = -1;
1647 goto out;
1648 } else {
1649 /*
1650 * Silently drop this PDU and let the
1651 * initiator plug the CmdSN gap.
1652 */
1653 pr_debug("Dropping NOPOUT"
1654 " Command CmdSN: 0x%08x due to"
1655 " DataCRC error.\n", hdr->cmdsn);
1656 ret = 0;
1657 goto out;
1658 }
1659 } else {
1660 pr_debug("Got CRC32C DataDigest"
1661 " 0x%08x for %u bytes of ping data.\n",
1662 checksum, payload_length);
1663 }
1664 }
1665
1666 ping_data[payload_length] = '\0';
1667 /*
1668 * Attach ping data to struct iscsi_cmd->buf_ptr.
1669 */
8359cf43 1670 cmd->buf_ptr = ping_data;
e48354ce
NB		 1671 cmd->buf_ptr_size = payload_length;
1672
1673 pr_debug("Got %u bytes of NOPOUT ping"
1674 " data.\n", payload_length);
1675 pr_debug("Ping Data: \"%s\"\n", ping_data);
1676 }
1677
66c7db68 1678 if (hdr->itt != RESERVED_ITT) {
e48354ce
NB		 1679 if (!cmd) {
1680 pr_err("Checking CmdSN for NOPOUT,"
1681 " but cmd is NULL!\n");
1682 return -1;
1683 }
1684 /*
1685 * Initiator is expecting a NopIN ping reply,
1686 */
1687 spin_lock_bh(&conn->cmd_lock);
2fbb471e 1688 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 1689 spin_unlock_bh(&conn->cmd_lock);
1690
50e5c87d 1691 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
e48354ce
NB		 1692
1693 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1694 iscsit_add_cmd_to_response_queue(cmd, conn,
1695 cmd->i_state);
1696 return 0;
1697 }
1698
1699 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1700 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1701 ret = 0;
1702 goto ping_out;
1703 }
1704 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1705 return iscsit_add_reject_from_cmd(
1706 ISCSI_REASON_PROTOCOL_ERROR,
1707 1, 0, buf, cmd);
1708
1709 return 0;
1710 }
1711
50e5c87d 1712 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
e48354ce
NB		 1713 /*
1714 * This was a response to a unsolicited NOPIN ping.
1715 */
3e1c81a9
NB		 1716 cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
1717 if (!cmd_p)
e48354ce
NB		 1718 return -1;
1719
1720 iscsit_stop_nopin_response_timer(conn);
1721
3e1c81a9
NB		 1722 cmd_p->i_state = ISTATE_REMOVE;
1723 iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
e48354ce
NB		 1724 iscsit_start_nopin_timer(conn);
1725 } else {
1726 /*
1727 * Initiator is not expecting a NOPIN is response.
1728 * Just ignore for now.
1729 *
1730 * iSCSI v19-91 10.18
1731 * "A NOP-OUT may also be used to confirm a changed
1732 * ExpStatSN if another PDU will not be available
1733 * for a long time."
1734 */
1735 ret = 0;
1736 goto out;
1737 }
1738
1739 return 0;
1740out:
1741 if (cmd)
1742 iscsit_release_cmd(cmd);
1743ping_out:
1744 kfree(ping_data);
1745 return ret;
1746}
3e1c81a9 1747EXPORT_SYMBOL(iscsit_handle_nop_out);
e48354ce 1748
3e1c81a9
NB		 1749int
1750iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
1751 unsigned char *buf)
e48354ce 1752{
e48354ce
NB		 1753 struct se_tmr_req *se_tmr;
1754 struct iscsi_tmr_req *tmr_req;
1755 struct iscsi_tm *hdr;
e48354ce
NB		 1756 int out_of_order_cmdsn = 0;
1757 int ret;
1758 u8 function;
1759
1760 hdr = (struct iscsi_tm *) buf;
e48354ce
NB		 1761 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1762 function = hdr->flags;
1763
1764 pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1765 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1766 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1767 hdr->rtt, hdr->refcmdsn, conn->cid);
1768
1769 if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1770 ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
66c7db68 1771 hdr->rtt != RESERVED_ITT)) {
e48354ce 1772 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
66c7db68 1773 hdr->rtt = RESERVED_ITT;
e48354ce
NB		 1774 }
1775
1776 if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1777 !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1778 pr_err("Task Management Request TASK_REASSIGN not"
1779 " issued as immediate command, bad iSCSI Initiator"
1780 "implementation\n");
3e1c81a9
NB		 1781 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
1782 1, 1, buf, cmd);
e48354ce
NB		 1783 }
1784 if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
50e5c87d
CH		 1785 be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
1786 hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
e48354ce 1787
d28b1169
AG		 1788 cmd->data_direction = DMA_NONE;
1789
1790 cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL);
1791 if (!cmd->tmr_req) {
1792 pr_err("Unable to allocate memory for"
1793 " Task Management command!\n");
1794 return iscsit_add_reject_from_cmd(
1795 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1796 1, 1, buf, cmd);
1797 }
1798
1799 /*
1800 * TASK_REASSIGN for ERL=2 / connection stays inside of
1801 * LIO-Target $FABRIC_MOD
1802 */
1803 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1804
1805 u8 tcm_function;
1806 int ret;
1807
1808 transport_init_se_cmd(&cmd->se_cmd,
1809 &lio_target_fabric_configfs->tf_ops,
1810 conn->sess->se_sess, 0, DMA_NONE,
9c58b7dd 1811 MSG_SIMPLE_TAG, cmd->sense_buffer + 2);
d28b1169
AG		 1812
1813 switch (function) {
1814 case ISCSI_TM_FUNC_ABORT_TASK:
1815 tcm_function = TMR_ABORT_TASK;
1816 break;
1817 case ISCSI_TM_FUNC_ABORT_TASK_SET:
1818 tcm_function = TMR_ABORT_TASK_SET;
1819 break;
1820 case ISCSI_TM_FUNC_CLEAR_ACA:
1821 tcm_function = TMR_CLEAR_ACA;
1822 break;
1823 case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1824 tcm_function = TMR_CLEAR_TASK_SET;
1825 break;
1826 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1827 tcm_function = TMR_LUN_RESET;
1828 break;
1829 case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1830 tcm_function = TMR_TARGET_WARM_RESET;
1831 break;
1832 case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1833 tcm_function = TMR_TARGET_COLD_RESET;
1834 break;
1835 default:
1836 pr_err("Unknown iSCSI TMR Function:"
1837 " 0x%02x\n", function);
1838 return iscsit_add_reject_from_cmd(
1839 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1840 1, 1, buf, cmd);
1841 }
1842
1843 ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req,
1844 tcm_function, GFP_KERNEL);
1845 if (ret < 0)
1846 return iscsit_add_reject_from_cmd(
1847 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1848 1, 1, buf, cmd);
1849
1850 cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
1851 }
e48354ce
NB		 1852
1853 cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC;
1854 cmd->i_state = ISTATE_SEND_TASKMGTRSP;
1855 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1856 cmd->init_task_tag = hdr->itt;
1857 cmd->targ_xfer_tag = 0xFFFFFFFF;
50e5c87d
CH		 1858 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
1859 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
e48354ce
NB		 1860 se_tmr = cmd->se_cmd.se_tmr_req;
1861 tmr_req = cmd->tmr_req;
1862 /*
1863 * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
1864 */
1865 if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
4f26998a
AG		 1866 ret = transport_lookup_tmr_lun(&cmd->se_cmd,
1867 scsilun_to_int(&hdr->lun));
e48354ce 1868 if (ret < 0) {
e48354ce
NB		 1869 se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
1870 goto attach;
1871 }
1872 }
1873
1874 switch (function) {
1875 case ISCSI_TM_FUNC_ABORT_TASK:
1876 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
de103c93 1877 if (se_tmr->response)
e48354ce 1878 goto attach;
e48354ce
NB		 1879 break;
1880 case ISCSI_TM_FUNC_ABORT_TASK_SET:
1881 case ISCSI_TM_FUNC_CLEAR_ACA:
1882 case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1883 case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1884 break;
1885 case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1886 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
e48354ce
NB		 1887 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1888 goto attach;
1889 }
1890 break;
1891 case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1892 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
e48354ce
NB		 1893 se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1894 goto attach;
1895 }
1896 break;
1897 case ISCSI_TM_FUNC_TASK_REASSIGN:
1898 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
1899 /*
1900 * Perform sanity checks on the ExpDataSN only if the
1901 * TASK_REASSIGN was successful.
1902 */
de103c93 1903 if (se_tmr->response)
e48354ce
NB		 1904 break;
1905
1906 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
1907 return iscsit_add_reject_from_cmd(
1908 ISCSI_REASON_BOOKMARK_INVALID, 1, 1,
1909 buf, cmd);
1910 break;
1911 default:
1912 pr_err("Unknown TMR function: 0x%02x, protocol"
1913 " error.\n", function);
e48354ce
NB		 1914 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
1915 goto attach;
1916 }
1917
1918 if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1919 (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
1920 se_tmr->call_transport = 1;
1921attach:
1922 spin_lock_bh(&conn->cmd_lock);
2fbb471e 1923 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 1924 spin_unlock_bh(&conn->cmd_lock);
1925
1926 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1927 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1928 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
1929 out_of_order_cmdsn = 1;
5a4c8666 1930 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
e48354ce 1931 return 0;
5a4c8666 1932 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
e48354ce
NB		 1933 return iscsit_add_reject_from_cmd(
1934 ISCSI_REASON_PROTOCOL_ERROR,
1935 1, 0, buf, cmd);
e48354ce 1936 }
50e5c87d 1937 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
e48354ce 1938
5a4c8666 1939 if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
e48354ce
NB		 1940 return 0;
1941 /*
1942 * Found the referenced task, send to transport for processing.
1943 */
1944 if (se_tmr->call_transport)
1945 return transport_generic_handle_tmr(&cmd->se_cmd);
1946
1947 /*
1948 * Could not find the referenced LUN, task, or Task Management
1949 * command not authorized or supported. Change state and
1950 * let the tx_thread send the response.
1951 *
1952 * For connection recovery, this is also the default action for
1953 * TMR TASK_REASSIGN.
1954 */
1955 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
1956 return 0;
1957}
3e1c81a9 1958EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
e48354ce
NB		 1959
1960/* #warning FIXME: Support Text Command parameters besides SendTargets */
1961static int iscsit_handle_text_cmd(
1962 struct iscsi_conn *conn,
1963 unsigned char *buf)
1964{
1965 char *text_ptr, *text_in;
1966 int cmdsn_ret, niov = 0, rx_got, rx_size;
1967 u32 checksum = 0, data_crc = 0, payload_length;
76f1928e 1968 u32 padding = 0, pad_bytes = 0, text_length = 0;
e48354ce
NB		 1969 struct iscsi_cmd *cmd;
1970 struct kvec iov[3];
1971 struct iscsi_text *hdr;
1972
1973 hdr = (struct iscsi_text *) buf;
1974 payload_length = ntoh24(hdr->dlength);
e48354ce 1975
21f5aa7e 1976 if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
e48354ce 1977 pr_err("Unable to accept text parameter length: %u"
21f5aa7e
NB		 1978 "greater than MaxXmitDataSegmentLength %u.\n",
1979 payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
e48354ce
NB		 1980 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1981 buf, conn);
1982 }
1983
1984 pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
1985 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
1986 hdr->exp_statsn, payload_length);
1987
1988 rx_size = text_length = payload_length;
1989 if (text_length) {
1990 text_in = kzalloc(text_length, GFP_KERNEL);
1991 if (!text_in) {
1992 pr_err("Unable to allocate memory for"
1993 " incoming text parameters\n");
1994 return -1;
1995 }
1996
1997 memset(iov, 0, 3 * sizeof(struct kvec));
1998 iov[niov].iov_base = text_in;
1999 iov[niov++].iov_len = text_length;
2000
2001 padding = ((-payload_length) & 3);
2002 if (padding != 0) {
76f1928e 2003 iov[niov].iov_base = &pad_bytes;
e48354ce
NB		 2004 iov[niov++].iov_len = padding;
2005 rx_size += padding;
2006 pr_debug("Receiving %u additional bytes"
2007 " for padding.\n", padding);
2008 }
2009 if (conn->conn_ops->DataDigest) {
2010 iov[niov].iov_base = &checksum;
2011 iov[niov++].iov_len = ISCSI_CRC_LEN;
2012 rx_size += ISCSI_CRC_LEN;
2013 }
2014
2015 rx_got = rx_data(conn, &iov[0], niov, rx_size);
2016 if (rx_got != rx_size) {
2017 kfree(text_in);
2018 return -1;
2019 }
2020
2021 if (conn->conn_ops->DataDigest) {
2022 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
2023 text_in, text_length,
76f1928e 2024 padding, (u8 *)&pad_bytes,
e48354ce
NB		 2025 (u8 *)&data_crc);
2026
2027 if (checksum != data_crc) {
2028 pr_err("Text data CRC32C DataDigest"
2029 " 0x%08x does not match computed"
2030 " 0x%08x\n", checksum, data_crc);
2031 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2032 pr_err("Unable to recover from"
2033 " Text Data digest failure while in"
2034 " ERL=0.\n");
2035 kfree(text_in);
2036 return -1;
2037 } else {
2038 /*
2039 * Silently drop this PDU and let the
2040 * initiator plug the CmdSN gap.
2041 */
2042 pr_debug("Dropping Text"
2043 " Command CmdSN: 0x%08x due to"
2044 " DataCRC error.\n", hdr->cmdsn);
2045 kfree(text_in);
2046 return 0;
2047 }
2048 } else {
2049 pr_debug("Got CRC32C DataDigest"
2050 " 0x%08x for %u bytes of text data.\n",
2051 checksum, text_length);
2052 }
2053 }
2054 text_in[text_length - 1] = '\0';
2055 pr_debug("Successfully read %d bytes of text"
2056 " data.\n", text_length);
2057
2058 if (strncmp("SendTargets", text_in, 11) != 0) {
2059 pr_err("Received Text Data that is not"
2060 " SendTargets, cannot continue.\n");
2061 kfree(text_in);
2062 return -1;
2063 }
2064 text_ptr = strchr(text_in, '=');
2065 if (!text_ptr) {
2066 pr_err("No \"=\" separator found in Text Data,"
2067 " cannot continue.\n");
2068 kfree(text_in);
2069 return -1;
2070 }
2071 if (strncmp("=All", text_ptr, 4) != 0) {
2072 pr_err("Unable to locate All value for"
2073 " SendTargets key, cannot continue.\n");
2074 kfree(text_in);
2075 return -1;
2076 }
2077/*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */
2078 kfree(text_in);
2079 }
2080
2081 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
2082 if (!cmd)
2083 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
2084 1, buf, conn);
2085
2086 cmd->iscsi_opcode = ISCSI_OP_TEXT;
2087 cmd->i_state = ISTATE_SEND_TEXTRSP;
2088 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2089 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
2090 cmd->targ_xfer_tag = 0xFFFFFFFF;
50e5c87d
CH		 2091 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
2092 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
e48354ce
NB		 2093 cmd->data_direction = DMA_NONE;
2094
2095 spin_lock_bh(&conn->cmd_lock);
2fbb471e 2096 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 2097 spin_unlock_bh(&conn->cmd_lock);
2098
50e5c87d 2099 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
e48354ce
NB		 2100
2101 if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
2102 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
2103 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
2104 return iscsit_add_reject_from_cmd(
2105 ISCSI_REASON_PROTOCOL_ERROR,
2106 1, 0, buf, cmd);
2107
2108 return 0;
2109 }
2110
2111 return iscsit_execute_cmd(cmd, 0);
2112}
2113
2114int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2115{
2116 struct iscsi_conn *conn_p;
2117 struct iscsi_session *sess = conn->sess;
2118
2119 pr_debug("Received logout request CLOSESESSION on CID: %hu"
2120 " for SID: %u.\n", conn->cid, conn->sess->sid);
2121
2122 atomic_set(&sess->session_logout, 1);
2123 atomic_set(&conn->conn_logout_remove, 1);
2124 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2125
2126 iscsit_inc_conn_usage_count(conn);
2127 iscsit_inc_session_usage_count(sess);
2128
2129 spin_lock_bh(&sess->conn_lock);
2130 list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2131 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2132 continue;
2133
2134 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2135 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2136 }
2137 spin_unlock_bh(&sess->conn_lock);
2138
2139 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2140
2141 return 0;
2142}
2143
2144int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2145{
2146 struct iscsi_conn *l_conn;
2147 struct iscsi_session *sess = conn->sess;
2148
2149 pr_debug("Received logout request CLOSECONNECTION for CID:"
2150 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2151
2152 /*
2153 * A Logout Request with a CLOSECONNECTION reason code for a CID
2154 * can arrive on a connection with a differing CID.
2155 */
2156 if (conn->cid == cmd->logout_cid) {
2157 spin_lock_bh(&conn->state_lock);
2158 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2159 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2160
2161 atomic_set(&conn->conn_logout_remove, 1);
2162 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2163 iscsit_inc_conn_usage_count(conn);
2164
2165 spin_unlock_bh(&conn->state_lock);
2166 } else {
2167 /*
2168 * Handle all different cid CLOSECONNECTION requests in
2169 * iscsit_logout_post_handler_diffcid() as to give enough
2170 * time for any non immediate command's CmdSN to be
2171 * acknowledged on the connection in question.
2172 *
2173 * Here we simply make sure the CID is still around.
2174 */
2175 l_conn = iscsit_get_conn_from_cid(sess,
2176 cmd->logout_cid);
2177 if (!l_conn) {
2178 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2179 iscsit_add_cmd_to_response_queue(cmd, conn,
2180 cmd->i_state);
2181 return 0;
2182 }
2183
2184 iscsit_dec_conn_usage_count(l_conn);
2185 }
2186
2187 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2188
2189 return 0;
2190}
2191
2192int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2193{
2194 struct iscsi_session *sess = conn->sess;
2195
2196 pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2197 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2198
2199 if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2200 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2201 " while ERL!=2.\n");
2202 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2203 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2204 return 0;
2205 }
2206
2207 if (conn->cid == cmd->logout_cid) {
2208 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2209 " with CID: %hu on CID: %hu, implementation error.\n",
2210 cmd->logout_cid, conn->cid);
2211 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2212 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2213 return 0;
2214 }
2215
2216 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2217
2218 return 0;
2219}
2220
3e1c81a9
NB		 2221int
2222iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2223 unsigned char *buf)
e48354ce
NB		 2224{
2225 int cmdsn_ret, logout_remove = 0;
2226 u8 reason_code = 0;
e48354ce
NB		 2227 struct iscsi_logout *hdr;
2228 struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2229
2230 hdr = (struct iscsi_logout *) buf;
2231 reason_code = (hdr->flags & 0x7f);
e48354ce
NB		 2232
2233 if (tiqn) {
2234 spin_lock(&tiqn->logout_stats.lock);
2235 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2236 tiqn->logout_stats.normal_logouts++;
2237 else
2238 tiqn->logout_stats.abnormal_logouts++;
2239 spin_unlock(&tiqn->logout_stats.lock);
2240 }
2241
2242 pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2243 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2244 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2245 hdr->cid, conn->cid);
2246
2247 if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2248 pr_err("Received logout request on connection that"
2249 " is not in logged in state, ignoring request.\n");
3e1c81a9 2250 iscsit_release_cmd(cmd);
e48354ce
NB		 2251 return 0;
2252 }
2253
e48354ce
NB		 2254 cmd->iscsi_opcode = ISCSI_OP_LOGOUT;
2255 cmd->i_state = ISTATE_SEND_LOGOUTRSP;
2256 cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2257 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
2258 cmd->targ_xfer_tag = 0xFFFFFFFF;
50e5c87d
CH		 2259 cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
2260 cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
2261 cmd->logout_cid = be16_to_cpu(hdr->cid);
e48354ce
NB		 2262 cmd->logout_reason = reason_code;
2263 cmd->data_direction = DMA_NONE;
2264
2265 /*
2266 * We need to sleep in these cases (by returning 1) until the Logout
2267 * Response gets sent in the tx thread.
2268 */
2269 if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2270 ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
50e5c87d 2271 be16_to_cpu(hdr->cid) == conn->cid))
e48354ce
NB		 2272 logout_remove = 1;
2273
2274 spin_lock_bh(&conn->cmd_lock);
2fbb471e 2275 list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
e48354ce
NB		 2276 spin_unlock_bh(&conn->cmd_lock);
2277
2278 if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
50e5c87d 2279 iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
e48354ce
NB		 2280
2281 /*
2282 * Immediate commands are executed, well, immediately.
2283 * Non-Immediate Logout Commands are executed in CmdSN order.
2284 */
c6037cc5 2285 if (cmd->immediate_cmd) {
e48354ce
NB		 2286 int ret = iscsit_execute_cmd(cmd, 0);
2287
2288 if (ret < 0)
2289 return ret;
2290 } else {
2291 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
2292 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2293 logout_remove = 0;
2294 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2295 return iscsit_add_reject_from_cmd(
2296 ISCSI_REASON_PROTOCOL_ERROR,
2297 1, 0, buf, cmd);
2298 }
2299 }
2300
2301 return logout_remove;
2302}
3e1c81a9 2303EXPORT_SYMBOL(iscsit_handle_logout_cmd);
e48354ce
NB		 2304
2305static int iscsit_handle_snack(
2306 struct iscsi_conn *conn,
2307 unsigned char *buf)
2308{
e48354ce
NB		 2309 struct iscsi_snack *hdr;
2310
2311 hdr = (struct iscsi_snack *) buf;
2312 hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
e48354ce
NB		 2313
2314 pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2315 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2316 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2317 hdr->begrun, hdr->runlength, conn->cid);
2318
2319 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2320 pr_err("Initiator sent SNACK request while in"
2321 " ErrorRecoveryLevel=0.\n");
2322 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2323 buf, conn);
2324 }
2325 /*
2326 * SNACK_DATA and SNACK_R2T are both 0, so check which function to
2327 * call from inside iscsi_send_recovery_datain_or_r2t().
2328 */
2329 switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2330 case 0:
2331 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
50e5c87d
CH		 2332 hdr->itt,
2333 be32_to_cpu(hdr->ttt),
2334 be32_to_cpu(hdr->begrun),
2335 be32_to_cpu(hdr->runlength));
e48354ce 2336 case ISCSI_FLAG_SNACK_TYPE_STATUS:
50e5c87d
CH		 2337 return iscsit_handle_status_snack(conn, hdr->itt,
2338 be32_to_cpu(hdr->ttt),
2339 be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
e48354ce 2340 case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
50e5c87d
CH		 2341 return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
2342 be32_to_cpu(hdr->begrun),
2343 be32_to_cpu(hdr->runlength));
e48354ce
NB		 2344 case ISCSI_FLAG_SNACK_TYPE_RDATA:
2345 /* FIXME: Support R-Data SNACK */
2346 pr_err("R-Data SNACK Not Supported.\n");
2347 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2348 buf, conn);
2349 default:
2350 pr_err("Unknown SNACK type 0x%02x, protocol"
2351 " error.\n", hdr->flags & 0x0f);
2352 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2353 buf, conn);
2354 }
2355
2356 return 0;
2357}
2358
2359static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2360{
2361 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2362 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2363 wait_for_completion_interruptible_timeout(
2364 &conn->rx_half_close_comp,
2365 ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2366 }
2367}
2368
2369static int iscsit_handle_immediate_data(
2370 struct iscsi_cmd *cmd,
3e1c81a9 2371 struct iscsi_scsi_req *hdr,
e48354ce
NB		 2372 u32 length)
2373{
2374 int iov_ret, rx_got = 0, rx_size = 0;
2375 u32 checksum, iov_count = 0, padding = 0;
2376 struct iscsi_conn *conn = cmd->conn;
2377 struct kvec *iov;
2378
2379 iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2380 if (iov_ret < 0)
2381 return IMMEDIATE_DATA_CANNOT_RECOVER;
2382
2383 rx_size = length;
2384 iov_count = iov_ret;
2385 iov = &cmd->iov_data[0];
2386
2387 padding = ((-length) & 3);
2388 if (padding != 0) {
2389 iov[iov_count].iov_base = cmd->pad_bytes;
2390 iov[iov_count++].iov_len = padding;
2391 rx_size += padding;
2392 }
2393
2394 if (conn->conn_ops->DataDigest) {
2395 iov[iov_count].iov_base = &checksum;
2396 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2397 rx_size += ISCSI_CRC_LEN;
2398 }
2399
2400 rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2401
2402 iscsit_unmap_iovec(cmd);
2403
2404 if (rx_got != rx_size) {
2405 iscsit_rx_thread_wait_for_tcp(conn);
2406 return IMMEDIATE_DATA_CANNOT_RECOVER;
2407 }
2408
2409 if (conn->conn_ops->DataDigest) {
2410 u32 data_crc;
2411
2412 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
2413 cmd->write_data_done, length, padding,
2414 cmd->pad_bytes);
2415
2416 if (checksum != data_crc) {
2417 pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2418 " does not match computed 0x%08x\n", checksum,
2419 data_crc);
2420
2421 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2422 pr_err("Unable to recover from"
2423 " Immediate Data digest failure while"
2424 " in ERL=0.\n");
2425 iscsit_add_reject_from_cmd(
2426 ISCSI_REASON_DATA_DIGEST_ERROR,
3e1c81a9 2427 1, 0, (unsigned char *)hdr, cmd);
e48354ce
NB		 2428 return IMMEDIATE_DATA_CANNOT_RECOVER;
2429 } else {
2430 iscsit_add_reject_from_cmd(
2431 ISCSI_REASON_DATA_DIGEST_ERROR,
3e1c81a9 2432 0, 0, (unsigned char *)hdr, cmd);
e48354ce
NB		 2433 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2434 }
2435 } else {
2436 pr_debug("Got CRC32C DataDigest 0x%08x for"
2437 " %u bytes of Immediate Data\n", checksum,
2438 length);
2439 }
2440 }
2441
2442 cmd->write_data_done += length;
2443
ebf1d95c 2444 if (cmd->write_data_done == cmd->se_cmd.data_length) {
e48354ce
NB		 2445 spin_lock_bh(&cmd->istate_lock);
2446 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2447 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2448 spin_unlock_bh(&cmd->istate_lock);
2449 }
2450
2451 return IMMEDIATE_DATA_NORMAL_OPERATION;
2452}
2453
2454/*
2455 * Called with sess->conn_lock held.
2456 */
2457/* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2458 with active network interface */
2459static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2460{
2461 struct iscsi_cmd *cmd;
2462 struct iscsi_conn *conn_p;
2463
2464 /*
2465 * Only send a Asynchronous Message on connections whos network
2466 * interface is still functional.
2467 */
2468 list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2469 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2470 iscsit_inc_conn_usage_count(conn_p);
2471 break;
2472 }
2473 }
2474
2475 if (!conn_p)
2476 return;
2477
3c989d76 2478 cmd = iscsit_allocate_cmd(conn_p, GFP_ATOMIC);
e48354ce
NB		 2479 if (!cmd) {
2480 iscsit_dec_conn_usage_count(conn_p);
2481 return;
2482 }
2483
2484 cmd->logout_cid = conn->cid;
2485 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2486 cmd->i_state = ISTATE_SEND_ASYNCMSG;
2487
2488 spin_lock_bh(&conn_p->cmd_lock);
2fbb471e 2489 list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
e48354ce
NB		 2490 spin_unlock_bh(&conn_p->cmd_lock);
2491
2492 iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2493 iscsit_dec_conn_usage_count(conn_p);
2494}
2495
2496static int iscsit_send_conn_drop_async_message(
2497 struct iscsi_cmd *cmd,
2498 struct iscsi_conn *conn)
2499{
2500 struct iscsi_async *hdr;
2501
2502 cmd->tx_size = ISCSI_HDR_LEN;
2503 cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2504
2505 hdr = (struct iscsi_async *) cmd->pdu;
2506 hdr->opcode = ISCSI_OP_ASYNC_EVENT;
2507 hdr->flags = ISCSI_FLAG_CMD_FINAL;
66c7db68 2508 cmd->init_task_tag = RESERVED_ITT;
e48354ce
NB		 2509 cmd->targ_xfer_tag = 0xFFFFFFFF;
2510 put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2511 cmd->stat_sn = conn->stat_sn++;
2512 hdr->statsn = cpu_to_be32(cmd->stat_sn);
2513 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
2514 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
2515 hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2516 hdr->param1 = cpu_to_be16(cmd->logout_cid);
2517 hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2518 hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2519
2520 if (conn->conn_ops->HeaderDigest) {
2521 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2522
2523 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2524 (unsigned char *)hdr, ISCSI_HDR_LEN,
2525 0, NULL, (u8 *)header_digest);
2526
2527 cmd->tx_size += ISCSI_CRC_LEN;
2528 pr_debug("Attaching CRC32C HeaderDigest to"
2529 " Async Message 0x%08x\n", *header_digest);
2530 }
2531
2532 cmd->iov_misc[0].iov_base = cmd->pdu;
2533 cmd->iov_misc[0].iov_len = cmd->tx_size;
2534 cmd->iov_misc_count = 1;
2535
2536 pr_debug("Sending Connection Dropped Async Message StatSN:"
2537 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2538 cmd->logout_cid, conn->cid);
2539 return 0;
2540}
2541
6f3c0e69
AG		 2542static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
2543{
2544 if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2545 (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2546 wait_for_completion_interruptible_timeout(
2547 &conn->tx_half_close_comp,
2548 ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
2549 }
2550}
2551
e48354ce
NB		 2552static int iscsit_send_data_in(
2553 struct iscsi_cmd *cmd,
6f3c0e69 2554 struct iscsi_conn *conn)
e48354ce
NB		 2555{
2556 int iov_ret = 0, set_statsn = 0;
2557 u32 iov_count = 0, tx_size = 0;
2558 struct iscsi_datain datain;
2559 struct iscsi_datain_req *dr;
2560 struct iscsi_data_rsp *hdr;
2561 struct kvec *iov;
6f3c0e69
AG		 2562 int eodr = 0;
2563 int ret;
e48354ce
NB		 2564
2565 memset(&datain, 0, sizeof(struct iscsi_datain));
2566 dr = iscsit_get_datain_values(cmd, &datain);
2567 if (!dr) {
2568 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2569 cmd->init_task_tag);
2570 return -1;
2571 }
2572
2573 /*
2574 * Be paranoid and double check the logic for now.
2575 */
ebf1d95c 2576 if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
e48354ce
NB		 2577 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2578 " datain.length: %u exceeds cmd->data_length: %u\n",
2579 cmd->init_task_tag, datain.offset, datain.length,
ebf1d95c 2580 cmd->se_cmd.data_length);
e48354ce
NB		 2581 return -1;
2582 }
2583
2584 spin_lock_bh(&conn->sess->session_stats_lock);
2585 conn->sess->tx_data_octets += datain.length;
2586 if (conn->sess->se_sess->se_node_acl) {
2587 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
2588 conn->sess->se_sess->se_node_acl->read_bytes += datain.length;
2589 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
2590 }
2591 spin_unlock_bh(&conn->sess->session_stats_lock);
2592 /*
2593 * Special case for successfully execution w/ both DATAIN
2594 * and Sense Data.
2595 */
2596 if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2597 (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2598 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2599 else {
2600 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2601 (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2602 iscsit_increment_maxcmdsn(cmd, conn->sess);
2603 cmd->stat_sn = conn->stat_sn++;
2604 set_statsn = 1;
2605 } else if (dr->dr_complete ==
2606 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2607 set_statsn = 1;
2608 }
2609
2610 hdr = (struct iscsi_data_rsp *) cmd->pdu;
2611 memset(hdr, 0, ISCSI_HDR_LEN);
2612 hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
2613 hdr->flags = datain.flags;
2614 if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2615 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2616 hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
7e46cf02 2617 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
e48354ce
NB		 2618 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2619 hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
7e46cf02 2620 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
e48354ce
NB		 2621 }
2622 }
2623 hton24(hdr->dlength, datain.length);
2624 if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2625 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2626 (struct scsi_lun *)&hdr->lun);
2627 else
2628 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2629
66c7db68 2630 hdr->itt = cmd->init_task_tag;
50e5c87d
CH		 2631
2632 if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2633 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
2634 else
2635 hdr->ttt = cpu_to_be32(0xFFFFFFFF);
2636 if (set_statsn)
2637 hdr->statsn = cpu_to_be32(cmd->stat_sn);
2638 else
2639 hdr->statsn = cpu_to_be32(0xFFFFFFFF);
2640
e48354ce
NB		 2641 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
2642 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
2643 hdr->datasn = cpu_to_be32(datain.data_sn);
2644 hdr->offset = cpu_to_be32(datain.offset);
2645
2646 iov = &cmd->iov_data[0];
2647 iov[iov_count].iov_base = cmd->pdu;
2648 iov[iov_count++].iov_len = ISCSI_HDR_LEN;
2649 tx_size += ISCSI_HDR_LEN;
2650
2651 if (conn->conn_ops->HeaderDigest) {
2652 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2653
2654 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2655 (unsigned char *)hdr, ISCSI_HDR_LEN,
2656 0, NULL, (u8 *)header_digest);
2657
2658 iov[0].iov_len += ISCSI_CRC_LEN;
2659 tx_size += ISCSI_CRC_LEN;
2660
2661 pr_debug("Attaching CRC32 HeaderDigest"
2662 " for DataIN PDU 0x%08x\n", *header_digest);
2663 }
2664
2665 iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length);
2666 if (iov_ret < 0)
2667 return -1;
2668
2669 iov_count += iov_ret;
2670 tx_size += datain.length;
2671
2672 cmd->padding = ((-datain.length) & 3);
2673 if (cmd->padding) {
2674 iov[iov_count].iov_base = cmd->pad_bytes;
2675 iov[iov_count++].iov_len = cmd->padding;
2676 tx_size += cmd->padding;
2677
2678 pr_debug("Attaching %u padding bytes\n",
2679 cmd->padding);
2680 }
2681 if (conn->conn_ops->DataDigest) {
2682 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd,
2683 datain.offset, datain.length, cmd->padding, cmd->pad_bytes);
2684
2685 iov[iov_count].iov_base = &cmd->data_crc;
2686 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2687 tx_size += ISCSI_CRC_LEN;
2688
2689 pr_debug("Attached CRC32C DataDigest %d bytes, crc"
2690 " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc);
2691 }
2692
2693 cmd->iov_data_count = iov_count;
2694 cmd->tx_size = tx_size;
2695
2696 pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2697 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2698 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2699 ntohl(hdr->offset), datain.length, conn->cid);
2700
6f3c0e69
AG		 2701 /* sendpage is preferred but can't insert markers */
2702 if (!conn->conn_ops->IFMarker)
2703 ret = iscsit_fe_sendpage_sg(cmd, conn);
2704 else
2705 ret = iscsit_send_tx_data(cmd, conn, 0);
2706
2707 iscsit_unmap_iovec(cmd);
2708
2709 if (ret < 0) {
2710 iscsit_tx_thread_wait_for_tcp(conn);
2711 return ret;
2712 }
2713
e48354ce 2714 if (dr->dr_complete) {
6f3c0e69 2715 eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
e48354ce
NB		 2716 2 : 1;
2717 iscsit_free_datain_req(cmd, dr);
2718 }
2719
6f3c0e69 2720 return eodr;
e48354ce
NB		 2721}
2722
2723static int iscsit_send_logout_response(
2724 struct iscsi_cmd *cmd,
2725 struct iscsi_conn *conn)
2726{
2727 int niov = 0, tx_size;
2728 struct iscsi_conn *logout_conn = NULL;
2729 struct iscsi_conn_recovery *cr = NULL;
2730 struct iscsi_session *sess = conn->sess;
2731 struct kvec *iov;
2732 struct iscsi_logout_rsp *hdr;
2733 /*
2734 * The actual shutting down of Sessions and/or Connections
2735 * for CLOSESESSION and CLOSECONNECTION Logout Requests
2736 * is done in scsi_logout_post_handler().
2737 */
2738 switch (cmd->logout_reason) {
2739 case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2740 pr_debug("iSCSI session logout successful, setting"
2741 " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2742 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2743 break;
2744 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2745 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2746 break;
2747 /*
2748 * For CLOSECONNECTION logout requests carrying
2749 * a matching logout CID -> local CID, the reference
2750 * for the local CID will have been incremented in
2751 * iscsi_logout_closeconnection().
2752 *
2753 * For CLOSECONNECTION logout requests carrying
2754 * a different CID than the connection it arrived
2755 * on, the connection responding to cmd->logout_cid
2756 * is stopped in iscsit_logout_post_handler_diffcid().
2757 */
2758
2759 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2760 " successful.\n", cmd->logout_cid, conn->cid);
2761 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2762 break;
2763 case ISCSI_LOGOUT_REASON_RECOVERY:
2764 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2765 (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2766 break;
2767 /*
2768 * If the connection is still active from our point of view
2769 * force connection recovery to occur.
2770 */
2771 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2772 cmd->logout_cid);
ee1b1b9c 2773 if (logout_conn) {
e48354ce
NB		 2774 iscsit_connection_reinstatement_rcfr(logout_conn);
2775 iscsit_dec_conn_usage_count(logout_conn);
2776 }
2777
2778 cr = iscsit_get_inactive_connection_recovery_entry(
2779 conn->sess, cmd->logout_cid);
2780 if (!cr) {
2781 pr_err("Unable to locate CID: %hu for"
2782 " REMOVECONNFORRECOVERY Logout Request.\n",
2783 cmd->logout_cid);
2784 cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2785 break;
2786 }
2787
2788 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2789
2790 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2791 " for recovery for CID: %hu on CID: %hu successful.\n",
2792 cmd->logout_cid, conn->cid);
2793 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2794 break;
2795 default:
2796 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2797 cmd->logout_reason);
2798 return -1;
2799 }
2800
2801 tx_size = ISCSI_HDR_LEN;
2802 hdr = (struct iscsi_logout_rsp *)cmd->pdu;
2803 memset(hdr, 0, ISCSI_HDR_LEN);
2804 hdr->opcode = ISCSI_OP_LOGOUT_RSP;
2805 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
2806 hdr->response = cmd->logout_response;
66c7db68 2807 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 2808 cmd->stat_sn = conn->stat_sn++;
2809 hdr->statsn = cpu_to_be32(cmd->stat_sn);
2810
2811 iscsit_increment_maxcmdsn(cmd, conn->sess);
2812 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
2813 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
2814
2815 iov = &cmd->iov_misc[0];
2816 iov[niov].iov_base = cmd->pdu;
2817 iov[niov++].iov_len = ISCSI_HDR_LEN;
2818
2819 if (conn->conn_ops->HeaderDigest) {
2820 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2821
2822 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2823 (unsigned char *)hdr, ISCSI_HDR_LEN,
2824 0, NULL, (u8 *)header_digest);
2825
2826 iov[0].iov_len += ISCSI_CRC_LEN;
2827 tx_size += ISCSI_CRC_LEN;
2828 pr_debug("Attaching CRC32C HeaderDigest to"
2829 " Logout Response 0x%08x\n", *header_digest);
2830 }
2831 cmd->iov_misc_count = niov;
2832 cmd->tx_size = tx_size;
2833
2834 pr_debug("Sending Logout Response ITT: 0x%08x StatSN:"
2835 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2836 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2837 cmd->logout_cid, conn->cid);
2838
2839 return 0;
2840}
2841
2842/*
2843 * Unsolicited NOPIN, either requesting a response or not.
2844 */
2845static int iscsit_send_unsolicited_nopin(
2846 struct iscsi_cmd *cmd,
2847 struct iscsi_conn *conn,
2848 int want_response)
2849{
2850 int tx_size = ISCSI_HDR_LEN;
2851 struct iscsi_nopin *hdr;
6f3c0e69 2852 int ret;
e48354ce
NB		 2853
2854 hdr = (struct iscsi_nopin *) cmd->pdu;
2855 memset(hdr, 0, ISCSI_HDR_LEN);
2856 hdr->opcode = ISCSI_OP_NOOP_IN;
2857 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
66c7db68 2858 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 2859 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
2860 cmd->stat_sn = conn->stat_sn;
2861 hdr->statsn = cpu_to_be32(cmd->stat_sn);
2862 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
2863 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
2864
2865 if (conn->conn_ops->HeaderDigest) {
2866 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2867
2868 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2869 (unsigned char *)hdr, ISCSI_HDR_LEN,
2870 0, NULL, (u8 *)header_digest);
2871
2872 tx_size += ISCSI_CRC_LEN;
2873 pr_debug("Attaching CRC32C HeaderDigest to"
2874 " NopIN 0x%08x\n", *header_digest);
2875 }
2876
2877 cmd->iov_misc[0].iov_base = cmd->pdu;
2878 cmd->iov_misc[0].iov_len = tx_size;
2879 cmd->iov_misc_count = 1;
2880 cmd->tx_size = tx_size;
2881
2882 pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
2883 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
2884
6f3c0e69
AG		 2885 ret = iscsit_send_tx_data(cmd, conn, 1);
2886 if (ret < 0) {
2887 iscsit_tx_thread_wait_for_tcp(conn);
2888 return ret;
2889 }
2890
2891 spin_lock_bh(&cmd->istate_lock);
2892 cmd->i_state = want_response ?
2893 ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
2894 spin_unlock_bh(&cmd->istate_lock);
2895
e48354ce
NB		 2896 return 0;
2897}
2898
2899static int iscsit_send_nopin_response(
2900 struct iscsi_cmd *cmd,
2901 struct iscsi_conn *conn)
2902{
2903 int niov = 0, tx_size;
2904 u32 padding = 0;
2905 struct kvec *iov;
2906 struct iscsi_nopin *hdr;
2907
2908 tx_size = ISCSI_HDR_LEN;
2909 hdr = (struct iscsi_nopin *) cmd->pdu;
2910 memset(hdr, 0, ISCSI_HDR_LEN);
2911 hdr->opcode = ISCSI_OP_NOOP_IN;
2912 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
2913 hton24(hdr->dlength, cmd->buf_ptr_size);
2914 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
66c7db68 2915 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 2916 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
2917 cmd->stat_sn = conn->stat_sn++;
2918 hdr->statsn = cpu_to_be32(cmd->stat_sn);
2919
2920 iscsit_increment_maxcmdsn(cmd, conn->sess);
2921 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
2922 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
2923
2924 iov = &cmd->iov_misc[0];
2925 iov[niov].iov_base = cmd->pdu;
2926 iov[niov++].iov_len = ISCSI_HDR_LEN;
2927
2928 if (conn->conn_ops->HeaderDigest) {
2929 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2930
2931 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2932 (unsigned char *)hdr, ISCSI_HDR_LEN,
2933 0, NULL, (u8 *)header_digest);
2934
2935 iov[0].iov_len += ISCSI_CRC_LEN;
2936 tx_size += ISCSI_CRC_LEN;
2937 pr_debug("Attaching CRC32C HeaderDigest"
2938 " to NopIn 0x%08x\n", *header_digest);
2939 }
2940
2941 /*
2942 * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
2943 * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
2944 */
2945 if (cmd->buf_ptr_size) {
2946 iov[niov].iov_base = cmd->buf_ptr;
2947 iov[niov++].iov_len = cmd->buf_ptr_size;
2948 tx_size += cmd->buf_ptr_size;
2949
2950 pr_debug("Echoing back %u bytes of ping"
2951 " data.\n", cmd->buf_ptr_size);
2952
2953 padding = ((-cmd->buf_ptr_size) & 3);
2954 if (padding != 0) {
2955 iov[niov].iov_base = &cmd->pad_bytes;
2956 iov[niov++].iov_len = padding;
2957 tx_size += padding;
2958 pr_debug("Attaching %u additional"
2959 " padding bytes.\n", padding);
2960 }
2961 if (conn->conn_ops->DataDigest) {
2962 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2963 cmd->buf_ptr, cmd->buf_ptr_size,
2964 padding, (u8 *)&cmd->pad_bytes,
2965 (u8 *)&cmd->data_crc);
2966
2967 iov[niov].iov_base = &cmd->data_crc;
2968 iov[niov++].iov_len = ISCSI_CRC_LEN;
2969 tx_size += ISCSI_CRC_LEN;
2970 pr_debug("Attached DataDigest for %u"
2971 " bytes of ping data, CRC 0x%08x\n",
2972 cmd->buf_ptr_size, cmd->data_crc);
2973 }
2974 }
2975
2976 cmd->iov_misc_count = niov;
2977 cmd->tx_size = tx_size;
2978
2979 pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:"
2980 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag,
2981 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
2982
2983 return 0;
2984}
2985
6f3c0e69 2986static int iscsit_send_r2t(
e48354ce
NB		 2987 struct iscsi_cmd *cmd,
2988 struct iscsi_conn *conn)
2989{
2990 int tx_size = 0;
2991 struct iscsi_r2t *r2t;
2992 struct iscsi_r2t_rsp *hdr;
6f3c0e69 2993 int ret;
e48354ce
NB		 2994
2995 r2t = iscsit_get_r2t_from_list(cmd);
2996 if (!r2t)
2997 return -1;
2998
2999 hdr = (struct iscsi_r2t_rsp *) cmd->pdu;
3000 memset(hdr, 0, ISCSI_HDR_LEN);
3001 hdr->opcode = ISCSI_OP_R2T;
3002 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3003 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
3004 (struct scsi_lun *)&hdr->lun);
66c7db68 3005 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 3006 spin_lock_bh(&conn->sess->ttt_lock);
3007 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
3008 if (r2t->targ_xfer_tag == 0xFFFFFFFF)
3009 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
3010 spin_unlock_bh(&conn->sess->ttt_lock);
3011 hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag);
3012 hdr->statsn = cpu_to_be32(conn->stat_sn);
3013 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3014 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
3015 hdr->r2tsn = cpu_to_be32(r2t->r2t_sn);
3016 hdr->data_offset = cpu_to_be32(r2t->offset);
3017 hdr->data_length = cpu_to_be32(r2t->xfer_len);
3018
3019 cmd->iov_misc[0].iov_base = cmd->pdu;
3020 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN;
3021 tx_size += ISCSI_HDR_LEN;
3022
3023 if (conn->conn_ops->HeaderDigest) {
3024 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3025
3026 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3027 (unsigned char *)hdr, ISCSI_HDR_LEN,
3028 0, NULL, (u8 *)header_digest);
3029
3030 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3031 tx_size += ISCSI_CRC_LEN;
3032 pr_debug("Attaching CRC32 HeaderDigest for R2T"
3033 " PDU 0x%08x\n", *header_digest);
3034 }
3035
3036 pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
3037 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
3038 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
3039 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
3040 r2t->offset, r2t->xfer_len, conn->cid);
3041
3042 cmd->iov_misc_count = 1;
3043 cmd->tx_size = tx_size;
3044
3045 spin_lock_bh(&cmd->r2t_lock);
3046 r2t->sent_r2t = 1;
3047 spin_unlock_bh(&cmd->r2t_lock);
3048
6f3c0e69
AG		 3049 ret = iscsit_send_tx_data(cmd, conn, 1);
3050 if (ret < 0) {
3051 iscsit_tx_thread_wait_for_tcp(conn);
3052 return ret;
3053 }
3054
3055 spin_lock_bh(&cmd->dataout_timeout_lock);
3056 iscsit_start_dataout_timer(cmd, conn);
3057 spin_unlock_bh(&cmd->dataout_timeout_lock);
3058
e48354ce
NB		 3059 return 0;
3060}
3061
3062/*
8b1e1244
AG		 3063 * @recovery: If called from iscsi_task_reassign_complete_write() for
3064 * connection recovery.
e48354ce
NB		 3065 */
3066int iscsit_build_r2ts_for_cmd(
e48354ce 3067 struct iscsi_conn *conn,
3e1c81a9 3068 struct iscsi_cmd *cmd,
8b1e1244 3069 bool recovery)
e48354ce
NB		 3070{
3071 int first_r2t = 1;
3072 u32 offset = 0, xfer_len = 0;
3073
3074 spin_lock_bh(&cmd->r2t_lock);
3075 if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
3076 spin_unlock_bh(&cmd->r2t_lock);
3077 return 0;
3078 }
3079
8b1e1244
AG		 3080 if (conn->sess->sess_ops->DataSequenceInOrder &&
3081 !recovery)
c6037cc5 3082 cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
e48354ce
NB		 3083
3084 while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
3085 if (conn->sess->sess_ops->DataSequenceInOrder) {
3086 offset = cmd->r2t_offset;
3087
8b1e1244
AG		 3088 if (first_r2t && recovery) {
3089 int new_data_end = offset +
3090 conn->sess->sess_ops->MaxBurstLength -
3091 cmd->next_burst_len;
3092
ebf1d95c
AG		 3093 if (new_data_end > cmd->se_cmd.data_length)
3094 xfer_len = cmd->se_cmd.data_length - offset;
8b1e1244
AG		 3095 else
3096 xfer_len =
3097 conn->sess->sess_ops->MaxBurstLength -
3098 cmd->next_burst_len;
e48354ce 3099 } else {
8b1e1244
AG		 3100 int new_data_end = offset +
3101 conn->sess->sess_ops->MaxBurstLength;
3102
ebf1d95c
AG		 3103 if (new_data_end > cmd->se_cmd.data_length)
3104 xfer_len = cmd->se_cmd.data_length - offset;
8b1e1244
AG		 3105 else
3106 xfer_len = conn->sess->sess_ops->MaxBurstLength;
e48354ce
NB		 3107 }
3108 cmd->r2t_offset += xfer_len;
3109
ebf1d95c 3110 if (cmd->r2t_offset == cmd->se_cmd.data_length)
e48354ce
NB		 3111 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3112 } else {
3113 struct iscsi_seq *seq;
3114
3115 seq = iscsit_get_seq_holder_for_r2t(cmd);
3116 if (!seq) {
3117 spin_unlock_bh(&cmd->r2t_lock);
3118 return -1;
3119 }
3120
3121 offset = seq->offset;
3122 xfer_len = seq->xfer_len;
3123
3124 if (cmd->seq_send_order == cmd->seq_count)
3125 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
3126 }
3127 cmd->outstanding_r2ts++;
3128 first_r2t = 0;
3129
3130 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
3131 spin_unlock_bh(&cmd->r2t_lock);
3132 return -1;
3133 }
3134
3135 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
3136 break;
3137 }
3138 spin_unlock_bh(&cmd->r2t_lock);
3139
3140 return 0;
3141}
3142
3143static int iscsit_send_status(
3144 struct iscsi_cmd *cmd,
3145 struct iscsi_conn *conn)
3146{
3147 u8 iov_count = 0, recovery;
3148 u32 padding = 0, tx_size = 0;
3149 struct iscsi_scsi_rsp *hdr;
3150 struct kvec *iov;
3151
3152 recovery = (cmd->i_state != ISTATE_SEND_STATUS);
3153 if (!recovery)
3154 cmd->stat_sn = conn->stat_sn++;
3155
3156 spin_lock_bh(&conn->sess->session_stats_lock);
3157 conn->sess->rsp_pdus++;
3158 spin_unlock_bh(&conn->sess->session_stats_lock);
3159
3160 hdr = (struct iscsi_scsi_rsp *) cmd->pdu;
3161 memset(hdr, 0, ISCSI_HDR_LEN);
3162 hdr->opcode = ISCSI_OP_SCSI_CMD_RSP;
3163 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3164 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3165 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
7e46cf02 3166 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
e48354ce
NB		 3167 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3168 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
7e46cf02 3169 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
e48354ce
NB		 3170 }
3171 hdr->response = cmd->iscsi_response;
3172 hdr->cmd_status = cmd->se_cmd.scsi_status;
66c7db68 3173 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 3174 hdr->statsn = cpu_to_be32(cmd->stat_sn);
3175
3176 iscsit_increment_maxcmdsn(cmd, conn->sess);
3177 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3178 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
3179
3180 iov = &cmd->iov_misc[0];
3181 iov[iov_count].iov_base = cmd->pdu;
3182 iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3183 tx_size += ISCSI_HDR_LEN;
3184
3185 /*
3186 * Attach SENSE DATA payload to iSCSI Response PDU
3187 */
3188 if (cmd->se_cmd.sense_buffer &&
3189 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3190 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
9c58b7dd
RD		 3191 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
3192 cmd->se_cmd.scsi_sense_length += sizeof (__be16);
3193
e48354ce 3194 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
50e5c87d 3195 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
9c58b7dd 3196 iov[iov_count].iov_base = cmd->sense_buffer;
e48354ce
NB		 3197 iov[iov_count++].iov_len =
3198 (cmd->se_cmd.scsi_sense_length + padding);
3199 tx_size += cmd->se_cmd.scsi_sense_length;
3200
3201 if (padding) {
9c58b7dd 3202 memset(cmd->sense_buffer +
e48354ce
NB		 3203 cmd->se_cmd.scsi_sense_length, 0, padding);
3204 tx_size += padding;
3205 pr_debug("Adding %u bytes of padding to"
3206 " SENSE.\n", padding);
3207 }
3208
3209 if (conn->conn_ops->DataDigest) {
3210 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
9c58b7dd 3211 cmd->sense_buffer,
e48354ce
NB		 3212 (cmd->se_cmd.scsi_sense_length + padding),
3213 0, NULL, (u8 *)&cmd->data_crc);
3214
3215 iov[iov_count].iov_base = &cmd->data_crc;
3216 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3217 tx_size += ISCSI_CRC_LEN;
3218
3219 pr_debug("Attaching CRC32 DataDigest for"
3220 " SENSE, %u bytes CRC 0x%08x\n",
3221 (cmd->se_cmd.scsi_sense_length + padding),
3222 cmd->data_crc);
3223 }
3224
3225 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3226 " Response PDU\n",
3227 cmd->se_cmd.scsi_sense_length);
3228 }
3229
3230 if (conn->conn_ops->HeaderDigest) {
3231 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3232
3233 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3234 (unsigned char *)hdr, ISCSI_HDR_LEN,
3235 0, NULL, (u8 *)header_digest);
3236
3237 iov[0].iov_len += ISCSI_CRC_LEN;
3238 tx_size += ISCSI_CRC_LEN;
3239 pr_debug("Attaching CRC32 HeaderDigest for Response"
3240 " PDU 0x%08x\n", *header_digest);
3241 }
3242
3243 cmd->iov_misc_count = iov_count;
3244 cmd->tx_size = tx_size;
3245
3246 pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3247 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3248 (!recovery) ? "" : "Recovery ", cmd->init_task_tag,
3249 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid);
3250
3251 return 0;
3252}
3253
3254static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3255{
3256 switch (se_tmr->response) {
3257 case TMR_FUNCTION_COMPLETE:
3258 return ISCSI_TMF_RSP_COMPLETE;
3259 case TMR_TASK_DOES_NOT_EXIST:
3260 return ISCSI_TMF_RSP_NO_TASK;
3261 case TMR_LUN_DOES_NOT_EXIST:
3262 return ISCSI_TMF_RSP_NO_LUN;
3263 case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3264 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3265 case TMR_FUNCTION_AUTHORIZATION_FAILED:
3266 return ISCSI_TMF_RSP_AUTH_FAILED;
3267 case TMR_FUNCTION_REJECTED:
3268 default:
3269 return ISCSI_TMF_RSP_REJECTED;
3270 }
3271}
3272
3273static int iscsit_send_task_mgt_rsp(
3274 struct iscsi_cmd *cmd,
3275 struct iscsi_conn *conn)
3276{
3277 struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3278 struct iscsi_tm_rsp *hdr;
3279 u32 tx_size = 0;
3280
3281 hdr = (struct iscsi_tm_rsp *) cmd->pdu;
3282 memset(hdr, 0, ISCSI_HDR_LEN);
3283 hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
7ae0b103 3284 hdr->flags = ISCSI_FLAG_CMD_FINAL;
e48354ce 3285 hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr);
66c7db68 3286 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 3287 cmd->stat_sn = conn->stat_sn++;
3288 hdr->statsn = cpu_to_be32(cmd->stat_sn);
3289
3290 iscsit_increment_maxcmdsn(cmd, conn->sess);
3291 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3292 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
3293
3294 cmd->iov_misc[0].iov_base = cmd->pdu;
3295 cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN;
3296 tx_size += ISCSI_HDR_LEN;
3297
3298 if (conn->conn_ops->HeaderDigest) {
3299 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3300
3301 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3302 (unsigned char *)hdr, ISCSI_HDR_LEN,
3303 0, NULL, (u8 *)header_digest);
3304
3305 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3306 tx_size += ISCSI_CRC_LEN;
3307 pr_debug("Attaching CRC32 HeaderDigest for Task"
3308 " Mgmt Response PDU 0x%08x\n", *header_digest);
3309 }
3310
3311 cmd->iov_misc_count = 1;
3312 cmd->tx_size = tx_size;
3313
3314 pr_debug("Built Task Management Response ITT: 0x%08x,"
3315 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3316 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3317
3318 return 0;
3319}
3320
2f9bc894
NB		 3321static bool iscsit_check_inaddr_any(struct iscsi_np *np)
3322{
3323 bool ret = false;
3324
3325 if (np->np_sockaddr.ss_family == AF_INET6) {
3326 const struct sockaddr_in6 sin6 = {
3327 .sin6_addr = IN6ADDR_ANY_INIT };
3328 struct sockaddr_in6 *sock_in6 =
3329 (struct sockaddr_in6 *)&np->np_sockaddr;
3330
3331 if (!memcmp(sock_in6->sin6_addr.s6_addr,
3332 sin6.sin6_addr.s6_addr, 16))
3333 ret = true;
3334 } else {
3335 struct sockaddr_in * sock_in =
3336 (struct sockaddr_in *)&np->np_sockaddr;
3337
cea0b4ce 3338 if (sock_in->sin_addr.s_addr == htonl(INADDR_ANY))
2f9bc894
NB		 3339 ret = true;
3340 }
3341
3342 return ret;
3343}
3344
8b1e1244
AG		 3345#define SENDTARGETS_BUF_LIMIT 32768U
3346
e48354ce
NB		 3347static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd)
3348{
3349 char *payload = NULL;
3350 struct iscsi_conn *conn = cmd->conn;
3351 struct iscsi_portal_group *tpg;
3352 struct iscsi_tiqn *tiqn;
3353 struct iscsi_tpg_np *tpg_np;
3354 int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
8b1e1244 3355 unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
e48354ce 3356
8b1e1244
AG		 3357 buffer_len = max(conn->conn_ops->MaxRecvDataSegmentLength,
3358 SENDTARGETS_BUF_LIMIT);
e48354ce
NB		 3359
3360 payload = kzalloc(buffer_len, GFP_KERNEL);
3361 if (!payload) {
3362 pr_err("Unable to allocate memory for sendtargets"
3363 " response.\n");
3364 return -ENOMEM;
3365 }
3366
3367 spin_lock(&tiqn_lock);
3368 list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3369 len = sprintf(buf, "TargetName=%s", tiqn->tiqn);
3370 len += 1;
3371
3372 if ((len + payload_len) > buffer_len) {
e48354ce
NB		 3373 end_of_buf = 1;
3374 goto eob;
3375 }
8359cf43 3376 memcpy(payload + payload_len, buf, len);
e48354ce
NB		 3377 payload_len += len;
3378
3379 spin_lock(&tiqn->tiqn_tpg_lock);
3380 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3381
3382 spin_lock(&tpg->tpg_state_lock);
3383 if ((tpg->tpg_state == TPG_STATE_FREE) ||
3384 (tpg->tpg_state == TPG_STATE_INACTIVE)) {
3385 spin_unlock(&tpg->tpg_state_lock);
3386 continue;
3387 }
3388 spin_unlock(&tpg->tpg_state_lock);
3389
3390 spin_lock(&tpg->tpg_np_lock);
3391 list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3392 tpg_np_list) {
2f9bc894
NB		 3393 struct iscsi_np *np = tpg_np->tpg_np;
3394 bool inaddr_any = iscsit_check_inaddr_any(np);
3395
e48354ce
NB		 3396 len = sprintf(buf, "TargetAddress="
3397 "%s%s%s:%hu,%hu",
2f9bc894
NB		 3398 (np->np_sockaddr.ss_family == AF_INET6) ?
3399 "[" : "", (inaddr_any == false) ?
3400 np->np_ip : conn->local_ip,
3401 (np->np_sockaddr.ss_family == AF_INET6) ?
3402 "]" : "", (inaddr_any == false) ?
3403 np->np_port : conn->local_port,
e48354ce
NB		 3404 tpg->tpgt);
3405 len += 1;
3406
3407 if ((len + payload_len) > buffer_len) {
3408 spin_unlock(&tpg->tpg_np_lock);
3409 spin_unlock(&tiqn->tiqn_tpg_lock);
3410 end_of_buf = 1;
3411 goto eob;
3412 }
8359cf43 3413 memcpy(payload + payload_len, buf, len);
e48354ce
NB		 3414 payload_len += len;
3415 }
3416 spin_unlock(&tpg->tpg_np_lock);
3417 }
3418 spin_unlock(&tiqn->tiqn_tpg_lock);
3419eob:
3420 if (end_of_buf)
3421 break;
3422 }
3423 spin_unlock(&tiqn_lock);
3424
3425 cmd->buf_ptr = payload;
3426
3427 return payload_len;
3428}
3429
3430/*
3431 * FIXME: Add support for F_BIT and C_BIT when the length is longer than
3432 * MaxRecvDataSegmentLength.
3433 */
3434static int iscsit_send_text_rsp(
3435 struct iscsi_cmd *cmd,
3436 struct iscsi_conn *conn)
3437{
3438 struct iscsi_text_rsp *hdr;
3439 struct kvec *iov;
3440 u32 padding = 0, tx_size = 0;
3441 int text_length, iov_count = 0;
3442
3443 text_length = iscsit_build_sendtargets_response(cmd);
3444 if (text_length < 0)
3445 return text_length;
3446
3447 padding = ((-text_length) & 3);
3448 if (padding != 0) {
3449 memset(cmd->buf_ptr + text_length, 0, padding);
3450 pr_debug("Attaching %u additional bytes for"
3451 " padding.\n", padding);
3452 }
3453
3454 hdr = (struct iscsi_text_rsp *) cmd->pdu;
3455 memset(hdr, 0, ISCSI_HDR_LEN);
3456 hdr->opcode = ISCSI_OP_TEXT_RSP;
3457 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3458 hton24(hdr->dlength, text_length);
66c7db68 3459 hdr->itt = cmd->init_task_tag;
e48354ce
NB		 3460 hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
3461 cmd->stat_sn = conn->stat_sn++;
3462 hdr->statsn = cpu_to_be32(cmd->stat_sn);
3463
3464 iscsit_increment_maxcmdsn(cmd, conn->sess);
3465 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3466 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
3467
3468 iov = &cmd->iov_misc[0];
3469
3470 iov[iov_count].iov_base = cmd->pdu;
3471 iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3472 iov[iov_count].iov_base = cmd->buf_ptr;
3473 iov[iov_count++].iov_len = text_length + padding;
3474
3475 tx_size += (ISCSI_HDR_LEN + text_length + padding);
3476
3477 if (conn->conn_ops->HeaderDigest) {
3478 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3479
3480 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3481 (unsigned char *)hdr, ISCSI_HDR_LEN,
3482 0, NULL, (u8 *)header_digest);
3483
3484 iov[0].iov_len += ISCSI_CRC_LEN;
3485 tx_size += ISCSI_CRC_LEN;
3486 pr_debug("Attaching CRC32 HeaderDigest for"
3487 " Text Response PDU 0x%08x\n", *header_digest);
3488 }
3489
3490 if (conn->conn_ops->DataDigest) {
3491 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3492 cmd->buf_ptr, (text_length + padding),
3493 0, NULL, (u8 *)&cmd->data_crc);
3494
3495 iov[iov_count].iov_base = &cmd->data_crc;
3496 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3497 tx_size += ISCSI_CRC_LEN;
3498
3499 pr_debug("Attaching DataDigest for %u bytes of text"
3500 " data, CRC 0x%08x\n", (text_length + padding),
3501 cmd->data_crc);
3502 }
3503
3504 cmd->iov_misc_count = iov_count;
3505 cmd->tx_size = tx_size;
3506
3507 pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x,"
3508 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn,
3509 text_length, conn->cid);
3510 return 0;
3511}
3512
3513static int iscsit_send_reject(
3514 struct iscsi_cmd *cmd,
3515 struct iscsi_conn *conn)
3516{
3517 u32 iov_count = 0, tx_size = 0;
3518 struct iscsi_reject *hdr;
3519 struct kvec *iov;
3520
3521 hdr = (struct iscsi_reject *) cmd->pdu;
3522 hdr->opcode = ISCSI_OP_REJECT;
3523 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3524 hton24(hdr->dlength, ISCSI_HDR_LEN);
50e5c87d 3525 hdr->ffffffff = cpu_to_be32(0xffffffff);
e48354ce
NB		 3526 cmd->stat_sn = conn->stat_sn++;
3527 hdr->statsn = cpu_to_be32(cmd->stat_sn);
3528 hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
3529 hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn);
3530
3531 iov = &cmd->iov_misc[0];
3532
3533 iov[iov_count].iov_base = cmd->pdu;
3534 iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3535 iov[iov_count].iov_base = cmd->buf_ptr;
3536 iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3537
3538 tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN);
3539
3540 if (conn->conn_ops->HeaderDigest) {
3541 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3542
3543 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3544 (unsigned char *)hdr, ISCSI_HDR_LEN,
3545 0, NULL, (u8 *)header_digest);
3546
3547 iov[0].iov_len += ISCSI_CRC_LEN;
3548 tx_size += ISCSI_CRC_LEN;
3549 pr_debug("Attaching CRC32 HeaderDigest for"
3550 " REJECT PDU 0x%08x\n", *header_digest);
3551 }
3552
3553 if (conn->conn_ops->DataDigest) {
3554 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3555 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN,
3556 0, NULL, (u8 *)&cmd->data_crc);
3557
3558 iov[iov_count].iov_base = &cmd->data_crc;
3559 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3560 tx_size += ISCSI_CRC_LEN;
3561 pr_debug("Attaching CRC32 DataDigest for REJECT"
3562 " PDU 0x%08x\n", cmd->data_crc);
3563 }
3564
3565 cmd->iov_misc_count = iov_count;
3566 cmd->tx_size = tx_size;
3567
3568 pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3569 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3570
3571 return 0;
3572}
3573
e48354ce
NB		 3574void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3575{
3576 struct iscsi_thread_set *ts = conn->thread_set;
3577 int ord, cpu;
3578 /*
3579 * thread_id is assigned from iscsit_global->ts_bitmap from
3580 * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
3581 *
3582 * Here we use thread_id to determine which CPU that this
3583 * iSCSI connection's iscsi_thread_set will be scheduled to
3584 * execute upon.
3585 */
3586 ord = ts->thread_id % cpumask_weight(cpu_online_mask);
e48354ce
NB		 3587 for_each_online_cpu(cpu) {
3588 if (ord-- == 0) {
3589 cpumask_set_cpu(cpu, conn->conn_cpumask);
3590 return;
3591 }
3592 }
3593 /*
3594 * This should never be reached..
3595 */
3596 dump_stack();
3597 cpumask_setall(conn->conn_cpumask);
3598}
3599
3600static inline void iscsit_thread_check_cpumask(
3601 struct iscsi_conn *conn,
3602 struct task_struct *p,
3603 int mode)
3604{
3605 char buf[128];
3606 /*
3607 * mode == 1 signals iscsi_target_tx_thread() usage.
3608 * mode == 0 signals iscsi_target_rx_thread() usage.
3609 */
3610 if (mode == 1) {
3611 if (!conn->conn_tx_reset_cpumask)
3612 return;
3613 conn->conn_tx_reset_cpumask = 0;
3614 } else {
3615 if (!conn->conn_rx_reset_cpumask)
3616 return;
3617 conn->conn_rx_reset_cpumask = 0;
3618 }
3619 /*
3620 * Update the CPU mask for this single kthread so that
3621 * both TX and RX kthreads are scheduled to run on the
3622 * same CPU.
3623 */
3624 memset(buf, 0, 128);
3625 cpumask_scnprintf(buf, 128, conn->conn_cpumask);
e48354ce
NB		 3626 set_cpus_allowed_ptr(p, conn->conn_cpumask);
3627}
3628
6f3c0e69 3629static int handle_immediate_queue(struct iscsi_conn *conn)
e48354ce 3630{
6f3c0e69
AG		 3631 struct iscsi_queue_req *qr;
3632 struct iscsi_cmd *cmd;
e48354ce 3633 u8 state;
6f3c0e69
AG		 3634 int ret;
3635
3636 while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
3637 atomic_set(&conn->check_immediate_queue, 0);
3638 cmd = qr->cmd;
3639 state = qr->state;
3640 kmem_cache_free(lio_qr_cache, qr);
3641
3642 switch (state) {
3643 case ISTATE_SEND_R2T:
3644 ret = iscsit_send_r2t(cmd, conn);
3645 if (ret < 0)
3646 goto err;
3647 break;
3648 case ISTATE_REMOVE:
3649 if (cmd->data_direction == DMA_TO_DEVICE)
3650 iscsit_stop_dataout_timer(cmd);
3651
3652 spin_lock_bh(&conn->cmd_lock);
3653 list_del(&cmd->i_conn_node);
3654 spin_unlock_bh(&conn->cmd_lock);
3655
3656 iscsit_free_cmd(cmd);
3657 continue;
3658 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3659 iscsit_mod_nopin_response_timer(conn);
3660 ret = iscsit_send_unsolicited_nopin(cmd,
3661 conn, 1);
3662 if (ret < 0)
3663 goto err;
3664 break;
3665 case ISTATE_SEND_NOPIN_NO_RESPONSE:
3666 ret = iscsit_send_unsolicited_nopin(cmd,
3667 conn, 0);
3668 if (ret < 0)
3669 goto err;
3670 break;
3671 default:
3672 pr_err("Unknown Opcode: 0x%02x ITT:"
3673 " 0x%08x, i_state: %d on CID: %hu\n",
3674 cmd->iscsi_opcode, cmd->init_task_tag, state,
3675 conn->cid);
3676 goto err;
3677 }
3678 }
3679
3680 return 0;
3681
3682err:
3683 return -1;
3684}
3685
3686static int handle_response_queue(struct iscsi_conn *conn)
3687{
3688 struct iscsi_queue_req *qr;
3689 struct iscsi_cmd *cmd;
3690 u8 state;
3691 int ret;
3692
3693 while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
3694 cmd = qr->cmd;
3695 state = qr->state;
3696 kmem_cache_free(lio_qr_cache, qr);
3697
3698check_rsp_state:
3699 switch (state) {
3700 case ISTATE_SEND_DATAIN:
3701 ret = iscsit_send_data_in(cmd, conn);
3702 if (ret < 0)
3703 goto err;
3704 else if (!ret)
3705 /* more drs */
3706 goto check_rsp_state;
3707 else if (ret == 1) {
3708 /* all done */
3709 spin_lock_bh(&cmd->istate_lock);
3710 cmd->i_state = ISTATE_SENT_STATUS;
3711 spin_unlock_bh(&cmd->istate_lock);
fd3a9025
NB		 3712
3713 if (atomic_read(&conn->check_immediate_queue))
3714 return 1;
3715
6f3c0e69
AG		 3716 continue;
3717 } else if (ret == 2) {
3718 /* Still must send status,
3719 SCF_TRANSPORT_TASK_SENSE was set */
3720 spin_lock_bh(&cmd->istate_lock);
3721 cmd->i_state = ISTATE_SEND_STATUS;
3722 spin_unlock_bh(&cmd->istate_lock);
3723 state = ISTATE_SEND_STATUS;
3724 goto check_rsp_state;
3725 }
3726
3727 break;
3728 case ISTATE_SEND_STATUS:
3729 case ISTATE_SEND_STATUS_RECOVERY:
3730 ret = iscsit_send_status(cmd, conn);
3731 break;
3732 case ISTATE_SEND_LOGOUTRSP:
3733 ret = iscsit_send_logout_response(cmd, conn);
3734 break;
3735 case ISTATE_SEND_ASYNCMSG:
3736 ret = iscsit_send_conn_drop_async_message(
3737 cmd, conn);
3738 break;
3739 case ISTATE_SEND_NOPIN:
3740 ret = iscsit_send_nopin_response(cmd, conn);
3741 break;
3742 case ISTATE_SEND_REJECT:
3743 ret = iscsit_send_reject(cmd, conn);
3744 break;
3745 case ISTATE_SEND_TASKMGTRSP:
3746 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3747 if (ret != 0)
3748 break;
3749 ret = iscsit_tmr_post_handler(cmd, conn);
3750 if (ret != 0)
3751 iscsit_fall_back_to_erl0(conn->sess);
3752 break;
3753 case ISTATE_SEND_TEXTRSP:
3754 ret = iscsit_send_text_rsp(cmd, conn);
3755 break;
3756 default:
3757 pr_err("Unknown Opcode: 0x%02x ITT:"
3758 " 0x%08x, i_state: %d on CID: %hu\n",
3759 cmd->iscsi_opcode, cmd->init_task_tag,
3760 state, conn->cid);
3761 goto err;
3762 }
3763 if (ret < 0)
3764 goto err;
3765
3766 if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
3767 iscsit_tx_thread_wait_for_tcp(conn);
3768 iscsit_unmap_iovec(cmd);
3769 goto err;
3770 }
3771 iscsit_unmap_iovec(cmd);
3772
3773 switch (state) {
3774 case ISTATE_SEND_LOGOUTRSP:
3775 if (!iscsit_logout_post_handler(cmd, conn))
3776 goto restart;
3777 /* fall through */
3778 case ISTATE_SEND_STATUS:
3779 case ISTATE_SEND_ASYNCMSG:
3780 case ISTATE_SEND_NOPIN:
3781 case ISTATE_SEND_STATUS_RECOVERY:
3782 case ISTATE_SEND_TEXTRSP:
3783 case ISTATE_SEND_TASKMGTRSP:
3784 spin_lock_bh(&cmd->istate_lock);
3785 cmd->i_state = ISTATE_SENT_STATUS;
3786 spin_unlock_bh(&cmd->istate_lock);
3787 break;
3788 case ISTATE_SEND_REJECT:
3789 if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
3790 cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
3791 complete(&cmd->reject_comp);
3792 goto err;
3793 }
3794 complete(&cmd->reject_comp);
3795 break;
3796 default:
3797 pr_err("Unknown Opcode: 0x%02x ITT:"
3798 " 0x%08x, i_state: %d on CID: %hu\n",
3799 cmd->iscsi_opcode, cmd->init_task_tag,
3800 cmd->i_state, conn->cid);
3801 goto err;
3802 }
3803
3804 if (atomic_read(&conn->check_immediate_queue))
fd3a9025 3805 return 1;
6f3c0e69
AG		 3806 }
3807
3808 return 0;
3809
3810err:
3811 return -1;
3812restart:
3813 return -EAGAIN;
3814}
3815
3816int iscsi_target_tx_thread(void *arg)
3817{
e48354ce 3818 int ret = 0;
e48354ce 3819 struct iscsi_conn *conn;
8359cf43 3820 struct iscsi_thread_set *ts = arg;
e48354ce
NB		 3821 /*
3822 * Allow ourselves to be interrupted by SIGINT so that a
3823 * connection recovery / failure event can be triggered externally.
3824 */
3825 allow_signal(SIGINT);
3826
3827restart:
3828 conn = iscsi_tx_thread_pre_handler(ts);
3829 if (!conn)
3830 goto out;
3831
6f3c0e69 3832 ret = 0;
e48354ce
NB		 3833
3834 while (!kthread_should_stop()) {
3835 /*
3836 * Ensure that both TX and RX per connection kthreads
3837 * are scheduled to run on the same CPU.
3838 */
3839 iscsit_thread_check_cpumask(conn, current, 1);
3840
d5627acb
RD		 3841 wait_event_interruptible(conn->queues_wq,
3842 !iscsit_conn_all_queues_empty(conn) ||
3843 ts->status == ISCSI_THREAD_SET_RESET);
e48354ce
NB		 3844
3845 if ((ts->status == ISCSI_THREAD_SET_RESET) ||
3846 signal_pending(current))
3847 goto transport_err;
3848
fd3a9025 3849get_immediate:
6f3c0e69
AG		 3850 ret = handle_immediate_queue(conn);
3851 if (ret < 0)
3852 goto transport_err;
e48354ce 3853
6f3c0e69 3854 ret = handle_response_queue(conn);
fd3a9025
NB		 3855 if (ret == 1)
3856 goto get_immediate;
3857 else if (ret == -EAGAIN)
6f3c0e69
AG		 3858 goto restart;
3859 else if (ret < 0)
3860 goto transport_err;
e48354ce
NB		 3861 }
3862
3863transport_err:
3864 iscsit_take_action_for_connection_exit(conn);
3865 goto restart;
3866out:
3867 return 0;
3868}
3869
3e1c81a9
NB		 3870static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
3871{
3872 struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
3873 struct iscsi_cmd *cmd;
3874 int ret = 0;
3875
3876 switch (hdr->opcode & ISCSI_OPCODE_MASK) {
3877 case ISCSI_OP_SCSI_CMD:
3878 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3879 if (!cmd)
3880 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
3881 1, buf, conn);
3882
3883 ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
3884 break;
3885 case ISCSI_OP_SCSI_DATA_OUT:
3886 ret = iscsit_handle_data_out(conn, buf);
3887 break;
3888 case ISCSI_OP_NOOP_OUT:
3889 cmd = NULL;
3890 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
3891 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3892 if (!cmd)
3893 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
3894 1, buf, conn);
3895 }
3896 ret = iscsit_handle_nop_out(conn, cmd, buf);
3897 break;
3898 case ISCSI_OP_SCSI_TMFUNC:
3899 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3900 if (!cmd)
3901 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
3902 1, buf, conn);
3903
3904 ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
3905 break;
3906 case ISCSI_OP_TEXT:
3907 ret = iscsit_handle_text_cmd(conn, buf);
3908 break;
3909 case ISCSI_OP_LOGOUT:
3910 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
3911 if (!cmd)
3912 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
3913 1, buf, conn);
3914
3915 ret = iscsit_handle_logout_cmd(conn, cmd, buf);
3916 if (ret > 0)
3917 wait_for_completion_timeout(&conn->conn_logout_comp,
3918 SECONDS_FOR_LOGOUT_COMP * HZ);
3919 break;
3920 case ISCSI_OP_SNACK:
3921 ret = iscsit_handle_snack(conn, buf);
3922 break;
3923 default:
3924 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
3925 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3926 pr_err("Cannot recover from unknown"
3927 " opcode while ERL=0, closing iSCSI connection.\n");
3928 return -1;
3929 }
3930 if (!conn->conn_ops->OFMarker) {
3931 pr_err("Unable to recover from unknown"
3932 " opcode while OFMarker=No, closing iSCSI"
3933 " connection.\n");
3934 return -1;
3935 }
3936 if (iscsit_recover_from_unknown_opcode(conn) < 0) {
3937 pr_err("Unable to recover from unknown"
3938 " opcode, closing iSCSI connection.\n");
3939 return -1;
3940 }
3941 break;
3942 }
3943
3944 return ret;
3945}
3946
e48354ce
NB		 3947int iscsi_target_rx_thread(void *arg)
3948{
3949 int ret;
3950 u8 buffer[ISCSI_HDR_LEN], opcode;
3951 u32 checksum = 0, digest = 0;
3952 struct iscsi_conn *conn = NULL;
8359cf43 3953 struct iscsi_thread_set *ts = arg;
e48354ce
NB		 3954 struct kvec iov;
3955 /*
3956 * Allow ourselves to be interrupted by SIGINT so that a
3957 * connection recovery / failure event can be triggered externally.
3958 */
3959 allow_signal(SIGINT);
3960
3961restart:
3962 conn = iscsi_rx_thread_pre_handler(ts);
3963 if (!conn)
3964 goto out;
3965
3e1c81a9
NB		 3966 if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) {
3967 struct completion comp;
3968 int rc;
3969
3970 init_completion(&comp);
3971 rc = wait_for_completion_interruptible(&comp);
3972 if (rc < 0)
3973 goto transport_err;
3974
3975 goto out;
3976 }
3977
e48354ce
NB		 3978 while (!kthread_should_stop()) {
3979 /*
3980 * Ensure that both TX and RX per connection kthreads
3981 * are scheduled to run on the same CPU.
3982 */
3983 iscsit_thread_check_cpumask(conn, current, 0);
3984
3985 memset(buffer, 0, ISCSI_HDR_LEN);
3986 memset(&iov, 0, sizeof(struct kvec));
3987
3988 iov.iov_base = buffer;
3989 iov.iov_len = ISCSI_HDR_LEN;
3990
3991 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3992 if (ret != ISCSI_HDR_LEN) {
3993 iscsit_rx_thread_wait_for_tcp(conn);
3994 goto transport_err;
3995 }
3996
3997 /*
3998 * Set conn->bad_hdr for use with REJECT PDUs.
3999 */
4000 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN);
4001
4002 if (conn->conn_ops->HeaderDigest) {
4003 iov.iov_base = &digest;
4004 iov.iov_len = ISCSI_CRC_LEN;
4005
4006 ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
4007 if (ret != ISCSI_CRC_LEN) {
4008 iscsit_rx_thread_wait_for_tcp(conn);
4009 goto transport_err;
4010 }
4011
4012 iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
4013 buffer, ISCSI_HDR_LEN,
4014 0, NULL, (u8 *)&checksum);
4015
4016 if (digest != checksum) {
4017 pr_err("HeaderDigest CRC32C failed,"
4018 " received 0x%08x, computed 0x%08x\n",
4019 digest, checksum);
4020 /*
4021 * Set the PDU to 0xff so it will intentionally
4022 * hit default in the switch below.
4023 */
4024 memset(buffer, 0xff, ISCSI_HDR_LEN);
4025 spin_lock_bh(&conn->sess->session_stats_lock);
4026 conn->sess->conn_digest_errors++;
4027 spin_unlock_bh(&conn->sess->session_stats_lock);
4028 } else {
4029 pr_debug("Got HeaderDigest CRC32C"
4030 " 0x%08x\n", checksum);
4031 }
4032 }
4033
4034 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
4035 goto transport_err;
4036
4037 opcode = buffer[0] & ISCSI_OPCODE_MASK;
4038
4039 if (conn->sess->sess_ops->SessionType &&
4040 ((!(opcode & ISCSI_OP_TEXT)) ||
4041 (!(opcode & ISCSI_OP_LOGOUT)))) {
4042 pr_err("Received illegal iSCSI Opcode: 0x%02x"
4043 " while in Discovery Session, rejecting.\n", opcode);
4044 iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
4045 buffer, conn);
4046 goto transport_err;
4047 }
4048
3e1c81a9
NB		 4049 ret = iscsi_target_rx_opcode(conn, buffer);
4050 if (ret < 0)
4051 goto transport_err;
e48354ce
NB		 4052 }
4053
4054transport_err:
4055 if (!signal_pending(current))
4056 atomic_set(&conn->transport_failed, 1);
4057 iscsit_take_action_for_connection_exit(conn);
4058 goto restart;
4059out:
4060 return 0;
4061}
4062
4063static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
4064{
4065 struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
4066 struct iscsi_session *sess = conn->sess;
e48354ce
NB		 4067 /*
4068 * We expect this function to only ever be called from either RX or TX
4069 * thread context via iscsit_close_connection() once the other context
4070 * has been reset -> returned sleeping pre-handler state.
4071 */
4072 spin_lock_bh(&conn->cmd_lock);
2fbb471e 4073 list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_conn_node) {
e48354ce 4074
2fbb471e 4075 list_del(&cmd->i_conn_node);
e48354ce
NB		 4076 spin_unlock_bh(&conn->cmd_lock);
4077
4078 iscsit_increment_maxcmdsn(cmd, sess);
e48354ce 4079
d270190a 4080 iscsit_free_cmd(cmd);
e48354ce
NB		 4081
4082 spin_lock_bh(&conn->cmd_lock);
4083 }
4084 spin_unlock_bh(&conn->cmd_lock);
4085}
4086
4087static void iscsit_stop_timers_for_cmds(
4088 struct iscsi_conn *conn)
4089{
4090 struct iscsi_cmd *cmd;
4091
4092 spin_lock_bh(&conn->cmd_lock);
2fbb471e 4093 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
e48354ce
NB		 4094 if (cmd->data_direction == DMA_TO_DEVICE)
4095 iscsit_stop_dataout_timer(cmd);
4096 }
4097 spin_unlock_bh(&conn->cmd_lock);
4098}
4099
4100int iscsit_close_connection(
4101 struct iscsi_conn *conn)
4102{
4103 int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
4104 struct iscsi_session *sess = conn->sess;
4105
4106 pr_debug("Closing iSCSI connection CID %hu on SID:"
4107 " %u\n", conn->cid, sess->sid);
4108 /*
4109 * Always up conn_logout_comp just in case the RX Thread is sleeping
4110 * and the logout response never got sent because the connection
4111 * failed.
4112 */
4113 complete(&conn->conn_logout_comp);
4114
4115 iscsi_release_thread_set(conn);
4116
4117 iscsit_stop_timers_for_cmds(conn);
4118 iscsit_stop_nopin_response_timer(conn);
4119 iscsit_stop_nopin_timer(conn);
4120 iscsit_free_queue_reqs_for_conn(conn);
4121
4122 /*
4123 * During Connection recovery drop unacknowledged out of order
4124 * commands for this connection, and prepare the other commands
4125 * for realligence.
4126 *
4127 * During normal operation clear the out of order commands (but
4128 * do not free the struct iscsi_ooo_cmdsn's) and release all
4129 * struct iscsi_cmds.
4130 */
4131 if (atomic_read(&conn->connection_recovery)) {
4132 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
4133 iscsit_prepare_cmds_for_realligance(conn);
4134 } else {
4135 iscsit_clear_ooo_cmdsns_for_conn(conn);
4136 iscsit_release_commands_from_conn(conn);
4137 }
4138
4139 /*
4140 * Handle decrementing session or connection usage count if
4141 * a logout response was not able to be sent because the
4142 * connection failed. Fall back to Session Recovery here.
4143 */
4144 if (atomic_read(&conn->conn_logout_remove)) {
4145 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4146 iscsit_dec_conn_usage_count(conn);
4147 iscsit_dec_session_usage_count(sess);
4148 }
4149 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4150 iscsit_dec_conn_usage_count(conn);
4151
4152 atomic_set(&conn->conn_logout_remove, 0);
4153 atomic_set(&sess->session_reinstatement, 0);
4154 atomic_set(&sess->session_fall_back_to_erl0, 1);
4155 }
4156
4157 spin_lock_bh(&sess->conn_lock);
4158 list_del(&conn->conn_list);
4159
4160 /*
4161 * Attempt to let the Initiator know this connection failed by
4162 * sending an Connection Dropped Async Message on another
4163 * active connection.
4164 */
4165 if (atomic_read(&conn->connection_recovery))
4166 iscsit_build_conn_drop_async_message(conn);
4167
4168 spin_unlock_bh(&sess->conn_lock);
4169
4170 /*
4171 * If connection reinstatement is being performed on this connection,
4172 * up the connection reinstatement semaphore that is being blocked on
4173 * in iscsit_cause_connection_reinstatement().
4174 */
4175 spin_lock_bh(&conn->state_lock);
4176 if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4177 spin_unlock_bh(&conn->state_lock);
4178 complete(&conn->conn_wait_comp);
4179 wait_for_completion(&conn->conn_post_wait_comp);
4180 spin_lock_bh(&conn->state_lock);
4181 }
4182
4183 /*
4184 * If connection reinstatement is being performed on this connection
4185 * by receiving a REMOVECONNFORRECOVERY logout request, up the
4186 * connection wait rcfr semaphore that is being blocked on
4187 * an iscsit_connection_reinstatement_rcfr().
4188 */
4189 if (atomic_read(&conn->connection_wait_rcfr)) {
4190 spin_unlock_bh(&conn->state_lock);
4191 complete(&conn->conn_wait_rcfr_comp);
4192 wait_for_completion(&conn->conn_post_wait_comp);
4193 spin_lock_bh(&conn->state_lock);
4194 }
4195 atomic_set(&conn->connection_reinstatement, 1);
4196 spin_unlock_bh(&conn->state_lock);
4197
4198 /*
4199 * If any other processes are accessing this connection pointer we
4200 * must wait until they have completed.
4201 */
4202 iscsit_check_conn_usage_count(conn);
4203
4204 if (conn->conn_rx_hash.tfm)
4205 crypto_free_hash(conn->conn_rx_hash.tfm);
4206 if (conn->conn_tx_hash.tfm)
4207 crypto_free_hash(conn->conn_tx_hash.tfm);
4208
4209 if (conn->conn_cpumask)
4210 free_cpumask_var(conn->conn_cpumask);
4211
4212 kfree(conn->conn_ops);
4213 conn->conn_ops = NULL;
4214
bf6932f4 4215 if (conn->sock)
e48354ce 4216 sock_release(conn->sock);
baa4d64b
NB		 4217
4218 if (conn->conn_transport->iscsit_free_conn)
4219 conn->conn_transport->iscsit_free_conn(conn);
4220
4221 iscsit_put_transport(conn->conn_transport);
4222
e48354ce
NB		 4223 conn->thread_set = NULL;
4224
4225 pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4226 conn->conn_state = TARG_CONN_STATE_FREE;
4227 kfree(conn);
4228
4229 spin_lock_bh(&sess->conn_lock);
4230 atomic_dec(&sess->nconn);
4231 pr_debug("Decremented iSCSI connection count to %hu from node:"
4232 " %s\n", atomic_read(&sess->nconn),
4233 sess->sess_ops->InitiatorName);
4234 /*
4235 * Make sure that if one connection fails in an non ERL=2 iSCSI
4236 * Session that they all fail.
4237 */
4238 if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4239 !atomic_read(&sess->session_logout))
4240 atomic_set(&sess->session_fall_back_to_erl0, 1);
4241
4242 /*
4243 * If this was not the last connection in the session, and we are
4244 * performing session reinstatement or falling back to ERL=0, call
4245 * iscsit_stop_session() without sleeping to shutdown the other
4246 * active connections.
4247 */
4248 if (atomic_read(&sess->nconn)) {
4249 if (!atomic_read(&sess->session_reinstatement) &&
4250 !atomic_read(&sess->session_fall_back_to_erl0)) {
4251 spin_unlock_bh(&sess->conn_lock);
4252 return 0;
4253 }
4254 if (!atomic_read(&sess->session_stop_active)) {
4255 atomic_set(&sess->session_stop_active, 1);
4256 spin_unlock_bh(&sess->conn_lock);
4257 iscsit_stop_session(sess, 0, 0);
4258 return 0;
4259 }
4260 spin_unlock_bh(&sess->conn_lock);
4261 return 0;
4262 }
4263
4264 /*
4265 * If this was the last connection in the session and one of the
4266 * following is occurring:
4267 *
4268 * Session Reinstatement is not being performed, and are falling back
4269 * to ERL=0 call iscsit_close_session().
4270 *
4271 * Session Logout was requested. iscsit_close_session() will be called
4272 * elsewhere.
4273 *
4274 * Session Continuation is not being performed, start the Time2Retain
4275 * handler and check if sleep_on_sess_wait_sem is active.
4276 */
4277 if (!atomic_read(&sess->session_reinstatement) &&
4278 atomic_read(&sess->session_fall_back_to_erl0)) {
4279 spin_unlock_bh(&sess->conn_lock);
99367f01 4280 target_put_session(sess->se_sess);
e48354ce
NB		 4281
4282 return 0;
4283 } else if (atomic_read(&sess->session_logout)) {
4284 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4285 sess->session_state = TARG_SESS_STATE_FREE;
4286 spin_unlock_bh(&sess->conn_lock);
4287
4288 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4289 complete(&sess->session_wait_comp);
4290
4291 return 0;
4292 } else {
4293 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4294 sess->session_state = TARG_SESS_STATE_FAILED;
4295
4296 if (!atomic_read(&sess->session_continuation)) {
4297 spin_unlock_bh(&sess->conn_lock);
4298 iscsit_start_time2retain_handler(sess);
4299 } else
4300 spin_unlock_bh(&sess->conn_lock);
4301
4302 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4303 complete(&sess->session_wait_comp);
4304
4305 return 0;
4306 }
4307 spin_unlock_bh(&sess->conn_lock);
4308
4309 return 0;
4310}
4311
4312int iscsit_close_session(struct iscsi_session *sess)
4313{
4314 struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
4315 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4316
4317 if (atomic_read(&sess->nconn)) {
4318 pr_err("%d connection(s) still exist for iSCSI session"
4319 " to %s\n", atomic_read(&sess->nconn),
4320 sess->sess_ops->InitiatorName);
4321 BUG();
4322 }
4323
4324 spin_lock_bh(&se_tpg->session_lock);
4325 atomic_set(&sess->session_logout, 1);
4326 atomic_set(&sess->session_reinstatement, 1);
4327 iscsit_stop_time2retain_timer(sess);
4328 spin_unlock_bh(&se_tpg->session_lock);
4329
4330 /*
4331 * transport_deregister_session_configfs() will clear the
4332 * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4333 * can be setting it again with __transport_register_session() in
4334 * iscsi_post_login_handler() again after the iscsit_stop_session()
4335 * completes in iscsi_np context.
4336 */
4337 transport_deregister_session_configfs(sess->se_sess);
4338
4339 /*
4340 * If any other processes are accessing this session pointer we must
4341 * wait until they have completed. If we are in an interrupt (the
4342 * time2retain handler) and contain and active session usage count we
4343 * restart the timer and exit.
4344 */
4345 if (!in_interrupt()) {
4346 if (iscsit_check_session_usage_count(sess) == 1)
4347 iscsit_stop_session(sess, 1, 1);
4348 } else {
4349 if (iscsit_check_session_usage_count(sess) == 2) {
4350 atomic_set(&sess->session_logout, 0);
4351 iscsit_start_time2retain_handler(sess);
4352 return 0;
4353 }
4354 }
4355
4356 transport_deregister_session(sess->se_sess);
4357
4358 if (sess->sess_ops->ErrorRecoveryLevel == 2)
4359 iscsit_free_connection_recovery_entires(sess);
4360
4361 iscsit_free_all_ooo_cmdsns(sess);
4362
4363 spin_lock_bh(&se_tpg->session_lock);
4364 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4365 sess->session_state = TARG_SESS_STATE_FREE;
4366 pr_debug("Released iSCSI session from node: %s\n",
4367 sess->sess_ops->InitiatorName);
4368 tpg->nsessions--;
4369 if (tpg->tpg_tiqn)
4370 tpg->tpg_tiqn->tiqn_nsessions--;
4371
4372 pr_debug("Decremented number of active iSCSI Sessions on"
4373 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4374
4375 spin_lock(&sess_idr_lock);
4376 idr_remove(&sess_idr, sess->session_index);
4377 spin_unlock(&sess_idr_lock);
4378
4379 kfree(sess->sess_ops);
4380 sess->sess_ops = NULL;
4381 spin_unlock_bh(&se_tpg->session_lock);
4382
4383 kfree(sess);
4384 return 0;
4385}
4386
4387static void iscsit_logout_post_handler_closesession(
4388 struct iscsi_conn *conn)
4389{
4390 struct iscsi_session *sess = conn->sess;
4391
4392 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4393 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4394
4395 atomic_set(&conn->conn_logout_remove, 0);
4396 complete(&conn->conn_logout_comp);
4397
4398 iscsit_dec_conn_usage_count(conn);
4399 iscsit_stop_session(sess, 1, 1);
4400 iscsit_dec_session_usage_count(sess);
99367f01 4401 target_put_session(sess->se_sess);
e48354ce
NB		 4402}
4403
4404static void iscsit_logout_post_handler_samecid(
4405 struct iscsi_conn *conn)
4406{
4407 iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4408 iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4409
4410 atomic_set(&conn->conn_logout_remove, 0);
4411 complete(&conn->conn_logout_comp);
4412
4413 iscsit_cause_connection_reinstatement(conn, 1);
4414 iscsit_dec_conn_usage_count(conn);
4415}
4416
4417static void iscsit_logout_post_handler_diffcid(
4418 struct iscsi_conn *conn,
4419 u16 cid)
4420{
4421 struct iscsi_conn *l_conn;
4422 struct iscsi_session *sess = conn->sess;
4423
4424 if (!sess)
4425 return;
4426
4427 spin_lock_bh(&sess->conn_lock);
4428 list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4429 if (l_conn->cid == cid) {
4430 iscsit_inc_conn_usage_count(l_conn);
4431 break;
4432 }
4433 }
4434 spin_unlock_bh(&sess->conn_lock);
4435
4436 if (!l_conn)
4437 return;
4438
4439 if (l_conn->sock)
4440 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4441
4442 spin_lock_bh(&l_conn->state_lock);
4443 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4444 l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4445 spin_unlock_bh(&l_conn->state_lock);
4446
4447 iscsit_cause_connection_reinstatement(l_conn, 1);
4448 iscsit_dec_conn_usage_count(l_conn);
4449}
4450
4451/*
4452 * Return of 0 causes the TX thread to restart.
4453 */
4454static int iscsit_logout_post_handler(
4455 struct iscsi_cmd *cmd,
4456 struct iscsi_conn *conn)
4457{
4458 int ret = 0;
4459
4460 switch (cmd->logout_reason) {
4461 case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4462 switch (cmd->logout_response) {
4463 case ISCSI_LOGOUT_SUCCESS:
4464 case ISCSI_LOGOUT_CLEANUP_FAILED:
4465 default:
4466 iscsit_logout_post_handler_closesession(conn);
4467 break;
4468 }
4469 ret = 0;
4470 break;
4471 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4472 if (conn->cid == cmd->logout_cid) {
4473 switch (cmd->logout_response) {
4474 case ISCSI_LOGOUT_SUCCESS:
4475 case ISCSI_LOGOUT_CLEANUP_FAILED:
4476 default:
4477 iscsit_logout_post_handler_samecid(conn);
4478 break;
4479 }
4480 ret = 0;
4481 } else {
4482 switch (cmd->logout_response) {
4483 case ISCSI_LOGOUT_SUCCESS:
4484 iscsit_logout_post_handler_diffcid(conn,
4485 cmd->logout_cid);
4486 break;
4487 case ISCSI_LOGOUT_CID_NOT_FOUND:
4488 case ISCSI_LOGOUT_CLEANUP_FAILED:
4489 default:
4490 break;
4491 }
4492 ret = 1;
4493 }
4494 break;
4495 case ISCSI_LOGOUT_REASON_RECOVERY:
4496 switch (cmd->logout_response) {
4497 case ISCSI_LOGOUT_SUCCESS:
4498 case ISCSI_LOGOUT_CID_NOT_FOUND:
4499 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4500 case ISCSI_LOGOUT_CLEANUP_FAILED:
4501 default:
4502 break;
4503 }
4504 ret = 1;
4505 break;
4506 default:
4507 break;
4508
4509 }
4510 return ret;
4511}
4512
4513void iscsit_fail_session(struct iscsi_session *sess)
4514{
4515 struct iscsi_conn *conn;
4516
4517 spin_lock_bh(&sess->conn_lock);
4518 list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4519 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4520 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4521 }
4522 spin_unlock_bh(&sess->conn_lock);
4523
4524 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4525 sess->session_state = TARG_SESS_STATE_FAILED;
4526}
4527
4528int iscsit_free_session(struct iscsi_session *sess)
4529{
4530 u16 conn_count = atomic_read(&sess->nconn);
4531 struct iscsi_conn *conn, *conn_tmp = NULL;
4532 int is_last;
4533
4534 spin_lock_bh(&sess->conn_lock);
4535 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4536
4537 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4538 conn_list) {
4539 if (conn_count == 0)
4540 break;
4541
4542 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4543 is_last = 1;
4544 } else {
4545 iscsit_inc_conn_usage_count(conn_tmp);
4546 is_last = 0;
4547 }
4548 iscsit_inc_conn_usage_count(conn);
4549
4550 spin_unlock_bh(&sess->conn_lock);
4551 iscsit_cause_connection_reinstatement(conn, 1);
4552 spin_lock_bh(&sess->conn_lock);
4553
4554 iscsit_dec_conn_usage_count(conn);
4555 if (is_last == 0)
4556 iscsit_dec_conn_usage_count(conn_tmp);
4557
4558 conn_count--;
4559 }
4560
4561 if (atomic_read(&sess->nconn)) {
4562 spin_unlock_bh(&sess->conn_lock);
4563 wait_for_completion(&sess->session_wait_comp);
4564 } else
4565 spin_unlock_bh(&sess->conn_lock);
4566
99367f01 4567 target_put_session(sess->se_sess);
e48354ce
NB		 4568 return 0;
4569}
4570
4571void iscsit_stop_session(
4572 struct iscsi_session *sess,
4573 int session_sleep,
4574 int connection_sleep)
4575{
4576 u16 conn_count = atomic_read(&sess->nconn);
4577 struct iscsi_conn *conn, *conn_tmp = NULL;
4578 int is_last;
4579
4580 spin_lock_bh(&sess->conn_lock);
4581 if (session_sleep)
4582 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4583
4584 if (connection_sleep) {
4585 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4586 conn_list) {
4587 if (conn_count == 0)
4588 break;
4589
4590 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4591 is_last = 1;
4592 } else {
4593 iscsit_inc_conn_usage_count(conn_tmp);
4594 is_last = 0;
4595 }
4596 iscsit_inc_conn_usage_count(conn);
4597
4598 spin_unlock_bh(&sess->conn_lock);
4599 iscsit_cause_connection_reinstatement(conn, 1);
4600 spin_lock_bh(&sess->conn_lock);
4601
4602 iscsit_dec_conn_usage_count(conn);
4603 if (is_last == 0)
4604 iscsit_dec_conn_usage_count(conn_tmp);
4605 conn_count--;
4606 }
4607 } else {
4608 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4609 iscsit_cause_connection_reinstatement(conn, 0);
4610 }
4611
4612 if (session_sleep && atomic_read(&sess->nconn)) {
4613 spin_unlock_bh(&sess->conn_lock);
4614 wait_for_completion(&sess->session_wait_comp);
4615 } else
4616 spin_unlock_bh(&sess->conn_lock);
4617}
4618
4619int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4620{
4621 struct iscsi_session *sess;
4622 struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4623 struct se_session *se_sess, *se_sess_tmp;
4624 int session_count = 0;
4625
4626 spin_lock_bh(&se_tpg->session_lock);
4627 if (tpg->nsessions && !force) {
4628 spin_unlock_bh(&se_tpg->session_lock);
4629 return -1;
4630 }
4631
4632 list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4633 sess_list) {
4634 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4635
4636 spin_lock(&sess->conn_lock);
4637 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4638 atomic_read(&sess->session_logout) ||
4639 (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4640 spin_unlock(&sess->conn_lock);
4641 continue;
4642 }
4643 atomic_set(&sess->session_reinstatement, 1);
4644 spin_unlock(&sess->conn_lock);
4645 spin_unlock_bh(&se_tpg->session_lock);
4646
4647 iscsit_free_session(sess);
4648 spin_lock_bh(&se_tpg->session_lock);
4649
4650 session_count++;
4651 }
4652 spin_unlock_bh(&se_tpg->session_lock);
4653
4654 pr_debug("Released %d iSCSI Session(s) from Target Portal"
4655 " Group: %hu\n", session_count, tpg->tpgt);
4656 return 0;
4657}
4658
4659MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4660MODULE_VERSION("4.1.x");
4661MODULE_AUTHOR("nab@Linux-iSCSI.org");
4662MODULE_LICENSE("GPL");
4663
4664module_init(iscsi_target_init_module);
4665module_exit(iscsi_target_cleanup_module);
kernel source for telekom puls (alcatel/mediatek)