return ret;
}
-static int
-vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *cmd,
- struct iovec *iov,
- int niov,
- bool write)
-{
- struct scatterlist *sg = cmd->tvc_sgl;
- unsigned int sgl_count = 0;
- int ret, i;
-
- for (i = 0; i < niov; i++)
- sgl_count += iov_num_pages(iov[i].iov_base, iov[i].iov_len);
-
- if (sgl_count > TCM_VHOST_PREALLOC_SGLS) {
- pr_err("vhost_scsi_map_iov_to_sgl() sgl_count: %u greater than"
- " preallocated TCM_VHOST_PREALLOC_SGLS: %u\n",
- sgl_count, TCM_VHOST_PREALLOC_SGLS);
- return -ENOBUFS;
- }
-
- pr_debug("%s sg %p sgl_count %u\n", __func__, sg, sgl_count);
- sg_init_table(sg, sgl_count);
- cmd->tvc_sgl_count = sgl_count;
-
- pr_debug("Mapping iovec %p for %u pages\n", &iov[0], sgl_count);
-
- for (i = 0; i < niov; i++) {
- ret = vhost_scsi_map_to_sgl(cmd, iov[i].iov_base, iov[i].iov_len,
- sg, write);
- if (ret < 0) {
- for (i = 0; i < cmd->tvc_sgl_count; i++) {
- struct page *page = sg_page(&cmd->tvc_sgl[i]);
- if (page)
- put_page(page);
- }
- cmd->tvc_sgl_count = 0;
- return ret;
- }
- sg += ret;
- sgl_count -= ret;
- }
- return 0;
-}
-
-static int
-vhost_scsi_map_iov_to_prot(struct tcm_vhost_cmd *cmd,
- struct iovec *iov,
- int niov,
- bool write)
-{
- struct scatterlist *prot_sg = cmd->tvc_prot_sgl;
- unsigned int prot_sgl_count = 0;
- int ret, i;
-
- for (i = 0; i < niov; i++)
- prot_sgl_count += iov_num_pages(iov[i].iov_base, iov[i].iov_len);
-
- if (prot_sgl_count > TCM_VHOST_PREALLOC_PROT_SGLS) {
- pr_err("vhost_scsi_map_iov_to_prot() sgl_count: %u greater than"
- " preallocated TCM_VHOST_PREALLOC_PROT_SGLS: %u\n",
- prot_sgl_count, TCM_VHOST_PREALLOC_PROT_SGLS);
- return -ENOBUFS;
- }
-
- pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,
- prot_sg, prot_sgl_count);
- sg_init_table(prot_sg, prot_sgl_count);
- cmd->tvc_prot_sgl_count = prot_sgl_count;
-
- for (i = 0; i < niov; i++) {
- ret = vhost_scsi_map_to_sgl(cmd, iov[i].iov_base, iov[i].iov_len,
- prot_sg, write);
- if (ret < 0) {
- for (i = 0; i < cmd->tvc_prot_sgl_count; i++) {
- struct page *page = sg_page(&cmd->tvc_prot_sgl[i]);
- if (page)
- put_page(page);
- }
- cmd->tvc_prot_sgl_count = 0;
- return ret;
- }
- prot_sg += ret;
- prot_sgl_count -= ret;
- }
- return 0;
-}
-
static int
vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)
{
static void
vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
{
- struct tcm_vhost_tpg **vs_tpg;
+ struct tcm_vhost_tpg **vs_tpg, *tpg;
struct virtio_scsi_cmd_req v_req;
struct virtio_scsi_cmd_req_pi v_req_pi;
- struct tcm_vhost_tpg *tpg;
struct tcm_vhost_cmd *cmd;
+ struct iov_iter out_iter, in_iter, prot_iter, data_iter;
u64 tag;
- u32 exp_data_len, data_first, data_num, data_direction, prot_first;
- unsigned out, in, i;
- int head, ret, data_niov, prot_niov, prot_bytes;
- size_t req_size;
+ u32 exp_data_len, data_direction;
+ unsigned out, in;
+ int head, ret, prot_bytes;
+ size_t req_size, rsp_size = sizeof(struct virtio_scsi_cmd_resp);
+ size_t out_size, in_size;
u16 lun;
u8 *target, *lunp, task_attr;
- bool hdr_pi;
+ bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);
void *req, *cdb;
mutex_lock(&vq->mutex);
for (;;) {
head = vhost_get_vq_desc(vq, vq->iov,
- ARRAY_SIZE(vq->iov), &out, &in,
- NULL, NULL);
+ ARRAY_SIZE(vq->iov), &out, &in,
+ NULL, NULL);
pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",
- head, out, in);
+ head, out, in);
/* On error, stop handling until the next kick. */
if (unlikely(head < 0))
break;
}
break;
}
-
- /* FIXME: BIDI operation */
- if (out == 1 && in == 1) {
- data_direction = DMA_NONE;
- data_first = 0;
- data_num = 0;
- } else if (out == 1 && in > 1) {
- data_direction = DMA_FROM_DEVICE;
- data_first = out + 1;
- data_num = in - 1;
- } else if (out > 1 && in == 1) {
- data_direction = DMA_TO_DEVICE;
- data_first = 1;
- data_num = out - 1;
- } else {
- vq_err(vq, "Invalid buffer layout out: %u in: %u\n",
- out, in);
- break;
- }
-
/*
- * Check for a sane resp buffer so we can report errors to
- * the guest.
+ * Check for a sane response buffer so we can report early
+ * errors back to the guest.
*/
- if (unlikely(vq->iov[out].iov_len !=
- sizeof(struct virtio_scsi_cmd_resp))) {
- vq_err(vq, "Expecting virtio_scsi_cmd_resp, got %zu"
- " bytes\n", vq->iov[out].iov_len);
+ if (unlikely(vq->iov[out].iov_len < rsp_size)) {
+ vq_err(vq, "Expecting at least virtio_scsi_cmd_resp"
+ " size, got %zu bytes\n", vq->iov[out].iov_len);
break;
}
-
- if (vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI)) {
+ /*
+ * Setup pointers and values based upon different virtio-scsi
+ * request header if T10_PI is enabled in KVM guest.
+ */
+ if (t10_pi) {
req = &v_req_pi;
+ req_size = sizeof(v_req_pi);
lunp = &v_req_pi.lun[0];
target = &v_req_pi.lun[1];
- req_size = sizeof(v_req_pi);
- hdr_pi = true;
} else {
req = &v_req;
+ req_size = sizeof(v_req);
lunp = &v_req.lun[0];
target = &v_req.lun[1];
- req_size = sizeof(v_req);
- hdr_pi = false;
}
+ /*
+ * FIXME: Not correct for BIDI operation
+ */
+ out_size = iov_length(vq->iov, out);
+ in_size = iov_length(&vq->iov[out], in);
- if (unlikely(vq->iov[0].iov_len < req_size)) {
- pr_err("Expecting virtio-scsi header: %zu, got %zu\n",
- req_size, vq->iov[0].iov_len);
- vhost_scsi_send_bad_target(vs, vq, head, out);
- continue;
- }
- ret = memcpy_fromiovecend(req, &vq->iov[0], 0, req_size);
- if (unlikely(ret)) {
- vq_err(vq, "Faulted on virtio_scsi_cmd_req\n");
+ /*
+ * Copy over the virtio-scsi request header, which for a
+ * ANY_LAYOUT enabled guest may span multiple iovecs, or a
+ * single iovec may contain both the header + outgoing
+ * WRITE payloads.
+ *
+ * copy_from_iter() will advance out_iter, so that it will
+ * point at the start of the outgoing WRITE payload, if
+ * DMA_TO_DEVICE is set.
+ */
+ iov_iter_init(&out_iter, WRITE, vq->iov, out, out_size);
+
+ ret = copy_from_iter(req, req_size, &out_iter);
+ if (unlikely(ret != req_size)) {
+ vq_err(vq, "Faulted on copy_from_iter\n");
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
-
/* virtio-scsi spec requires byte 0 of the lun to be 1 */
if (unlikely(*lunp != 1)) {
+ vq_err(vq, "Illegal virtio-scsi lun: %u\n", *lunp);
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
tpg = ACCESS_ONCE(vs_tpg[*target]);
-
- /* Target does not exist, fail the request */
if (unlikely(!tpg)) {
+ /* Target does not exist, fail the request */
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
-
- data_niov = data_num;
- prot_niov = prot_first = prot_bytes = 0;
/*
- * Determine if any protection information iovecs are preceeding
- * the actual data payload, and adjust data_first + data_niov
- * values accordingly for vhost_scsi_map_iov_to_sgl() below.
+ * Determine data_direction by calculating the total outgoing
+ * iovec sizes + incoming iovec sizes vs. virtio-scsi request +
+ * response headers respectively.
*
- * Also extract virtio_scsi header bits for vhost_scsi_get_tag()
+ * For DMA_TO_DEVICE this is out_iter, which is already pointing
+ * to the right place.
+ *
+ * For DMA_FROM_DEVICE, the iovec will be just past the end
+ * of the virtio-scsi response header in either the same
+ * or immediately following iovec.
+ *
+ * Any associated T10_PI bytes for the outgoing / incoming
+ * payloads are included in calculation of exp_data_len here.
+ */
+ prot_bytes = 0;
+
+ if (out_size > req_size) {
+ data_direction = DMA_TO_DEVICE;
+ exp_data_len = out_size - req_size;
+ data_iter = out_iter;
+ } else if (in_size > rsp_size) {
+ data_direction = DMA_FROM_DEVICE;
+ exp_data_len = in_size - rsp_size;
+
+ iov_iter_init(&in_iter, READ, &vq->iov[out], in,
+ rsp_size + exp_data_len);
+ iov_iter_advance(&in_iter, rsp_size);
+ data_iter = in_iter;
+ } else {
+ data_direction = DMA_NONE;
+ exp_data_len = 0;
+ }
+ /*
+ * If T10_PI header + payload is present, setup prot_iter values
+ * and recalculate data_iter for vhost_scsi_mapal() mapping to
+ * host scatterlists via get_user_pages_fast().
*/
- if (hdr_pi) {
+ if (t10_pi) {
if (v_req_pi.pi_bytesout) {
if (data_direction != DMA_TO_DEVICE) {
- vq_err(vq, "Received non zero do_pi_niov"
- ", but wrong data_direction\n");
+ vq_err(vq, "Received non zero pi_bytesout,"
+ " but wrong data_direction\n");
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);
} else if (v_req_pi.pi_bytesin) {
if (data_direction != DMA_FROM_DEVICE) {
- vq_err(vq, "Received non zero di_pi_niov"
- ", but wrong data_direction\n");
+ vq_err(vq, "Received non zero pi_bytesin,"
+ " but wrong data_direction\n");
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);
}
+ /*
+ * Set prot_iter to data_iter, and advance past any
+ * preceeding prot_bytes that may be present.
+ *
+ * Also fix up the exp_data_len to reflect only the
+ * actual data payload length.
+ */
if (prot_bytes) {
- int tmp = 0;
-
- for (i = 0; i < data_num; i++) {
- tmp += vq->iov[data_first + i].iov_len;
- prot_niov++;
- if (tmp >= prot_bytes)
- break;
- }
- prot_first = data_first;
- data_first += prot_niov;
- data_niov = data_num - prot_niov;
+ exp_data_len -= prot_bytes;
+ prot_iter = data_iter;
+ iov_iter_advance(&data_iter, prot_bytes);
}
tag = vhost64_to_cpu(vq, v_req_pi.tag);
task_attr = v_req_pi.task_attr;
cdb = &v_req.cdb[0];
lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF;
}
- exp_data_len = 0;
- for (i = 0; i < data_niov; i++)
- exp_data_len += vq->iov[data_first + i].iov_len;
/*
- * Check that the recieved CDB size does not exceeded our
- * hardcoded max for vhost-scsi
+ * Check that the received CDB size does not exceeded our
+ * hardcoded max for vhost-scsi, then get a pre-allocated
+ * cmd descriptor for the new virtio-scsi tag.
*
* TODO what if cdb was too small for varlen cdb header?
*/
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
-
cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr,
exp_data_len + prot_bytes,
data_direction);
if (IS_ERR(cmd)) {
vq_err(vq, "vhost_scsi_get_tag failed %ld\n",
- PTR_ERR(cmd));
+ PTR_ERR(cmd));
vhost_scsi_send_bad_target(vs, vq, head, out);
continue;
}
-
- pr_debug("Allocated tv_cmd: %p exp_data_len: %d, data_direction"
- ": %d\n", cmd, exp_data_len, data_direction);
-
cmd->tvc_vhost = vs;
cmd->tvc_vq = vq;
cmd->tvc_resp_iov = &vq->iov[out];
cmd->tvc_in_iovs = in;
pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",
- cmd->tvc_cdb[0], cmd->tvc_lun);
+ cmd->tvc_cdb[0], cmd->tvc_lun);
+ pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"
+ " %d\n", cmd, exp_data_len, prot_bytes, data_direction);
- if (prot_niov) {
- ret = vhost_scsi_map_iov_to_prot(cmd,
- &vq->iov[prot_first], prot_niov,
- data_direction == DMA_FROM_DEVICE);
- if (unlikely(ret)) {
- vq_err(vq, "Failed to map iov to"
- " prot_sgl\n");
- tcm_vhost_release_cmd(&cmd->tvc_se_cmd);
- vhost_scsi_send_bad_target(vs, vq, head, out);
- continue;
- }
- }
if (data_direction != DMA_NONE) {
- ret = vhost_scsi_map_iov_to_sgl(cmd,
- &vq->iov[data_first], data_niov,
- data_direction == DMA_FROM_DEVICE);
+ ret = vhost_scsi_mapal(cmd,
+ prot_bytes, &prot_iter,
+ exp_data_len, &data_iter);
if (unlikely(ret)) {
vq_err(vq, "Failed to map iov to sgl\n");
tcm_vhost_release_cmd(&cmd->tvc_se_cmd);
/*
* Save the descriptor from vhost_get_vq_desc() to be used to
* complete the virtio-scsi request in TCM callback context via
- * tcm_vhost_queue_data_in() and tcm_vhost_queue_status()
+ * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()
*/
cmd->tvc_vq_desc = head;
/*
- * Dispatch tv_cmd descriptor for cmwq execution in process
- * context provided by tcm_vhost_workqueue. This also ensures
- * tv_cmd is executed on the same kworker CPU as this vhost
- * thread to gain positive L2 cache locality effects..
+ * Dispatch cmd descriptor for cmwq execution in process
+ * context provided by vhost_scsi_workqueue. This also ensures
+ * cmd is executed on the same kworker CPU as this vhost
+ * thread to gain positive L2 cache locality effects.
*/
INIT_WORK(&cmd->work, tcm_vhost_submission_work);
queue_work(tcm_vhost_workqueue, &cmd->work);
projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / commitdiff