[GitHub/mt8127/android_kernel_alcatel_ttab.git] / drivers / s390 / block / scm_blk.c

/*
 * Block driver for s390 storage class memory.
 *
 * Copyright IBM Corp. 2012
 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
 */

#define KMSG_COMPONENT "scm_block"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/genhd.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <asm/eadm.h>
#include "scm_blk.h"

debug_info_t *scm_debug;
static int scm_major;
static DEFINE_SPINLOCK(list_lock);
static LIST_HEAD(inactive_requests);
static unsigned int nr_requests = 64;
static atomic_t nr_devices = ATOMIC_INIT(0);
module_param(nr_requests, uint, S_IRUGO);
MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");

MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
MODULE_LICENSE("GPL");
MODULE_ALIAS("scm:scmdev*");

static void __scm_free_rq(struct scm_request *scmrq)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);

	free_page((unsigned long) scmrq->aob);
	free_page((unsigned long) scmrq->aidaw);
	__scm_free_rq_cluster(scmrq);
	kfree(aobrq);
}

static void scm_free_rqs(void)
{
	struct list_head *iter, *safe;
	struct scm_request *scmrq;

	spin_lock_irq(&list_lock);
	list_for_each_safe(iter, safe, &inactive_requests) {
		scmrq = list_entry(iter, struct scm_request, list);
		list_del(&scmrq->list);
		__scm_free_rq(scmrq);
	}
	spin_unlock_irq(&list_lock);
}

static int __scm_alloc_rq(void)
{
	struct aob_rq_header *aobrq;
	struct scm_request *scmrq;

	aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
	if (!aobrq)
		return -ENOMEM;

	scmrq = (void *) aobrq->data;
	scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
	if (!scmrq->aob || !scmrq->aidaw) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	if (__scm_alloc_rq_cluster(scmrq)) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	INIT_LIST_HEAD(&scmrq->list);
	spin_lock_irq(&list_lock);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irq(&list_lock);

	return 0;
}

static int scm_alloc_rqs(unsigned int nrqs)
{
	int ret = 0;

	while (nrqs-- && !ret)
		ret = __scm_alloc_rq();

	return ret;
}

static struct scm_request *scm_request_fetch(void)
{
	struct scm_request *scmrq = NULL;

	spin_lock(&list_lock);
	if (list_empty(&inactive_requests))
		goto out;
	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	list_del(&scmrq->list);
out:
	spin_unlock(&list_lock);
	return scmrq;
}

static void scm_request_done(struct scm_request *scmrq)
{
	unsigned long flags;

	spin_lock_irqsave(&list_lock, flags);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irqrestore(&list_lock, flags);
}

static int scm_open(struct block_device *blkdev, fmode_t mode)
{
	return scm_get_ref();
}

static int scm_release(struct gendisk *gendisk, fmode_t mode)
{
	scm_put_ref();
	return 0;
}

static const struct block_device_operations scm_blk_devops = {
	.owner = THIS_MODULE,
	.open = scm_open,
	.release = scm_release,
};

static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
{
	return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
}

static void scm_request_prepare(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	struct scm_device *scmdev = bdev->gendisk->private_data;
	struct aidaw *aidaw = scmrq->aidaw;
	struct msb *msb = &scmrq->aob->msb[0];
	struct req_iterator iter;
	struct bio_vec *bv;

	msb->bs = MSB_BS_4K;
	scmrq->aob->request.msb_count = 1;
	msb->scm_addr = scmdev->address +
		((u64) blk_rq_pos(scmrq->request) << 9);
	msb->oc = (rq_data_dir(scmrq->request) == READ) ?
		MSB_OC_READ : MSB_OC_WRITE;
	msb->flags |= MSB_FLAG_IDA;
	msb->data_addr = (u64) aidaw;

	rq_for_each_segment(bv, scmrq->request, iter) {
		WARN_ON(bv->bv_offset);
		msb->blk_count += bv->bv_len >> 12;
		aidaw->data_addr = (u64) page_address(bv->bv_page);
		aidaw++;
	}
}

static inline void scm_request_init(struct scm_blk_dev *bdev,
				    struct scm_request *scmrq,
				    struct request *req)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	struct aob *aob = scmrq->aob;

	memset(aob, 0, sizeof(*aob));
	memset(scmrq->aidaw, 0, PAGE_SIZE);
	aobrq->scmdev = bdev->scmdev;
	aob->request.cmd_code = ARQB_CMD_MOVE;
	aob->request.data = (u64) aobrq;
	scmrq->request = req;
	scmrq->bdev = bdev;
	scmrq->retries = 4;
	scmrq->error = 0;
	scm_request_cluster_init(scmrq);
}

static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
{
	if (atomic_read(&bdev->queued_reqs)) {
		/* Queue restart is triggered by the next interrupt. */
		return;
	}
	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
}

void scm_request_requeue(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;

	scm_release_cluster(scmrq);
	blk_requeue_request(bdev->rq, scmrq->request);
	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
	scm_ensure_queue_restart(bdev);
}

void scm_request_finish(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;

	scm_release_cluster(scmrq);
	blk_end_request_all(scmrq->request, scmrq->error);
	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
}

static void scm_blk_request(struct request_queue *rq)
{
	struct scm_device *scmdev = rq->queuedata;
	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
	struct scm_request *scmrq;
	struct request *req;
	int ret;

	while ((req = blk_peek_request(rq))) {
		if (req->cmd_type != REQ_TYPE_FS)
			continue;

		if (!scm_permit_request(bdev, req)) {
			scm_ensure_queue_restart(bdev);
			return;
		}
		scmrq = scm_request_fetch();
		if (!scmrq) {
			SCM_LOG(5, "no request");
			scm_ensure_queue_restart(bdev);
			return;
		}
		scm_request_init(bdev, scmrq, req);
		if (!scm_reserve_cluster(scmrq)) {
			SCM_LOG(5, "cluster busy");
			scm_request_done(scmrq);
			return;
		}
		if (scm_need_cluster_request(scmrq)) {
			atomic_inc(&bdev->queued_reqs);
			blk_start_request(req);
			scm_initiate_cluster_request(scmrq);
			return;
		}
		scm_request_prepare(scmrq);
		atomic_inc(&bdev->queued_reqs);
		blk_start_request(req);

		ret = scm_start_aob(scmrq->aob);
		if (ret) {
			SCM_LOG(5, "no subchannel");
			scm_request_requeue(scmrq);
			return;
		}
	}
}

static void __scmrq_log_error(struct scm_request *scmrq)
{
	struct aob *aob = scmrq->aob;

	if (scmrq->error == -ETIMEDOUT)
		SCM_LOG(1, "Request timeout");
	else {
		SCM_LOG(1, "Request error");
		SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	}
	if (scmrq->retries)
		SCM_LOG(1, "Retry request");
	else
		pr_err("An I/O operation to SCM failed with rc=%d\n",
		       scmrq->error);
}

void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
{
	struct scm_request *scmrq = data;
	struct scm_blk_dev *bdev = scmrq->bdev;

	scmrq->error = error;
	if (error)
		__scmrq_log_error(scmrq);

	spin_lock(&bdev->lock);
	list_add_tail(&scmrq->list, &bdev->finished_requests);
	spin_unlock(&bdev->lock);
	tasklet_hi_schedule(&bdev->tasklet);
}

static void scm_blk_handle_error(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	unsigned long flags;

	if (scmrq->error != -EIO)
		goto restart;

	/* For -EIO the response block is valid. */
	switch (scmrq->aob->response.eqc) {
	case EQC_WR_PROHIBIT:
		spin_lock_irqsave(&bdev->lock, flags);
		if (bdev->state != SCM_WR_PROHIBIT)
			pr_info("%lx: Write access to the SCM increment is suspended\n",
				(unsigned long) bdev->scmdev->address);
		bdev->state = SCM_WR_PROHIBIT;
		spin_unlock_irqrestore(&bdev->lock, flags);
		goto requeue;
	default:
		break;
	}

restart:
	if (!scm_start_aob(scmrq->aob))
		return;

requeue:
	spin_lock_irqsave(&bdev->rq_lock, flags);
	scm_request_requeue(scmrq);
	spin_unlock_irqrestore(&bdev->rq_lock, flags);
}

static void scm_blk_tasklet(struct scm_blk_dev *bdev)
{
	struct scm_request *scmrq;
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	while (!list_empty(&bdev->finished_requests)) {
		scmrq = list_first_entry(&bdev->finished_requests,
					 struct scm_request, list);
		list_del(&scmrq->list);
		spin_unlock_irqrestore(&bdev->lock, flags);

		if (scmrq->error && scmrq->retries-- > 0) {
			scm_blk_handle_error(scmrq);

			/* Request restarted or requeued, handle next. */
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		if (scm_test_cluster_request(scmrq)) {
			scm_cluster_request_irq(scmrq);
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		scm_request_finish(scmrq);
		spin_lock_irqsave(&bdev->lock, flags);
	}
	spin_unlock_irqrestore(&bdev->lock, flags);
	/* Look out for more requests. */
	blk_run_queue(bdev->rq);
}

int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
{
	struct request_queue *rq;
	int len, ret = -ENOMEM;
	unsigned int devindex, nr_max_blk;

	devindex = atomic_inc_return(&nr_devices) - 1;
	/* scma..scmz + scmaa..scmzz */
	if (devindex > 701) {
		ret = -ENODEV;
		goto out;
	}

	bdev->scmdev = scmdev;
	bdev->state = SCM_OPER;
	spin_lock_init(&bdev->rq_lock);
	spin_lock_init(&bdev->lock);
	INIT_LIST_HEAD(&bdev->finished_requests);
	atomic_set(&bdev->queued_reqs, 0);
	tasklet_init(&bdev->tasklet,
		     (void (*)(unsigned long)) scm_blk_tasklet,
		     (unsigned long) bdev);

	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	if (!rq)
		goto out;

	bdev->rq = rq;
	nr_max_blk = min(scmdev->nr_max_block,
			 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));

	blk_queue_logical_block_size(rq, 1 << 12);
	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	blk_queue_max_segments(rq, nr_max_blk);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
	scm_blk_dev_cluster_setup(bdev);

	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	if (!bdev->gendisk)
		goto out_queue;

	rq->queuedata = scmdev;
	bdev->gendisk->driverfs_dev = &scmdev->dev;
	bdev->gendisk->private_data = scmdev;
	bdev->gendisk->fops = &scm_blk_devops;
	bdev->gendisk->queue = rq;
	bdev->gendisk->major = scm_major;
	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;

	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	if (devindex > 25) {
		len += snprintf(bdev->gendisk->disk_name + len,
				DISK_NAME_LEN - len, "%c",
				'a' + (devindex / 26) - 1);
		devindex = devindex % 26;
	}
	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
		 'a' + devindex);

	/* 512 byte sectors */
	set_capacity(bdev->gendisk, scmdev->size >> 9);
	add_disk(bdev->gendisk);
	return 0;

out_queue:
	blk_cleanup_queue(rq);
out:
	atomic_dec(&nr_devices);
	return ret;
}

void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
{
	tasklet_kill(&bdev->tasklet);
	del_gendisk(bdev->gendisk);
	blk_cleanup_queue(bdev->gendisk->queue);
	put_disk(bdev->gendisk);
}

void scm_blk_set_available(struct scm_blk_dev *bdev)
{
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	if (bdev->state == SCM_WR_PROHIBIT)
		pr_info("%lx: Write access to the SCM increment is restored\n",
			(unsigned long) bdev->scmdev->address);
	bdev->state = SCM_OPER;
	spin_unlock_irqrestore(&bdev->lock, flags);
}

static int __init scm_blk_init(void)
{
	int ret = -EINVAL;

	if (!scm_cluster_size_valid())
		goto out;

	ret = register_blkdev(0, "scm");
	if (ret < 0)
		goto out;

	scm_major = ret;
	ret = scm_alloc_rqs(nr_requests);
	if (ret)
		goto out_unreg;

	scm_debug = debug_register("scm_log", 16, 1, 16);
	if (!scm_debug) {
		ret = -ENOMEM;
		goto out_free;
	}

	debug_register_view(scm_debug, &debug_hex_ascii_view);
	debug_set_level(scm_debug, 2);

	ret = scm_drv_init();
	if (ret)
		goto out_dbf;

	return ret;

out_dbf:
	debug_unregister(scm_debug);
out_free:
	scm_free_rqs();
out_unreg:
	unregister_blkdev(scm_major, "scm");
out:
	return ret;
}
module_init(scm_blk_init);

static void __exit scm_blk_cleanup(void)
{
	scm_drv_cleanup();
	debug_unregister(scm_debug);
	scm_free_rqs();
	unregister_blkdev(scm_major, "scm");
}
module_exit(scm_blk_cleanup);
Commit	Line	Data
f30664e2 SO	1	/*
	2	* Block driver for s390 storage class memory.
	3	*
	4	* Copyright IBM Corp. 2012
	5	* Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
	6	*/
	7
	8	#define KMSG_COMPONENT "scm_block"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
	11	#include <linux/interrupt.h>
	12	#include <linux/spinlock.h>
	13	#include <linux/module.h>
	14	#include <linux/blkdev.h>
	15	#include <linux/genhd.h>
	16	#include <linux/slab.h>
	17	#include <linux/list.h>
	18	#include <asm/eadm.h>
	19	#include "scm_blk.h"
	20
	21	debug_info_t *scm_debug;
	22	static int scm_major;
	23	static DEFINE_SPINLOCK(list_lock);
	24	static LIST_HEAD(inactive_requests);
	25	static unsigned int nr_requests = 64;
	26	static atomic_t nr_devices = ATOMIC_INIT(0);
	27	module_param(nr_requests, uint, S_IRUGO);
	28	MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
	29
	30	MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
	31	MODULE_LICENSE("GPL");
	32	MODULE_ALIAS("scm:scmdev*");
	33
	34	static void __scm_free_rq(struct scm_request *scmrq)
	35	{
	36	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	37
	38	free_page((unsigned long) scmrq->aob);
	39	free_page((unsigned long) scmrq->aidaw);
0d804b20	40	__scm_free_rq_cluster(scmrq);
f30664e2 SO	41	kfree(aobrq);
	42	}
	43
	44	static void scm_free_rqs(void)
	45	{
	46	struct list_head iter, safe;
	47	struct scm_request *scmrq;
	48
	49	spin_lock_irq(&list_lock);
	50	list_for_each_safe(iter, safe, &inactive_requests) {
	51	scmrq = list_entry(iter, struct scm_request, list);
	52	list_del(&scmrq->list);
	53	__scm_free_rq(scmrq);
	54	}
	55	spin_unlock_irq(&list_lock);
	56	}
	57
	58	static int __scm_alloc_rq(void)
	59	{
	60	struct aob_rq_header *aobrq;
	61	struct scm_request *scmrq;
	62
	63	aobrq = kzalloc(sizeof(aobrq) + sizeof(scmrq), GFP_KERNEL);
	64	if (!aobrq)
	65	return -ENOMEM;
	66
	67	scmrq = (void *) aobrq->data;
	68	scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
	69	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
	70	if (!scmrq->aob \|\| !scmrq->aidaw) {
	71	__scm_free_rq(scmrq);
	72	return -ENOMEM;
	73	}
0d804b20 SO	74
	75	if (__scm_alloc_rq_cluster(scmrq)) {
	76	__scm_free_rq(scmrq);
	77	return -ENOMEM;
	78	}
	79
f30664e2 SO	80	INIT_LIST_HEAD(&scmrq->list);
	81	spin_lock_irq(&list_lock);
	82	list_add(&scmrq->list, &inactive_requests);
	83	spin_unlock_irq(&list_lock);
	84
	85	return 0;
	86	}
	87
	88	static int scm_alloc_rqs(unsigned int nrqs)
	89	{
	90	int ret = 0;
	91
	92	while (nrqs-- && !ret)
	93	ret = __scm_alloc_rq();
	94
	95	return ret;
	96	}
	97
	98	static struct scm_request *scm_request_fetch(void)
	99	{
	100	struct scm_request *scmrq = NULL;
	101
	102	spin_lock(&list_lock);
	103	if (list_empty(&inactive_requests))
	104	goto out;
	105	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	106	list_del(&scmrq->list);
	107	out:
	108	spin_unlock(&list_lock);
	109	return scmrq;
	110	}
	111
	112	static void scm_request_done(struct scm_request *scmrq)
	113	{
	114	unsigned long flags;
	115
	116	spin_lock_irqsave(&list_lock, flags);
	117	list_add(&scmrq->list, &inactive_requests);
	118	spin_unlock_irqrestore(&list_lock, flags);
	119	}
	120
	121	static int scm_open(struct block_device *blkdev, fmode_t mode)
	122	{
	123	return scm_get_ref();
	124	}
	125
	126	static int scm_release(struct gendisk *gendisk, fmode_t mode)
	127	{
	128	scm_put_ref();
	129	return 0;
	130	}
	131
	132	static const struct block_device_operations scm_blk_devops = {
	133	.owner = THIS_MODULE,
	134	.open = scm_open,
	135	.release = scm_release,
	136	};
	137
4fa3c019 SO	138	static bool scm_permit_request(struct scm_blk_dev bdev, struct request req)
	139	{
	140	return rq_data_dir(req) != WRITE \|\| bdev->state != SCM_WR_PROHIBIT;
	141	}
	142
f30664e2 SO	143	static void scm_request_prepare(struct scm_request *scmrq)
	144	{
	145	struct scm_blk_dev *bdev = scmrq->bdev;
	146	struct scm_device *scmdev = bdev->gendisk->private_data;
	147	struct aidaw *aidaw = scmrq->aidaw;
	148	struct msb *msb = &scmrq->aob->msb[0];
	149	struct req_iterator iter;
	150	struct bio_vec *bv;
	151
	152	msb->bs = MSB_BS_4K;
	153	scmrq->aob->request.msb_count = 1;
	154	msb->scm_addr = scmdev->address +
	155	((u64) blk_rq_pos(scmrq->request) << 9);
	156	msb->oc = (rq_data_dir(scmrq->request) == READ) ?
	157	MSB_OC_READ : MSB_OC_WRITE;
	158	msb->flags \|= MSB_FLAG_IDA;
	159	msb->data_addr = (u64) aidaw;
	160
	161	rq_for_each_segment(bv, scmrq->request, iter) {
	162	WARN_ON(bv->bv_offset);
	163	msb->blk_count += bv->bv_len >> 12;
	164	aidaw->data_addr = (u64) page_address(bv->bv_page);
	165	aidaw++;
	166	}
	167	}
	168
	169	static inline void scm_request_init(struct scm_blk_dev *bdev,
	170	struct scm_request *scmrq,
	171	struct request *req)
	172	{
	173	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	174	struct aob *aob = scmrq->aob;
	175
	176	memset(aob, 0, sizeof(*aob));
	177	memset(scmrq->aidaw, 0, PAGE_SIZE);
	178	aobrq->scmdev = bdev->scmdev;
	179	aob->request.cmd_code = ARQB_CMD_MOVE;
	180	aob->request.data = (u64) aobrq;
	181	scmrq->request = req;
	182	scmrq->bdev = bdev;
	183	scmrq->retries = 4;
	184	scmrq->error = 0;
0d804b20	185	scm_request_cluster_init(scmrq);
f30664e2 SO	186	}
	187
	188	static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
	189	{
	190	if (atomic_read(&bdev->queued_reqs)) {
	191	/* Queue restart is triggered by the next interrupt. */
	192	return;
	193	}
	194	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
	195	}
	196
0d804b20	197	void scm_request_requeue(struct scm_request *scmrq)
f30664e2 SO	198	{
	199	struct scm_blk_dev *bdev = scmrq->bdev;
	200
0d804b20	201	scm_release_cluster(scmrq);
f30664e2	202	blk_requeue_request(bdev->rq, scmrq->request);
8360cb5f	203	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	204	scm_request_done(scmrq);
	205	scm_ensure_queue_restart(bdev);
	206	}
	207
0d804b20	208	void scm_request_finish(struct scm_request *scmrq)
f30664e2	209	{
8360cb5f SO	210	struct scm_blk_dev *bdev = scmrq->bdev;
8360cb5f SO	211
0d804b20	212	scm_release_cluster(scmrq);
f30664e2	213	blk_end_request_all(scmrq->request, scmrq->error);
8360cb5f	214	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	215	scm_request_done(scmrq);
	216	}
	217
	218	static void scm_blk_request(struct request_queue *rq)
	219	{
	220	struct scm_device *scmdev = rq->queuedata;
	221	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
	222	struct scm_request *scmrq;
	223	struct request *req;
	224	int ret;
	225
	226	while ((req = blk_peek_request(rq))) {
	227	if (req->cmd_type != REQ_TYPE_FS)
	228	continue;
	229
4fa3c019 SO	230	if (!scm_permit_request(bdev, req)) {
	231	scm_ensure_queue_restart(bdev);
	232	return;
	233	}
f30664e2 SO	234	scmrq = scm_request_fetch();
	235	if (!scmrq) {
	236	SCM_LOG(5, "no request");
	237	scm_ensure_queue_restart(bdev);
	238	return;
	239	}
	240	scm_request_init(bdev, scmrq, req);
0d804b20 SO	241	if (!scm_reserve_cluster(scmrq)) {
	242	SCM_LOG(5, "cluster busy");
	243	scm_request_done(scmrq);
	244	return;
	245	}
	246	if (scm_need_cluster_request(scmrq)) {
8360cb5f	247	atomic_inc(&bdev->queued_reqs);
0d804b20 SO	248	blk_start_request(req);
	249	scm_initiate_cluster_request(scmrq);
	250	return;
	251	}
f30664e2	252	scm_request_prepare(scmrq);
8360cb5f	253	atomic_inc(&bdev->queued_reqs);
f30664e2 SO	254	blk_start_request(req);
	255
	256	ret = scm_start_aob(scmrq->aob);
	257	if (ret) {
	258	SCM_LOG(5, "no subchannel");
	259	scm_request_requeue(scmrq);
	260	return;
	261	}
f30664e2 SO	262	}
	263	}
	264
	265	static void __scmrq_log_error(struct scm_request *scmrq)
	266	{
	267	struct aob *aob = scmrq->aob;
	268
	269	if (scmrq->error == -ETIMEDOUT)
	270	SCM_LOG(1, "Request timeout");
	271	else {
	272	SCM_LOG(1, "Request error");
	273	SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	274	}
	275	if (scmrq->retries)
	276	SCM_LOG(1, "Retry request");
	277	else
	278	pr_err("An I/O operation to SCM failed with rc=%d\n",
	279	scmrq->error);
	280	}
	281
	282	void scm_blk_irq(struct scm_device scmdev, void data, int error)
	283	{
	284	struct scm_request *scmrq = data;
	285	struct scm_blk_dev *bdev = scmrq->bdev;
	286
	287	scmrq->error = error;
	288	if (error)
	289	__scmrq_log_error(scmrq);
	290
	291	spin_lock(&bdev->lock);
	292	list_add_tail(&scmrq->list, &bdev->finished_requests);
	293	spin_unlock(&bdev->lock);
	294	tasklet_hi_schedule(&bdev->tasklet);
	295	}
	296
4fa3c019 SO	297	static void scm_blk_handle_error(struct scm_request *scmrq)
	298	{
	299	struct scm_blk_dev *bdev = scmrq->bdev;
	300	unsigned long flags;
	301
	302	if (scmrq->error != -EIO)
	303	goto restart;
	304
	305	/* For -EIO the response block is valid. */
	306	switch (scmrq->aob->response.eqc) {
	307	case EQC_WR_PROHIBIT:
	308	spin_lock_irqsave(&bdev->lock, flags);
	309	if (bdev->state != SCM_WR_PROHIBIT)
3bff6038	310	pr_info("%lx: Write access to the SCM increment is suspended\n",
4fa3c019 SO	311	(unsigned long) bdev->scmdev->address);
	312	bdev->state = SCM_WR_PROHIBIT;
	313	spin_unlock_irqrestore(&bdev->lock, flags);
	314	goto requeue;
	315	default:
	316	break;
	317	}
	318
	319	restart:
	320	if (!scm_start_aob(scmrq->aob))
	321	return;
	322
	323	requeue:
	324	spin_lock_irqsave(&bdev->rq_lock, flags);
	325	scm_request_requeue(scmrq);
	326	spin_unlock_irqrestore(&bdev->rq_lock, flags);
	327	}
	328
f30664e2 SO	329	static void scm_blk_tasklet(struct scm_blk_dev *bdev)
	330	{
	331	struct scm_request *scmrq;
	332	unsigned long flags;
	333
	334	spin_lock_irqsave(&bdev->lock, flags);
	335	while (!list_empty(&bdev->finished_requests)) {
	336	scmrq = list_first_entry(&bdev->finished_requests,
	337	struct scm_request, list);
	338	list_del(&scmrq->list);
	339	spin_unlock_irqrestore(&bdev->lock, flags);
	340
	341	if (scmrq->error && scmrq->retries-- > 0) {
4fa3c019 SO	342	scm_blk_handle_error(scmrq);
4fa3c019 SO	343
f30664e2 SO	344	/* Request restarted or requeued, handle next. */
	345	spin_lock_irqsave(&bdev->lock, flags);
	346	continue;
	347	}
0d804b20 SO	348
	349	if (scm_test_cluster_request(scmrq)) {
	350	scm_cluster_request_irq(scmrq);
	351	spin_lock_irqsave(&bdev->lock, flags);
	352	continue;
	353	}
	354
f30664e2	355	scm_request_finish(scmrq);
f30664e2 SO	356	spin_lock_irqsave(&bdev->lock, flags);
	357	}
	358	spin_unlock_irqrestore(&bdev->lock, flags);
	359	/* Look out for more requests. */
	360	blk_run_queue(bdev->rq);
	361	}
	362
	363	int scm_blk_dev_setup(struct scm_blk_dev bdev, struct scm_device scmdev)
	364	{
	365	struct request_queue *rq;
	366	int len, ret = -ENOMEM;
	367	unsigned int devindex, nr_max_blk;
	368
	369	devindex = atomic_inc_return(&nr_devices) - 1;
	370	/* scma..scmz + scmaa..scmzz */
	371	if (devindex > 701) {
	372	ret = -ENODEV;
	373	goto out;
	374	}
	375
	376	bdev->scmdev = scmdev;
4fa3c019	377	bdev->state = SCM_OPER;
f30664e2 SO	378	spin_lock_init(&bdev->rq_lock);
	379	spin_lock_init(&bdev->lock);
	380	INIT_LIST_HEAD(&bdev->finished_requests);
	381	atomic_set(&bdev->queued_reqs, 0);
	382	tasklet_init(&bdev->tasklet,
	383	(void (*)(unsigned long)) scm_blk_tasklet,
	384	(unsigned long) bdev);
	385
	386	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	387	if (!rq)
	388	goto out;
	389
	390	bdev->rq = rq;
	391	nr_max_blk = min(scmdev->nr_max_block,
	392	(unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
	393
	394	blk_queue_logical_block_size(rq, 1 << 12);
	395	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	396	blk_queue_max_segments(rq, nr_max_blk);
	397	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
0d804b20	398	scm_blk_dev_cluster_setup(bdev);
f30664e2 SO	399
	400	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	401	if (!bdev->gendisk)
	402	goto out_queue;
	403
	404	rq->queuedata = scmdev;
	405	bdev->gendisk->driverfs_dev = &scmdev->dev;
	406	bdev->gendisk->private_data = scmdev;
	407	bdev->gendisk->fops = &scm_blk_devops;
	408	bdev->gendisk->queue = rq;
	409	bdev->gendisk->major = scm_major;
	410	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
	411
	412	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	413	if (devindex > 25) {
	414	len += snprintf(bdev->gendisk->disk_name + len,
	415	DISK_NAME_LEN - len, "%c",
	416	'a' + (devindex / 26) - 1);
	417	devindex = devindex % 26;
	418	}
	419	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
	420	'a' + devindex);
	421
	422	/* 512 byte sectors */
	423	set_capacity(bdev->gendisk, scmdev->size >> 9);
	424	add_disk(bdev->gendisk);
	425	return 0;
	426
	427	out_queue:
	428	blk_cleanup_queue(rq);
	429	out:
	430	atomic_dec(&nr_devices);
	431	return ret;
	432	}
	433
	434	void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
	435	{
	436	tasklet_kill(&bdev->tasklet);
	437	del_gendisk(bdev->gendisk);
	438	blk_cleanup_queue(bdev->gendisk->queue);
	439	put_disk(bdev->gendisk);
	440	}
	441
4fa3c019 SO	442	void scm_blk_set_available(struct scm_blk_dev *bdev)
	443	{
	444	unsigned long flags;
	445
	446	spin_lock_irqsave(&bdev->lock, flags);
	447	if (bdev->state == SCM_WR_PROHIBIT)
3bff6038	448	pr_info("%lx: Write access to the SCM increment is restored\n",
4fa3c019 SO	449	(unsigned long) bdev->scmdev->address);
	450	bdev->state = SCM_OPER;
	451	spin_unlock_irqrestore(&bdev->lock, flags);
	452	}
	453
f30664e2 SO	454	static int __init scm_blk_init(void)
f30664e2 SO	455	{
0d804b20 SO	456	int ret = -EINVAL;
	457
	458	if (!scm_cluster_size_valid())
	459	goto out;
f30664e2 SO	460
	461	ret = register_blkdev(0, "scm");
	462	if (ret < 0)
	463	goto out;
	464
	465	scm_major = ret;
94f9852d WY	466	ret = scm_alloc_rqs(nr_requests);
94f9852d WY	467	if (ret)
f30664e2 SO	468	goto out_unreg;
	469
	470	scm_debug = debug_register("scm_log", 16, 1, 16);
94f9852d WY	471	if (!scm_debug) {
94f9852d WY	472	ret = -ENOMEM;
f30664e2	473	goto out_free;
94f9852d	474	}
f30664e2 SO	475
	476	debug_register_view(scm_debug, &debug_hex_ascii_view);
	477	debug_set_level(scm_debug, 2);
	478
	479	ret = scm_drv_init();
	480	if (ret)
	481	goto out_dbf;
	482
	483	return ret;
	484
	485	out_dbf:
	486	debug_unregister(scm_debug);
	487	out_free:
	488	scm_free_rqs();
	489	out_unreg:
	490	unregister_blkdev(scm_major, "scm");
	491	out:
	492	return ret;
	493	}
	494	module_init(scm_blk_init);
	495
	496	static void __exit scm_blk_cleanup(void)
	497	{
	498	scm_drv_cleanup();
	499	debug_unregister(scm_debug);
	500	scm_free_rqs();
	501	unregister_blkdev(scm_major, "scm");
	502	}
	503	module_exit(scm_blk_cleanup);