Commit 6b7e9d0a authored by GangCao's avatar GangCao Committed by Tomasz Zawadzki
Browse files

Lib/iSCSI: add the LUN Resize support 



From SAM-4, section 5.13 (Sense Data);
“When a command terminates with a CHECK CONDITION status, sense data shall be returned
in the same I_T_L_Q nexus transaction (see 3.1.50) as the CHECK CONDITION status. After
the sense data is returned, it shall be cleared except when it is associated with a unit
attention condition and the UA_INTLCK_CTRL field in the Control mode page (see SPC-4)
contains 10b or 11b.”

SPDK does not set UA_INTLCK_CTRL to 10b or 11b, so we set the unit attention condition
immediately against a single IO or Admin IO after reporting it via a CHECK CONDITION.
Once the failed IO received at iSCSI initiator side, it will be retried. In the case of
resize operation, if there is no IO from iSCSI initiator side, the unit attention
condition will be delayed to report until the first IO is received at the iSCSI target
side.

Meanwhile, we clear the resizing (newly added) flag on our SCSI LUN structure after
first time we report the resize unit attention condition.

The kernel initiator won’t actually resize the corresponding block device automatically.
It will report a uevent, and then you can set up udev rules to trigger a rescan. SPDK
iSCSI initiator will automatically report the LUN size change.

Change-Id: Ifc85b8d4d3fbea13e76fb5d1faf1ac6c8f662e6c
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11086


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarXiaodong Liu <xiaodong.liu@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
parent 2da24e2c

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -58,6 +58,11 @@ Added 3 experimental APIs to handle PCI device interrupts (`spdk_pci_device_enab 
Added a 'subsystem' parameter to spdk_nvmf_transport_stop_listen_async. When not NULL,

it will only disconnect qpairs for controllers associated with the specified subsystem.



### scsi



Structure `spdk_scsi_lun` has been extended with new member `resizing` so that SCSI layer now reports

unit attention for disk resize.



## v21.10



Structure `spdk_nvmf_target_opts` has been extended with new member `discovery_filter` which allows to specify

include/spdk/scsi_spec.h

+2 −0
Original line number Diff line number Diff line
@@ -101,6 +101,7 @@ enum spdk_scsi_asc { 
	SPDK_SCSI_ASC_INVALID_FIELD_IN_CDB = 0x24,

	SPDK_SCSI_ASC_LOGICAL_UNIT_NOT_SUPPORTED = 0x25,

	SPDK_SCSI_ASC_WRITE_PROTECTED = 0x27,

	SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED = 0x2a,

	SPDK_SCSI_ASC_FORMAT_COMMAND_FAILED = 0x31,

	SPDK_SCSI_ASC_SAVING_PARAMETERS_NOT_SUPPORTED = 0x39,

	SPDK_SCSI_ASC_INTERNAL_TARGET_FAILURE = 0x44,
@@ -116,6 +117,7 @@ enum spdk_scsi_ascq { 
	SPDK_SCSI_ASCQ_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED = 0x03,

	SPDK_SCSI_ASCQ_POWER_LOSS_EXPECTED = 0x08,

	SPDK_SCSI_ASCQ_INVALID_LU_IDENTIFIER = 0x09,

	SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED = 0x09,

};



enum spdk_spc_opcode {

lib/scsi/lun.c

+30 −4
Original line number Diff line number Diff line
@@ -125,6 +125,23 @@ scsi_lun_append_mgmt_task(struct spdk_scsi_lun *lun, 
	TAILQ_INSERT_TAIL(&lun->pending_mgmt_tasks, task, scsi_link);

}



static bool

_scsi_lun_handle_unit_attention(struct spdk_scsi_task *task)

{

	uint8_t *cdb = task->cdb;



	assert(task->cdb);



	switch (cdb[0]) {

	case SPDK_SPC_INQUIRY:

	case SPDK_SPC_REPORT_LUNS:

	case SPDK_SPC_REQUEST_SENSE:

		return false;

	default:

		return true;

	}

}



static void

_scsi_lun_execute_mgmt_task(struct spdk_scsi_lun *lun)

{
@@ -195,7 +212,17 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) 
	task->status = SPDK_SCSI_STATUS_GOOD;

	spdk_trace_record(TRACE_SCSI_TASK_START, lun->dev->id, task->length, (uintptr_t)task);

	TAILQ_INSERT_TAIL(&lun->tasks, task, scsi_link);

	if (!lun->removed) {

	if (spdk_unlikely(lun->removed)) {

		spdk_scsi_task_process_abort(task);

		rc = SPDK_SCSI_TASK_COMPLETE;

	} else if (spdk_unlikely(lun->resizing) && _scsi_lun_handle_unit_attention(task)) {

		spdk_scsi_task_set_status(task, SPDK_SCSI_STATUS_CHECK_CONDITION,

					  SPDK_SCSI_SENSE_UNIT_ATTENTION,

					  SPDK_SCSI_ASC_CAPACITY_DATA_HAS_CHANGED,

					  SPDK_SCSI_ASCQ_CAPACITY_DATA_HAS_CHANGED);

		lun->resizing = false;

		rc = SPDK_SCSI_TASK_COMPLETE;

	} else {

		/* Check the command is allowed or not when reservation is exist */

		if (spdk_unlikely(lun->reservation.flags & SCSI_SPC2_RESERVE)) {

			rc = scsi2_reserve_check(task);
@@ -208,9 +235,6 @@ _scsi_lun_execute_task(struct spdk_scsi_lun *lun, struct spdk_scsi_task *task) 
		} else {

			rc = bdev_scsi_execute(task);

		}

	} else {

		spdk_scsi_task_process_abort(task);

		rc = SPDK_SCSI_TASK_COMPLETE;

	}



	switch (rc) {
@@ -405,6 +429,7 @@ bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, 
		break;

	case SPDK_BDEV_EVENT_RESIZE:

		SPDK_NOTICELOG("bdev name (%s) received event(SPDK_BDEV_EVENT_RESIZE)\n", spdk_bdev_get_name(bdev));

		lun->resizing = true;

		if (lun->resize_cb) {

			lun->resize_cb(lun, lun->resize_ctx);

		}
@@ -466,6 +491,7 @@ struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name, 


	lun->resize_cb = resize_cb;

	lun->resize_ctx = resize_ctx;

	lun->resizing = false;



	TAILQ_INIT(&lun->open_descs);

	TAILQ_INIT(&lun->reg_head);

lib/scsi/scsi_internal.h

+3 −0
Original line number Diff line number Diff line
@@ -178,6 +178,9 @@ struct spdk_scsi_lun { 


	/** A structure to connect LUNs in a list. */

	TAILQ_ENTRY(spdk_scsi_lun) tailq;



	/** The LUN is resizing */

	bool resizing;

};



struct spdk_scsi_lun *scsi_lun_construct(const char *bdev_name,

module/bdev/iscsi/bdev_iscsi.c

+109 −1
Original line number Diff line number Diff line
@@ -72,12 +72,16 @@ struct bdev_iscsi_lun; 
#define BDEV_ISCSI_MAX_UNMAP_BLOCK_DESCS_COUNT (1)



static int bdev_iscsi_initialize(void);

static void bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun);

static void _bdev_iscsi_submit_request(void *_bdev_io);



static TAILQ_HEAD(, bdev_iscsi_conn_req) g_iscsi_conn_req = TAILQ_HEAD_INITIALIZER(

			g_iscsi_conn_req);

static struct spdk_poller *g_conn_poller = NULL;



struct bdev_iscsi_io {

	struct spdk_thread *submit_td;

	struct bdev_iscsi_lun *lun;

	enum spdk_bdev_io_status status;

	int scsi_status;

	enum spdk_scsi_sense sk;
@@ -217,12 +221,26 @@ bdev_iscsi_io_complete(struct bdev_iscsi_io *iscsi_io, enum spdk_bdev_io_status 
	}

}



static bool

_bdev_iscsi_is_size_change(int status, struct scsi_task *task)

{

	if (status == SPDK_SCSI_STATUS_CHECK_CONDITION &&

	    (uint8_t)task->sense.key == SPDK_SCSI_SENSE_UNIT_ATTENTION &&

	    task->sense.ascq == 0x2a09) {

		/* ASCQ: SCSI_SENSE_ASCQ_CAPACITY_DATA_HAS_CHANGED (0x2a09) */

		return true;

	}



	return false;

}



/* Common call back function for read/write/flush command */

static void

bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, void *_iscsi_io)

{

	struct scsi_task *task = _task;

	struct bdev_iscsi_io *iscsi_io = _iscsi_io;

	struct spdk_bdev_io *bdev_io;



	iscsi_io->scsi_status = status;

	iscsi_io->sk = (uint8_t)task->sense.key;
@@ -230,8 +248,87 @@ bdev_iscsi_command_cb(struct iscsi_context *context, int status, void *_task, vo 
	iscsi_io->ascq = task->sense.ascq & 0xFF;



	scsi_free_scsi_task(task);



	if (_bdev_iscsi_is_size_change(status, task)) {

		bdev_iscsi_readcapacity16(context, iscsi_io->lun);



		/* Retry this failed IO immediately */

		bdev_io = spdk_bdev_io_from_ctx(iscsi_io);

		if (iscsi_io->submit_td != NULL) {

			spdk_thread_send_msg(iscsi_io->lun->main_td,

					     _bdev_iscsi_submit_request, bdev_io);

		} else {

			_bdev_iscsi_submit_request(bdev_io);

		}

	} else {

		bdev_iscsi_io_complete(iscsi_io, SPDK_BDEV_IO_STATUS_SUCCESS);

	}

}



static int

bdev_iscsi_resize(struct spdk_bdev *bdev, const uint64_t new_size_in_block)

{

	int rc;



	assert(bdev->module == &g_iscsi_bdev_module);



	if (new_size_in_block <= bdev->blockcnt) {

		SPDK_ERRLOG("The new bdev size must be larger than current bdev size.\n");

		return -EINVAL;

	}



	rc = spdk_bdev_notify_blockcnt_change(bdev, new_size_in_block);

	if (rc != 0) {

		SPDK_ERRLOG("failed to notify block cnt change.\n");

		return rc;

	}



	return 0;

}



static void

bdev_iscsi_readcapacity16_cb(struct iscsi_context *context, int status, void *_task,

			     void *private_data)

{

	struct bdev_iscsi_lun *lun = private_data;

	struct scsi_readcapacity16 *readcap16;

	struct scsi_task *task = _task;

	uint64_t size_in_block = 0;

	int rc;



	if (status != SPDK_SCSI_STATUS_GOOD) {

		SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(context));

		goto ret;

	}



	readcap16 = scsi_datain_unmarshall(task);

	if (!readcap16) {

		SPDK_ERRLOG("Read capacity error\n");

		goto ret;

	}



	size_in_block = readcap16->returned_lba + 1;



	rc = bdev_iscsi_resize(&lun->bdev, size_in_block);

	if (rc != 0) {

		SPDK_ERRLOG("Bdev (%s) resize error: %d\n", lun->bdev.name, rc);

	}



ret:

	scsi_free_scsi_task(task);

}



static void

bdev_iscsi_readcapacity16(struct iscsi_context *context, struct bdev_iscsi_lun *lun)

{

	struct scsi_task *task;



	task = iscsi_readcapacity16_task(context, lun->lun_id,

					 bdev_iscsi_readcapacity16_cb, lun);

	if (task == NULL) {

		SPDK_ERRLOG("failed to get readcapacity16_task\n");

	}

}



static void

bdev_iscsi_readv(struct bdev_iscsi_lun *lun, struct bdev_iscsi_io *iscsi_io,
@@ -510,6 +607,8 @@ static void bdev_iscsi_submit_request(struct spdk_io_channel *_ch, struct spdk_b 
	struct bdev_iscsi_io *iscsi_io = (struct bdev_iscsi_io *)bdev_io->driver_ctx;

	struct bdev_iscsi_lun *lun = (struct bdev_iscsi_lun *)bdev_io->bdev->ctxt;



	iscsi_io->lun = lun;



	if (lun->main_td != submit_td) {

		iscsi_io->submit_td = submit_td;

		spdk_thread_send_msg(lun->main_td, _bdev_iscsi_submit_request, bdev_io);
@@ -722,9 +821,18 @@ iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status, 
	struct scsi_readcapacity16 *readcap16;

	struct spdk_bdev *bdev = NULL;

	struct scsi_task *task = command_data;

	struct scsi_task *retry_task = NULL;



	if (status != SPDK_SCSI_STATUS_GOOD) {

		SPDK_ERRLOG("iSCSI error: %s\n", iscsi_get_error(iscsi));

		if (_bdev_iscsi_is_size_change(status, task)) {

			scsi_free_scsi_task(task);

			retry_task = iscsi_readcapacity16_task(iscsi, req->lun,

							       iscsi_readcapacity16_cb, req);

			if (retry_task) {

				return;

			}

		}

		goto ret;

	}