Commit cefab5d6 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Ben Walker
Browse files

iscsi: Support hot removal of LUN based on LUN open/close 



When hot removal of a LUN is started, callback is called for each
iSCSI connection which accesses the LUN. Callback checks all transfer
tasks complete and then close the LUN.

If the connection clears all transfer tasks before getting all responses
to them from the initiator, the initiator continues to retry data write.
Hence the connection have to wait until all transfer tasks complete.

Change-Id: Iad9063673cfedbd78758890d55a4254512e4fca4
Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-on: https://review.gerrithub.io/417199


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent fec7c88e

include/spdk/scsi.h

+9 −0
Original line number Diff line number Diff line
@@ -190,6 +190,15 @@ const char *spdk_scsi_lun_get_bdev_name(const struct spdk_scsi_lun *lun); 
 */

const struct spdk_scsi_dev *spdk_scsi_lun_get_dev(const struct spdk_scsi_lun *lun);



/**

 * Check if the logical unit is hot removing.

 *

 * \param lun Logical unit

 *

 * \return true if removing, false otherwise.

 */

bool spdk_scsi_lun_is_removing(const struct spdk_scsi_lun *lun);



/**

 * Get the name of the given SCSI device.

 *

lib/iscsi/conn.c

+72 −7
Original line number Diff line number Diff line
@@ -50,7 +50,6 @@ 
#include "iscsi/conn.h"

#include "iscsi/tgt_node.h"

#include "iscsi/portal_grp.h"

#include "spdk/scsi.h"



#define SPDK_ISCSI_CONNECTION_MEMSET(conn)		\

	memset(&(conn)->portal, 0, sizeof(*(conn)) -	\
@@ -280,6 +279,7 @@ spdk_iscsi_conn_construct(struct spdk_iscsi_portal *portal, 
	TAILQ_INIT(&conn->queued_r2t_tasks);

	TAILQ_INIT(&conn->active_r2t_tasks);

	TAILQ_INIT(&conn->queued_datain_tasks);

	memset(&conn->open_lun_descs, 0, sizeof(conn->open_lun_descs));



	rc = spdk_sock_getaddr(sock, conn->target_addr,

			       sizeof conn->target_addr,
@@ -583,14 +583,81 @@ spdk_iscsi_conn_check_shutdown(void *arg) 


	if (spdk_iscsi_get_active_conns() == 0) {

		spdk_poller_unregister(&g_shutdown_timer);

		event = spdk_event_allocate(spdk_env_get_current_core(), spdk_iscsi_conn_check_shutdown_cb, NULL,

					    NULL);

		event = spdk_event_allocate(spdk_env_get_current_core(),

					    spdk_iscsi_conn_check_shutdown_cb, NULL, NULL);

		spdk_event_call(event);

	}



	return -1;

}



static void

spdk_iscsi_conn_close_lun(struct spdk_iscsi_conn *conn, int lun_id)

{

	struct spdk_scsi_desc *desc;



	desc = conn->open_lun_descs[lun_id];

	if (desc != NULL) {

		spdk_scsi_lun_free_io_channel(desc);

		spdk_scsi_lun_close(desc);

		conn->open_lun_descs[lun_id] = NULL;

	}

}



static void

spdk_iscsi_conn_close_luns(struct spdk_iscsi_conn *conn)

{

	int i;



	for (i = 0; i < SPDK_SCSI_DEV_MAX_LUN; i++) {

		spdk_iscsi_conn_close_lun(conn, i);

	}

}



static void

spdk_iscsi_conn_remove_lun(struct spdk_scsi_lun *lun, void *remove_ctx)

{

	struct spdk_iscsi_conn *conn = remove_ctx;

	int lun_id = spdk_scsi_lun_get_id(lun);



	spdk_clear_all_transfer_task(conn, lun);



	spdk_iscsi_conn_close_lun(conn, lun_id);

}



static void

spdk_iscsi_conn_open_luns(struct spdk_iscsi_conn *conn)

{

	int i, rc;

	struct spdk_scsi_lun *lun;

	struct spdk_scsi_desc *desc;



	for (i = 0; i < SPDK_SCSI_DEV_MAX_LUN; i++) {

		lun = spdk_scsi_dev_get_lun(conn->dev, i);

		if (lun == NULL) {

			continue;

		}



		rc = spdk_scsi_lun_open(lun, spdk_iscsi_conn_remove_lun, conn, &desc);

		if (rc != 0) {

			goto error;

		}



		rc = spdk_scsi_lun_allocate_io_channel(desc);

		if (rc != 0) {

			spdk_scsi_lun_close(desc);

			goto error;

		}



		conn->open_lun_descs[i] = desc;

	}



	return;



error:

	spdk_iscsi_conn_close_luns(conn);

}



/**

 *  This function will stop executing the specified connection.

 */
@@ -607,8 +674,7 @@ spdk_iscsi_conn_stop(struct spdk_iscsi_conn *conn) 
		target->num_active_conns--;

		pthread_mutex_unlock(&target->mutex);



		assert(conn->dev != NULL);

		spdk_scsi_dev_free_io_channels(conn->dev);

		spdk_iscsi_conn_close_luns(conn);

	}



	__sync_fetch_and_sub(&g_num_connections[spdk_env_get_current_core()], 1);
@@ -1171,8 +1237,7 @@ spdk_iscsi_conn_full_feature_migrate(void *arg1, void *arg2) 
	struct spdk_iscsi_conn *conn = arg1;



	if (conn->sess->session_type == SESSION_TYPE_NORMAL) {

		assert(conn->dev != NULL);

		spdk_scsi_dev_allocate_io_channels(conn->dev);

		spdk_iscsi_conn_open_luns(conn);

	}



	/* The poller has been unregistered, so now we can re-register it on the new core. */

lib/iscsi/conn.h

+3 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 
#include "iscsi/iscsi.h"

#include "spdk/queue.h"

#include "spdk/cpuset.h"

#include "spdk/scsi.h"



/*

 * MAX_CONNECTION_PARAMS: The numbers of the params in conn_param_table
@@ -164,6 +165,8 @@ struct spdk_iscsi_conn { 
	TAILQ_HEAD(queued_r2t_tasks, spdk_iscsi_task)	queued_r2t_tasks;

	TAILQ_HEAD(active_r2t_tasks, spdk_iscsi_task)	active_r2t_tasks;

	TAILQ_HEAD(queued_datain_tasks, spdk_iscsi_task)	queued_datain_tasks;



	struct spdk_scsi_desc	*open_lun_descs[SPDK_SCSI_DEV_MAX_LUN];

};



extern struct spdk_iscsi_conn *g_conns_array;

lib/iscsi/iscsi.c

+8 −3
Original line number Diff line number Diff line
@@ -3558,7 +3558,8 @@ void spdk_del_transfer_task(struct spdk_iscsi_conn *conn, uint32_t task_tag) 
}



static void

spdk_del_connection_queued_task(void *tailq, struct spdk_scsi_lun *lun)

spdk_del_connection_queued_task(struct spdk_iscsi_conn *conn, void *tailq,

				struct spdk_scsi_lun *lun)

{

	struct spdk_iscsi_task *task, *task_tmp;

	/*
@@ -3571,6 +3572,10 @@ spdk_del_connection_queued_task(void *tailq, struct spdk_scsi_lun *lun) 
	TAILQ_FOREACH_SAFE(task, head, link, task_tmp) {

		if (lun == NULL || lun == task->scsi.lun) {

			TAILQ_REMOVE(head, task, link);

			if (lun != NULL && spdk_scsi_lun_is_removing(lun)) {

				spdk_scsi_task_process_null_lun(&task->scsi);

				spdk_iscsi_task_response(conn, task);

			}

			spdk_iscsi_task_put(task);

		}

	}
@@ -3608,8 +3613,8 @@ void spdk_clear_all_transfer_task(struct spdk_iscsi_conn *conn, 
		}

	}



	spdk_del_connection_queued_task(&conn->active_r2t_tasks, lun);

	spdk_del_connection_queued_task(&conn->queued_r2t_tasks, lun);

	spdk_del_connection_queued_task(conn, &conn->active_r2t_tasks, lun);

	spdk_del_connection_queued_task(conn, &conn->queued_r2t_tasks, lun);



	spdk_start_queued_transfer_tasks(conn);

}

lib/scsi/lun.c

+6 −0
Original line number Diff line number Diff line
@@ -444,3 +444,9 @@ spdk_scsi_lun_has_pending_tasks(const struct spdk_scsi_lun *lun) 
{

	return !TAILQ_EMPTY(&lun->tasks);

}



bool

spdk_scsi_lun_is_removing(const struct spdk_scsi_lun *lun)

{

	return lun->removed;

}