Commit f167fac3 authored by Jim Harris's avatar Jim Harris
Browse files

iscsi: start all sessions for a target node on the same lcore 



This ensures against races, when an existing session to a target node
stalls, causing the initiator to create a new session.  These new
session's connection may get migrated to a different core than the
core of the stalled session.

In practice, this does not happen, but is a common occurrence when
debugging the iSCSI target using gdb.

Signed-off-by: default avatarJim Harris <james.r.harris@intel.com>
Change-Id: I1864c2ca0c330dc4faeeb1312adac7a02c8281dc
parent 2029c9fa

lib/iscsi/conn.c

+44 −10
Original line number Diff line number Diff line
@@ -92,7 +92,7 @@ void spdk_iscsi_conn_idle_do_work(void *arg); 


static void spdk_iscsi_conn_full_feature_migrate(struct spdk_event *event);

static struct spdk_event *spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn,

		int lcore);

		int *lcore);

static void spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after_stop,

					int lcore);
@@ -630,9 +630,38 @@ spdk_iscsi_conn_check_shutdown(struct rte_timer *timer, void *arg) 
}



static struct spdk_event *

spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn, int lcore)

spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn, int *_lcore)

{

	struct spdk_event *event;

	struct spdk_iscsi_tgt_node *target;

	int lcore;



	lcore = spdk_iscsi_conn_allocate_reactor(conn->portal->cpumask);

	if (conn->sess->session_type == SESSION_TYPE_NORMAL) {

		target = conn->sess->target;

		pthread_mutex_lock(&target->mutex);

		target->num_active_conns++;

		if (target->num_active_conns == 1) {

			/**

			 * This is the only active connection for this target node.

			 *  Save the lcore in the target node so it can be used for

			 *  any other connections to this target node.

			 */

			target->lcore = lcore;

		} else {

			/**

			 * There are other active connections for this target node.

			 *  Ignore the lcore specified by the allocator and use the

			 *  the target node's lcore to ensure this connection runs on

			 *  the same lcore as other connections for this target node.

			 */

			lcore = target->lcore;

		}

		pthread_mutex_unlock(&target->mutex);

	}

	if (_lcore != NULL) {

		*_lcore = lcore;

	}



	event = spdk_event_allocate(lcore, spdk_iscsi_conn_full_feature_migrate, conn, NULL, NULL);
@@ -647,7 +676,14 @@ static void 
spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after_stop, int lcore)

{

	struct spdk_event *event;

	struct spdk_iscsi_tgt_node *target;



	if (conn->sess->session_type == SESSION_TYPE_NORMAL) {

		target = conn->sess->target;

		pthread_mutex_lock(&target->mutex);

		target->num_active_conns--;

		pthread_mutex_unlock(&target->mutex);

	}

	rte_atomic32_dec(&g_num_connections[spdk_app_get_current_core()]);

	spdk_net_framework_clear_socket_association(conn->sock);

	event = spdk_event_allocate(lcore, fn_after_stop, conn, NULL, NULL);
@@ -664,7 +700,7 @@ void spdk_shutdown_iscsi_conns(void) 
	 */

	STAILQ_FOREACH_SAFE(conn, &g_idle_conn_list_head, link, tmp) {

		STAILQ_REMOVE(&g_idle_conn_list_head, conn, spdk_iscsi_conn, link);

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, rte_get_master_lcore()));

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));

		conn->is_idle = 0;

		del_idle_conn(conn);

	}
@@ -1255,8 +1291,7 @@ spdk_iscsi_conn_login_do_work(void *arg) 
	 * did, migrate it to a dedicated reactor for the target node.

	 */

	if (conn->login_phase == ISCSI_FULL_FEATURE_PHASE) {

		lcore = spdk_iscsi_conn_allocate_reactor(conn->portal->cpumask);

		event = spdk_iscsi_conn_get_migrate_event(conn, lcore);

		event = spdk_iscsi_conn_get_migrate_event(conn, &lcore);

		rte_atomic32_dec(&g_num_connections[spdk_app_get_current_core()]);

		rte_atomic32_inc(&g_num_connections[lcore]);

		spdk_net_framework_clear_socket_association(conn->sock);
@@ -1321,7 +1356,7 @@ void spdk_iscsi_conn_idle_do_work(void *arg) 
		}



		if (tconn->pending_activate_event) {

			uint32_t lcore;

			int lcore;



			spdk_trace_record(TRACE_ISCSI_CONN_ACTIVE, tconn->id, 0, 0, 0);
@@ -1332,10 +1367,9 @@ void spdk_iscsi_conn_idle_do_work(void *arg) 
			tconn->is_idle = 0;

			del_idle_conn(tconn);

			/* migrate work item to new core */

			lcore = spdk_iscsi_conn_allocate_reactor(tconn->portal->cpumask);

			rte_atomic32_inc(&g_num_connections[lcore]);

			spdk_net_framework_clear_socket_association(tconn->sock);

			spdk_event_call(spdk_iscsi_conn_get_migrate_event(tconn, lcore));

			spdk_event_call(spdk_iscsi_conn_get_migrate_event(tconn, &lcore));

			rte_atomic32_inc(&g_num_connections[lcore]);

			SPDK_TRACELOG(SPDK_TRACE_DEBUG, "add conn id = %d, cid = %d poller = %p to lcore = %d active\n",

				      tconn->id, tconn->cid, &tconn->poller, lcore);

		}
@@ -1355,7 +1389,7 @@ __add_idle_conn(spdk_event_t e) 
	 *  process.

	 */

	if (conn->state == ISCSI_CONN_STATE_EXITING) {

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, rte_get_master_lcore()));

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));

		return;

	}

lib/iscsi/tgt_node.h

+6 −0
Original line number Diff line number Diff line
@@ -67,6 +67,12 @@ struct spdk_iscsi_tgt_node { 
	int queue_depth;



	struct spdk_scsi_dev *dev;

	/**

	 * Counts number of active iSCSI connections associated with this

	 *  target node.

	 */

	uint32_t num_active_conns;

	int lcore;



	int maxmap;

	struct spdk_iscsi_tgt_node_map map[MAX_TARGET_MAP];