Commit e99c5e99 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Konrad Sztyber
Browse files

iscsi: Schedule active target to idlest poll group 



We used round robin to schedule target connections. However, this
had problems so that targets are not balanced equally among poll groups
when target connection up/down was repeated.

Fix the problem by scheduling target connections to the idlest poll
group.

target->num_active_targets is incremented only if target has the first
active connection. If connection is deleted before becoming active,
target->num_active_conns is not incremented. Add a variable scheduled to
iscsi_conn structure to cover such case. Get the global mutex to decrement
pg->num_active_targets and keep the same ordering, i.e. get first and free
last global mutex, as iscsi_conn_schedule().

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Signed-off-by: default avatarChangqi Lu <luchangqi.123@bytedance.com>
Signed-off-by: default avatarzhenwei pi <pizhenwei@bytedance.com>
Suggested-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: I0671583424d9f08e93839718c543ee3325f179b4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24611


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 148b4bf9

lib/iscsi/conn.c

+32 −9
Original line number Diff line number Diff line
@@ -581,9 +581,18 @@ iscsi_conn_stop(struct spdk_iscsi_conn *conn) 
	    conn->sess->session_type == SESSION_TYPE_NORMAL &&

	    conn->full_feature) {

		target = conn->sess->target;



		pthread_mutex_lock(&g_iscsi.mutex);

		pthread_mutex_lock(&target->mutex);

		if (conn->scheduled != 0) {

			target->num_active_conns--;

			if (target->num_active_conns == 0) {

				assert(target->pg != NULL);

				target->pg->num_active_targets--;

			}

		}

		pthread_mutex_unlock(&target->mutex);

		pthread_mutex_unlock(&g_iscsi.mutex);



		iscsi_conn_close_luns(conn);

	}
@@ -1520,7 +1529,23 @@ iscsi_conn_full_feature_migrate(void *arg) 
	iscsi_poll_group_add_conn(conn->pg, conn);

}



static struct spdk_iscsi_poll_group *g_next_pg = NULL;

static struct spdk_iscsi_poll_group *

iscsi_get_idlest_poll_group(void)

{

	struct spdk_iscsi_poll_group *pg, *idle_pg = NULL;

	uint32_t min_num_targets = UINT32_MAX;



	TAILQ_FOREACH(pg, &g_iscsi.poll_group_head, link) {

		if (pg->num_active_targets == 0) {

			return pg;

		} else if (pg->num_active_targets < min_num_targets) {

			min_num_targets = pg->num_active_targets;

			idle_pg = pg;

		}

	}



	return idle_pg;

}



void

iscsi_conn_schedule(struct spdk_iscsi_conn *conn)
@@ -1541,15 +1566,12 @@ iscsi_conn_schedule(struct spdk_iscsi_conn *conn) 
	if (target->num_active_conns == 1) {

		/**

		 * This is the only active connection for this target node.

		 *  Pick a poll group using round-robin.

		 *  Pick the idlest poll group.

		 */

		if (g_next_pg == NULL) {

			g_next_pg = TAILQ_FIRST(&g_iscsi.poll_group_head);

			assert(g_next_pg != NULL);

		}

		pg = iscsi_get_idlest_poll_group();

		assert(pg != NULL);



		pg = g_next_pg;

		g_next_pg = TAILQ_NEXT(g_next_pg, link);

		pg->num_active_targets++;



		/* Save the pg in the target node so it can be used for any other connections to this target node. */

		target->pg = pg;
@@ -1570,6 +1592,7 @@ iscsi_conn_schedule(struct spdk_iscsi_conn *conn) 
	iscsi_poll_group_remove_conn(conn->pg, conn);



	conn->pg = pg;

	conn->scheduled = 1;



	spdk_thread_send_msg(spdk_io_channel_get_thread(spdk_io_channel_from_ctx(pg)),

			     iscsi_conn_full_feature_migrate, conn);

lib/iscsi/conn.h

+1 −0
Original line number Diff line number Diff line
@@ -130,6 +130,7 @@ struct spdk_iscsi_conn { 
	int header_digest;

	int data_digest;

	int full_feature;

	int scheduled;



	struct iscsi_param *params;

	bool sess_param_state_negotiated[MAX_SESSION_PARAMS];

lib/iscsi/iscsi.h

+1 −0
Original line number Diff line number Diff line
@@ -288,6 +288,7 @@ struct spdk_iscsi_poll_group { 
	STAILQ_HEAD(connections, spdk_iscsi_conn)	connections;

	struct spdk_sock_group				*sock_group;

	TAILQ_ENTRY(spdk_iscsi_poll_group)		link;

	uint32_t					num_active_targets;

};



struct spdk_iscsi_opts {