Commit 4d48d87a authored by Jim Harris's avatar Jim Harris
Browse files

iscsi: remove idle connection handling 



Some upcoming changes will effectively render this moot
anyways by adding an epoll/kqueue descriptor to poll
on in all cases (not just connections that have been
idle for 5ms).

The epoll/kqueue code was just ifdef'd out instead
of removed - some of this code will be useful
and reusable with minimal changes in the upcoming
patches.

Signed-off-by: default avatarJim Harris <james.r.harris@intel.com>
Change-Id: I0c354390537e6369cb3c32e78a59c300dec6d098

Reviewed-on: https://review.gerrithub.io/395553


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatar <shuhei.matsumoto.xt@hitachi.com>
parent 8c2b8ca7

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -95,6 +95,11 @@ Previously a part of the bdev_virtio module, now a separate library. Virtio is n 
via `spdk_internal/virtio.h` file. This is an internal interface to be used when implementing

new Virtio backends, namely Virtio-BLK.



### iSCSI



The MinConnectionIdleInterval parameter has been removed, and connections are no longer migrated

to an epoll/kqueue descriptor on the master core when idle.



## v17.10: Logical Volumes



### New dependencies

etc/spdk/iscsi.conf.in

+0 −5
Original line number Diff line number Diff line
@@ -37,11 +37,6 @@ 
  AuthFile /usr/local/etc/spdk/auth.conf



  MinConnectionsPerCore 4

  # Power saving related variable, this parameter defines how long an iSCSI

  # connection must be idle before moving it to a state where it will consume

  # less power. This variable is defined in terms of microseconds. We set default

  # value as 5ms.

  MinConnectionIdleInterval 5000



  # Socket I/O timeout sec. (0 is infinite)

  Timeout 30

include/spdk/net.h

+0 −1
Original line number Diff line number Diff line
@@ -55,7 +55,6 @@ const char *spdk_net_framework_get_name(void); 
int spdk_net_framework_start(void);

void spdk_net_framework_clear_socket_association(int sock);

void spdk_net_framework_fini(void);

int spdk_net_framework_idle_time(void);



#define SPDK_IFNAMSIZE		32

#define SPDK_MAX_IP_PER_IFC	32

lib/iscsi/conn.c

+9 −151
Original line number Diff line number Diff line
@@ -61,9 +61,6 @@ 
	memset(&(conn)->portal, 0, sizeof(*(conn)) -	\

		offsetof(struct spdk_iscsi_conn, portal));



#define MICROSECOND_TO_TSC(x) ((x) * spdk_get_ticks_hz()/1000000)

static int64_t g_conn_idle_interval_in_tsc = -1;



#define DEFAULT_CONNECTIONS_PER_LCORE	4

#define SPDK_MAX_POLLERS_PER_CORE	4096

static int g_connections_per_lcore = DEFAULT_CONNECTIONS_PER_LCORE;
@@ -77,16 +74,9 @@ static pthread_mutex_t g_conns_mutex; 
static struct spdk_poller *g_shutdown_timer = NULL;



static uint32_t spdk_iscsi_conn_allocate_reactor(uint64_t cpumask);

static void __add_idle_conn(void *arg1, void *arg2);



/** Global variables used for managing idle connections. */

static int g_poll_fd = 0;

static struct spdk_poller *g_idle_conn_poller;

static STAILQ_HEAD(idle_list, spdk_iscsi_conn) g_idle_conn_list_head;



void spdk_iscsi_conn_login_do_work(void *arg);

void spdk_iscsi_conn_full_feature_do_work(void *arg);

void spdk_iscsi_conn_idle_do_work(void *arg);



static void spdk_iscsi_conn_full_feature_migrate(void *arg1, void *arg2);

static struct spdk_event *spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_conn *conn,
@@ -94,11 +84,6 @@ static struct spdk_event *spdk_iscsi_conn_get_migrate_event(struct spdk_iscsi_co 
static void spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after_stop,

					int lcore);



void spdk_iscsi_set_min_conn_idle_interval(int interval_in_us)

{

	g_conn_idle_interval_in_tsc = MICROSECOND_TO_TSC(interval_in_us);

}



static struct spdk_iscsi_conn *

allocate_conn(void)

{
@@ -138,6 +123,11 @@ spdk_find_iscsi_connection_by_id(int cid) 
	}

}



/*

 * Some of this code may be useful once we add back an epoll/kqueue descriptor

 * for normal processing.  So just #if 0 it out for now.

 */

#if 0

#if defined(__FreeBSD__)



static int
@@ -335,6 +325,7 @@ check_idle_conns(void) 
	}

}



#endif

#endif



int spdk_initialize_iscsi_conns(void)
@@ -383,17 +374,6 @@ int spdk_initialize_iscsi_conns(void) 
		return -1;

	}



	if (g_conn_idle_interval_in_tsc == -1) {

		spdk_iscsi_set_min_conn_idle_interval(spdk_net_framework_idle_time());

	}



	STAILQ_INIT(&g_idle_conn_list_head);

	if (init_idle_conns() < 0) {

		return -1;

	}



	g_idle_conn_poller = spdk_poller_register(spdk_iscsi_conn_idle_do_work, NULL, 0);



	return 0;

}
@@ -504,11 +484,9 @@ error_return: 
		free_conn(conn);

		return -1;

	}

	conn->is_idle = 0;

	conn->logout_timer = NULL;

	conn->shutdown_timer = NULL;

	SPDK_NOTICELOG("Launching connection on acceptor thread\n");

	conn->last_activity_tsc = spdk_get_ticks();

	conn->pending_task_cnt = 0;

	conn->pending_activate_event = false;
@@ -845,19 +823,9 @@ spdk_iscsi_conn_stop_poller(struct spdk_iscsi_conn *conn, spdk_event_fn fn_after 


void spdk_shutdown_iscsi_conns(void)

{

	struct spdk_iscsi_conn	*conn, *tmp;

	struct spdk_iscsi_conn	*conn;

	int			i;



	/* cleanup - move conns from list back into ring

	   where they will get cleaned up

	 */

	STAILQ_FOREACH_SAFE(conn, &g_idle_conn_list_head, link, tmp) {

		STAILQ_REMOVE(&g_idle_conn_list_head, conn, spdk_iscsi_conn, link);

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));

		conn->is_idle = 0;

		del_idle_conn(conn);

	}



	pthread_mutex_lock(&g_conns_mutex);



	for (i = 0; i < MAX_ISCSI_CONNECTIONS; i++) {
@@ -989,7 +957,6 @@ spdk_iscsi_task_mgmt_cpl(struct spdk_scsi_task *scsi_task) 
{

	struct spdk_iscsi_task *task = spdk_iscsi_task_from_scsi_task(scsi_task);



	task->conn->last_activity_tsc = spdk_get_ticks();

	spdk_iscsi_task_mgmt_response(task->conn, task);

	spdk_iscsi_task_put(task);

}
@@ -1063,7 +1030,6 @@ spdk_iscsi_task_cpl(struct spdk_scsi_task *scsi_task) 
	struct spdk_iscsi_conn *conn = task->conn;



	spdk_trace_record(TRACE_ISCSI_TASK_DONE, conn->id, 0, (uintptr_t)task, 0);

	conn->last_activity_tsc = spdk_get_ticks();



	primary = spdk_iscsi_task_get_primary(task);
@@ -1361,19 +1327,6 @@ spdk_iscsi_conn_handle_incoming_pdus(struct spdk_iscsi_conn *conn) 
	return i;

}



static void spdk_iscsi_conn_handle_idle(struct spdk_iscsi_conn *conn)

{

	uint64_t current_tsc = spdk_get_ticks();



	if (g_conn_idle_interval_in_tsc > 0 &&

	    ((int64_t)(current_tsc - conn->last_activity_tsc)) >= g_conn_idle_interval_in_tsc &&

	    conn->pending_task_cnt == 0) {



		spdk_trace_record(TRACE_ISCSI_CONN_IDLE, conn->id, 0, 0, 0);

		spdk_iscsi_conn_stop_poller(conn, __add_idle_conn, spdk_env_get_first_core());

	}

}



static int

spdk_iscsi_conn_execute(struct spdk_iscsi_conn *conn)

{
@@ -1471,103 +1424,8 @@ void 
spdk_iscsi_conn_full_feature_do_work(void *arg)

{

	struct spdk_iscsi_conn	*conn = arg;

	int				rc = 0;



	rc = spdk_iscsi_conn_execute(conn);

	if (rc < 0) {

		return;

	} else if (rc > 0) {

		conn->last_activity_tsc = spdk_get_ticks();

	}



	/* Check if the session was idle during this access pass. If it was,

	   and it was idle longer than the configured timeout, migrate this

	   session to the first core. */

	spdk_iscsi_conn_handle_idle(conn);

}



/**

 * \brief This is the main routine for the iSCSI 'idle' connection

 * work item.

 *

 * This function handles processing of connecitons whose state have

 * been determined as 'idle' for lack of activity.  These connections

 * no longer reside in the reactor's poller ring, instead they have

 * been staged into an idle list.  This function utilizes the use of

 * epoll as a non-blocking means to test for new socket connection

 * events that indicate the connection should be moved back into the

 * active ring.

 *

 * While in the idle list, this function must scan these connections

 * to process required timer based actions that must be maintained

 * even though the connection is considered 'idle'.

 */

void spdk_iscsi_conn_idle_do_work(void *arg)

{

	uint64_t	tsc;

	struct spdk_iscsi_conn *tconn;



	check_idle_conns();



	/* Now walk the idle list to process timer based actions */

	STAILQ_FOREACH(tconn, &g_idle_conn_list_head, link) {



		assert(tconn->is_idle == 1);



		if (tconn->pending_activate_event == false) {

			tsc = spdk_get_ticks();

			if (tsc - tconn->last_nopin > tconn->nopininterval) {

				tconn->pending_activate_event = true;

			}

		}



		if (tconn->pending_activate_event) {

			int lcore;



			spdk_trace_record(TRACE_ISCSI_CONN_ACTIVE, tconn->id, 0, 0, 0);



			/* remove connection from idle list */

			STAILQ_REMOVE(&g_idle_conn_list_head, tconn, spdk_iscsi_conn, link);

			tconn->last_activity_tsc = spdk_get_ticks();

			tconn->pending_activate_event = false;

			tconn->is_idle = 0;

			del_idle_conn(tconn);

			/* migrate work item to new core */

			spdk_net_framework_clear_socket_association(tconn->sock);

			spdk_event_call(spdk_iscsi_conn_get_migrate_event(tconn, &lcore));

			__sync_fetch_and_add(&g_num_connections[lcore], 1);

			SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "add conn id = %d, cid = %d poller = %p to lcore = %d active\n",

				      tconn->id, tconn->cid, &tconn->poller, lcore);

		}

	} /* for each conn in idle list */

}



static void

__add_idle_conn(void *arg1, void *arg2)

{

	struct spdk_iscsi_conn *conn = arg1;

	int rc;



	/*

	 * The iSCSI target may have started shutting down when this connection was

	 *  determined as idle.  In that case, do not append the connection to the

	 *  idle list - just start the work item again so it can start its shutdown

	 *  process.

	 */

	if (conn->state == ISCSI_CONN_STATE_EXITING) {

		spdk_event_call(spdk_iscsi_conn_get_migrate_event(conn, NULL));

		return;

	}



	rc = add_idle_conn(conn);

	if (rc == 0) {

		SPDK_DEBUGLOG(SPDK_LOG_ISCSI, "add conn id = %d, cid = %d poller = %p to idle\n",

			      conn->id, conn->cid, conn->poller);

		conn->is_idle = 1;

		STAILQ_INSERT_TAIL(&g_idle_conn_list_head, conn, link);

	} else {

		SPDK_ERRLOG("add_idle_conn() failed\n");

	}

	spdk_iscsi_conn_execute(conn);

}



void

lib/iscsi/conn.h

+0 −3
Original line number Diff line number Diff line
@@ -70,7 +70,6 @@ struct spdk_poller; 
struct spdk_iscsi_conn {

	int				id;

	int				is_valid;

	int				is_idle;

	/*

	 * All fields below this point are reinitialized each time the

	 *  connection object is allocated.  Make sure to update the
@@ -136,7 +135,6 @@ struct spdk_iscsi_conn { 
	int authenticated;

	int req_auth;

	int req_mutual;

	uint64_t last_activity_tsc;

	uint32_t pending_task_cnt;

	uint32_t data_out_cnt;

	uint32_t data_in_cnt;
@@ -177,7 +175,6 @@ void spdk_iscsi_conn_logout(struct spdk_iscsi_conn *conn); 
int spdk_iscsi_drop_conns(struct spdk_iscsi_conn *conn,

			  const char *conn_match, int drop_all);

void spdk_iscsi_conn_set_min_per_core(int count);

void spdk_iscsi_set_min_conn_idle_interval(int interval_in_us);



int spdk_iscsi_conn_read_data(struct spdk_iscsi_conn *conn, int len,

			      void *buf);