Commit fdc12784 authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvmf: simplify session and connection cleanup 



The whole cleanup process is now started by
spdk_shutdown_nvmf_subsystems().  Each subsystem will clean up its
session, if any, and each session will clean up its connections.

Change-Id: I9915d4547751ed4ffc4baa2c45c628698dd0b881
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 9903050d

app/nvmf_tgt/nvmf_tgt.c

+1 −2
Original line number Diff line number Diff line
@@ -44,7 +44,6 @@ 


#include "spdk/event.h"



#include "nvmf/conn.h"

#include "nvmf/rdma.h"

#include "nvmf/port.h"

#include "nvmf/host.h"
@@ -61,7 +60,7 @@ static void 
spdk_nvmf_shutdown_cb(void)

{

	nvmf_acceptor_stop();

	spdk_shutdown_nvmf_conns();

	spdk_app_stop(0);



	fprintf(stdout, "\n=========================\n");

	fprintf(stdout, "   NVMF shutdown signal\n");

lib/nvmf/conn.c

+7 −152
Original line number Diff line number Diff line
@@ -59,84 +59,21 @@ 


*/



static int g_max_conns;

static struct spdk_nvmf_conn *g_conns_array;



static pthread_mutex_t g_conns_mutex;



static struct rte_timer g_shutdown_timer;



static int nvmf_allocate_reactor(uint64_t cpumask);

static void spdk_nvmf_conn_do_work(void *arg);



static struct spdk_nvmf_conn *

allocate_conn(void)

{

	struct spdk_nvmf_conn	*conn;

	int				i;



	pthread_mutex_lock(&g_conns_mutex);

	for (i = 0; i < g_max_conns; i++) {

		conn = &g_conns_array[i];

		if (!conn->is_valid) {

			memset(conn, 0, sizeof(*conn));

			conn->is_valid = 1;

			pthread_mutex_unlock(&g_conns_mutex);

			return conn;

		}

	}

	pthread_mutex_unlock(&g_conns_mutex);



	return NULL;

}



static void

free_conn(struct spdk_nvmf_conn *conn)

{

	conn->sess = NULL;

	conn->is_valid = 0;

}



int spdk_initialize_nvmf_conns(int max_connections)

{

	int rc;



	rc = pthread_mutex_init(&g_conns_mutex, NULL);

	if (rc != 0) {

		SPDK_ERRLOG("mutex_init() failed\n");

		return -1;

	}



	g_max_conns = max_connections;

	g_conns_array = calloc(g_max_conns, sizeof(struct spdk_nvmf_conn));



	return 0;

}



struct spdk_nvmf_conn *

spdk_nvmf_allocate_conn(void)

{

	struct spdk_nvmf_conn *conn;



	conn = allocate_conn();

	conn = calloc(1, sizeof(struct spdk_nvmf_conn));

	if (conn == NULL) {

		SPDK_ERRLOG("Could not allocate new connection.\n");

		goto err0;

		return NULL;

	}



	/* all new connections initially default as AQ until nvmf connect */

	conn->type = CONN_TYPE_AQ;



	/* no session association until nvmf connect */

	conn->sess = NULL;



	conn->state = CONN_STATE_INVALID;

	conn->sq_head = 0;



	return conn;



err0:

	return NULL;

}



/**
@@ -175,100 +112,18 @@ spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn) 


	return 0;

err0:

	free_conn(conn);

	free(conn);

	return -1;

}



static void

_conn_destruct(spdk_event_t event)

void

spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)

{

	struct spdk_nvmf_conn *conn = spdk_event_get_arg1(event);

	spdk_poller_unregister(&conn->poller, NULL);



	/*

	 * Notify NVMf library of the fabric connection

	 * going away.  If this is the AQ connection then

	 * set state for other connections to abort.

	 */

	nvmf_disconnect(conn->sess, conn);



	if (conn->type == CONN_TYPE_AQ) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "AQ connection destruct, trigger session closure\n");

		/* Trigger all I/O connections to shutdown */

		conn->state = CONN_STATE_FABRIC_DISCONNECT;

	}



	nvmf_rdma_conn_cleanup(conn);



	pthread_mutex_lock(&g_conns_mutex);

	free_conn(conn);

	pthread_mutex_unlock(&g_conns_mutex);

}



static void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn)

{

	struct spdk_event *event;



	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "conn %p\n", conn);

	conn->state = CONN_STATE_INVALID;



	event = spdk_event_allocate(rte_lcore_id(), _conn_destruct, conn, NULL, NULL);

	spdk_poller_unregister(&conn->poller, event);

}



static int

spdk_nvmf_get_active_conns(void)

{

	struct spdk_nvmf_conn *conn;

	int num = 0;

	int i;



	pthread_mutex_lock(&g_conns_mutex);

	for (i = 0; i < g_max_conns; i++) {

		conn = &g_conns_array[i];

		if (!conn->is_valid)

			continue;

		num++;

	}

	pthread_mutex_unlock(&g_conns_mutex);

	return num;

}



static void

spdk_nvmf_cleanup_conns(void)

{

	free(g_conns_array);

}



static void

spdk_nvmf_conn_check_shutdown(struct rte_timer *timer, void *arg)

{

	if (spdk_nvmf_get_active_conns() == 0) {

		RTE_VERIFY(timer == &g_shutdown_timer);

		rte_timer_stop(timer);

		spdk_nvmf_cleanup_conns();

		spdk_app_stop(0);

	}

}



void spdk_shutdown_nvmf_conns(void)

{

	struct spdk_nvmf_conn	*conn;

	int				i;



	pthread_mutex_lock(&g_conns_mutex);



	for (i = 0; i < g_max_conns; i++) {

		conn = &g_conns_array[i];

		if (!conn->is_valid)

			continue;

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Set conn %d state to exiting\n", i);

		conn->state = CONN_STATE_EXITING;

	}



	pthread_mutex_unlock(&g_conns_mutex);

	rte_timer_init(&g_shutdown_timer);

	rte_timer_reset(&g_shutdown_timer, rte_get_timer_hz() / 1000, PERIODICAL,

			rte_get_master_lcore(), spdk_nvmf_conn_check_shutdown, NULL);

	free(conn);

}



static void

lib/nvmf/conn.h

+1 −6
Original line number Diff line number Diff line
@@ -55,8 +55,6 @@ enum conn_type { 
};



struct spdk_nvmf_conn {

	uint32_t			is_valid;



	struct nvmf_session		*sess;



	uint16_t			qid;
@@ -72,13 +70,10 @@ struct spdk_nvmf_conn { 
	struct spdk_poller		poller;

};



int spdk_initialize_nvmf_conns(int max_connections);



void spdk_shutdown_nvmf_conns(void);



struct spdk_nvmf_conn *

spdk_nvmf_allocate_conn(void);



int spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn);

void spdk_nvmf_conn_destruct(struct spdk_nvmf_conn *conn);



#endif /* NVMF_CONN_H */

lib/nvmf/nvmf.c

+1 −9
Original line number Diff line number Diff line
@@ -77,10 +77,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) 
#define SPDK_NVMF_DESC_POOL_SIZE(spdk)	(SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \

					 SPDK_NVMF_IOQ_POOL_SIZE(spdk))



#define SPDK_NVMF_MAX_CONNECTIONS(spdk)	(MAX_SUBSYSTEMS * \

					 ((spdk)->MaxSessionsPerSubsystem) * \

					 ((spdk)->MaxConnectionsPerSession))



struct spdk_nvmf_globals g_nvmf_tgt;



extern struct rte_mempool *request_mempool;
@@ -238,11 +234,7 @@ nvmf_tgt_subsystem_initialize(void) 
		SPDK_ERRLOG("spdk_initialize_nvmf_subsystems failed\n");

		return rc;

	}

	rc = spdk_initialize_nvmf_conns(SPDK_NVMF_MAX_CONNECTIONS(&g_nvmf_tgt));

	if (rc < 0) {

		SPDK_ERRLOG("spdk_initialize_nvmf_conns() failed\n");

		return rc;

	}



	return rc;

}

lib/nvmf/session.c

+14 −10
Original line number Diff line number Diff line
@@ -195,10 +195,18 @@ nvmf_create_session(const char *subnqn) 
	return session;

}



static void

nvmf_delete_session(struct nvmf_session	*session)

void

spdk_nvmf_session_destruct(struct nvmf_session *session)

{

	session->subsys->session = NULL;



	while (!TAILQ_EMPTY(&session->connections)) {

		struct spdk_nvmf_conn *conn = TAILQ_FIRST(&session->connections);



		TAILQ_REMOVE(&session->connections, conn, link);

		spdk_nvmf_conn_destruct(conn);

	}



	free(session);

}
@@ -268,15 +276,11 @@ void 
nvmf_disconnect(struct nvmf_session *session,

		struct spdk_nvmf_conn *conn)

{

	if (session) {

		if (session->num_connections > 0) {

	session->num_connections--;

	TAILQ_REMOVE(&session->connections, conn, link);

		}



		if (session->num_connections == 0) {

			nvmf_delete_session(session);

		}

	if (session->num_connections == 0 || conn->type == CONN_TYPE_AQ) {

		spdk_nvmf_session_destruct(session);

	}

}