Commit 8b487155 authored by Ben Walker's avatar Ben Walker
Browse files

nvmf: Fail when sessions span RNICs 



The shared memory pool for a session is associated with
a particular RNIC via the protection domain. New connections
attempting to join a session that came in on a different RNIC
can't use that memory, so must be rejected.

Change-Id: Ibd79fe90566a231f76b7472e5e9b484c3e528454
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 349295ca

lib/nvmf/rdma.c

+48 −17
Original line number Diff line number Diff line
@@ -134,6 +134,8 @@ static TAILQ_HEAD(, spdk_nvmf_rdma_conn) g_pending_conns = TAILQ_HEAD_INITIALIZE 
struct spdk_nvmf_rdma_session {

	SLIST_HEAD(, spdk_nvmf_rdma_buf)	data_buf_pool;



	struct ibv_context			*verbs;



	uint8_t					*buf;

	struct ibv_mr				*buf_mr;

};
@@ -1169,10 +1171,9 @@ spdk_nvmf_rdma_discover(struct spdk_nvmf_listen_addr *listen_addr, 
}



static int

spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn)

spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session)

{

	struct spdk_nvmf_rdma_session	*rdma_sess;

	struct spdk_nvmf_rdma_conn	*rdma_conn = get_rdma_conn(conn);

	int				i;

	struct spdk_nvmf_rdma_buf	*buf;
@@ -1193,27 +1194,13 @@ spdk_nvmf_rdma_session_init(struct spdk_nvmf_session *session, struct spdk_nvmf_ 
		return -1;

	}



	rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,

				       g_rdma.max_queue_depth * g_rdma.max_io_size,

				       IBV_ACCESS_LOCAL_WRITE);

	if (!rdma_sess->buf_mr) {

		SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",

			    g_rdma.max_queue_depth, g_rdma.max_io_size);

		spdk_free(rdma_sess->buf);

		free(rdma_sess);

		return -1;

	}



	SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",

		      rdma_sess->buf,  g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);



	SLIST_INIT(&rdma_sess->data_buf_pool);

	for (i = 0; i < g_rdma.max_queue_depth; i++) {

		buf = (struct spdk_nvmf_rdma_buf *)(rdma_sess->buf + (i * g_rdma.max_io_size));

		SLIST_INSERT_HEAD(&rdma_sess->data_buf_pool, buf, link);

	}



	session->transport = conn->transport;

	session->transport = &spdk_nvmf_transport_rdma;

	session->trctx = rdma_sess;



	return 0;
@@ -1234,6 +1221,48 @@ spdk_nvmf_rdma_session_fini(struct spdk_nvmf_session *session) 
	session->trctx = NULL;

}



static int

spdk_nvmf_rdma_session_add_conn(struct spdk_nvmf_session *session,

				struct spdk_nvmf_conn *conn)

{

	struct spdk_nvmf_rdma_session	*rdma_sess = session->trctx;

	struct spdk_nvmf_rdma_conn	*rdma_conn = get_rdma_conn(conn);



	if (rdma_sess->verbs != NULL) {

		if (rdma_sess->verbs != rdma_conn->cm_id->verbs) {

			SPDK_ERRLOG("Two connections belonging to the same session cannot connect using different RDMA devices.\n");

			return -1;

		}



		/* Nothing else to do. */

		return 0;

	}



	rdma_sess->verbs = rdma_conn->cm_id->verbs;

	rdma_sess->buf_mr = ibv_reg_mr(rdma_conn->cm_id->pd, rdma_sess->buf,

				       g_rdma.max_queue_depth * g_rdma.max_io_size,

				       IBV_ACCESS_LOCAL_WRITE);

	if (!rdma_sess->buf_mr) {

		SPDK_ERRLOG("Large buffer pool registration failed (%d x %d)\n",

			    g_rdma.max_queue_depth, g_rdma.max_io_size);

		spdk_free(rdma_sess->buf);

		free(rdma_sess);

		return -1;

	}



	SPDK_TRACELOG(SPDK_TRACE_RDMA, "Session Shared Data Pool: %p Length: %x LKey: %x\n",

		      rdma_sess->buf,  g_rdma.max_queue_depth * g_rdma.max_io_size, rdma_sess->buf_mr->lkey);



	return 0;

}



static int

spdk_nvmf_rdma_session_remove_conn(struct spdk_nvmf_session *session,

				   struct spdk_nvmf_conn *conn)

{

	return 0;

}



static int

spdk_nvmf_rdma_request_complete(struct spdk_nvmf_request *req)

{
@@ -1412,6 +1441,8 @@ const struct spdk_nvmf_transport spdk_nvmf_transport_rdma = { 


	.session_init = spdk_nvmf_rdma_session_init,

	.session_fini = spdk_nvmf_rdma_session_fini,

	.session_add_conn = spdk_nvmf_rdma_session_add_conn,

	.session_remove_conn = spdk_nvmf_rdma_session_remove_conn,



	.req_complete = spdk_nvmf_rdma_request_complete,

	.req_release = spdk_nvmf_rdma_request_release,

lib/nvmf/session.c

+15 −1
Original line number Diff line number Diff line
@@ -259,7 +259,14 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, 
		session->num_connections = 0;

		session->subsys = subsystem;

		session->max_connections_allowed = g_nvmf_tgt.max_queues_per_session;

		if (conn->transport->session_init(session, conn)) {

		if (conn->transport->session_init(session)) {

			rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			conn->transport->session_fini(session);

			free(session);

			return;

		}



		if (conn->transport->session_add_conn(session, conn)) {

			rsp->status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			conn->transport->session_fini(session);

			free(session);
@@ -319,6 +326,11 @@ spdk_nvmf_session_connect(struct spdk_nvmf_conn *conn, 
			rsp->status.sc = SPDK_NVMF_FABRIC_SC_CONTROLLER_BUSY;

			return;

		}



		if (conn->transport->session_add_conn(session, conn)) {

			INVALID_CONNECT_CMD(qid);

			return;

		}

	}



	session->num_connections++;
@@ -339,6 +351,8 @@ spdk_nvmf_session_disconnect(struct spdk_nvmf_conn *conn) 
	assert(session != NULL);

	session->num_connections--;

	TAILQ_REMOVE(&session->connections, conn, link);



	conn->transport->session_remove_conn(session, conn);

	conn->transport->conn_fini(conn);



	if (session->num_connections == 0) {

lib/nvmf/transport.h

+11 −1
Original line number Diff line number Diff line
@@ -75,13 +75,23 @@ struct spdk_nvmf_transport { 
	/**

	 * Initialize the transport for the given session

	 */

	int (*session_init)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);

	int (*session_init)(struct spdk_nvmf_session *session);



	/**

	 * Deinitiallize the transport for the given session

	 */

	void (*session_fini)(struct spdk_nvmf_session *session);



	/**

	 * Add a connection to a session

	 */

	int (*session_add_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);



	/**

	 * Remove a connection from a session

	 */

	int (*session_remove_conn)(struct spdk_nvmf_session *session, struct spdk_nvmf_conn *conn);



	/*

	 * Signal request completion, which sends a response

	 * to the originator. A request can either