Commit 96a54158 authored by Ben Walker's avatar Ben Walker
Browse files

nvmf: Assign each subsystem to an lcore 



Each subsystem will run on a single core, which is more than enough
to fully saturate a device and a NIC. For now, all subsystems
run on the master lcore.

Change-Id: I95340a262d70fd346fa81fe519e7d4190a369e64
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
parent ca7a61e1

lib/nvmf/conf.c

+4 −1
Original line number Diff line number Diff line
@@ -35,6 +35,9 @@ 
#include <stdlib.h>

#include <string.h>



#include <rte_config.h>

#include <rte_lcore.h>



#include "conf.h"

#include "controller.h"

#include "host.h"
@@ -427,7 +430,7 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp) 
		return -1;

	}



	subsystem = nvmf_create_subsystem(sp->num, nqn, SPDK_NVMF_SUB_NVME);

	subsystem = nvmf_create_subsystem(sp->num, nqn, SPDK_NVMF_SUB_NVME, rte_get_master_lcore());

	if (subsystem == NULL) {

		return -1;

	}

lib/nvmf/conn.c

+2 −32
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ 
#include "rdma.h"

#include "request.h"

#include "session.h"

#include "subsystem.h"

#include "spdk/queue.h"

#include "spdk/log.h"

#include "spdk/trace.h"
@@ -59,41 +60,16 @@ 


*/



static int nvmf_allocate_reactor(uint64_t cpumask);

static void spdk_nvmf_conn_do_work(void *arg);



/**



\brief Create an NVMf fabric connection from the given parameters and schedule it

       on a reactor thread.



\code



# identify reactor where the new connections work item will be scheduled

reactor = nvmf_allocate_reactor()

schedule fabric connection work item on reactor



\endcode



*/

int

spdk_nvmf_startup_conn(struct spdk_nvmf_conn *conn)

{

	int lcore;

	uint64_t nvmf_session_core = spdk_app_get_core_mask();



	lcore = nvmf_allocate_reactor(nvmf_session_core);

	if (lcore < 0) {

		SPDK_ERRLOG("Unable to find core to launch connection.\n");

		return -1;

	}



	conn->state = CONN_STATE_RUNNING;

	SPDK_NOTICELOG("Launching nvmf connection on core: %d\n", lcore);

	conn->poller.fn = spdk_nvmf_conn_do_work;

	conn->poller.arg = conn;



	spdk_poller_register(&conn->poller, lcore, NULL);

	spdk_poller_register(&conn->poller, conn->sess->subsys->lcore, NULL);



	return 0;

}
@@ -136,9 +112,3 @@ spdk_nvmf_conn_do_work(void *arg) 
		}

	}

}
 


static int

nvmf_allocate_reactor(uint64_t cpumask)

{

	return rte_get_master_lcore();

}

lib/nvmf/request.c

+10 −9
Original line number Diff line number Diff line
@@ -398,6 +398,7 @@ nvmf_handle_connect(spdk_event_t event) 
			req->data;

	struct spdk_nvmf_fabric_connect_rsp *response = &req->rsp->connect_rsp;

	struct spdk_nvmf_conn *conn = req->conn;

	int rc;



	spdk_nvmf_session_connect(conn, connect, connect_data, response);
@@ -410,6 +411,15 @@ nvmf_handle_connect(spdk_event_t event) 
		return;

	}



	/* Start the connection poller */

	rc = spdk_nvmf_startup_conn(conn);

	if (rc) {

		SPDK_ERRLOG("Unable to start connection poller\n");

		req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

		spdk_nvmf_request_complete(req);

		return;

	}



	SPDK_TRACELOG(SPDK_TRACE_NVMF, "connect capsule response: cntlid = 0x%04x\n",

		      response->status_code_specific.success.cntlid);
@@ -421,7 +431,6 @@ static bool 
nvmf_process_connect(struct spdk_nvmf_request *req)

{

	struct spdk_nvmf_conn *conn = req->conn;

	int rc;

	spdk_event_t event;



	if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) {
@@ -430,14 +439,6 @@ nvmf_process_connect(struct spdk_nvmf_request *req) 
		return true;

	}



	/* Start the connection poller */

	rc = spdk_nvmf_startup_conn(conn);

	if (rc) {

		req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

		SPDK_ERRLOG("Unable to start connection poller\n");

		return true;

	}



	/* Pass an event to the lcore that owns this connection */

	event = spdk_event_allocate(conn->poller.lcore, nvmf_handle_connect, req, NULL, NULL);

	spdk_event_call(event);

lib/nvmf/subsystem.c

+5 −2
Original line number Diff line number Diff line
@@ -67,7 +67,9 @@ nvmf_find_subsystem(const char *subnqn) 
}



struct spdk_nvmf_subsystem *

nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types sub_type)

nvmf_create_subsystem(int num, const char *name,

		      enum spdk_nvmf_subsystem_types sub_type,

		      uint32_t lcore)

{

	struct spdk_nvmf_subsystem	*subsystem;
@@ -81,6 +83,7 @@ nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types 
	subsystem->num = num;

	subsystem->subtype = sub_type;

	snprintf(subsystem->subnqn, sizeof(subsystem->subnqn), "%s", name);

	subsystem->lcore = lcore;



	TAILQ_INSERT_HEAD(&g_subsystems, subsystem, entries);
@@ -175,7 +178,7 @@ spdk_add_nvmf_discovery_subsystem(void) 
		return -1;

	}



	subsystem = nvmf_create_subsystem(0, name, SPDK_NVMF_SUB_DISCOVERY);

	subsystem = nvmf_create_subsystem(0, name, SPDK_NVMF_SUB_DISCOVERY, rte_get_master_lcore());

	if (subsystem == NULL) {

		SPDK_ERRLOG("Failed creating discovery nvmf library subsystem\n");

		free(name);

lib/nvmf/subsystem.h

+4 −1
Original line number Diff line number Diff line
@@ -59,6 +59,7 @@ struct spdk_nvmf_subsystem { 
	struct nvmf_session *session;

	struct spdk_nvme_ctrlr *ctrlr;

	struct spdk_nvme_qpair *io_qpair;

	uint32_t		lcore;



	int map_count;

	struct spdk_nvmf_access_map map[MAX_PER_SUBSYSTEM_ACCESS_MAP];
@@ -67,7 +68,9 @@ struct spdk_nvmf_subsystem { 
};



struct spdk_nvmf_subsystem *

nvmf_create_subsystem(int num, const char *name, enum spdk_nvmf_subsystem_types sub_type);

nvmf_create_subsystem(int num, const char *name,

		      enum spdk_nvmf_subsystem_types sub_type,

		      uint32_t lcore);



int

nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem);