Commit 8a23223e authored by Changpeng Liu's avatar Changpeng Liu Committed by Daniel Verkamp
Browse files

nvmf: Allow users to configure which lcore each subsystem runs on 



Users can specify the core for each subsystem and the acceptor listen routine
to run on different cores for performance consideration.

Change-Id: I4bd1a96f39194c870863b4b778e6ea7cf8fc1a2d
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent 077fe1da

etc/spdk/nvmf.conf.in

+8 −0
Original line number Diff line number Diff line
@@ -39,8 +39,14 @@ 
  # Set the maximum I/O size. Must be a multiple of 4096.

  #MaxIOSize 131072



  # Set the global acceptor lcore ID, lcores are numbered starting at 0.

  #AcceptorCore 0



# Define an NVMf Subsystem.

# - NQN is required and must be unique.

# - Core may be set or not. If set, the specified subsystem will run on

#   it, otherwise each subsystem will use a round-robin method to allocate

#   core from available cores,  lcores are numbered starting at 0.

# - Mode may be either "Direct" or "Virtual". Direct means that physical

#   devices attached to the target will be presented to hosts as if they

#   were directly attached to the host. No software emulation or command
@@ -59,6 +65,7 @@ 
#   any PCI device.

[Subsystem1]

  NQN nqn.2016-06.io.spdk:cnode1

  Core 0

  Mode Direct

  Listen RDMA 15.15.15.2:4420

  Host nqn.2016-06.io.spdk:init
@@ -67,6 +74,7 @@ 
# Multiple subsystems are allowed.

[Subsystem2]

  NQN nqn.2016-06.io.spdk:cnode2

  Core 0

  Mode Direct

  Listen RDMA 192.168.2.21:4420

  Host nqn.2016-06.io.spdk:init

lib/nvmf/conf.c

+17 −7
Original line number Diff line number Diff line
@@ -100,6 +100,7 @@ spdk_nvmf_parse_nvmf_tgt(void) 
	int max_queues_per_sess;

	int in_capsule_data_size;

	int max_io_size;

	int acceptor_lcore;

	int rc;



	sp = spdk_conf_find_section(NULL, "Nvmf");
@@ -142,7 +143,13 @@ spdk_nvmf_parse_nvmf_tgt(void) 
	max_io_size = nvmf_max(max_io_size, SPDK_NVMF_CONFIG_MAX_IO_SIZE_MIN);

	max_io_size = nvmf_min(max_io_size, SPDK_NVMF_CONFIG_MAX_IO_SIZE_MAX);



	rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size);

	acceptor_lcore = spdk_conf_section_get_intval(sp, "AcceptorCore");

	if (acceptor_lcore < 0) {

		acceptor_lcore = rte_lcore_id();

	}



	rc = nvmf_tgt_init(max_queue_depth, max_queues_per_sess, in_capsule_data_size, max_io_size,

			   acceptor_lcore);

	if (rc != 0) {

		SPDK_ERRLOG("nvmf_tgt_init() failed\n");

		return rc;
@@ -360,7 +367,7 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp) 
	struct spdk_nvmf_subsystem *subsystem;

	int i, ret;

	uint64_t mask;

	uint32_t lcore;

	int lcore = 0;



	nqn = spdk_conf_section_get_val(sp, "NQN");

	if (nqn == NULL) {
@@ -372,13 +379,16 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp) 
		return -1;

	}



	/* Determine which core to assign to the subsystem using round robin */

	/* Determine which core to assign to the subsystem */

	mask = spdk_app_get_core_mask();

	lcore = spdk_conf_section_get_intval(sp, "Core");

	if (lcore < 0) {

		lcore = 0;

		for (i = 0; i < sp->num; i++) {

			lcore = spdk_nvmf_allocate_lcore(mask, lcore);

			lcore++;

		}

	}

	lcore = spdk_nvmf_allocate_lcore(mask, lcore);



	subsystem = nvmf_create_subsystem(sp->num, nqn, SPDK_NVMF_SUBTYPE_NVME, lcore);

lib/nvmf/nvmf.c

+4 −1
Original line number Diff line number Diff line
@@ -117,7 +117,8 @@ spdk_nvmf_check_pools(void) 


int

nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess,

	      uint32_t in_capsule_data_size, uint32_t max_io_size)

	      uint32_t in_capsule_data_size, uint32_t max_io_size,

	      uint32_t acceptor_lcore)

{

	int rc;
@@ -125,11 +126,13 @@ nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_queues_per_sess, 
	g_nvmf_tgt.max_queue_depth = max_queue_depth;

	g_nvmf_tgt.in_capsule_data_size = in_capsule_data_size;

	g_nvmf_tgt.max_io_size = max_io_size;

	g_nvmf_tgt.acceptor_lcore = acceptor_lcore;



	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queues Per Session: %d\n", max_queues_per_sess);

	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max Queue Depth: %d\n", max_queue_depth);

	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max In Capsule Data: %d bytes\n", in_capsule_data_size);

	SPDK_TRACELOG(SPDK_TRACE_NVMF, "Max I/O Size: %d bytes\n", max_io_size);

	SPDK_TRACELOG(SPDK_TRACE_NVMF, "NVMf Acceptor lcore: %d \n", acceptor_lcore);



	/* init nvmf specific config options */

	if (!g_nvmf_tgt.sin_port) {

lib/nvmf/nvmf_internal.h

+4 −1
Original line number Diff line number Diff line
@@ -69,11 +69,14 @@ struct spdk_nvmf_globals { 
	uint32_t in_capsule_data_size;

	uint32_t max_io_size;



	uint32_t acceptor_lcore;



	uint16_t	   sin_port;

};



int nvmf_tgt_init(uint16_t max_queue_depth, uint16_t max_conn_per_sess,

		  uint32_t in_capsule_data_size, uint32_t max_io_size);

		  uint32_t in_capsule_data_size, uint32_t max_io_size,

		  uint32_t acceptor_lcore);



static inline uint32_t

nvmf_u32log2(uint32_t x)

lib/nvmf/rdma.c

+2 −2
Original line number Diff line number Diff line
@@ -1018,8 +1018,8 @@ spdk_nvmf_rdma_acceptor_start(void) 
	sin_port = ntohs(rdma_get_src_port(g_rdma.acceptor_listen_id));

	SPDK_NOTICELOG("*** NVMf Target Listening on port %d ***\n", sin_port);



	spdk_poller_register(&g_rdma.acceptor_poller, nvmf_rdma_accept, NULL, rte_lcore_id(), NULL,

			     ACCEPT_TIMEOUT_US);

	spdk_poller_register(&g_rdma.acceptor_poller, nvmf_rdma_accept, NULL, g_nvmf_tgt.acceptor_lcore,

			     NULL, ACCEPT_TIMEOUT_US);

	return rc;



listen_error: