Commit 98e119f7 authored by GangCao's avatar GangCao Committed by Jim Harris
Browse files

lib/nvmf: add the nvmf qpair to the available poll group 



In the case that the subsystem in the related poll group has
NULL IO channel assigned due to some problem like out of resource,
for example, the NVMe SSD hardware itself has limited number
of IO qpairs. The subsystems in the particular poll group
could have zero valid channels. In this case, the creation of
assoicated poll group will fail and when adding the new qpair
to the specified poll group, needs to have a check and pick the
available poll group.

Change-Id: Iedee2a6375e48eb7bf899cfb0542c565c7ebd231
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.gerrithub.io/423646


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 85bc2bbe

lib/event/subsystems/nvmf/nvmf_tgt.c

+49 −5
Original line number Diff line number Diff line
@@ -137,6 +137,36 @@ spdk_nvmf_get_core_rr(void) 
	return core;

}



static void

nvmf_tgt_remove_host_trid(struct spdk_nvmf_qpair *qpair)

{

	struct spdk_nvme_transport_id trid_to_remove;

	struct nvmf_tgt_host_trid *trid = NULL, *tmp_trid = NULL;



	if (g_spdk_nvmf_tgt_conf->conn_sched != CONNECT_SCHED_HOST_IP) {

		return;

	}



	if (spdk_nvmf_qpair_get_peer_trid(qpair, &trid_to_remove) != 0) {

		return;

	}



	TAILQ_FOREACH_SAFE(trid, &g_nvmf_tgt_host_trids, link, tmp_trid) {

		if (trid && !strncmp(trid->host_trid.traddr,

				     trid_to_remove.traddr, SPDK_NVMF_TRADDR_MAX_LEN + 1)) {

			trid->ref--;

			if (trid->ref == 0) {

				TAILQ_REMOVE(&g_nvmf_tgt_host_trids, trid, link);

				free(trid);

			}



			break;

		}

	}



	return;

}



static uint32_t

nvmf_tgt_get_qpair_core(struct spdk_nvmf_qpair *qpair)

{
@@ -190,15 +220,28 @@ new_qpair(struct spdk_nvmf_qpair *qpair) 
	struct spdk_event *event;

	struct nvmf_tgt_poll_group *pg;

	uint32_t core;

	uint32_t attempts;



	if (g_tgt_state != NVMF_TGT_RUNNING) {

		spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);

		return;

	}



	for (attempts = 0; attempts < g_num_poll_groups; attempts++) {

		core = nvmf_tgt_get_qpair_core(qpair);



		pg = &g_poll_groups[core];

		if (pg->group != NULL) {

			break;

		} else {

			nvmf_tgt_remove_host_trid(qpair);

		}

	}



	if (attempts == g_num_poll_groups) {

		SPDK_ERRLOG("No poll groups exist.\n");

		spdk_nvmf_qpair_disconnect(qpair, NULL, NULL);

		return;

	}



	event = spdk_event_allocate(core, nvmf_tgt_poll_group_add, qpair, pg);

	spdk_event_call(event);
@@ -228,9 +271,11 @@ nvmf_tgt_destroy_poll_group(void *ctx) 


	pg = &g_poll_groups[spdk_env_get_current_core()];



	if (pg->group) {

		spdk_nvmf_poll_group_destroy(pg->group);

		pg->group = NULL;

	}

}



static void

nvmf_tgt_create_poll_group_done(void *ctx)
@@ -247,7 +292,6 @@ nvmf_tgt_create_poll_group(void *ctx) 
	pg = &g_poll_groups[spdk_env_get_current_core()];



	pg->group = spdk_nvmf_poll_group_create(g_spdk_nvmf_tgt);

	assert(pg->group != NULL);

}



static void

lib/nvmf/nvmf.c

+9 −3
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) 
#define SPDK_NVMF_DEFAULT_IO_UNIT_SIZE 131072



typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status);

static void spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf);



/* supplied to a single call to nvmf_qpair_disconnect */

struct nvmf_qpair_disconnect_ctx {
@@ -149,7 +150,10 @@ spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 
			continue;

		}



		spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL);

		if (spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) {

			spdk_nvmf_tgt_destroy_poll_group(io_device, ctx_buf);

			return -1;

		}

	}



	group->poller = spdk_poller_register(spdk_nvmf_poll_group_poll, group, 0);
@@ -963,7 +967,7 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 
	return poll_group_update_subsystem(group, subsystem);

}



void

int

spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,

				   struct spdk_nvmf_subsystem *subsystem,

				   spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)
@@ -975,7 +979,7 @@ spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 


	rc = poll_group_update_subsystem(group, subsystem);

	if (rc) {

		sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE;

		spdk_nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL);

		goto fini;

	}
@@ -984,6 +988,8 @@ fini: 
	if (cb_fn) {

		cb_fn(cb_arg, rc);

	}



	return rc;

}



static void

lib/nvmf/nvmf_internal.h

+3 −3
Original line number Diff line number Diff line
@@ -263,7 +263,7 @@ int spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 
				       struct spdk_nvmf_transport *transport);

int spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,

		struct spdk_nvmf_subsystem *subsystem);

void spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,

int spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,

				       struct spdk_nvmf_subsystem *subsystem,

				       spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg);

void spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,

test/unit/lib/nvmf/ctrlr_discovery.c/ctrlr_discovery_ut.c

+2 −1
Original line number Diff line number Diff line
@@ -179,11 +179,12 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 
	return 0;

}



void

int

spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,

				   struct spdk_nvmf_subsystem *subsystem,

				   spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)

{

	return 0;

}



void

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

+2 −1
Original line number Diff line number Diff line
@@ -135,11 +135,12 @@ spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 
	return 0;

}



void

int

spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,

				   struct spdk_nvmf_subsystem *subsystem,

				   spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg)

{

	return 0;

}



void