Commit 87e9ae40 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

bdev/nvme: Poll group manages ctrlr channels for qpair disconnect 



Poll group has a list of the associated ctrlr channels to update
them when the corresponding I/O qpair is disconnected to do path
failover.

Another idea is that poll group has a list of the associated bdev
channels. However, two or more bdev channels may share a single
ctrlr channel and I/O qpair is per ctrlr channel. What we want to do
by this addition is to stop I/O submission to any failed I/O qpair
and choose alternative I/O qpair. Hence we take the first idea.

Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Ia287bd1b803313e66b8505a19694a40133b675f1
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8124


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent cf0dbbb9

module/bdev/nvme/bdev_nvme.c

+7 −0
Original line number Diff line number Diff line
@@ -1284,6 +1284,7 @@ bdev_nvme_create_ctrlr_channel_cb(void *io_device, void *ctx_buf) 
	}



	ctrlr_ch->group = spdk_io_channel_get_ctx(pg_ch);

	TAILQ_INSERT_TAIL(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq);



#ifdef SPDK_CONFIG_VTUNE

	ctrlr_ch->group->collect_spin_stat = true;
@@ -1315,6 +1316,8 @@ bdev_nvme_destroy_ctrlr_channel_cb(void *io_device, void *ctx_buf) 


	bdev_nvme_destroy_qpair(ctrlr_ch);



	TAILQ_REMOVE(&ctrlr_ch->group->ctrlr_ch_list, ctrlr_ch, tailq);



	spdk_put_io_channel(spdk_io_channel_from_ctx(ctrlr_ch->group));

}
@@ -1349,6 +1352,8 @@ bdev_nvme_create_poll_group_cb(void *io_device, void *ctx_buf) 
{

	struct nvme_poll_group *group = ctx_buf;



	TAILQ_INIT(&group->ctrlr_ch_list);



	group->group = spdk_nvme_poll_group_create(group, &g_bdev_nvme_accel_fn_table);

	if (group->group == NULL) {

		return -1;
@@ -1378,6 +1383,8 @@ bdev_nvme_destroy_poll_group_cb(void *io_device, void *ctx_buf) 
{

	struct nvme_poll_group *group = ctx_buf;



	assert(TAILQ_EMPTY(&group->ctrlr_ch_list));



	if (group->accel_channel) {

		spdk_put_io_channel(group->accel_channel);

	}

module/bdev/nvme/bdev_nvme.h

+12 −10
Original line number Diff line number Diff line
@@ -140,20 +140,11 @@ struct nvme_bdev { 
	bool			opal;

};



struct nvme_poll_group {

	struct spdk_nvme_poll_group		*group;

	struct spdk_io_channel			*accel_channel;

	struct spdk_poller			*poller;

	bool					collect_spin_stat;

	uint64_t				spin_ticks;

	uint64_t				start_ticks;

	uint64_t				end_ticks;

};



struct nvme_ctrlr_channel {

	struct spdk_nvme_qpair		*qpair;

	struct nvme_poll_group		*group;

	TAILQ_HEAD(, spdk_bdev_io)	pending_resets;

	TAILQ_ENTRY(nvme_ctrlr_channel)	tailq;

};



#define nvme_ctrlr_channel_get_ctrlr(ctrlr_ch)	\
@@ -164,6 +155,17 @@ struct nvme_bdev_channel { 
	struct nvme_ctrlr_channel	*ctrlr_ch;

};



struct nvme_poll_group {

	struct spdk_nvme_poll_group		*group;

	struct spdk_io_channel			*accel_channel;

	struct spdk_poller			*poller;

	bool					collect_spin_stat;

	uint64_t				spin_ticks;

	uint64_t				start_ticks;

	uint64_t				end_ticks;

	TAILQ_HEAD(, nvme_ctrlr_channel)	ctrlr_ch_list;

};



struct nvme_ctrlr *nvme_ctrlr_get_by_name(const char *name);



typedef void (*nvme_ctrlr_for_each_fn)(struct nvme_ctrlr *nvme_ctrlr, void *ctx);