Commit fc86e792 authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

lib/nvme: switch poll group to use connect/disconnect semantics. 



This makes more sense within the context of the nvme driver and
helps us avoid the awkward situation of getting a failed_qp callback
on a qpair that simply hasn't been connected.

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: Ibac83c87c514ddcf7bd360af10fab462ae011112
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1734


Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent 300583e2

include/spdk/nvme.h

+10 −9
Original line number Diff line number Diff line
@@ -1961,10 +1961,11 @@ struct spdk_nvme_poll_group; 




/**

 * This function alerts the user to failed qpairs when calling

 * This function alerts the user to disconnected qpairs when calling

 * spdk_nvme_poll_group_process_completions.

 */

typedef void (*spdk_nvme_failed_qpair_cb)(struct spdk_nvme_qpair *qpair, void *poll_group_ctx);

typedef void (*spdk_nvme_disconnected_qpair_cb)(struct spdk_nvme_qpair *qpair,

		void *poll_group_ctx);



/**

 * Create a new poll group.
@@ -2011,19 +2012,19 @@ int spdk_nvme_poll_group_destroy(struct spdk_nvme_poll_group *group); 
/**

 * Poll for completions on all qpairs in this poll group.

 *

 * the failed_qpair_cb will be called for all failed qpairs in the poll group

 * the disconnected_qpair_cb will be called for all disconnected qpairs in the poll group

 * including qpairs which fail within the context of this call.

 * The user is responsible for trying to reconnect or destroy those qpairs.

 *

 * \param group The group on which to poll for completions.

 * \param completions_per_qpair The maximum number of completions per qpair.

 * \param failed_qpair_cb A callback function of type spdk_nvme_failed_qpair_cb. Must be non-NULL.

 * \param disconnected_qpair_cb A callback function of type spdk_nvme_disconnected_qpair_cb. Must be non-NULL.

 *

 * return The number of completions across all qpairs, -EINVAL if no failed_qpair_cb is passed, or

 * return The number of completions across all qpairs, -EINVAL if no disconnected_qpair_cb is passed, or

 * -EIO if the shared completion queue cannot be polled for the RDMA transport.

 */

int64_t spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,

		uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb);

		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);



/**

 * Retrieve the user context for this specific poll group.
@@ -3118,12 +3119,12 @@ struct spdk_nvme_transport_ops { 
	int (*poll_group_remove)(struct spdk_nvme_transport_poll_group *tgroup,

				 struct spdk_nvme_qpair *qpair);



	int (*poll_group_activate_qpair)(struct spdk_nvme_qpair *qpair);

	int (*poll_group_connect_qpair)(struct spdk_nvme_qpair *qpair);



	int (*poll_group_deactivate_qpair)(struct spdk_nvme_qpair *qpair);

	int (*poll_group_disconnect_qpair)(struct spdk_nvme_qpair *qpair);



	int64_t (*poll_group_process_completions)(struct spdk_nvme_transport_poll_group *tgroup,

			uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb);

			uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);



	int (*poll_group_destroy)(struct spdk_nvme_transport_poll_group *tgroup);

};

lib/nvme/nvme_internal.h

+7 −7
Original line number Diff line number Diff line
@@ -430,8 +430,8 @@ struct spdk_nvme_poll_group { 
struct spdk_nvme_transport_poll_group {

	struct spdk_nvme_poll_group			*group;

	const struct spdk_nvme_transport		*transport;

	STAILQ_HEAD(, spdk_nvme_qpair)			active_qpairs;

	STAILQ_HEAD(, spdk_nvme_qpair)			failed_qpairs;

	STAILQ_HEAD(, spdk_nvme_qpair)			connected_qpairs;

	STAILQ_HEAD(, spdk_nvme_qpair)			disconnected_qpairs;

	STAILQ_ENTRY(spdk_nvme_transport_poll_group)	link;

};
@@ -830,8 +830,8 @@ nvme_robust_mutex_unlock(pthread_mutex_t *mtx) 
}



/* Poll group management functions. */

int nvme_poll_group_activate_qpair(struct spdk_nvme_qpair *qpair);

int nvme_poll_group_deactivate_qpair(struct spdk_nvme_qpair *qpair);

int nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);

int nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);



/* Admin functions */

int	nvme_ctrlr_cmd_identify(struct spdk_nvme_ctrlr *ctrlr,
@@ -1185,10 +1185,10 @@ int nvme_transport_poll_group_add(struct spdk_nvme_transport_poll_group *tgroup, 
				  struct spdk_nvme_qpair *qpair);

int nvme_transport_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup,

				     struct spdk_nvme_qpair *qpair);

int nvme_transport_poll_group_deactivate_qpair(struct spdk_nvme_qpair *qpair);

int nvme_transport_poll_group_activate_qpair(struct spdk_nvme_qpair *qpair);

int nvme_transport_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair);

int nvme_transport_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair);

int64_t nvme_transport_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,

		uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb);

		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb);

int nvme_transport_poll_group_destroy(struct spdk_nvme_transport_poll_group *tgroup);

/*

 * Below ref related functions must be called with the global

lib/nvme/nvme_pcie.c

+5 −5
Original line number Diff line number Diff line
@@ -2436,13 +2436,13 @@ nvme_pcie_poll_group_create(void) 
}



static int

nvme_pcie_poll_group_activate_qpair(struct spdk_nvme_qpair *qpair)

nvme_pcie_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)

{

	return 0;

}



static int

nvme_pcie_poll_group_deactivate_qpair(struct spdk_nvme_qpair *qpair)

nvme_pcie_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)

{

	return 0;

}
@@ -2463,7 +2463,7 @@ nvme_pcie_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup, 


static int64_t

nvme_pcie_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,

		uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb)

		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)

{

	return -ENOTSUP;

}
@@ -2506,8 +2506,8 @@ const struct spdk_nvme_transport_ops pcie_ops = { 
	.admin_qpair_abort_aers = nvme_pcie_admin_qpair_abort_aers,



	.poll_group_create = nvme_pcie_poll_group_create,

	.poll_group_activate_qpair = nvme_pcie_poll_group_activate_qpair,

	.poll_group_deactivate_qpair = nvme_pcie_poll_group_deactivate_qpair,

	.poll_group_connect_qpair = nvme_pcie_poll_group_connect_qpair,

	.poll_group_disconnect_qpair = nvme_pcie_poll_group_disconnect_qpair,

	.poll_group_add = nvme_pcie_poll_group_add,

	.poll_group_remove = nvme_pcie_poll_group_remove,

	.poll_group_process_completions = nvme_pcie_poll_group_process_completions,

lib/nvme/nvme_poll_group.c

+7 −7
Original line number Diff line number Diff line
@@ -101,31 +101,31 @@ spdk_nvme_poll_group_remove(struct spdk_nvme_poll_group *group, struct spdk_nvme 
}



int

nvme_poll_group_activate_qpair(struct spdk_nvme_qpair *qpair)

nvme_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)

{

	return nvme_transport_poll_group_activate_qpair(qpair);

	return nvme_transport_poll_group_connect_qpair(qpair);

}



int

nvme_poll_group_deactivate_qpair(struct spdk_nvme_qpair *qpair)

nvme_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)

{

	return nvme_transport_poll_group_deactivate_qpair(qpair);

	return nvme_transport_poll_group_disconnect_qpair(qpair);

}



int64_t

spdk_nvme_poll_group_process_completions(struct spdk_nvme_poll_group *group,

		uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb)

		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)

{

	struct spdk_nvme_transport_poll_group *tgroup;

	int64_t local_completions = 0, error_reason = 0, num_completions = 0;



	if (failed_qpair_cb == NULL) {

	if (disconnected_qpair_cb == NULL) {

		return -EINVAL;

	}



	STAILQ_FOREACH(tgroup, &group->tgroups, link) {

		local_completions = nvme_transport_poll_group_process_completions(tgroup, completions_per_qpair,

				    failed_qpair_cb);

				    disconnected_qpair_cb);

		if (local_completions < 0 && error_reason == 0) {

			error_reason = local_completions;

		} else {

lib/nvme/nvme_rdma.c

+5 −5
Original line number Diff line number Diff line
@@ -2154,13 +2154,13 @@ nvme_rdma_poll_group_create(void) 
}



static int

nvme_rdma_poll_group_activate_qpair(struct spdk_nvme_qpair *qpair)

nvme_rdma_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)

{

	return 0;

}



static int

nvme_rdma_poll_group_deactivate_qpair(struct spdk_nvme_qpair *qpair)

nvme_rdma_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)

{

	return 0;

}
@@ -2181,7 +2181,7 @@ nvme_rdma_poll_group_remove(struct spdk_nvme_transport_poll_group *tgroup, 


static int64_t

nvme_rdma_poll_group_process_completions(struct spdk_nvme_transport_poll_group *tgroup,

		uint32_t completions_per_qpair, spdk_nvme_failed_qpair_cb failed_qpair_cb)

		uint32_t completions_per_qpair, spdk_nvme_disconnected_qpair_cb disconnected_qpair_cb)

{

	return -ENOTSUP;

}
@@ -2226,8 +2226,8 @@ const struct spdk_nvme_transport_ops rdma_ops = { 
	.admin_qpair_abort_aers = nvme_rdma_admin_qpair_abort_aers,



	.poll_group_create = nvme_rdma_poll_group_create,

	.poll_group_activate_qpair = nvme_rdma_poll_group_activate_qpair,

	.poll_group_deactivate_qpair = nvme_rdma_poll_group_deactivate_qpair,

	.poll_group_connect_qpair = nvme_rdma_poll_group_connect_qpair,

	.poll_group_disconnect_qpair = nvme_rdma_poll_group_disconnect_qpair,

	.poll_group_add = nvme_rdma_poll_group_add,

	.poll_group_remove = nvme_rdma_poll_group_remove,

	.poll_group_process_completions = nvme_rdma_poll_group_process_completions,