Commit 2c13441b authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Ben Walker
Browse files

nvme_rdma: Destroy qpair after qpair is actually disconnected 



The RDMA transport can disconnect qpair asynchronously now.

Previously, we tried to release the resource of the qpair after disconnected.
However it did not work because it was done when deleting the qpair.
The admin qpair was not deleted in a ctrlr reset sequence.

This patch tries to satisfy the same aim again but by a different way.

Previously, we released the resource of the qpair before starting
actual disconnection process. This patch release the resource of the
qpair after the qpair is actually disconnected.

The related patches are:
b9518a55
eb09178a

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: Id6a814895a35b1589b781a91744ef872b42aaa69
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11783


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 4b732235

lib/nvme/nvme_rdma.c

+29 −27
Original line number Diff line number Diff line
@@ -1950,6 +1950,34 @@ nvme_rdma_ctrlr_create_qpair(struct spdk_nvme_ctrlr *ctrlr, 
static void

nvme_rdma_qpair_destroy(struct nvme_rdma_qpair *rqpair)

{

	struct spdk_nvme_qpair *qpair = &rqpair->qpair;

	struct nvme_rdma_ctrlr *rctrlr;

	struct nvme_rdma_cm_event_entry *entry, *tmp;



	spdk_rdma_free_mem_map(&rqpair->mr_map);

	nvme_rdma_unregister_reqs(rqpair);

	nvme_rdma_unregister_rsps(rqpair);



	if (rqpair->evt) {

		rdma_ack_cm_event(rqpair->evt);

		rqpair->evt = NULL;

	}



	/*

	 * This works because we have the controller lock both in

	 * this function and in the function where we add new events.

	 */

	if (qpair->ctrlr != NULL) {

		rctrlr = nvme_rdma_ctrlr(qpair->ctrlr);

		STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) {

			if (nvme_rdma_qpair(entry->evt->id->context) == rqpair) {

				STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link);

				rdma_ack_cm_event(entry->evt);

				STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link);

			}

		}

	}



	if (rqpair->cm_id) {

		if (rqpair->rdma_qp) {

			spdk_rdma_qp_destroy(rqpair->rdma_qp);
@@ -2021,42 +2049,16 @@ _nvme_rdma_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvm 
				  nvme_rdma_cm_event_cb disconnected_qpair_cb)

{

	struct nvme_rdma_qpair *rqpair = nvme_rdma_qpair(qpair);

	struct nvme_rdma_ctrlr *rctrlr = NULL;

	struct nvme_rdma_cm_event_entry *entry, *tmp;

	int rc;



	assert(disconnected_qpair_cb != NULL);



	rqpair->state = NVME_RDMA_QPAIR_STATE_EXITING;



	spdk_rdma_free_mem_map(&rqpair->mr_map);

	nvme_rdma_unregister_reqs(rqpair);

	nvme_rdma_unregister_rsps(rqpair);



	if (rqpair->evt) {

		rdma_ack_cm_event(rqpair->evt);

		rqpair->evt = NULL;

	}



	/*

	 * This works because we have the controller lock both in

	 * this function and in the function where we add new events.

	 */

	if (qpair->ctrlr != NULL) {

		rctrlr = nvme_rdma_ctrlr(qpair->ctrlr);

		STAILQ_FOREACH_SAFE(entry, &rctrlr->pending_cm_events, link, tmp) {

			if (nvme_rdma_qpair(entry->evt->id->context) == rqpair) {

				STAILQ_REMOVE(&rctrlr->pending_cm_events, entry, nvme_rdma_cm_event_entry, link);

				rdma_ack_cm_event(entry->evt);

				STAILQ_INSERT_HEAD(&rctrlr->free_cm_events, entry, link);

			}

		}

	}



	if (rqpair->cm_id) {

		if (rqpair->rdma_qp) {

			rc = spdk_rdma_qp_disconnect(rqpair->rdma_qp);

			if ((rctrlr != NULL) && (rc == 0)) {

			if ((qpair->ctrlr != NULL) && (rc == 0)) {

				rc = nvme_rdma_process_event_start(rqpair, RDMA_CM_EVENT_DISCONNECTED,

								   disconnected_qpair_cb);

				if (rc == 0) {