Commit 736b9da0 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

nvme: Do Controller Level Reset when disconnecting adminq for PCIe 



As described in the previous patches, we need to delete all I/O
SQ/CQs before aborting trackers when disconnecting a controller.

The following patches reorder the operations. This patch changes
adminq disconnection to initiate a Controller Level Reset and
adminq completion processes it if ctrlr->is_disconnecting is true.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I64f06bae2ce8a9127124029fd042db0028198e3c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12560


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarMichael Haeuptle <michaelhaeuptle@gmail.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 813756e7

lib/nvme/nvme_pcie_common.c

+20 −2
Original line number Diff line number Diff line
@@ -602,7 +602,16 @@ nvme_pcie_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qp 
void

nvme_pcie_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)

{

	if (!nvme_qpair_is_admin_queue(qpair) || !ctrlr->is_disconnecting) {

		nvme_transport_ctrlr_disconnect_qpair_done(qpair);

	} else {

		/* If this function is called for the admin qpair via spdk_nvme_ctrlr_reset()

		 * or spdk_nvme_ctrlr_disconnect(), initiate a Controller Level Reset.

		 * Then we can abort trackers safely because the Controller Level Reset deletes

		 * all I/O SQ/CQs.

		 */

		nvme_ctrlr_disable(ctrlr);

	}

}



/* Used when dst points to MMIO (i.e. CMB) in a virtual machine - in these cases we must
@@ -980,10 +989,19 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_ 
		nvme_pcie_qpair_check_timeout(qpair);

	}



	/* Before returning, complete any pending admin request. */

	/* Before returning, complete any pending admin request or

	 * process the admin qpair disconnection.

	 */

	if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {

		nvme_pcie_qpair_complete_pending_admin_request(qpair);



		if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {

			rc = nvme_ctrlr_disable_poll(qpair->ctrlr);

			if (rc == 0) {

				nvme_transport_ctrlr_disconnect_qpair_done(qpair);

			}

		}



		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);

	}

test/common/lib/nvme/common_stubs.h

+2 −0
Original line number Diff line number Diff line
@@ -92,6 +92,8 @@ DEFINE_STUB(nvme_get_transport, const struct spdk_nvme_transport *, (const char 
	    NULL);

DEFINE_STUB(spdk_nvme_qpair_process_completions, int32_t, (struct spdk_nvme_qpair *qpair,

		uint32_t max_completions), 0);

DEFINE_STUB_V(nvme_ctrlr_disable, (struct spdk_nvme_ctrlr *ctrlr));

DEFINE_STUB(nvme_ctrlr_disable_poll, int, (struct spdk_nvme_ctrlr *ctrlr), 0);



/* Fabric transports only */

DEFINE_STUB_V(nvme_ctrlr_disconnect_qpair, (struct spdk_nvme_qpair *qpair));

test/unit/lib/nvme/nvme_pcie_common.c/nvme_pcie_common_ut.c

+4 −0
Original line number Diff line number Diff line
@@ -75,6 +75,10 @@ DEFINE_STUB(nvme_request_check_timeout, int, (struct nvme_request *req, uint16_t 
		struct spdk_nvme_ctrlr_process *active_proc, uint64_t now_tick), 0);

DEFINE_STUB(spdk_strerror, const char *, (int errnum), NULL);



DEFINE_STUB_V(nvme_ctrlr_disable, (struct spdk_nvme_ctrlr *ctrlr));



DEFINE_STUB(nvme_ctrlr_disable_poll, int, (struct spdk_nvme_ctrlr *ctrlr), 0);



DEFINE_STUB_V(nvme_transport_ctrlr_disconnect_qpair_done, (struct spdk_nvme_qpair *qpair));



int nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id,