Commit f9fba507 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

bdev/nvme: Redirect the reset ctrlr operation into nvme_ctrlr->thread 



In the following patches, we want to retry reconnect if reconnect failed
in a reset ctrlr sequence but we want to delay the retry. While
we wait the delayed retry, we want to quiesce ctrlr completely.

As part of quiesce ctrlr operations, we want to pause adminq poller but
we need to do it on the nvme_ctrlr->thread.

If a reset ctrlr sequence runs on the nvme_ctrlr->thread, we can avoid
redirecting the pending destruct request at completion too.

So we redirect the reset ctrlr sequence into the nvme_ctrlr->thread.

Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I538b962e2a7b5cf00ebbac2a1e888482ddeeee61
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10075


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
parent 74dcf4aa

module/bdev/nvme/bdev_nvme.c

+23 −17
Original line number Diff line number Diff line
@@ -515,10 +515,8 @@ nvme_ctrlr_unregister_cb(void *io_device) 
}



static void

nvme_ctrlr_unregister(void *ctx)

nvme_ctrlr_unregister(struct nvme_ctrlr *nvme_ctrlr)

{

	struct nvme_ctrlr *nvme_ctrlr = ctx;



	spdk_io_device_unregister(nvme_ctrlr, nvme_ctrlr_unregister_cb);

}
@@ -1217,6 +1215,8 @@ _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status) 
	void *reset_cb_arg = nvme_ctrlr->reset_cb_arg;

	bool complete_pending_destruct = false;



	assert(nvme_ctrlr->thread == spdk_get_thread());



	nvme_ctrlr->reset_cb_fn = NULL;

	nvme_ctrlr->reset_cb_arg = NULL;
@@ -1249,8 +1249,7 @@ _bdev_nvme_reset_complete(struct spdk_io_channel_iter *i, int status) 
	}



	if (complete_pending_destruct) {

		spdk_thread_send_msg(nvme_ctrlr->thread, nvme_ctrlr_unregister,

				     nvme_ctrlr);

		nvme_ctrlr_unregister(nvme_ctrlr);

	}

}
@@ -1343,6 +1342,23 @@ bdev_nvme_reset_destroy_qpair(struct spdk_io_channel_iter *i) 
	spdk_for_each_channel_continue(i, 0);

}



static void

_bdev_nvme_reset(void *ctx)

{

	struct nvme_ctrlr *nvme_ctrlr = ctx;



	assert(nvme_ctrlr->resetting == true);

	assert(nvme_ctrlr->thread == spdk_get_thread());



	spdk_nvme_ctrlr_prepare_for_reset(nvme_ctrlr->ctrlr);



	/* First, delete all NVMe I/O queue pairs. */

	spdk_for_each_channel(nvme_ctrlr,

			      bdev_nvme_reset_destroy_qpair,

			      NULL,

			      bdev_nvme_reset_ctrlr);

}



static int

bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr)

{
@@ -1360,14 +1376,8 @@ bdev_nvme_reset(struct nvme_ctrlr *nvme_ctrlr) 


	nvme_ctrlr->resetting = true;

	pthread_mutex_unlock(&nvme_ctrlr->mutex);

	spdk_nvme_ctrlr_prepare_for_reset(nvme_ctrlr->ctrlr);



	/* First, delete all NVMe I/O queue pairs. */

	spdk_for_each_channel(nvme_ctrlr,

			      bdev_nvme_reset_destroy_qpair,

			      NULL,

			      bdev_nvme_reset_ctrlr);



	spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);

	return 0;

}
@@ -1563,11 +1573,7 @@ bdev_nvme_failover(struct nvme_ctrlr *nvme_ctrlr, bool remove) 


	rc = bdev_nvme_failover_start(nvme_ctrlr, remove);

	if (rc == 0) {

		/* First, delete all NVMe I/O queue pairs. */

		spdk_for_each_channel(nvme_ctrlr,

				      bdev_nvme_reset_destroy_qpair,

				      NULL,

				      bdev_nvme_reset_ctrlr);

		spdk_thread_send_msg(nvme_ctrlr->thread, _bdev_nvme_reset, nvme_ctrlr);

	} else if (rc != -EALREADY) {

		return rc;

	}

test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c

+12 −16
Original line number Diff line number Diff line
@@ -1290,7 +1290,7 @@ test_reset_ctrlr(void) 
	CU_ASSERT(ctrlr_ch1->qpair != NULL);

	CU_ASSERT(ctrlr_ch2->qpair != NULL);



	poll_thread_times(0, 1);

	poll_thread_times(0, 3);

	CU_ASSERT(ctrlr_ch1->qpair == NULL);

	CU_ASSERT(ctrlr_ch2->qpair != NULL);
@@ -1299,7 +1299,7 @@ test_reset_ctrlr(void) 
	CU_ASSERT(ctrlr_ch2->qpair == NULL);

	CU_ASSERT(ctrlr.is_failed == true);



	poll_thread_times(1, 1);

	poll_thread_times(0, 1);

	CU_ASSERT(ctrlr.is_failed == false);



	poll_thread_times(0, 1);
@@ -1312,13 +1312,11 @@ test_reset_ctrlr(void) 
	CU_ASSERT(nvme_ctrlr->resetting == true);

	CU_ASSERT(curr_trid->is_failed == true);



	poll_thread_times(1, 1);

	CU_ASSERT(nvme_ctrlr->resetting == true);

	poll_thread_times(0, 1);

	poll_thread_times(0, 2);

	CU_ASSERT(nvme_ctrlr->resetting == true);

	poll_thread_times(1, 1);

	CU_ASSERT(nvme_ctrlr->resetting == true);

	poll_thread_times(1, 1);

	poll_thread_times(0, 1);

	CU_ASSERT(nvme_ctrlr->resetting == false);

	CU_ASSERT(curr_trid->is_failed == false);
@@ -2829,13 +2827,13 @@ test_reconnect_qpair(void) 
	CU_ASSERT(ctrlr_ch2->qpair == NULL);

	CU_ASSERT(nvme_ctrlr->resetting == true);



	poll_thread_times(0, 1);

	poll_thread_times(0, 2);

	poll_thread_times(1, 1);

	CU_ASSERT(ctrlr_ch1->qpair == NULL);

	CU_ASSERT(ctrlr_ch2->qpair == NULL);

	CU_ASSERT(ctrlr->is_failed == true);



	poll_thread_times(1, 1);

	poll_thread_times(0, 1);

	CU_ASSERT(ctrlr->is_failed == false);



	poll_thread_times(0, 1);
@@ -2844,10 +2842,9 @@ test_reconnect_qpair(void) 
	CU_ASSERT(ctrlr_ch2->qpair != NULL);

	CU_ASSERT(nvme_ctrlr->resetting == true);



	poll_thread_times(0, 2);

	poll_thread_times(1, 1);

	poll_thread_times(0, 1);

	poll_thread_times(1, 1);

	poll_thread_times(1, 1);

	CU_ASSERT(nvme_ctrlr->resetting == false);



	poll_threads();
@@ -2864,16 +2861,15 @@ test_reconnect_qpair(void) 
	CU_ASSERT(ctrlr_ch2->qpair == NULL);

	CU_ASSERT(nvme_ctrlr->resetting == true);



	poll_thread_times(0, 1);

	poll_thread_times(0, 2);

	poll_thread_times(1, 1);

	CU_ASSERT(ctrlr_ch1->qpair == NULL);

	CU_ASSERT(ctrlr_ch2->qpair == NULL);

	CU_ASSERT(ctrlr->is_failed == true);



	poll_thread_times(0, 2);

	poll_thread_times(1, 1);

	poll_thread_times(0, 1);

	poll_thread_times(1, 1);

	poll_thread_times(1, 1);

	CU_ASSERT(ctrlr->is_failed == true);

	CU_ASSERT(nvme_ctrlr->resetting == false);

	CU_ASSERT(ctrlr_ch1->qpair == NULL);
@@ -3654,7 +3650,7 @@ test_reset_bdev_ctrlr(void) 
	CU_ASSERT(nvme_ctrlr1->resetting == true);

	CU_ASSERT(nvme_ctrlr1->reset_cb_arg == first_bio);



	poll_thread_times(0, 1);

	poll_thread_times(0, 2);

	CU_ASSERT(io_path11->ctrlr_ch->qpair == NULL);

	CU_ASSERT(io_path21->ctrlr_ch->qpair != NULL);
@@ -3686,7 +3682,7 @@ test_reset_bdev_ctrlr(void) 
	CU_ASSERT(first_bio->io_path == io_path12);

	CU_ASSERT(nvme_ctrlr2->resetting == true);



	poll_thread_times(0, 1);

	poll_thread_times(0, 2);

	CU_ASSERT(io_path12->ctrlr_ch->qpair == NULL);

	CU_ASSERT(io_path22->ctrlr_ch->qpair != NULL);
@@ -3927,7 +3923,7 @@ test_retry_io_if_ctrlr_is_resetting(void) 
	ctrlr_ch->qpair->is_connected = false;

	ctrlr->is_failed = true;



	poll_thread_times(0, 3);

	poll_thread_times(0, 4);



	CU_ASSERT(ctrlr_ch->qpair == NULL);

	CU_ASSERT(nvme_ctrlr->resetting == true);