Commit 07034ca3 authored by Artsiom Koltun's avatar Artsiom Koltun Committed by Tomasz Zawadzki
Browse files

bdev/nvme: wait for detached nvme controller in rpc 



rpc_bdev_nvme_detach_controller did not wait for an operation
completion. If a sequence attach-detach-attach with the same
name is run, the script might fail if the detach is not
complete on an active path before the second attach.
Wait for a path does not exist before sending back a json rpc
response.

Change-Id: Id0e1fb49e69745acf1479188307aba20bf1b2020
Signed-off-by: default avatarArtsiom Koltun <artsiom.koltun@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/19340


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 829f837c

module/bdev/nvme/bdev_nvme.c

+122 −14
Original line number Diff line number Diff line
@@ -5972,8 +5972,26 @@ bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
	return 0;

}



struct bdev_nvme_delete_ctx {

	char                        *name;

	struct nvme_path_id         path_id;

	bdev_nvme_delete_done_fn    delete_done;

	void                        *delete_done_ctx;

	uint64_t                    timeout_ticks;

	struct spdk_poller          *poller;

};



static void

free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx)

{

	if (ctx != NULL) {

		free(ctx->name);

		free(ctx);

	}

}



static bool

nvme_path_should_delete(struct nvme_path_id *p, const struct nvme_path_id *path_id)

nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id)

{

	if (path_id->trid.trtype != 0) {

		if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
@@ -6026,6 +6044,59 @@ nvme_path_should_delete(struct nvme_path_id *p, const struct nvme_path_id *path_ 
	return true;

}



static bool

nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id)

{

	struct nvme_bdev_ctrlr  *nbdev_ctrlr;

	struct nvme_ctrlr       *ctrlr;

	struct nvme_path_id     *p;



	pthread_mutex_lock(&g_bdev_nvme_mutex);

	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);

	if (!nbdev_ctrlr) {

		pthread_mutex_unlock(&g_bdev_nvme_mutex);

		return false;

	}



	TAILQ_FOREACH(ctrlr, &nbdev_ctrlr->ctrlrs, tailq) {

		pthread_mutex_lock(&ctrlr->mutex);

		TAILQ_FOREACH(p, &ctrlr->trids, link) {

			if (nvme_path_id_compare(p, path_id)) {

				pthread_mutex_unlock(&ctrlr->mutex);

				pthread_mutex_unlock(&g_bdev_nvme_mutex);

				return true;

			}

		}

		pthread_mutex_unlock(&ctrlr->mutex);

	}

	pthread_mutex_unlock(&g_bdev_nvme_mutex);



	return false;

}



static int

bdev_nvme_delete_complete_poll(void *arg)

{

	struct bdev_nvme_delete_ctx     *ctx = arg;

	int                             rc = 0;



	if (nvme_path_id_exists(ctx->name, &ctx->path_id)) {

		if (ctx->timeout_ticks > spdk_get_ticks()) {

			return SPDK_POLLER_BUSY;

		}



		SPDK_ERRLOG("NVMe path '%s' still exists after delete\n", ctx->name);

		rc = -ETIMEDOUT;

	}



	spdk_poller_unregister(&ctx->poller);



	ctx->delete_done(ctx->delete_done_ctx, rc);

	free_bdev_nvme_delete_ctx(ctx);



	return SPDK_POLLER_BUSY;

}



static int

_bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)

{
@@ -6040,7 +6111,7 @@ _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path 
			break;

		}



		if (!nvme_path_should_delete(p, path_id)) {

		if (!nvme_path_id_compare(p, path_id)) {

			continue;

		}
@@ -6050,7 +6121,7 @@ _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path 
		rc = 0;

	}



	if (p == NULL || !nvme_path_should_delete(p, path_id)) {

	if (p == NULL || !nvme_path_id_compare(p, path_id)) {

		pthread_mutex_unlock(&nvme_ctrlr->mutex);

		return rc;

	}
@@ -6082,14 +6153,17 @@ _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path 
}



int

bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id)

bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,

		 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx)

{

	struct nvme_bdev_ctrlr		*nbdev_ctrlr;

	struct nvme_ctrlr		*nvme_ctrlr, *tmp_nvme_ctrlr;

	struct bdev_nvme_delete_ctx     *ctx = NULL;

	int				rc = -ENXIO, _rc;



	if (name == NULL || path_id == NULL) {

		return -EINVAL;

		rc = -EINVAL;

		goto exit;

	}



	pthread_mutex_lock(&g_bdev_nvme_mutex);
@@ -6099,15 +6173,16 @@ bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id) 
		pthread_mutex_unlock(&g_bdev_nvme_mutex);



		SPDK_ERRLOG("Failed to find NVMe bdev controller\n");

		return -ENODEV;

		rc = -ENODEV;

		goto exit;

	}



	TAILQ_FOREACH_SAFE(nvme_ctrlr, &nbdev_ctrlr->ctrlrs, tailq, tmp_nvme_ctrlr) {

		_rc = _bdev_nvme_delete(nvme_ctrlr, path_id);

		if (_rc < 0 && _rc != -ENXIO) {

			pthread_mutex_unlock(&g_bdev_nvme_mutex);



			return _rc;

			rc = _rc;

			goto exit;

		} else if (_rc == 0) {

			/* We traverse all remaining nvme_ctrlrs even if one nvme_ctrlr

			 * was deleted successfully. To remember the successful deletion,
@@ -6119,7 +6194,40 @@ bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id) 


	pthread_mutex_unlock(&g_bdev_nvme_mutex);



	/* All nvme_ctrlrs were deleted or no nvme_ctrlr which had the trid was found. */

	if (rc != 0 || delete_done == NULL) {

		goto exit;

	}



	ctx = calloc(1, sizeof(*ctx));

	if (ctx == NULL) {

		SPDK_ERRLOG("Failed to allocate context for bdev_nvme_delete\n");

		rc = -ENOMEM;

		goto exit;

	}



	ctx->name = strdup(name);

	if (ctx->name == NULL) {

		SPDK_ERRLOG("Failed to copy controller name for deletion\n");

		rc = -ENOMEM;

		goto exit;

	}



	ctx->delete_done = delete_done;

	ctx->delete_done_ctx = delete_done_ctx;

	ctx->path_id = *path_id;

	ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz();

	ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000);

	if (ctx->poller == NULL) {

		SPDK_ERRLOG("Failed to register bdev_nvme_delete poller\n");

		rc = -ENOMEM;

		goto exit;

	}



exit:

	if (rc != 0) {

		free_bdev_nvme_delete_ctx(ctx);

	}



	return rc;

}
@@ -6271,7 +6379,7 @@ _stop_discovery(void *_ctx) 


		entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);

		path.trid = entry_ctx->trid;

		bdev_nvme_delete(entry_ctx->name, &path);

		bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

		TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);

		free(entry_ctx);

	}
@@ -6361,7 +6469,7 @@ discovery_remove_controllers(struct discovery_ctx *ctx) 
					  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);



			path.trid = entry_ctx->trid;

			bdev_nvme_delete(entry_ctx->name, &path);

			bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

			TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);

			free(entry_ctx);

		}

module/bdev/nvme/bdev_nvme.h

+10 −2
Original line number Diff line number Diff line
@@ -334,15 +334,23 @@ void bdev_nvme_mdns_discovery_config_json(struct spdk_json_write_ctx *w); 


struct spdk_nvme_ctrlr *bdev_nvme_get_ctrlr(struct spdk_bdev *bdev);



typedef void (*bdev_nvme_delete_done_fn)(void *ctx, int rc);



/**

 * Delete NVMe controller with all bdevs on top of it, or delete the specified path

 * if there is any alternative path. Requires to pass name of NVMe controller.

 *

 * \param name NVMe controller name

 * \param path_id The specified path to remove (optional)

 * \return zero on success, -EINVAL on wrong parameters or -ENODEV if controller is not found

 * \param delete_done Callback function on delete complete (optional)

 * \param delete_done_ctx Context passed to callback (optional)

 * \return zero on success,

 *		-EINVAL on wrong parameters or

 *		-ENODEV if controller is not found or

 *		-ENOMEM on no memory

 */

int bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id);

int bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,

		     bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx);



enum nvme_ctrlr_op {

	NVME_CTRLR_OP_RESET = 1,

module/bdev/nvme/bdev_nvme_rpc.c

+13 −4
Original line number Diff line number Diff line
@@ -709,6 +709,18 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_detach_controller_dec 
	{"hostsvcid", offsetof(struct rpc_bdev_nvme_detach_controller, hostsvcid), spdk_json_decode_string, true},

};



static void

rpc_bdev_nvme_detach_controller_done(void *arg, int rc)

{

	struct spdk_jsonrpc_request *request = arg;



	if (rc == 0) {

		spdk_jsonrpc_send_bool_response(request, true);

	} else {

		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));

	}

}



static void

rpc_bdev_nvme_detach_controller(struct spdk_jsonrpc_request *request,

				const struct spdk_json_val *params)
@@ -810,15 +822,12 @@ rpc_bdev_nvme_detach_controller(struct spdk_jsonrpc_request *request, 
		snprintf(path.hostid.hostsvcid, maxlen, "%s", req.hostsvcid);

	}



	rc = bdev_nvme_delete(req.name, &path);

	rc = bdev_nvme_delete(req.name, &path, rpc_bdev_nvme_detach_controller_done, request);



	if (rc != 0) {

		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));

		goto cleanup;

	}



	spdk_jsonrpc_send_bool_response(request, true);



cleanup:

	free_rpc_bdev_nvme_detach_controller(&req);

}

test/unit/lib/bdev/nvme/bdev_nvme.c/bdev_nvme_ut.c

+82 −46
Original line number Diff line number Diff line
@@ -1367,7 +1367,7 @@ test_create_ctrlr(void) 


	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") != NULL);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") != NULL);
@@ -1520,7 +1520,7 @@ test_reset_ctrlr(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -1568,7 +1568,7 @@ test_race_between_reset_and_destruct_ctrlr(void) 
	/* Try destructing ctrlr while ctrlr is being reset, but it will be deferred. */

	set_thread(0);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);

	CU_ASSERT(nvme_ctrlr->destruct == true);
@@ -1727,7 +1727,7 @@ test_failover_ctrlr(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -1835,7 +1835,7 @@ test_race_between_failover_and_add_secondary_trid(void) 


	set_thread(0);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -1975,7 +1975,7 @@ test_pending_reset(void) 


	set_thread(0);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2040,7 +2040,7 @@ test_attach_ctrlr(void) 
	SPDK_CU_ASSERT_FATAL(nvme_ctrlr != NULL);

	CU_ASSERT(nvme_ctrlr->ctrlr == ctrlr);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2075,7 +2075,7 @@ test_attach_ctrlr(void) 
	SPDK_CU_ASSERT_FATAL(nbdev != NULL);

	CU_ASSERT(bdev_nvme_get_ctrlr(&nbdev->disk) == ctrlr);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2106,7 +2106,7 @@ test_attach_ctrlr(void) 


	CU_ASSERT(attached_names[0] == NULL);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2206,7 +2206,7 @@ test_aer_cb(void) 
	CU_ASSERT(nvme_ctrlr_get_ns(nvme_ctrlr, 2)->ana_state == SPDK_NVME_ANA_INACCESSIBLE_STATE);

	CU_ASSERT(nvme_ctrlr_get_ns(nvme_ctrlr, 4)->ana_state == SPDK_NVME_ANA_CHANGE_STATE);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2405,7 +2405,7 @@ test_submit_nvme_cmd(void) 


	set_thread(1);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2470,11 +2470,11 @@ test_add_remove_trid(void) 
	CU_ASSERT(ctrid != NULL);



	/* trid3 is not in the registered list. */

	rc = bdev_nvme_delete("nvme0", &path3);

	rc = bdev_nvme_delete("nvme0", &path3, NULL, NULL);

	CU_ASSERT(rc == -ENXIO);



	/* trid2 is not used, and simply removed. */

	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);

	TAILQ_FOREACH(ctrid, &nvme_ctrlr->trids, link) {
@@ -2525,7 +2525,7 @@ test_add_remove_trid(void) 
	SPDK_CU_ASSERT_FATAL(ctrid != NULL);

	CU_ASSERT(spdk_nvme_transport_id_compare(&ctrid->trid, &path3.trid) == 0);



	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);

	TAILQ_FOREACH(ctrid, &nvme_ctrlr->trids, link) {
@@ -2535,7 +2535,7 @@ test_add_remove_trid(void) 
	/* path1 is currently used and path3 is an alternative path.

	 * If we remove path1, path is changed to path3.

	 */

	rc = bdev_nvme_delete("nvme0", &path1);

	rc = bdev_nvme_delete("nvme0", &path1, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);

	CU_ASSERT(nvme_ctrlr->resetting == true);
@@ -2553,7 +2553,7 @@ test_add_remove_trid(void) 
	/* path3 is the current and only path. If we remove path3, the corresponding

	 * nvme_ctrlr is removed.

	 */

	rc = bdev_nvme_delete("nvme0", &path3);

	rc = bdev_nvme_delete("nvme0", &path3, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);
@@ -2597,7 +2597,7 @@ test_add_remove_trid(void) 
	CU_ASSERT(ctrid != NULL);



	/* If trid is not specified, nvme_ctrlr itself is removed. */

	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == nvme_ctrlr);
@@ -2845,7 +2845,7 @@ test_abort(void) 


	set_thread(1);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -2887,7 +2887,7 @@ test_get_io_qpair(void) 


	spdk_put_io_channel(ch);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3078,7 +3078,7 @@ test_init_ana_log_page(void) 
	CU_ASSERT(nvme_ctrlr_get_ns(nvme_ctrlr, 4)->bdev != NULL);

	CU_ASSERT(nvme_ctrlr_get_ns(nvme_ctrlr, 5)->bdev != NULL);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3312,7 +3312,7 @@ test_reconnect_qpair(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3395,7 +3395,7 @@ test_create_bdev_ctrlr(void) 
	CU_ASSERT(nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, &path2.trid) != NULL);



	/* Delete two ctrlrs at once. */

	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	CU_ASSERT(nvme_bdev_ctrlr_get_by_name("nvme0") == nbdev_ctrlr);
@@ -3438,7 +3438,7 @@ test_create_bdev_ctrlr(void) 
	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name("nvme0");

	SPDK_CU_ASSERT_FATAL(nbdev_ctrlr != NULL);



	rc = bdev_nvme_delete("nvme0", &path1);

	rc = bdev_nvme_delete("nvme0", &path1, NULL, NULL);

	CU_ASSERT(rc == 0);



	CU_ASSERT(nvme_bdev_ctrlr_get_by_name("nvme0") == nbdev_ctrlr);
@@ -3453,7 +3453,7 @@ test_create_bdev_ctrlr(void) 
	CU_ASSERT(nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, &path1.trid) == NULL);

	CU_ASSERT(nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, &path2.trid) != NULL);



	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);



	CU_ASSERT(nvme_bdev_ctrlr_get_by_name("nvme0") == nbdev_ctrlr);
@@ -3593,7 +3593,7 @@ test_add_multi_ns_to_bdev(void) 
	CU_ASSERT(bdev4->ref == 1);



	/* Test if nvme_bdevs can be deleted by deleting ctrlr one by one. */

	rc = bdev_nvme_delete("nvme0", &path1);

	rc = bdev_nvme_delete("nvme0", &path1, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3604,7 +3604,7 @@ test_add_multi_ns_to_bdev(void) 
	CU_ASSERT(nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, &path1.trid) == NULL);

	CU_ASSERT(nvme_bdev_ctrlr_get_ctrlr(nbdev_ctrlr, &path2.trid) == nvme_ctrlr2);



	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3810,7 +3810,7 @@ test_add_multi_io_paths_to_nbdev_ch(void) 
	SPDK_CU_ASSERT_FATAL(io_path3 != NULL);



	/* Check if I/O path is dynamically deleted from nvme_bdev_channel. */

	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3833,7 +3833,7 @@ test_add_multi_io_paths_to_nbdev_ch(void) 


	set_thread(1);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -3943,7 +3943,7 @@ test_admin_path(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -4211,7 +4211,7 @@ test_reset_bdev_ctrlr(void) 


	set_thread(0);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -4403,7 +4403,7 @@ test_retry_io_if_ana_state_is_updating(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -4609,7 +4609,7 @@ test_retry_io_for_io_path_error(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -4803,7 +4803,7 @@ test_retry_io_count(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -4894,7 +4894,7 @@ test_concurrent_read_ana_log_page(void) 


	set_thread(0);



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -5041,7 +5041,7 @@ test_retry_io_for_ana_error(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -5236,7 +5236,7 @@ test_retry_io_if_ctrlr_is_resetting(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -5523,7 +5523,7 @@ test_retry_failover_ctrlr(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -5918,7 +5918,7 @@ test_set_preferred_path(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6187,7 +6187,7 @@ test_disable_auto_failback(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6314,7 +6314,7 @@ test_set_multipath_policy(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6529,7 +6529,7 @@ test_retry_io_to_same_path(void) 
	CU_ASSERT(bio->io_path == io_path2);



	/* Detach ctrlr2 dynamically. */

	rc = bdev_nvme_delete("nvme0", &path2);

	rc = bdev_nvme_delete("nvme0", &path2, NULL, NULL);

	CU_ASSERT(rc == 0);



	spdk_delay_us(1000);
@@ -6562,7 +6562,7 @@ test_retry_io_to_same_path(void) 
	spdk_delay_us(1);

	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6706,7 +6706,7 @@ test_race_between_reset_and_disconnected(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6827,7 +6827,7 @@ test_ctrlr_op_rpc(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -6989,7 +6989,7 @@ test_bdev_ctrlr_op_rpc(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -7172,7 +7172,7 @@ test_disable_enable_ctrlr(void) 


	poll_threads();



	rc = bdev_nvme_delete("nvme0", &g_any_path);

	rc = bdev_nvme_delete("nvme0", &g_any_path, NULL, NULL);

	CU_ASSERT(rc == 0);



	poll_threads();
@@ -7182,6 +7182,41 @@ test_disable_enable_ctrlr(void) 
	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == NULL);

}



static void

ut_delete_done(void *ctx, int rc)

{

	int *delete_done_rc = ctx;

	*delete_done_rc = rc;

}



static void

test_delete_ctrlr_done(void)

{

	struct spdk_nvme_transport_id trid = {};

	struct spdk_nvme_ctrlr ctrlr = {};

	int delete_done_rc = 0xDEADBEEF;

	int rc;



	ut_init_trid(&trid);



	nvme_ctrlr_create(&ctrlr, "nvme0", &trid, NULL);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") != NULL);



	rc = bdev_nvme_delete("nvme0", &g_any_path, ut_delete_done, &delete_done_rc);

	CU_ASSERT(rc == 0);



	for (int i = 0; i < 20; i++) {

		poll_threads();

		if (delete_done_rc == 0) {

			break;

		}

		spdk_delay_us(1000);

	}



	CU_ASSERT(delete_done_rc == 0);

	CU_ASSERT(nvme_ctrlr_get_by_name("nvme0") == NULL);

}



int

main(int argc, char **argv)

{
@@ -7238,6 +7273,7 @@ main(int argc, char **argv) 
	CU_ADD_TEST(suite, test_ctrlr_op_rpc);

	CU_ADD_TEST(suite, test_bdev_ctrlr_op_rpc);

	CU_ADD_TEST(suite, test_disable_enable_ctrlr);

	CU_ADD_TEST(suite, test_delete_ctrlr_done);



	allocate_threads(3);

	set_thread(0);