Commit 58cc695e authored by Saravanan D's avatar Saravanan D Committed by Tomasz Zawadzki
Browse files

bdev/nvme: bdev nvme delete public api 



Introduced public api `spdk_bdev_nvme_delete`,
and forward declared `struct spdk_nvme_path_id`
in public header /module/bdev/nvme.h
Replaced internal function `bdev_nvme_delete` and `struct
nvme_path_id`.
`struct spdk_nvme_path_id` definition is retained in the private
header /module/bdev/bdev_nvme.h
Minor update to `struct bdev_nvme_delete_ctx` with
`spdk_bdev_nvme_delete_cb`
Updated `bdev_nvme_ut.c`
Updated CHANGELOG.md

Change-Id: I2887f1416fe07bbb72942c15b8766bfcede24c00
Signed-off-by: default avatarSaravanan D <saravanan.d@oracle.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/25419


Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarJacek Kalwas <jacek.kalwas@nutanix.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Community-CI: Mellanox Build Bot
parent c4ba0294

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -19,6 +19,8 @@ If multipathing shall be enabled for nvme bdev, `bdev_opts.multipath` shall be s 
Added public APIs `spdk_bdev_nvme_get_opts` and `spdk_bdev_nvme_set_opts` to get default bdev nvme

options and set them respectively.



Added public API `spdk_bdev_nvme_delete` to delete the specified NVMe controller or one of its paths.



### env



Added 3 APIs to handle multiple interrupts for PCI device `spdk_pci_device_enable_interrupts()`,

include/spdk/module/bdev/nvme.h

+25 −0
Original line number Diff line number Diff line
@@ -18,6 +18,7 @@ extern "C" { 


typedef void (*spdk_bdev_nvme_create_cb)(void *ctx, size_t bdev_count, int rc);

typedef void (*spdk_bdev_nvme_set_multipath_policy_cb)(void *cb_arg, int rc);

typedef void (*spdk_bdev_nvme_delete_cb)(void *ctx, int rc);



enum spdk_bdev_nvme_multipath_policy {

	BDEV_NVME_MP_POLICY_ACTIVE_PASSIVE,
@@ -49,6 +50,8 @@ struct spdk_bdev_nvme_ctrlr_opts { 
	bool multipath;

};



struct spdk_nvme_path_id;



enum spdk_bdev_timeout_action {

	SPDK_BDEV_NVME_TIMEOUT_ACTION_NONE = 0,

	SPDK_BDEV_NVME_TIMEOUT_ACTION_RESET,
@@ -132,6 +135,28 @@ int spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
			  struct spdk_nvme_ctrlr_opts *drv_opts,

			  struct spdk_bdev_nvme_ctrlr_opts *bdev_opts);



/**

 * Delete the specified NVMe controller, or one of its paths.

 *

 * NOTE: When path_id is specified and it is the only path_id associated with NVMe controller

 * the path is removed and the NVMe controller gets deleted. (Optional) callback

 * function gets executed on delete complete in caller's thread. When the (optional)

 * callback is not provided, the control is returned back at the time delete is initiated,

 * not when it is completed. When NVMe controller deletion is already in progress state,

 * this function returns success.

 *

 * \param name NVMe controller name.

 * \param path_id The specified path to remove (optional).

 * \param delete_cb	Callback function on delete complete (optional).

 * \param cb_ctx Context passed to callback (optional).

 * \return zero on success,

 *		-EINVAL on wrong parameters or

 *		-ENODEV if controller is not found or

 *		-ENOMEM on no memory

 */

int spdk_bdev_nvme_delete(const char *name, const struct spdk_nvme_path_id *path_id,

			  spdk_bdev_nvme_delete_cb delete_cb, void *cb_ctx);



/**

 * Set multipath policy of the NVMe bdev.

 *

module/bdev/nvme/bdev_nvme.c

+27 −27
Original line number Diff line number Diff line
@@ -617,7 +617,7 @@ nvme_bdev_ctrlr_delete(struct nvme_bdev_ctrlr *nbdev_ctrlr, 
static void

_nvme_ctrlr_delete(struct nvme_ctrlr *nvme_ctrlr)

{

	struct nvme_path_id *path_id, *tmp_path;

	struct spdk_nvme_path_id *path_id, *tmp_path;

	struct nvme_ns *ns, *tmp_ns;



	free(nvme_ctrlr->copied_ana_desc);
@@ -2032,7 +2032,7 @@ bdev_nvme_complete_pending_resets(struct nvme_ctrlr *nvme_ctrlr, bool success) 
static bool

bdev_nvme_failover_trid(struct nvme_ctrlr *nvme_ctrlr, bool remove, bool start)

{

	struct nvme_path_id *path_id, *next_path;

	struct spdk_nvme_path_id *path_id, *next_path;

	int rc __attribute__((unused));



	path_id = TAILQ_FIRST(&nvme_ctrlr->trids);
@@ -3952,7 +3952,7 @@ nvme_ctrlr_info_json(struct spdk_json_write_ctx *w, struct nvme_ctrlr *nvme_ctrl 
	struct spdk_nvme_transport_id *trid;

	const struct spdk_nvme_ctrlr_opts *opts;

	const struct spdk_nvme_ctrlr_data *cdata;

	struct nvme_path_id *path_id;

	struct spdk_nvme_path_id *path_id;

	int32_t numa_id;



	spdk_json_write_object_begin(w);
@@ -5871,7 +5871,7 @@ nvme_ctrlr_create(struct spdk_nvme_ctrlr *ctrlr, 
		  struct nvme_async_probe_ctx *ctx)

{

	struct nvme_ctrlr *nvme_ctrlr;

	struct nvme_path_id *path_id;

	struct spdk_nvme_path_id *path_id;

	const struct spdk_nvme_ctrlr_data *cdata;

	struct spdk_event_handler_opts opts = {

		.opts_size = SPDK_SIZEOF(&opts, fd_type),
@@ -6445,7 +6445,7 @@ bdev_nvme_check_secondary_trid(struct nvme_ctrlr *nvme_ctrlr, 
			       struct spdk_nvme_ctrlr *new_ctrlr,

			       struct spdk_nvme_transport_id *trid)

{

	struct nvme_path_id *tmp_trid;

	struct spdk_nvme_path_id *tmp_trid;



	if (trid->trtype == SPDK_NVME_TRANSPORT_PCIE) {

		NVME_CTRLR_ERRLOG(nvme_ctrlr, "PCIe failover is not supported.\n");
@@ -6508,7 +6508,7 @@ static int 
_bdev_nvme_add_secondary_trid(struct nvme_ctrlr *nvme_ctrlr,

			      struct spdk_nvme_transport_id *trid)

{

	struct nvme_path_id *active_id, *new_trid, *tmp_trid;

	struct spdk_nvme_path_id *active_id, *new_trid, *tmp_trid;



	new_trid = calloc(1, sizeof(*new_trid));

	if (new_trid == NULL) {
@@ -6838,9 +6838,9 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 


struct bdev_nvme_delete_ctx {

	char                        *name;

	struct nvme_path_id         path_id;

	bdev_nvme_delete_done_fn    delete_done;

	void                        *delete_done_ctx;

	struct spdk_nvme_path_id    path_id;

	spdk_bdev_nvme_delete_cb    delete_cb;

	void                        *delete_cb_ctx;

	uint64_t                    timeout_ticks;

	struct spdk_poller          *poller;

};
@@ -6855,7 +6855,7 @@ free_bdev_nvme_delete_ctx(struct bdev_nvme_delete_ctx *ctx) 
}



static bool

nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id)

nvme_path_id_compare(struct spdk_nvme_path_id *p, const struct spdk_nvme_path_id *path_id)

{

	if (path_id->trid.trtype != 0) {

		if (path_id->trid.trtype == SPDK_NVME_TRANSPORT_CUSTOM) {
@@ -6909,11 +6909,11 @@ nvme_path_id_compare(struct nvme_path_id *p, const struct nvme_path_id *path_id) 
}



static bool

nvme_path_id_exists(const char *name, const struct nvme_path_id *path_id)

nvme_path_id_exists(const char *name, const struct spdk_nvme_path_id *path_id)

{

	struct nvme_bdev_ctrlr  *nbdev_ctrlr;

	struct nvme_ctrlr       *ctrlr;

	struct nvme_path_id     *p;

	struct spdk_nvme_path_id     *p;



	pthread_mutex_lock(&g_bdev_nvme_mutex);

	nbdev_ctrlr = nvme_bdev_ctrlr_get_by_name(name);
@@ -6955,16 +6955,16 @@ bdev_nvme_delete_complete_poll(void *arg) 


	spdk_poller_unregister(&ctx->poller);



	ctx->delete_done(ctx->delete_done_ctx, rc);

	ctx->delete_cb(ctx->delete_cb_ctx, rc);

	free_bdev_nvme_delete_ctx(ctx);



	return SPDK_POLLER_BUSY;

}



static int

_bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path_id)

_bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct spdk_nvme_path_id *path_id)

{

	struct nvme_path_id	*p, *t;

	struct spdk_nvme_path_id	*p, *t;

	spdk_msg_fn		msg_fn;

	int			rc = -ENXIO;
@@ -7017,8 +7017,8 @@ _bdev_nvme_delete(struct nvme_ctrlr *nvme_ctrlr, const struct nvme_path_id *path 
}



int

bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,

		 bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx)

spdk_bdev_nvme_delete(const char *name, const struct spdk_nvme_path_id *path_id,

		      spdk_bdev_nvme_delete_cb delete_cb, void *cb_ctx)

{

	struct nvme_bdev_ctrlr		*nbdev_ctrlr;

	struct nvme_ctrlr		*nvme_ctrlr, *tmp_nvme_ctrlr;
@@ -7058,7 +7058,7 @@ bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 


	pthread_mutex_unlock(&g_bdev_nvme_mutex);



	if (rc != 0 || delete_done == NULL) {

	if (rc != 0 || delete_cb == NULL) {

		goto exit;

	}
@@ -7076,8 +7076,8 @@ bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id, 
		goto exit;

	}



	ctx->delete_done = delete_done;

	ctx->delete_done_ctx = delete_done_ctx;

	ctx->delete_cb = delete_cb;

	ctx->delete_cb_ctx = cb_ctx;

	ctx->path_id = *path_id;

	ctx->timeout_ticks = spdk_get_ticks() + 10 * spdk_get_ticks_hz();

	ctx->poller = SPDK_POLLER_REGISTER(bdev_nvme_delete_complete_poll, ctx, 1000);
@@ -7239,11 +7239,11 @@ _stop_discovery(void *_ctx) 


	while (!TAILQ_EMPTY(&ctx->nvm_entry_ctxs)) {

		struct discovery_entry_ctx *entry_ctx;

		struct nvme_path_id path = {};

		struct spdk_nvme_path_id path = {};



		entry_ctx = TAILQ_FIRST(&ctx->nvm_entry_ctxs);

		path.trid = entry_ctx->trid;

		bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

		spdk_bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

		TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);

		free(entry_ctx);

	}
@@ -7278,7 +7278,7 @@ static void 
remove_discovery_entry(struct nvme_ctrlr *nvme_ctrlr)

{

	struct discovery_ctx *d_ctx;

	struct nvme_path_id *path_id;

	struct spdk_nvme_path_id *path_id;

	struct spdk_nvme_transport_id trid = {};

	struct discovery_entry_ctx *entry_ctx, *tmp;
@@ -7327,13 +7327,13 @@ discovery_remove_controllers(struct discovery_ctx *ctx) 
			}

		}

		if (!found) {

			struct nvme_path_id path = {};

			struct spdk_nvme_path_id path = {};



			DISCOVERY_INFOLOG(ctx, "NVM %s:%s:%s not found\n",

					  old_trid.subnqn, old_trid.traddr, old_trid.trsvcid);



			path.trid = entry_ctx->trid;

			bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

			spdk_bdev_nvme_delete(entry_ctx->name, &path, NULL, NULL);

			TAILQ_REMOVE(&ctx->nvm_entry_ctxs, entry_ctx, tailq);

			free(entry_ctx);

		}
@@ -8987,7 +8987,7 @@ nvme_ctrlr_cuse_config_json(struct spdk_json_write_ctx *w, 
static void

nvme_ctrlr_config_json(struct spdk_json_write_ctx *w,

		       struct nvme_ctrlr *nvme_ctrlr,

		       struct nvme_path_id *path_id)

		       struct spdk_nvme_path_id *path_id)

{

	struct spdk_nvme_transport_id	*trid;

	const struct spdk_nvme_ctrlr_opts *opts;
@@ -9067,7 +9067,7 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w) 
	struct nvme_bdev_ctrlr	*nbdev_ctrlr;

	struct nvme_ctrlr	*nvme_ctrlr;

	struct discovery_ctx	*ctx;

	struct nvme_path_id	*path_id;

	struct spdk_nvme_path_id	*path_id;



	bdev_nvme_opts_config_json(w);

module/bdev/nvme/bdev_nvme.h

+4 −22
Original line number Diff line number Diff line
@@ -75,10 +75,10 @@ struct nvme_io_path; 
struct nvme_ctrlr_channel_iter;

struct nvme_bdev_channel_iter;



struct nvme_path_id {

struct spdk_nvme_path_id {

	struct spdk_nvme_transport_id		trid;

	struct spdk_nvme_host_id		hostid;

	TAILQ_ENTRY(nvme_path_id)		link;

	TAILQ_ENTRY(spdk_nvme_path_id)		link;

	uint64_t				last_failed_tsc;

};
@@ -92,7 +92,7 @@ struct nvme_ctrlr { 
	 *  target for CONTROLLER IDENTIFY command during initialization

	 */

	struct spdk_nvme_ctrlr			*ctrlr;

	struct nvme_path_id			*active_path_id;

	struct spdk_nvme_path_id		*active_path_id;

	int					ref;



	uint32_t				resetting : 1;
@@ -133,7 +133,7 @@ struct nvme_ctrlr { 
	TAILQ_ENTRY(nvme_ctrlr)			tailq;

	struct nvme_bdev_ctrlr			*nbdev_ctrlr;



	TAILQ_HEAD(nvme_paths, nvme_path_id)	trids;

	TAILQ_HEAD(nvme_paths, spdk_nvme_path_id)	trids;



	uint32_t				max_ana_log_page_size;

	struct spdk_nvme_ana_page		*ana_log_page;
@@ -309,24 +309,6 @@ int bdev_nvme_set_keys(const char *name, const char *dhchap_key, const char *dhc 


struct spdk_nvme_ctrlr *bdev_nvme_get_ctrlr(struct spdk_bdev *bdev);



typedef void (*bdev_nvme_delete_done_fn)(void *ctx, int rc);



/**

 * Delete NVMe controller with all bdevs on top of it, or delete the specified path

 * if there is any alternative path. Requires to pass name of NVMe controller.

 *

 * \param name NVMe controller name

 * \param path_id The specified path to remove (optional)

 * \param delete_done Callback function on delete complete (optional)

 * \param delete_done_ctx Context passed to callback (optional)

 * \return zero on success,

 *		-EINVAL on wrong parameters or

 *		-ENODEV if controller is not found or

 *		-ENOMEM on no memory

 */

int bdev_nvme_delete(const char *name, const struct nvme_path_id *path_id,

		     bdev_nvme_delete_done_fn delete_done, void *delete_done_ctx);



enum nvme_ctrlr_op {

	NVME_CTRLR_OP_RESET = 1,

	NVME_CTRLR_OP_ENABLE,

module/bdev/nvme/bdev_nvme_rpc.c

+2 −2
Original line number Diff line number Diff line
@@ -758,7 +758,7 @@ rpc_bdev_nvme_detach_controller(struct spdk_jsonrpc_request *request, 
				const struct spdk_json_val *params)

{

	struct rpc_bdev_nvme_detach_controller req = {NULL};

	struct nvme_path_id path = {};

	struct spdk_nvme_path_id path = {};

	size_t len, maxlen;

	int rc = 0;
@@ -854,7 +854,7 @@ rpc_bdev_nvme_detach_controller(struct spdk_jsonrpc_request *request, 
		snprintf(path.hostid.hostsvcid, maxlen, "%s", req.hostsvcid);

	}



	rc = bdev_nvme_delete(req.name, &path, rpc_bdev_nvme_detach_controller_done, request);

	rc = spdk_bdev_nvme_delete(req.name, &path, rpc_bdev_nvme_detach_controller_done, request);



	if (rc != 0) {

		spdk_jsonrpc_send_error_response(request, rc, spdk_strerror(-rc));