Commit b3be320d authored by GangCao's avatar GangCao Committed by Tomasz Zawadzki
Browse files

vbdev: close the base bdev on its opened thread 



With the JSON configure, the base device will be opened on the
same thread (RPC thread) to handle the JSON operation. Later the
virtual device upon the base device can be opened on another
thread. At the time of virtual device destruction, the base
device is also closed at the thread where opening the virtual
device, it is actually different than the original thread where
this base device is opened through the JSON configure.

Add a thread here to record the exact thread where the base
device is opened and then later route it back to handle the base
device close operation.

Change-Id: Id1299bb0d86db57bce48b9787d7f248d29a6f345
Signed-off-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/974


Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent ae86baeb

lib/bdev/part.c

+19 −2
Original line number Diff line number Diff line
@@ -39,6 +39,7 @@ 
#include "spdk/likely.h"

#include "spdk/log.h"

#include "spdk/string.h"

#include "spdk/thread.h"



#include "spdk/bdev_module.h"
@@ -55,6 +56,7 @@ struct spdk_bdev_part_base { 
	struct bdev_part_tailq		*tailq;

	spdk_io_channel_create_cb	ch_create_cb;

	spdk_io_channel_destroy_cb	ch_destroy_cb;

	struct spdk_thread		*thread;

};



struct spdk_bdev *
@@ -87,12 +89,24 @@ spdk_bdev_part_base_get_bdev_name(struct spdk_bdev_part_base *part_base) 
	return part_base->bdev->name;

}



static void

_spdk_bdev_part_base_free(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



void

spdk_bdev_part_base_free(struct spdk_bdev_part_base *base)

{

	if (base->desc) {

		/* Close the underlying bdev on its same opened thread. */

		if (base->thread && base->thread != spdk_get_thread()) {

			spdk_thread_send_msg(base->thread, _spdk_bdev_part_base_free, base->desc);

		} else {

			spdk_bdev_close(base->desc);

		base->desc = NULL;

		}

	}



	if (base->base_free_fn != NULL) {
@@ -442,6 +456,9 @@ struct spdk_bdev_part_base * 
		return NULL;

	}



	/* Save the thread where the base device is opened */

	base->thread = spdk_get_thread();



	return base;

}

module/bdev/compress/vbdev_compress.c

+56 −3
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ struct vbdev_compress { 
	bool				orphaned;	/* base bdev claimed but comp_bdev not registered */

	TAILQ_HEAD(, vbdev_comp_op)	queued_comp_ops;

	TAILQ_ENTRY(vbdev_compress)	link;

	struct spdk_thread		*thread;	/* thread where base device is opened */

};

static TAILQ_HEAD(, vbdev_compress) g_vbdev_comp = TAILQ_HEAD_INITIALIZER(g_vbdev_comp);
@@ -909,6 +910,14 @@ _device_unregister_cb(void *io_device) 
	free(comp_bdev);

}



static void

_vbdev_compress_destruct_cb(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



static void

vbdev_compress_destruct_cb(void *cb_arg, int reduce_errno)

{
@@ -919,7 +928,12 @@ vbdev_compress_destruct_cb(void *cb_arg, int reduce_errno) 
	} else {

		TAILQ_REMOVE(&g_vbdev_comp, comp_bdev, link);

		spdk_bdev_module_release_bdev(comp_bdev->base_bdev);

		/* Close the underlying bdev on its same opened thread. */

		if (comp_bdev->thread && comp_bdev->thread != spdk_get_thread()) {

			spdk_thread_send_msg(comp_bdev->thread, _vbdev_compress_destruct_cb, comp_bdev->base_desc);

		} else {

			spdk_bdev_close(comp_bdev->base_desc);

		}

		comp_bdev->vol = NULL;

		if (comp_bdev->orphaned == false) {

			spdk_io_device_unregister(comp_bdev, _device_unregister_cb);
@@ -1076,6 +1090,14 @@ vbdev_compress_config_json(struct spdk_json_write_ctx *w) 
	return 0;

}



static void

_vbdev_reduce_init_cb(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



/* Callback from reduce for when init is complete. We'll pass the vbdev_comp struct

 * used for initial metadata operations to claim where it will be further filled out

 * and added to the global list.
@@ -1087,7 +1109,12 @@ vbdev_reduce_init_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno 


	/* We're done with metadata operations */

	spdk_put_io_channel(meta_ctx->base_ch);

	/* Close the underlying bdev on its same opened thread. */

	if (meta_ctx->thread && meta_ctx->thread != spdk_get_thread()) {

		spdk_thread_send_msg(meta_ctx->thread, _vbdev_reduce_init_cb, meta_ctx->base_desc);

	} else {

		spdk_bdev_close(meta_ctx->base_desc);

	}

	meta_ctx->base_desc = NULL;



	if (reduce_errno == 0) {
@@ -1311,6 +1338,10 @@ vbdev_init_reduce(struct spdk_bdev *bdev, const char *pm_path) 
		free(meta_ctx);

		return -EINVAL;

	}



	/* Save the thread where the base device is opened */

	meta_ctx->thread = spdk_get_thread();



	meta_ctx->base_ch = spdk_bdev_get_io_channel(meta_ctx->base_desc);



	spdk_reduce_vol_init(&meta_ctx->params, &meta_ctx->backing_dev,
@@ -1569,6 +1600,9 @@ vbdev_compress_claim(struct vbdev_compress *comp_bdev) 
		goto error_open;

	}



	/* Save the thread where the base device is opened */

	comp_bdev->thread = spdk_get_thread();



	spdk_io_device_register(comp_bdev, comp_bdev_ch_create_cb, comp_bdev_ch_destroy_cb,

				sizeof(struct comp_io_channel),

				comp_bdev->comp_bdev.name);
@@ -1632,6 +1666,14 @@ bdev_compress_delete(const char *name, spdk_delete_compress_complete cb_fn, void 
	}

}



static void

_vbdev_reduce_load_cb(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



/* Callback from reduce for then load is complete. We'll pass the vbdev_comp struct

 * used for initial metadata operations to claim where it will be further filled out

 * and added to the global list.
@@ -1644,7 +1686,12 @@ vbdev_reduce_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno 


	/* Done with metadata operations */

	spdk_put_io_channel(meta_ctx->base_ch);

	/* Close the underlying bdev on its same opened thread. */

	if (meta_ctx->thread && meta_ctx->thread != spdk_get_thread()) {

		spdk_thread_send_msg(meta_ctx->thread, _vbdev_reduce_load_cb, meta_ctx->base_desc);

	} else {

		spdk_bdev_close(meta_ctx->base_desc);

	}

	meta_ctx->base_desc = NULL;



	if (reduce_errno != 0 && reduce_errno != -ENOENT) {
@@ -1676,6 +1723,9 @@ vbdev_reduce_load_cb(void *cb_arg, struct spdk_reduce_vol *vol, int reduce_errno 
			goto err;

		}



		/* Save the thread where the base device is opened */

		meta_ctx->thread = spdk_get_thread();



		meta_ctx->comp_bdev.module = &compress_if;

		pthread_mutex_init(&meta_ctx->reduce_lock, NULL);

		rc = spdk_bdev_module_claim_bdev(meta_ctx->base_bdev, meta_ctx->base_desc,
@@ -1736,6 +1786,9 @@ vbdev_compress_examine(struct spdk_bdev *bdev) 
		return;

	}



	/* Save the thread where the base device is opened */

	meta_ctx->thread = spdk_get_thread();



	meta_ctx->base_ch = spdk_bdev_get_io_channel(meta_ctx->base_desc);

	spdk_reduce_vol_load(&meta_ctx->backing_dev, vbdev_reduce_load_cb, meta_ctx);

}

module/bdev/crypto/vbdev_crypto.c

+19 −2
Original line number Diff line number Diff line
@@ -173,6 +173,7 @@ struct vbdev_crypto { 
	struct rte_cryptodev_sym_session *session_decrypt;	/* decryption session for this bdev */

	struct rte_crypto_sym_xform	cipher_xform;		/* crypto control struct for this bdev */

	TAILQ_ENTRY(vbdev_crypto)	link;

	struct spdk_thread		*thread;		/* thread where base device is opened */

};

static TAILQ_HEAD(, vbdev_crypto) g_vbdev_crypto = TAILQ_HEAD_INITIALIZER(g_vbdev_crypto);
@@ -1202,6 +1203,15 @@ _device_unregister_cb(void *io_device) 
	free(crypto_bdev);

}



/* Wrapper for the bdev close operation. */

static void

_vbdev_crypto_destruct(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



/* Called after we've unregistered following a hot remove callback.

 * Our finish entry point will be called next.

 */
@@ -1216,8 +1226,12 @@ vbdev_crypto_destruct(void *ctx) 
	/* Unclaim the underlying bdev. */

	spdk_bdev_module_release_bdev(crypto_bdev->base_bdev);



	/* Close the underlying bdev. */

	/* Close the underlying bdev on its same opened thread. */

	if (crypto_bdev->thread && crypto_bdev->thread != spdk_get_thread()) {

		spdk_thread_send_msg(crypto_bdev->thread, _vbdev_crypto_destruct, crypto_bdev->base_desc);

	} else {

		spdk_bdev_close(crypto_bdev->base_desc);

	}



	/* Unregister the io_device. */

	spdk_io_device_unregister(crypto_bdev, _device_unregister_cb);
@@ -1826,6 +1840,9 @@ vbdev_crypto_claim(struct spdk_bdev *bdev) 
			goto error_open;

		}



		/* Save the thread where the base device is opened */

		vbdev->thread = spdk_get_thread();



		rc = spdk_bdev_module_claim_bdev(bdev, vbdev->base_desc, vbdev->crypto_bdev.module);

		if (rc) {

			SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev));

module/bdev/delay/vbdev_delay.c

+18 −2
Original line number Diff line number Diff line
@@ -87,6 +87,7 @@ struct vbdev_delay { 
	uint64_t			average_write_latency_ticks; /* the average write delay */

	uint64_t			p99_write_latency_ticks; /* the p99 write delay */

	TAILQ_ENTRY(vbdev_delay)	link;

	struct spdk_thread		*thread;    /* thread where base device is opened */

};

static TAILQ_HEAD(, vbdev_delay) g_delay_nodes = TAILQ_HEAD_INITIALIZER(g_delay_nodes);
@@ -129,6 +130,14 @@ _device_unregister_cb(void *io_device) 
	free(delay_node);

}



static void

_vbdev_delay_destruct(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



static int

vbdev_delay_destruct(void *ctx)

{
@@ -143,8 +152,12 @@ vbdev_delay_destruct(void *ctx) 
	/* Unclaim the underlying bdev. */

	spdk_bdev_module_release_bdev(delay_node->base_bdev);



	/* Close the underlying bdev. */

	/* Close the underlying bdev on its same opened thread. */

	if (delay_node->thread && delay_node->thread != spdk_get_thread()) {

		spdk_thread_send_msg(delay_node->thread, _vbdev_delay_destruct, delay_node->base_desc);

	} else {

		spdk_bdev_close(delay_node->base_desc);

	}



	/* Unregister the io_device. */

	spdk_io_device_unregister(delay_node, _device_unregister_cb);
@@ -653,6 +666,9 @@ vbdev_delay_register(struct spdk_bdev *bdev) 
			goto error_unregister;

		}



		/* Save the thread where the base device is opened */

		delay_node->thread = spdk_get_thread();



		rc = spdk_bdev_module_claim_bdev(bdev, delay_node->base_desc, delay_node->delay_bdev.module);

		if (rc) {

			SPDK_ERRLOG("could not claim bdev %s\n", spdk_bdev_get_name(bdev));

module/bdev/nvme/bdev_nvme_rpc.c

+18 −1
Original line number Diff line number Diff line
@@ -49,6 +49,7 @@ struct open_descriptors { 
	void *desc;

	struct  spdk_bdev *bdev;

	TAILQ_ENTRY(open_descriptors) tqlst;

	struct spdk_thread *thread;

};

typedef TAILQ_HEAD(, open_descriptors) open_descriptors_t;
@@ -515,6 +516,14 @@ struct firmware_update_info { 
	struct rpc_apply_firmware	*req;

};



static void

_apply_firmware_cleanup(void *ctx)

{

	struct spdk_bdev_desc *desc = ctx;



	spdk_bdev_close(desc);

}



static void

apply_firmware_cleanup(void *cb_arg)

{
@@ -540,7 +549,12 @@ apply_firmware_cleanup(void *cb_arg) 


	TAILQ_FOREACH_SAFE(opt, &firm_ctx->desc_head, tqlst, tmp) {

		TAILQ_REMOVE(&firm_ctx->desc_head, opt, tqlst);

		/* Close the underlying bdev on its same opened thread. */

		if (opt->thread && opt->thread != spdk_get_thread()) {

			spdk_thread_send_msg(opt->thread, _apply_firmware_cleanup, opt->desc);

		} else {

			spdk_bdev_close(opt->desc);

		}

		free(opt);

	}

	free(firm_ctx);
@@ -715,6 +729,9 @@ spdk_rpc_bdev_nvme_apply_firmware(struct spdk_jsonrpc_request *request, 
			return;

		}



		/* Save the thread where the base device is opened */

		opt->thread = spdk_get_thread();



		opt->desc = desc;

		opt->bdev = bdev;

		TAILQ_INSERT_TAIL(&firm_ctx->desc_head, opt, tqlst);