Commit ebf0312e authored by Tomasz Zawadzki's avatar Tomasz Zawadzki Committed by Jim Harris
Browse files

lvol: Add vbdev_lvs_destruct() and change unload behaviour [3/3] 



This patch adds new API to remove logical volume store
from device it is on. It is only used from RPC, when
user explicitly requests. It allows to use the device to
use as any other bdev.

vbdev_lvs_unload() is now only called from hotremove and
during application shutdown. Which makes it possible to
load it again during application start up.

Signed-off-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: If6452ecc3fff99237d1704ff7cd8de4d7133221d
Reviewed-on: https://review.gerrithub.io/382021


Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent d189b8ee

include/spdk_internal/lvolstore.h

+1 −0
Original line number Diff line number Diff line
@@ -85,6 +85,7 @@ struct spdk_lvol_store { 
	uint64_t			total_blocks;

	int				lvol_count;

	int				lvols_opened;

	bool				destruct;

	TAILQ_HEAD(, spdk_lvol)		lvols;

	bool				on_list;

	TAILQ_ENTRY(spdk_lvol_store)	link;

lib/bdev/lvol/vbdev_lvol.c

+60 −7
Original line number Diff line number Diff line
@@ -53,7 +53,7 @@ vbdev_lvs_hotremove_cb(void *ctx) 
	TAILQ_FOREACH_SAFE(lvs_bdev, &g_spdk_lvol_pairs, lvol_stores, tmp) {

		if (lvs_bdev) {

			if (lvs_bdev->bdev == bdev) {

				vbdev_lvs_destruct(lvs_bdev->lvs, NULL, NULL);

				vbdev_lvs_unload(lvs_bdev->lvs, NULL, NULL);

			}

		}

	}
@@ -155,6 +155,60 @@ vbdev_lvs_create(struct spdk_bdev *base_bdev, uint32_t cluster_sz, 
	return 0;

}



static void

_vbdev_lvs_unload_cb(void *cb_arg, int lvserrno)

{

	struct spdk_lvs_req *req = cb_arg;



	SPDK_INFOLOG(SPDK_TRACE_VBDEV_LVOL, "Lvol store bdev unloaded\n");



	if (req->cb_fn != NULL)

		req->cb_fn(req->cb_arg, lvserrno);

	free(req);

}



void

vbdev_lvs_unload(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *cb_arg)

{

	struct spdk_lvs_req *req;

	struct lvol_store_bdev *lvs_bdev;

	struct spdk_lvol *lvol, *tmp;



	lvs_bdev = vbdev_get_lvs_bdev_by_lvs(lvs);

	TAILQ_REMOVE(&g_spdk_lvol_pairs, lvs_bdev, lvol_stores);



	req = calloc(1, sizeof(*req));

	if (!req) {

		SPDK_ERRLOG("Cannot alloc memory for vbdev lvol store request pointer\n");

		if (cb_fn != NULL)

			cb_fn(cb_arg, -ENOMEM);

		return;

	}



	req->cb_fn = cb_fn;

	req->cb_arg = cb_arg;



	if (TAILQ_EMPTY(&lvs->lvols)) {

		spdk_lvs_unload(lvs, _vbdev_lvs_unload_cb, req);

	} else {

		lvs->destruct_req = calloc(1, sizeof(*lvs->destruct_req));

		if (!lvs->destruct_req) {

			SPDK_ERRLOG("Cannot alloc memory for vbdev lvol store request pointer\n");

			_vbdev_lvs_unload_cb(req, -ENOMEM);

			return;

		}

		lvs->destruct_req->cb_fn = _vbdev_lvs_unload_cb;

		lvs->destruct_req->cb_arg = req;

		lvs->destruct = false;

		TAILQ_FOREACH_SAFE(lvol, &lvs->lvols, link, tmp) {

			lvol->close_only = true;

			spdk_vbdev_unregister(lvol->bdev, NULL, NULL);

		}

	}



	free(lvs_bdev);

}



static void

_vbdev_lvs_destruct_cb(void *cb_arg, int lvserrno)

{
@@ -168,10 +222,8 @@ _vbdev_lvs_destruct_cb(void *cb_arg, int lvserrno) 
}



void

vbdev_lvs_destruct(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn,

		   void *cb_arg)

vbdev_lvs_destruct(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *cb_arg)

{



	struct spdk_lvs_req *req;

	struct lvol_store_bdev *lvs_bdev;

	struct spdk_lvol *lvol, *tmp;
@@ -198,7 +250,7 @@ vbdev_lvs_destruct(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, 
	}



	if (all_lvols_closed == true) {

		spdk_lvs_unload(lvs, _vbdev_lvs_destruct_cb, req);

		spdk_lvs_destroy(lvs, false, _vbdev_lvs_destruct_cb, req);

	} else {

		lvs->destruct_req = calloc(1, sizeof(*lvs->destruct_req));

		if (!lvs->destruct_req) {
@@ -208,8 +260,9 @@ vbdev_lvs_destruct(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, 
		}

		lvs->destruct_req->cb_fn = _vbdev_lvs_destruct_cb;

		lvs->destruct_req->cb_arg = req;

		lvs->destruct = true;

		TAILQ_FOREACH_SAFE(lvol, &lvs->lvols, link, tmp) {

			lvol->close_only = true;

			lvol->close_only = false;

			spdk_vbdev_unregister(lvol->bdev, NULL, NULL);

		}

	}
@@ -686,7 +739,7 @@ vbdev_lvs_fini(void) 
	struct lvol_store_bdev *lvs_bdev, *tmp;



	TAILQ_FOREACH_SAFE(lvs_bdev, &g_spdk_lvol_pairs, lvol_stores, tmp) {

		vbdev_lvs_destruct(lvs_bdev->lvs, NULL, NULL);

		vbdev_lvs_unload(lvs_bdev->lvs, NULL, NULL);

	}

}

lib/bdev/lvol/vbdev_lvol.h

+1 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ struct lvol_store_bdev { 
int vbdev_lvs_create(struct spdk_bdev *base_bdev, uint32_t cluster_sz,

		     spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg);

void vbdev_lvs_destruct(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *cb_arg);

void vbdev_lvs_unload(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *cb_arg);



int vbdev_lvol_create(uuid_t uuid, size_t sz, spdk_lvol_op_with_handle_complete cb_fn,

		      void *cb_arg);

lib/lvol/lvol.c

+8 −3
Original line number Diff line number Diff line
@@ -575,6 +575,7 @@ spdk_lvs_init(struct spdk_bs_dev *bs_dev, struct spdk_lvs_opts *o, 
	lvs_req->cb_arg = cb_arg;

	lvs_req->lvol_store = lvs;

	lvs->bs_dev = bs_dev;

	lvs->destruct = false;



	strncpy(opts.bstype.bstype, "LVOLSTORE", SPDK_BLOBSTORE_TYPE_LENGTH);
@@ -751,8 +752,10 @@ _spdk_lvol_close_blob_cb(void *cb_arg, int lvolerrno) 
	SPDK_INFOLOG(SPDK_TRACE_LVOL, "Lvol %s closed\n", lvol->old_name);



	if (lvol->lvol_store->destruct_req && all_lvols_closed == true) {

		if (!lvol->lvol_store->destruct) {

			spdk_lvs_unload(lvol->lvol_store, _spdk_lvs_destruct_cb, lvol->lvol_store->destruct_req);

		}

	}



end:

	req->cb_fn(req->cb_arg, lvolerrno);
@@ -773,8 +776,11 @@ _spdk_lvol_delete_blob_cb(void *cb_arg, int lvolerrno) 
		goto end;

	}



	TAILQ_REMOVE(&lvol->lvol_store->lvols, lvol, link);



	if (lvol->lvol_store->destruct_req && TAILQ_EMPTY(&lvol->lvol_store->lvols)) {

		spdk_lvs_unload(lvol->lvol_store, _spdk_lvs_destruct_cb, lvol->lvol_store->destruct_req);

		if (lvol->lvol_store->destruct)

			spdk_lvs_destroy(lvol->lvol_store, false, _spdk_lvs_destruct_cb, lvol->lvol_store->destruct_req);

	}



	SPDK_INFOLOG(SPDK_TRACE_LVOL, "Lvol %s deleted\n", lvol->old_name);
@@ -1065,7 +1071,6 @@ spdk_lvol_destroy(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn, void *cb_ 
	req->cb_arg = lvol_req;

	req->lvol = lvol;



	TAILQ_REMOVE(&lvol->lvol_store->lvols, lvol, link);

	spdk_bs_md_delete_blob(bs, lvol->blob_id, _spdk_lvol_delete_blob_cb, req);

}

test/unit/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut.c

+81 −8
Original line number Diff line number Diff line
@@ -183,15 +183,35 @@ spdk_lvs_init(struct spdk_bs_dev *bs_dev, struct spdk_lvs_opts *o, 
int

spdk_lvs_unload(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *cb_arg)

{

	struct spdk_lvol *lvol;

	struct spdk_lvol *lvol, *tmp;



	while (!TAILQ_EMPTY(&lvs->lvols)) {

		lvol = TAILQ_FIRST(&lvs->lvols);

	TAILQ_FOREACH_SAFE(lvol, &lvs->lvols, link, tmp) {

		TAILQ_REMOVE(&lvs->lvols, lvol, link);

		free(lvol->old_name);

		free(lvol);

	}

	g_lvol_store = NULL;

	free(lvs);



	g_bs_dev->destroy(g_bs_dev);



	if (cb_fn != NULL)

		cb_fn(cb_arg, 0);



	return 0;

}



int

spdk_lvs_destroy(struct spdk_lvol_store *lvs,  bool unmap_device, spdk_lvs_op_complete cb_fn,

		 void *cb_arg)

{

	struct spdk_lvol *lvol, *tmp;



	TAILQ_FOREACH_SAFE(lvol, &lvs->lvols, link, tmp) {

		TAILQ_REMOVE(&lvs->lvols, lvol, link);

		free(lvol->old_name);

		free(lvol);

	}

	g_lvol_store = NULL;

	free(lvs);
@@ -204,8 +224,7 @@ spdk_lvs_unload(struct spdk_lvol_store *lvs, spdk_lvs_op_complete cb_fn, void *c 
}



int

spdk_lvol_resize(struct spdk_lvol *lvol, size_t sz,

		 spdk_lvol_op_complete cb_fn, void *cb_arg)

spdk_lvol_resize(struct spdk_lvol *lvol, size_t sz,  spdk_lvol_op_complete cb_fn, void *cb_arg)

{

	cb_fn(cb_arg, 0);
@@ -245,9 +264,11 @@ spdk_lvol_close(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn, void *cb_ar 


	destruct_req = lvol->lvol_store->destruct_req;

	if (destruct_req && all_lvols_closed == true) {

		if (!lvol->lvol_store->destruct) {

			spdk_lvs_unload(lvol->lvol_store, destruct_req->cb_fn, destruct_req->cb_arg);

			free(destruct_req);

		}

	}



	cb_fn(cb_arg, 0);

}
@@ -255,10 +276,28 @@ spdk_lvol_close(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn, void *cb_ar 
void

spdk_lvol_destroy(struct spdk_lvol *lvol, spdk_lvol_op_complete cb_fn, void *cb_arg)

{

	struct spdk_lvs_req *destruct_req;



	SPDK_CU_ASSERT_FATAL(lvol == g_lvol);



	if (lvol->ref_count != 0) {

		cb_fn(cb_arg, -ENODEV);

	}



	TAILQ_REMOVE(&lvol->lvol_store->lvols, lvol, link);



	destruct_req = lvol->lvol_store->destruct_req;

	if (destruct_req && TAILQ_EMPTY(&lvol->lvol_store->lvols)) {

		if (!lvol->lvol_store->destruct) {

			spdk_lvs_unload(lvol->lvol_store, destruct_req->cb_fn, destruct_req->cb_arg);

		} else {

			spdk_lvs_destroy(lvol->lvol_store, false, destruct_req->cb_fn, destruct_req->cb_arg);

			free(destruct_req);

		}

	}

	g_lvol = NULL;

	free(lvol->old_name);

	free(lvol);

	g_lvol = NULL;



	cb_fn(cb_arg, 0);

}
@@ -673,6 +712,39 @@ ut_lvol_resize(void) 
	free(g_base_bdev);

}



static void

ut_lvs_unload(void)

{

	int rc = 0;

	int sz = 10;

	struct spdk_lvol_store *lvs;



	/* Lvol store is succesfully created */

	rc = vbdev_lvs_create(&g_bdev, 0, lvol_store_op_with_handle_complete, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(g_lvserrno == 0);

	SPDK_CU_ASSERT_FATAL(g_lvol_store != NULL);

	CU_ASSERT(g_bs_dev != NULL);



	lvs = g_lvol_store;

	g_lvol_store = NULL;



	uuid_generate_time(lvs->uuid);



	/* Suuccessfully create lvol, which should be destroyed with lvs later */

	g_lvolerrno = -1;

	rc = vbdev_lvol_create(lvs->uuid, sz, vbdev_lvol_create_complete, NULL);

	CU_ASSERT(rc == 0);

	CU_ASSERT(g_lvolerrno == 0);

	SPDK_CU_ASSERT_FATAL(g_lvol != NULL);



	/* Unload lvol store */

	vbdev_lvs_unload(lvs, lvol_store_op_complete, NULL);

	CU_ASSERT(g_lvserrno == 0);

	CU_ASSERT(g_lvol_store == NULL);

	CU_ASSERT(g_lvol != NULL);

}



static void

ut_lvs_init(void)

{
@@ -868,6 +940,7 @@ int main(int argc, char **argv) 
		CU_add_test(suite, "ut_lvs_init", ut_lvs_init) == NULL ||

		CU_add_test(suite, "ut_lvol_init", ut_lvol_init) == NULL ||

		CU_add_test(suite, "ut_lvs_destroy", ut_lvs_destroy) == NULL ||

		CU_add_test(suite, "ut_lvs_unload", ut_lvs_unload) == NULL ||

		CU_add_test(suite, "ut_lvol_resize", ut_lvol_resize) == NULL ||

		CU_add_test(suite, "lvol_hotremove", ut_lvol_hotremove) == NULL ||

		CU_add_test(suite, "ut_vbdev_lvol_get_io_channel", ut_vbdev_lvol_get_io_channel) == NULL ||