Commit 60920573 authored by Mike Gerdts's avatar Mike Gerdts Committed by Jim Harris
Browse files

lvol: add support for external snapshots 



This provides the lib/lvol wrapper around blobstore's external
snapshots. Later commits make this work with vbdev_lvol.

The blobstore external snapshot implementation stores an opaque
identifier in an internal xattr. Lvstore uses this to store the
stringified UUID of the bdev that will act as the external snapshot.
This is used by the newly introduced spdk_lvol_create_esnap_clone() to
store the bdev UUID in the blob's metadata.

Change-Id: I58c7b32b656ad1d21a446e3b91e59e655efac7e4
Signed-off-by: default avatarMike Gerdts <mgerdts@nvidia.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/14977


Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Community-CI: Mellanox Build Bot
parent c894388d

include/spdk/lvol.h

+40 −2
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (C) 2017 Intel Corporation.

 *   All rights reserved.

 *   Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 *   Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 */



/** \file
@@ -62,8 +62,14 @@ struct spdk_lvs_opts { 
	 * values. After that, new added fields should be put in the end of the struct.

	 */

	uint32_t		opts_size;



	/**

	 * A function to be called to load external snapshots. If this is NULL while the lvolstore

	 * is being loaded, the lvolstore will not support external snapshots.

	 */

	spdk_bs_esnap_dev_create esnap_bs_dev_create;

} __attribute__((packed));

SPDK_STATIC_ASSERT(sizeof(struct spdk_lvs_opts) == 80, "Incorrect size");

SPDK_STATIC_ASSERT(sizeof(struct spdk_lvs_opts) == 88, "Incorrect size");



/**

 * Initialize an spdk_lvs_opts structure to the defaults.
@@ -200,6 +206,24 @@ void spdk_lvol_create_snapshot(struct spdk_lvol *lvol, const char *snapshot_name 
void spdk_lvol_create_clone(struct spdk_lvol *lvol, const char *clone_name,

			    spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg);



/**

 * Create clone of given non-lvol device.

 *

 * The bdev that is being cloned is commonly called an external snapshot or esnap. The clone is

 * commonly called an esnap clone.

 *

 * \param esnap_id The identifier that will be passed to the spdk_bs_esnap_dev_create callback.

 * \param id_len The length of esnap_id, in bytes.

 * \param size_bytes The size of the external snapshot device, in bytes.

 * \param lvs Handle to lvolstore.

 * \param clone_name Name of created clone.

 * \param cb_fn Completion callback.

 * \param cb_arg Completion callback custom arguments.

 */

int spdk_lvol_create_esnap_clone(const void *esnap_id, uint32_t id_len, uint64_t size_bytes,

				 struct spdk_lvol_store *lvs, const char *clone_name,

				 spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg);



/**

 * Rename lvol with new_name.

 *
@@ -255,6 +279,20 @@ struct spdk_io_channel *spdk_lvol_get_io_channel(struct spdk_lvol *lvol); 
void spdk_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn,

		   void *cb_arg);



/**

 * Load lvolstore from the given blobstore device with options.

 *

 * If lvs_opts is not NULL, it should be initalized with spdk_lvs_opts_init().

 *

 * \param bs_dev Pointer to the blobstore device.

 * \param lvs_opts lvolstore options.

 * \param cb_fn Completion callback.

 * \param cb_arg Completion callback custom arguments.

 * blobstore.

 */

void spdk_lvs_load_ext(struct spdk_bs_dev *bs_dev, const struct spdk_lvs_opts *lvs_opts,

		       spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg);



/**

 * Grow a lvstore to fill the underlying device

 *

lib/lvol/lvol.c

+117 −17
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ static TAILQ_HEAD(, spdk_lvol_store) g_lvol_stores = TAILQ_HEAD_INITIALIZER(g_lv 
static pthread_mutex_t g_lvol_stores_mutex = PTHREAD_MUTEX_INITIALIZER;



static inline int lvs_opts_copy(const struct spdk_lvs_opts *src, struct spdk_lvs_opts *dst);



static int

add_lvs_to_list(struct spdk_lvol_store *lvs)

{
@@ -394,11 +395,13 @@ lvs_bs_opts_init(struct spdk_bs_opts *opts) 
	opts->max_channel_ops = SPDK_LVOL_BLOB_OPTS_CHANNEL_OPS;

}



void

spdk_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)

static void

lvs_load(struct spdk_bs_dev *bs_dev, const struct spdk_lvs_opts *_lvs_opts,

	 spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)

{

	struct spdk_lvs_with_handle_req *req;

	struct spdk_bs_opts opts = {};

	struct spdk_bs_opts bs_opts = {};

	struct spdk_lvs_opts lvs_opts;



	assert(cb_fn != NULL);
@@ -408,6 +411,14 @@ spdk_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn 
		return;

	}



	spdk_lvs_opts_init(&lvs_opts);

	if (_lvs_opts != NULL) {

		if (lvs_opts_copy(_lvs_opts, &lvs_opts) != 0) {

			SPDK_ERRLOG("Invalid options\n");

			cb_fn(cb_arg, NULL, -EINVAL);

		}

	}



	req = calloc(1, sizeof(*req));

	if (req == NULL) {

		SPDK_ERRLOG("Cannot alloc memory for request structure\n");
@@ -426,10 +437,28 @@ spdk_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn 
	req->cb_arg = cb_arg;

	req->bs_dev = bs_dev;



	lvs_bs_opts_init(&opts);

	snprintf(opts.bstype.bstype, sizeof(opts.bstype.bstype), "LVOLSTORE");

	lvs_bs_opts_init(&bs_opts);

	snprintf(bs_opts.bstype.bstype, sizeof(bs_opts.bstype.bstype), "LVOLSTORE");



	if (lvs_opts.esnap_bs_dev_create != NULL) {

		bs_opts.esnap_bs_dev_create = lvs_opts.esnap_bs_dev_create;

		bs_opts.esnap_ctx = req->lvol_store;

	}



	spdk_bs_load(bs_dev, &bs_opts, lvs_load_cb, req);

}



	spdk_bs_load(bs_dev, &opts, lvs_load_cb, req);

void

spdk_lvs_load(struct spdk_bs_dev *bs_dev, spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)

{

	lvs_load(bs_dev, NULL, cb_fn, cb_arg);

}



void

spdk_lvs_load_ext(struct spdk_bs_dev *bs_dev, const struct spdk_lvs_opts *opts,

		  spdk_lvs_op_with_handle_complete cb_fn, void *cb_arg)

{

	lvs_load(bs_dev, opts, cb_fn, cb_arg);

}



static void
@@ -591,12 +620,13 @@ lvs_opts_copy(const struct spdk_lvs_opts *src, struct spdk_lvs_opts *dst) 
	}

	SET_FIELD(num_md_pages_per_cluster_ratio);

	SET_FIELD(opts_size);

	SET_FIELD(esnap_bs_dev_create);



	dst->opts_size = src->opts_size;



	/* You should not remove this statement, but need to update the assert statement

	 * if you add a new field, and also add a corresponding SET_FIELD statement */

	SPDK_STATIC_ASSERT(sizeof(struct spdk_lvs_opts) == 80, "Incorrect size");

	SPDK_STATIC_ASSERT(sizeof(struct spdk_lvs_opts) == 88, "Incorrect size");



#undef FIELD_OK

#undef SET_FIELD
@@ -605,13 +635,17 @@ lvs_opts_copy(const struct spdk_lvs_opts *src, struct spdk_lvs_opts *dst) 
}



static void

setup_lvs_opts(struct spdk_bs_opts *bs_opts, struct spdk_lvs_opts *o, uint32_t total_clusters)

setup_lvs_opts(struct spdk_bs_opts *bs_opts, struct spdk_lvs_opts *o, uint32_t total_clusters,

	       void *esnap_ctx)

{

	assert(o != NULL);

	lvs_bs_opts_init(bs_opts);

	bs_opts->cluster_sz = o->cluster_sz;

	bs_opts->clear_method = (enum bs_clear_method)o->clear_method;

	bs_opts->num_md_pages = (o->num_md_pages_per_cluster_ratio * total_clusters) / 100;

	bs_opts->esnap_bs_dev_create = o->esnap_bs_dev_create;

	bs_opts->esnap_ctx = esnap_ctx;

	snprintf(bs_opts->bstype.bstype, sizeof(bs_opts->bstype.bstype), "LVOLSTORE");

}



int
@@ -648,24 +682,26 @@ spdk_lvs_init(struct spdk_bs_dev *bs_dev, struct spdk_lvs_opts *o, 
	}

	total_clusters = bs_dev->blockcnt / (lvs_opts.cluster_sz / bs_dev->blocklen);



	setup_lvs_opts(&opts, o, total_clusters);

	lvs = lvs_alloc();

	if (!lvs) {

		SPDK_ERRLOG("Cannot alloc memory for lvol store base pointer\n");

		return -ENOMEM;

	}



	setup_lvs_opts(&opts, o, total_clusters, lvs);



	if (strnlen(lvs_opts.name, SPDK_LVS_NAME_MAX) == SPDK_LVS_NAME_MAX) {

		SPDK_ERRLOG("Name has no null terminator.\n");

		lvs_free(lvs);

		return -EINVAL;

	}



	if (strnlen(lvs_opts.name, SPDK_LVS_NAME_MAX) == 0) {

		SPDK_ERRLOG("No name specified.\n");

		lvs_free(lvs);

		return -EINVAL;

	}



	lvs = lvs_alloc();

	if (!lvs) {

		SPDK_ERRLOG("Cannot alloc memory for lvol store base pointer\n");

		return -ENOMEM;

	}



	spdk_uuid_generate(&lvs->uuid);

	snprintf(lvs->name, sizeof(lvs->name), "%s", lvs_opts.name);
@@ -689,8 +725,6 @@ spdk_lvs_init(struct spdk_bs_dev *bs_dev, struct spdk_lvs_opts *o, 
	lvs_req->lvol_store = lvs;

	lvs->bs_dev = bs_dev;



	snprintf(opts.bstype.bstype, sizeof(opts.bstype.bstype), "LVOLSTORE");



	SPDK_INFOLOG(lvol, "Initializing lvol store\n");

	spdk_bs_init(bs_dev, &opts, lvs_init_cb, lvs_req);
@@ -1017,6 +1051,16 @@ lvol_create_cb(void *cb_arg, spdk_blob_id blobid, int lvolerrno) 


	spdk_blob_open_opts_init(&opts, sizeof(opts));

	opts.clear_method = req->lvol->clear_method;

	/*

	 * If the lvol that is being created is an esnap clone, the blobstore needs to be able to

	 * pass the lvol to the esnap_bs_dev_create callback. In order for that to happen, we need

	 * to pass it here.

	 *

	 * This does set ensap_ctx in cases where it's not needed, but we don't know that it's not

	 * needed until after the blob is open. When the blob is not an esnap clone, a reference to

	 * the value stored in opts.esnap_ctx is not retained by the blobstore.

	 */

	opts.esnap_ctx = req->lvol;

	bs = req->lvol->lvol_store->blobstore;



	spdk_bs_open_blob_ext(bs, blobid, &opts, lvol_create_open_cb, req);
@@ -1128,6 +1172,62 @@ spdk_lvol_create(struct spdk_lvol_store *lvs, const char *name, uint64_t sz, 
	return 0;

}



int

spdk_lvol_create_esnap_clone(const void *esnap_id, uint32_t id_len, uint64_t size_bytes,

			     struct spdk_lvol_store *lvs, const char *clone_name,

			     spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg)

{

	struct spdk_lvol_with_handle_req *req;

	struct spdk_blob_store *bs;

	struct spdk_lvol *lvol;

	struct spdk_blob_opts opts;

	char *xattr_names[] = {LVOL_NAME, "uuid"};

	int rc;



	if (lvs == NULL) {

		SPDK_ERRLOG("lvol store does not exist\n");

		return -EINVAL;

	}



	rc = lvs_verify_lvol_name(lvs, clone_name);

	if (rc < 0) {

		return rc;

	}



	bs = lvs->blobstore;



	req = calloc(1, sizeof(*req));

	if (!req) {

		SPDK_ERRLOG("Cannot alloc memory for lvol request pointer\n");

		return -ENOMEM;

	}

	req->cb_fn = cb_fn;

	req->cb_arg = cb_arg;



	lvol = lvol_alloc(lvs, clone_name, true, LVOL_CLEAR_WITH_DEFAULT);

	if (!lvol) {

		free(req);

		SPDK_ERRLOG("Cannot alloc memory for lvol base pointer\n");

		return -ENOMEM;

	}

	req->lvol = lvol;



	spdk_blob_opts_init(&opts, sizeof(opts));

	opts.esnap_id = esnap_id;

	opts.esnap_id_len = id_len;

	opts.thin_provision = true;

	opts.num_clusters = spdk_divide_round_up(size_bytes, spdk_bs_get_cluster_size(bs));

	opts.clear_method = lvol->clear_method;

	opts.xattrs.count = SPDK_COUNTOF(xattr_names);

	opts.xattrs.names = xattr_names;

	opts.xattrs.ctx = lvol;

	opts.xattrs.get_value = lvol_get_xattr_value;



	spdk_bs_create_blob_ext(lvs->blobstore, &opts, lvol_create_cb, req);



	return 0;

}



void

spdk_lvol_create_snapshot(struct spdk_lvol *origlvol, const char *snapshot_name,

			  spdk_lvol_op_with_handle_complete cb_fn, void *cb_arg)

lib/lvol/spdk_lvol.map

+2 −0
Original line number Diff line number Diff line
@@ -17,9 +17,11 @@ 
	spdk_lvol_close;

	spdk_lvol_get_io_channel;

	spdk_lvs_load;

	spdk_lvs_load_ext;

	spdk_lvol_open;

	spdk_lvol_inflate;

	spdk_lvol_decouple_parent;

	spdk_lvol_create_esnap_clone;



	# internal functions

	spdk_lvol_resize;

mk/spdk.lib_deps.mk

+1 −1
Original line number Diff line number Diff line 
#  SPDX-License-Identifier: BSD-3-Clause

#  Copyright (C) 2015 Intel Corporation.

#  Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES

#  All rights reserved.

#  Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#



# A quick note on organization:

test/unit/lib/bdev/vbdev_lvol.c/vbdev_lvol_ut.c

+7 −1
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (C) 2017 Intel Corporation.

 *   All rights reserved.

 *   Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 *   Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 */



#include "spdk_cunit.h"
@@ -52,6 +52,12 @@ spdk_bdev_get_md_size(const struct spdk_bdev *bdev) 
	return bdev->md_len;

}



const struct spdk_uuid *

spdk_bdev_get_uuid(const struct spdk_bdev *bdev)

{

	return &bdev->uuid;

}



int

spdk_bdev_alias_add(struct spdk_bdev *bdev, const char *alias)

{