Commit 57d174ff authored by Jim Harris's avatar Jim Harris
Browse files

bdev: add spdk_bdev_open/close 



Retire the old claim/unclaim semantics in favor of
open/close.  Clients must now open a bdev to get
an spdk_bdev_desc, then pass this desc to get an
I/O channel.

This allows multiple clients to open a bdev,
although only one may open a bdev with write
access.

Signed-off-by: default avatarJim Harris <james.r.harris@intel.com>
Change-Id: I4d319f1278170124169a8a75fd791e926b3f7171

Reviewed-on: https://review.gerrithub.io/367611


Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 7347136f

include/spdk/bdev.h

+23 −21
Original line number Diff line number Diff line
@@ -63,8 +63,7 @@ struct spdk_json_write_ctx; 
/** Blockdev status */

enum spdk_bdev_status {

	SPDK_BDEV_STATUS_INVALID,

	SPDK_BDEV_STATUS_UNCLAIMED,

	SPDK_BDEV_STATUS_CLAIMED,

	SPDK_BDEV_STATUS_READY,

	SPDK_BDEV_STATUS_REMOVING,

};
@@ -75,6 +74,11 @@ enum spdk_bdev_status { 
 */

struct spdk_bdev;



/**

 * \brief Handle to an opened SPDK block device.

 */

struct spdk_bdev_desc;



/** Blockdev I/O type */

enum spdk_bdev_io_type {

	SPDK_BDEV_IO_TYPE_READ = 1,
@@ -139,29 +143,26 @@ struct spdk_bdev *spdk_bdev_next(struct spdk_bdev *prev); 
 */

struct spdk_bdev *spdk_bdev_first_leaf(void);

struct spdk_bdev *spdk_bdev_next_leaf(struct spdk_bdev *prev);



/**

 * Claim ownership of a block device.

 * Open a block device for I/O operations.

 *

 * User applications and virtual blockdevs may use this to mediate access to bdevs.

 *

 * When the ownership of the bdev is no longer needed, the user should call spdk_bdev_unclaim().

 *

 * \param bdev Block device to claim.

 * \param bdev Block device to open.

 * \param write true is read/write access requested, false if read-only

 * \param remove_cb callback function for hot remove the device.

 * \param remove_ctx param for hot removal callback function.

 * \return true if the caller claimed the bdev, or false if it was already claimed by another user.

 * \param desc output parameter for the descriptor when operation is successful

 * \return 0 if operation is successful, suitable errno value otherwise

 */

bool spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb, void *remove_ctx);

int spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,

		   void *remove_ctx, struct spdk_bdev_desc **desc);



/**

 * Release claim of ownership of a block device.

 * Close a previously opened block device.

 *

 * When a bdev reference acquired with spdk_bdev_claim() is no longer needed, the user should

 * release the claim using spdk_bdev_unclaim().

 *

 * \param bdev Block device to release.

 * \param desc Block device descriptor to close.

 */

void spdk_bdev_unclaim(struct spdk_bdev *bdev);

void spdk_bdev_close(struct spdk_bdev_desc *desc);



bool spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type);
@@ -229,15 +230,16 @@ size_t spdk_bdev_get_buf_align(const struct spdk_bdev *bdev); 
bool spdk_bdev_has_write_cache(const struct spdk_bdev *bdev);



/**

 * Obtain an I/O channel for this block device. I/O channels are bound to threads,

 * so the resulting I/O channel may only be used from the thread it was originally

 * obtained from.

 * Obtain an I/O channel for the block device opened by the specified

 * descriptor. I/O channels are bound to threads, so the resulting I/O

 * channel may only be used from the thread it was originally obtained

 * from.

 *

 * \param bdev Block device

 * \param desc Block device descriptor

 *

 * \return A handle to the I/O channel or NULL on failure.

 */

struct spdk_io_channel *spdk_bdev_get_io_channel(struct spdk_bdev *bdev);

struct spdk_io_channel *spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc);



/**

 * Submit a read request to the bdev on the given channel.

include/spdk/nvmf.h

+1 −0
Original line number Diff line number Diff line
@@ -113,6 +113,7 @@ struct spdk_nvmf_subsystem { 
		struct {

			char	sn[MAX_SN_LEN + 1];

			struct spdk_bdev *ns_list[MAX_VIRTUAL_NAMESPACE];

			struct spdk_bdev_desc *desc[MAX_VIRTUAL_NAMESPACE];

			struct spdk_io_channel *ch[MAX_VIRTUAL_NAMESPACE];

			uint32_t max_nsid;

		} virt;

include/spdk_internal/bdev.h

+5 −4
Original line number Diff line number Diff line
@@ -211,11 +211,12 @@ struct spdk_bdev { 


	TAILQ_ENTRY(spdk_bdev) vbdev_link;



	/** Remove callback function pointer to upper level stack */

	spdk_bdev_remove_cb_t remove_cb;

	bool bdev_opened_for_write;



	/** Callback context for hot remove the device */

	void *remove_ctx;

	uint32_t vbdevs_opened_for_write;



	/** List of open descriptors for this block device. */

	TAILQ_HEAD(, spdk_bdev_desc) open_descs;



	TAILQ_ENTRY(spdk_bdev) link;

lib/bdev/bdev.c

+97 −38
Original line number Diff line number Diff line
@@ -96,6 +96,14 @@ struct spdk_bdev_mgmt_channel { 
	need_buf_tailq_t need_buf_large;

};



struct spdk_bdev_desc {

	struct spdk_bdev		*bdev;

	spdk_bdev_remove_cb_t		remove_cb;

	void				*remove_ctx;

	bool				write;

	TAILQ_ENTRY(spdk_bdev_desc)	link;

};



struct spdk_bdev_channel {

	struct spdk_bdev	*bdev;
@@ -702,9 +710,9 @@ spdk_bdev_channel_destroy(void *io_device, void *ctx_buf) 
}



struct spdk_io_channel *

spdk_bdev_get_io_channel(struct spdk_bdev *bdev)

spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)

{

	return spdk_get_io_channel(bdev);

	return spdk_get_io_channel(desc->bdev);

}



const char *
@@ -785,7 +793,6 @@ spdk_bdev_read(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {

		return -EINVAL;

	}
@@ -825,7 +832,6 @@ spdk_bdev_readv(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {

		return -EINVAL;

	}
@@ -862,7 +868,6 @@ spdk_bdev_write(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	if (spdk_bdev_io_valid(bdev, offset, nbytes) != 0) {

		return -EINVAL;

	}
@@ -902,7 +907,6 @@ spdk_bdev_writev(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	if (spdk_bdev_io_valid(bdev, offset, len) != 0) {

		return -EINVAL;

	}
@@ -940,7 +944,6 @@ spdk_bdev_unmap(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	if (bdesc_count == 0) {

		SPDK_ERRLOG("Invalid bdesc_count 0\n");

		return -EINVAL;
@@ -982,7 +985,6 @@ spdk_bdev_flush(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);

	int rc;



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);

	bdev_io = spdk_bdev_get_io();

	if (!bdev_io) {

		SPDK_ERRLOG("bdev_io memory allocation failed duing flush\n");
@@ -1070,8 +1072,6 @@ spdk_bdev_reset(struct spdk_bdev *bdev, struct spdk_io_channel *ch, 
	struct spdk_bdev_io *bdev_io;

	struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);



	assert(bdev->status != SPDK_BDEV_STATUS_UNCLAIMED);



	bdev_io = spdk_bdev_get_io();

	if (!bdev_io) {

		SPDK_ERRLOG("bdev_io memory allocation failed duing reset\n");
@@ -1360,8 +1360,13 @@ _spdk_bdev_register(struct spdk_bdev *bdev) 
{

	struct spdk_bdev_module_if *vbdev_module;



	bdev->status = SPDK_BDEV_STATUS_READY;



	/* initialize the reset generation value to zero */

	bdev->gencnt = 0;

	TAILQ_INIT(&bdev->open_descs);

	bdev->bdev_opened_for_write = false;

	bdev->vbdevs_opened_for_write = 0;



	TAILQ_INIT(&bdev->vbdevs);

	TAILQ_INIT(&bdev->base_bdevs);
@@ -1373,7 +1378,6 @@ _spdk_bdev_register(struct spdk_bdev *bdev) 
				sizeof(struct spdk_bdev_channel));



	pthread_mutex_init(&bdev->mutex, NULL);

	bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;

	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Inserting bdev %s into list\n", bdev->name);

	TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, link);
@@ -1406,23 +1410,28 @@ spdk_vbdev_register(struct spdk_bdev *vbdev, struct spdk_bdev **base_bdevs, int 
void

spdk_bdev_unregister(struct spdk_bdev *bdev)

{

	struct spdk_bdev_desc	*desc, *tmp;

	int			rc;



	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "Removing bdev %s from list\n", bdev->name);



	pthread_mutex_lock(&bdev->mutex);

	assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_UNCLAIMED);

	if (bdev->status == SPDK_BDEV_STATUS_CLAIMED) {

		if (bdev->remove_cb) {



	bdev->status = SPDK_BDEV_STATUS_REMOVING;



	TAILQ_FOREACH_SAFE(desc, &bdev->open_descs, link, tmp) {

		if (desc->remove_cb) {

			pthread_mutex_unlock(&bdev->mutex);

			bdev->remove_cb(bdev->remove_ctx);

			return;

		} else {

			bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;

			desc->remove_cb(desc->remove_ctx);

			pthread_mutex_lock(&bdev->mutex);

		}

	}



	if (!TAILQ_EMPTY(&bdev->open_descs)) {

		pthread_mutex_unlock(&bdev->mutex);

		return;

	}



	TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, link);

	pthread_mutex_unlock(&bdev->mutex);
@@ -1448,43 +1457,93 @@ spdk_vbdev_unregister(struct spdk_bdev *vbdev) 
	spdk_bdev_unregister(vbdev);

}



bool

spdk_bdev_claim(struct spdk_bdev *bdev, spdk_bdev_remove_cb_t remove_cb,

		void *remove_ctx)

static bool

__is_bdev_opened_for_write(struct spdk_bdev *bdev)

{

	bool success;

	struct spdk_bdev *base;



	if (bdev->bdev_opened_for_write) {

		return true;

	}



	TAILQ_FOREACH(base, &bdev->base_bdevs, base_bdev_link) {

		if (__is_bdev_opened_for_write(base)) {

			return true;

		}

	}



	return false;

}



static void

__modify_write_counts(struct spdk_bdev *bdev, int mod)

{

	struct spdk_bdev *base;



	TAILQ_FOREACH(base, &bdev->base_bdevs, base_bdev_link) {

		base->vbdevs_opened_for_write += mod;

		__modify_write_counts(base, mod);

	}

}



int

spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,

	       void *remove_ctx, struct spdk_bdev_desc **_desc)

{

	struct spdk_bdev_desc *desc;



	desc = calloc(1, sizeof(*desc));

	if (desc == NULL) {

		return -ENOMEM;

	}



	pthread_mutex_lock(&bdev->mutex);



	if (bdev->status != SPDK_BDEV_STATUS_CLAIMED) {

		/* Take ownership of bdev. */

		bdev->remove_cb = remove_cb;

		bdev->remove_ctx = remove_ctx;

		bdev->status = SPDK_BDEV_STATUS_CLAIMED;

		success = true;

	} else {

		/* bdev is already claimed. */

		success = false;

	if (write && (__is_bdev_opened_for_write(bdev) || bdev->vbdevs_opened_for_write > 0)) {

		SPDK_ERRLOG("failed, %s (or one of its virtual bdevs) already opened for write\n", bdev->name);

		free(desc);

		pthread_mutex_unlock(&bdev->mutex);

		return -EPERM;

	}



	TAILQ_INSERT_TAIL(&bdev->open_descs, desc, link);



	if (write) {

		bdev->bdev_opened_for_write = true;

		__modify_write_counts(bdev, 1);

	}



	desc->bdev = bdev;

	desc->remove_cb = remove_cb;

	desc->remove_ctx = remove_ctx;

	desc->write = write;

	*_desc = desc;



	pthread_mutex_unlock(&bdev->mutex);



	return success;

	return 0;

}



void

spdk_bdev_unclaim(struct spdk_bdev *bdev)

spdk_bdev_close(struct spdk_bdev_desc *desc)

{

	struct spdk_bdev *bdev = desc->bdev;

	bool do_unregister = false;



	pthread_mutex_lock(&bdev->mutex);

	assert(bdev->status == SPDK_BDEV_STATUS_CLAIMED || bdev->status == SPDK_BDEV_STATUS_REMOVING);



	if (desc->write) {

		assert(bdev->bdev_opened_for_write);

		bdev->bdev_opened_for_write = false;

		__modify_write_counts(bdev, -1);

	}



	TAILQ_REMOVE(&bdev->open_descs, desc, link);

	free(desc);



	if (bdev->status == SPDK_BDEV_STATUS_REMOVING) {

		do_unregister = true;

	}

	bdev->remove_cb = NULL;

	bdev->remove_ctx = NULL;

	bdev->status = SPDK_BDEV_STATUS_UNCLAIMED;

	pthread_mutex_unlock(&bdev->mutex);



	if (do_unregister == true) {

lib/bdev/error/vbdev_error.c

+0 −6
Original line number Diff line number Diff line
@@ -157,7 +157,6 @@ vbdev_error_free(struct vbdev_error_disk *error_disk) 


	TAILQ_REMOVE(&g_vbdev_error_disks, error_disk, tailq);



	spdk_bdev_unclaim(error_disk->base_bdev);

	vbdev_error_disk_free(error_disk);

}
@@ -217,11 +216,6 @@ spdk_vbdev_error_create(struct spdk_bdev *base_bdev) 
	struct vbdev_error_disk *disk;

	int rc;



	if (!spdk_bdev_claim(base_bdev, NULL, NULL)) {

		SPDK_ERRLOG("Error bdev %s is already claimed\n", base_bdev->name);

		return -1;

	}



	disk = calloc(1, sizeof(*disk));

	if (!disk) {

		SPDK_ERRLOG("Memory allocation failure\n");