Commit b55efd2a authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

bdev: finish encapsulating members of spdk_bdev_io 



Change-Id: Ic3e1c535d8d6323990bc353305d617b3fd0fa471
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/416233


Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
parent a32e9c35

include/spdk/bdev_module.h

+6 −6
Original line number Diff line number Diff line
@@ -299,12 +299,6 @@ struct spdk_bdev_io { 
	/** The block device that this I/O belongs to. */

	struct spdk_bdev *bdev;



	/** The bdev I/O channel that this was handled on. */

	struct spdk_bdev_channel *ch;



	/** The bdev I/O channel that this was submitted on. */

	struct spdk_bdev_channel *io_submit_ch;



	/** Error information from a device */

	union {

		/** Only valid when status is SPDK_BDEV_IO_STATUS_NVME_ERROR */
@@ -388,6 +382,12 @@ struct spdk_bdev_io { 
	 *  must not read or write to these fields.

	 */

	struct __bdev_io_internal_fields {

		/** The bdev I/O channel that this was handled on. */

		struct spdk_bdev_channel *ch;



		/** The bdev I/O channel that this was submitted on. */

		struct spdk_bdev_channel *io_submit_ch;



		/** User function that will be called when this completes */

		spdk_bdev_io_completion_cb cb;

lib/bdev/bdev.c

+50 −50
Original line number Diff line number Diff line
@@ -387,7 +387,7 @@ spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf) 
	bdev_io->internal.buf = buf;

	bdev_io->u.bdev.iovs[0].iov_base = (void *)((unsigned long)((char *)buf + 512) & ~511UL);

	bdev_io->u.bdev.iovs[0].iov_len = bdev_io->internal.buf_len;

	bdev_io->internal.get_buf_cb(bdev_io->ch->channel, bdev_io);

	bdev_io->internal.get_buf_cb(bdev_io->internal.ch->channel, bdev_io);

}



static void
@@ -402,7 +402,7 @@ spdk_bdev_io_put_buf(struct spdk_bdev_io *bdev_io) 
	assert(bdev_io->u.bdev.iovcnt == 1);



	buf = bdev_io->internal.buf;

	ch = bdev_io->ch->shared_resource->mgmt_ch;

	ch = bdev_io->internal.ch->shared_resource->mgmt_ch;



	if (bdev_io->internal.buf_len <= SPDK_BDEV_SMALL_BUF_MAX_SIZE) {

		pool = g_bdev_mgr.buf_small_pool;
@@ -434,12 +434,12 @@ spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, u 


	if (spdk_unlikely(bdev_io->u.bdev.iovs[0].iov_base != NULL)) {

		/* Buffer already present */

		cb(bdev_io->ch->channel, bdev_io);

		cb(bdev_io->internal.ch->channel, bdev_io);

		return;

	}



	assert(len <= SPDK_BDEV_LARGE_BUF_MAX_SIZE);

	mgmt_ch = bdev_io->ch->shared_resource->mgmt_ch;

	mgmt_ch = bdev_io->internal.ch->shared_resource->mgmt_ch;



	bdev_io->internal.buf_len = len;

	bdev_io->internal.get_buf_cb = cb;
@@ -942,7 +942,7 @@ spdk_bdev_get_io(struct spdk_bdev_channel *channel) 
void

spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)

{

	struct spdk_bdev_mgmt_channel *ch = bdev_io->ch->shared_resource->mgmt_ch;

	struct spdk_bdev_mgmt_channel *ch = bdev_io->internal.ch->shared_resource->mgmt_ch;



	assert(bdev_io != NULL);

	assert(bdev_io->internal.status != SPDK_BDEV_IO_STATUS_PENDING);
@@ -1022,7 +1022,7 @@ _spdk_bdev_io_submit(void *ctx) 
{

	struct spdk_bdev_io *bdev_io = ctx;

	struct spdk_bdev *bdev = bdev_io->bdev;

	struct spdk_bdev_channel *bdev_ch = bdev_io->ch;

	struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;

	struct spdk_io_channel *ch = bdev_ch->channel;

	struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
@@ -1056,16 +1056,16 @@ static void 
spdk_bdev_io_submit(struct spdk_bdev_io *bdev_io)

{

	struct spdk_bdev *bdev = bdev_io->bdev;

	struct spdk_thread *thread = spdk_io_channel_get_thread(bdev_io->ch->channel);

	struct spdk_thread *thread = spdk_io_channel_get_thread(bdev_io->internal.ch->channel);



	assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);



	if (bdev_io->ch->flags & BDEV_CH_QOS_ENABLED) {

	if (bdev_io->internal.ch->flags & BDEV_CH_QOS_ENABLED) {

		if (thread == bdev->qos->thread) {

			_spdk_bdev_io_submit(bdev_io);

		} else {

			bdev_io->io_submit_ch = bdev_io->ch;

			bdev_io->ch = bdev->qos->ch;

			bdev_io->internal.io_submit_ch = bdev_io->internal.ch;

			bdev_io->internal.ch = bdev->qos->ch;

			spdk_thread_send_msg(bdev->qos->thread, _spdk_bdev_io_submit, bdev_io);

		}

	} else {
@@ -1077,7 +1077,7 @@ static void 
spdk_bdev_io_submit_reset(struct spdk_bdev_io *bdev_io)

{

	struct spdk_bdev *bdev = bdev_io->bdev;

	struct spdk_bdev_channel *bdev_ch = bdev_io->ch;

	struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;

	struct spdk_io_channel *ch = bdev_ch->channel;



	assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
@@ -1098,7 +1098,7 @@ spdk_bdev_io_init(struct spdk_bdev_io *bdev_io, 
	bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;

	bdev_io->internal.in_submit_request = false;

	bdev_io->internal.buf = NULL;

	bdev_io->io_submit_ch = NULL;

	bdev_io->internal.io_submit_ch = NULL;

}



bool
@@ -1330,7 +1330,7 @@ _spdk_bdev_abort_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_channel *ch) 
	while (!STAILQ_EMPTY(queue)) {

		bdev_io = STAILQ_FIRST(queue);

		STAILQ_REMOVE_HEAD(queue, internal.buf_link);

		if (bdev_io->ch == ch) {

		if (bdev_io->internal.ch == ch) {

			spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);

		} else {

			STAILQ_INSERT_TAIL(&tmp, bdev_io, internal.buf_link);
@@ -1350,7 +1350,7 @@ _spdk_bdev_abort_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_channel *ch) 
	struct spdk_bdev_io *bdev_io, *tmp;



	TAILQ_FOREACH_SAFE(bdev_io, queue, link, tmp) {

		if (bdev_io->ch == ch) {

		if (bdev_io->internal.ch == ch) {

			TAILQ_REMOVE(queue, bdev_io, link);

			/*

			 * spdk_bdev_io_complete() assumes that the completed I/O had
@@ -1666,7 +1666,7 @@ spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_READ;

	bdev_io->u.bdev.iov.iov_base = buf;

	bdev_io->u.bdev.iov.iov_len = num_blocks * bdev->blocklen;
@@ -1713,7 +1713,7 @@ int spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel * 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_READ;

	bdev_io->u.bdev.iovs = iov;

	bdev_io->u.bdev.iovcnt = iovcnt;
@@ -1761,7 +1761,7 @@ spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;

	bdev_io->u.bdev.iov.iov_base = buf;

	bdev_io->u.bdev.iov.iov_len = num_blocks * bdev->blocklen;
@@ -1813,7 +1813,7 @@ spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;

	bdev_io->u.bdev.iovs = iov;

	bdev_io->u.bdev.iovcnt = iovcnt;
@@ -1864,7 +1864,7 @@ spdk_bdev_write_zeroes_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channe 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->u.bdev.offset_blocks = offset_blocks;



	if (spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES)) {
@@ -1944,7 +1944,7 @@ spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP;

	bdev_io->u.bdev.iov.iov_base = NULL;

	bdev_io->u.bdev.iov.iov_len = 0;
@@ -1994,7 +1994,7 @@ spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH;

	bdev_io->u.bdev.iovs = NULL;

	bdev_io->u.bdev.iovcnt = 0;
@@ -2100,7 +2100,7 @@ spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_RESET;

	bdev_io->u.reset.ch_ref = NULL;

	spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
@@ -2194,7 +2194,7 @@ spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channe 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_ADMIN;

	bdev_io->u.nvme_passthru.cmd = *cmd;

	bdev_io->u.nvme_passthru.buf = buf;
@@ -2231,7 +2231,7 @@ spdk_bdev_nvme_io_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel * 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO;

	bdev_io->u.nvme_passthru.cmd = *cmd;

	bdev_io->u.nvme_passthru.buf = buf;
@@ -2268,7 +2268,7 @@ spdk_bdev_nvme_io_passthru_md(struct spdk_bdev_desc *desc, struct spdk_io_channe 
		return -ENOMEM;

	}



	bdev_io->ch = channel;

	bdev_io->internal.ch = channel;

	bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO_MD;

	bdev_io->u.nvme_passthru.cmd = *cmd;

	bdev_io->u.nvme_passthru.buf = buf;
@@ -2325,10 +2325,10 @@ _spdk_bdev_ch_retry_io(struct spdk_bdev_channel *bdev_ch) 
	while (!TAILQ_EMPTY(&shared_resource->nomem_io)) {

		bdev_io = TAILQ_FIRST(&shared_resource->nomem_io);

		TAILQ_REMOVE(&shared_resource->nomem_io, bdev_io, link);

		bdev_io->ch->io_outstanding++;

		bdev_io->internal.ch->io_outstanding++;

		shared_resource->io_outstanding++;

		bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;

		bdev->fn_table->submit_request(bdev_io->ch->channel, bdev_io);

		bdev->fn_table->submit_request(bdev_io->internal.ch->channel, bdev_io);

		if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {

			break;

		}
@@ -2340,21 +2340,21 @@ _spdk_bdev_io_complete(void *ctx) 
{

	struct spdk_bdev_io *bdev_io = ctx;



	if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->io_submit_ch)) {

	if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->internal.io_submit_ch)) {

		/*

		 * Send the completion to the thread that originally submitted the I/O,

		 * which may not be the current thread in the case of QoS.

		 */

		if (bdev_io->io_submit_ch) {

			bdev_io->ch = bdev_io->io_submit_ch;

			bdev_io->io_submit_ch = NULL;

		if (bdev_io->internal.io_submit_ch) {

			bdev_io->internal.ch = bdev_io->internal.io_submit_ch;

			bdev_io->internal.io_submit_ch = NULL;

		}



		/*

		 * Defer completion to avoid potential infinite recursion if the

		 * user's completion callback issues a new I/O.

		 */

		spdk_thread_send_msg(spdk_io_channel_get_thread(bdev_io->ch->channel),

		spdk_thread_send_msg(spdk_io_channel_get_thread(bdev_io->internal.ch->channel),

				     _spdk_bdev_io_complete, bdev_io);

		return;

	}
@@ -2362,14 +2362,14 @@ _spdk_bdev_io_complete(void *ctx) 
	if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {

		switch (bdev_io->type) {

		case SPDK_BDEV_IO_TYPE_READ:

			bdev_io->ch->stat.bytes_read += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;

			bdev_io->ch->stat.num_read_ops++;

			bdev_io->ch->stat.read_latency_ticks += (spdk_get_ticks() - bdev_io->internal.submit_tsc);

			bdev_io->internal.ch->stat.bytes_read += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;

			bdev_io->internal.ch->stat.num_read_ops++;

			bdev_io->internal.ch->stat.read_latency_ticks += (spdk_get_ticks() - bdev_io->internal.submit_tsc);

			break;

		case SPDK_BDEV_IO_TYPE_WRITE:

			bdev_io->ch->stat.bytes_written += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;

			bdev_io->ch->stat.num_write_ops++;

			bdev_io->ch->stat.write_latency_ticks += (spdk_get_ticks() - bdev_io->internal.submit_tsc);

			bdev_io->internal.ch->stat.bytes_written += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;

			bdev_io->internal.ch->stat.num_write_ops++;

			bdev_io->internal.ch->stat.write_latency_ticks += (spdk_get_ticks() - bdev_io->internal.submit_tsc);

			break;

		default:

			break;
@@ -2378,26 +2378,26 @@ _spdk_bdev_io_complete(void *ctx) 


#ifdef SPDK_CONFIG_VTUNE

	uint64_t now_tsc = spdk_get_ticks();

	if (now_tsc > (bdev_io->ch->start_tsc + bdev_io->ch->interval_tsc)) {

	if (now_tsc > (bdev_io->internal.ch->start_tsc + bdev_io->internal.ch->interval_tsc)) {

		uint64_t data[5];



		data[0] = bdev_io->ch->stat.num_read_ops - bdev_io->ch->prev_stat.num_read_ops;

		data[1] = bdev_io->ch->stat.bytes_read - bdev_io->ch->prev_stat.bytes_read;

		data[2] = bdev_io->ch->stat.num_write_ops - bdev_io->ch->prev_stat.num_write_ops;

		data[3] = bdev_io->ch->stat.bytes_written - bdev_io->ch->prev_stat.bytes_written;

		data[0] = bdev_io->internal.ch->stat.num_read_ops - bdev_io->internal.ch->prev_stat.num_read_ops;

		data[1] = bdev_io->internal.ch->stat.bytes_read - bdev_io->internal.ch->prev_stat.bytes_read;

		data[2] = bdev_io->internal.ch->stat.num_write_ops - bdev_io->internal.ch->prev_stat.num_write_ops;

		data[3] = bdev_io->internal.ch->stat.bytes_written - bdev_io->internal.ch->prev_stat.bytes_written;

		data[4] = bdev_io->bdev->fn_table->get_spin_time ?

			  bdev_io->bdev->fn_table->get_spin_time(bdev_io->ch->channel) : 0;

			  bdev_io->bdev->fn_table->get_spin_time(bdev_io->internal.ch->channel) : 0;



		__itt_metadata_add(g_bdev_mgr.domain, __itt_null, bdev_io->ch->handle,

		__itt_metadata_add(g_bdev_mgr.domain, __itt_null, bdev_io->internal.ch->handle,

				   __itt_metadata_u64, 5, data);



		bdev_io->ch->prev_stat = bdev_io->ch->stat;

		bdev_io->ch->start_tsc = now_tsc;

		bdev_io->internal.ch->prev_stat = bdev_io->internal.ch->stat;

		bdev_io->internal.ch->start_tsc = now_tsc;

	}

#endif



	assert(bdev_io->internal.cb != NULL);

	assert(spdk_get_thread() == spdk_io_channel_get_thread(bdev_io->ch->channel));

	assert(spdk_get_thread() == spdk_io_channel_get_thread(bdev_io->internal.ch->channel));



	bdev_io->internal.cb(bdev_io, bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,

			     bdev_io->internal.caller_ctx);
@@ -2434,7 +2434,7 @@ void 
spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status)

{

	struct spdk_bdev *bdev = bdev_io->bdev;

	struct spdk_bdev_channel *bdev_ch = bdev_io->ch;

	struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;

	struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;



	bdev_io->internal.status = status;
@@ -2570,7 +2570,7 @@ spdk_bdev_io_get_nvme_status(const struct spdk_bdev_io *bdev_io, int *sct, int * 
struct spdk_thread *

spdk_bdev_io_get_thread(struct spdk_bdev_io *bdev_io)

{

	return spdk_io_channel_get_thread(bdev_io->ch->channel);

	return spdk_io_channel_get_thread(bdev_io->internal.ch->channel);

}



static void