Commit 299394f7 authored by Ben Walker's avatar Ben Walker Committed by Konrad Sztyber
Browse files

bdev: Move spdk_bdev_io splitting parameters to internal area 



These are only used internally by the bdev layer and should never be
accessed by modules.

This also gets struct spdk_bdev_io_block_params down to just 2 cache
lines, and only the first line is accessed commonly.

Change-Id: Ic2866a3d0629fa2efab821c8ceb5e35add9a22e4
Signed-off-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21942


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 0a889254

include/spdk/bdev_module.h

+12 −12
Original line number Diff line number Diff line
@@ -820,18 +820,6 @@ struct spdk_bdev_io_block_params { 
	/** Number of iovecs in fused_iovs. */

	int fused_iovcnt;



	/** stored user callback in case we split the I/O and use a temporary callback */

	spdk_bdev_io_completion_cb stored_user_cb;



	/** number of blocks remaining in a split i/o */

	uint64_t split_remaining_num_blocks;



	/** current offset of the split I/O in the bdev */

	uint64_t split_current_offset_blocks;



	/** count of outstanding batched split I/Os */

	uint32_t split_outstanding;



	/** Specify whether each DIF check type is enabled. */

	uint32_t dif_check_flags;
@@ -1015,6 +1003,18 @@ struct spdk_bdev_io { 
		/** Indicates that the IO is associated with an accel sequence */

		bool has_accel_sequence;



		/** stored user callback in case we split the I/O and use a temporary callback */

		spdk_bdev_io_completion_cb stored_user_cb;



		/** number of blocks remaining in a split i/o */

		uint64_t split_remaining_num_blocks;



		/** current offset of the split I/O in the bdev */

		uint64_t split_current_offset_blocks;



		/** count of outstanding batched split I/Os */

		uint32_t split_outstanding;



		/** bdev allocated memory associated with this request */

		void *buf;

lib/bdev/bdev.c

+27 −27
Original line number Diff line number Diff line
@@ -3013,7 +3013,7 @@ bdev_io_split_submit(struct spdk_bdev_io *bdev_io, struct iovec *iov, int iovcnt 
	current_offset = *offset;

	current_remaining = *remaining;



	bdev_io->u.bdev.split_outstanding++;

	bdev_io->internal.split_outstanding++;



	io_wait_fn = _bdev_rw_split;

	switch (bdev_io->type) {
@@ -3071,20 +3071,20 @@ bdev_io_split_submit(struct spdk_bdev_io *bdev_io, struct iovec *iov, int iovcnt 
	if (rc == 0) {

		current_offset += num_blocks;

		current_remaining -= num_blocks;

		bdev_io->u.bdev.split_current_offset_blocks = current_offset;

		bdev_io->u.bdev.split_remaining_num_blocks = current_remaining;

		bdev_io->internal.split_current_offset_blocks = current_offset;

		bdev_io->internal.split_remaining_num_blocks = current_remaining;

		*offset = current_offset;

		*remaining = current_remaining;

	} else {

		bdev_io->u.bdev.split_outstanding--;

		bdev_io->internal.split_outstanding--;

		if (rc == -ENOMEM) {

			if (bdev_io->u.bdev.split_outstanding == 0) {

			if (bdev_io->internal.split_outstanding == 0) {

				/* No I/O is outstanding. Hence we should wait here. */

				bdev_queue_io_wait_with_cb(bdev_io, io_wait_fn);

			}

		} else {

			bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;

			if (bdev_io->u.bdev.split_outstanding == 0) {

			if (bdev_io->internal.split_outstanding == 0) {

				bdev_ch_remove_from_io_submitted(bdev_io);

				spdk_trace_record(TRACE_BDEV_IO_DONE, bdev_io->internal.ch->trace_id,

						  0, (uintptr_t)bdev_io, bdev_io->internal.caller_ctx,
@@ -3128,8 +3128,8 @@ _bdev_rw_split(void *_bdev_io) 
		io_boundary = UINT32_MAX;

	}



	remaining = bdev_io->u.bdev.split_remaining_num_blocks;

	current_offset = bdev_io->u.bdev.split_current_offset_blocks;

	remaining = bdev_io->internal.split_remaining_num_blocks;

	current_offset = bdev_io->internal.split_current_offset_blocks;

	parent_offset = bdev_io->u.bdev.offset_blocks;

	parent_iov_offset = (current_offset - parent_offset) * blocklen;

	parent_iovcnt = bdev_io->u.bdev.iovcnt;
@@ -3210,7 +3210,7 @@ _bdev_rw_split(void *_bdev_io) 
							 * If the first child IO of any split round is less than

							 * a block size, an error exit.

							 */

							if (bdev_io->u.bdev.split_outstanding == 0) {

							if (bdev_io->internal.split_outstanding == 0) {

								SPDK_ERRLOG("The first child io was less than a block size\n");

								bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;

								bdev_ch_remove_from_io_submitted(bdev_io);
@@ -3253,8 +3253,8 @@ bdev_unmap_split(struct spdk_bdev_io *bdev_io) 
	uint32_t num_children_reqs = 0;

	int rc;



	offset = bdev_io->u.bdev.split_current_offset_blocks;

	remaining = bdev_io->u.bdev.split_remaining_num_blocks;

	offset = bdev_io->internal.split_current_offset_blocks;

	remaining = bdev_io->internal.split_remaining_num_blocks;

	max_unmap_blocks = bdev_io->bdev->max_unmap * bdev_io->bdev->max_unmap_segments;



	while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {
@@ -3277,8 +3277,8 @@ bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io) 
	uint32_t num_children_reqs = 0;

	int rc;



	offset = bdev_io->u.bdev.split_current_offset_blocks;

	remaining = bdev_io->u.bdev.split_remaining_num_blocks;

	offset = bdev_io->internal.split_current_offset_blocks;

	remaining = bdev_io->internal.split_remaining_num_blocks;



	while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {

		write_zeroes_blocks = spdk_min(remaining, bdev_io->bdev->max_write_zeroes);
@@ -3300,8 +3300,8 @@ bdev_copy_split(struct spdk_bdev_io *bdev_io) 
	uint32_t num_children_reqs = 0;

	int rc;



	offset = bdev_io->u.bdev.split_current_offset_blocks;

	remaining = bdev_io->u.bdev.split_remaining_num_blocks;

	offset = bdev_io->internal.split_current_offset_blocks;

	remaining = bdev_io->internal.split_remaining_num_blocks;



	assert(bdev_io->bdev->max_copy != 0);

	while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_COPY_REQS)) {
@@ -3357,18 +3357,18 @@ bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 
	if (!success) {

		parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;

		/* If any child I/O failed, stop further splitting process. */

		parent_io->u.bdev.split_current_offset_blocks += parent_io->u.bdev.split_remaining_num_blocks;

		parent_io->u.bdev.split_remaining_num_blocks = 0;

		parent_io->internal.split_current_offset_blocks += parent_io->internal.split_remaining_num_blocks;

		parent_io->internal.split_remaining_num_blocks = 0;

	}

	parent_io->u.bdev.split_outstanding--;

	if (parent_io->u.bdev.split_outstanding != 0) {

	parent_io->internal.split_outstanding--;

	if (parent_io->internal.split_outstanding != 0) {

		return;

	}



	/*

	 * Parent I/O finishes when all blocks are consumed.

	 */

	if (parent_io->u.bdev.split_remaining_num_blocks == 0) {

	if (parent_io->internal.split_remaining_num_blocks == 0) {

		assert(parent_io->internal.cb != bdev_io_split_done);

		bdev_ch_remove_from_io_submitted(parent_io);

		spdk_trace_record(TRACE_BDEV_IO_DONE, parent_io->internal.ch->trace_id,
@@ -3423,9 +3423,9 @@ bdev_io_split(struct spdk_bdev_io *bdev_io) 
{

	assert(bdev_io_should_split(bdev_io));



	bdev_io->u.bdev.split_current_offset_blocks = bdev_io->u.bdev.offset_blocks;

	bdev_io->u.bdev.split_remaining_num_blocks = bdev_io->u.bdev.num_blocks;

	bdev_io->u.bdev.split_outstanding = 0;

	bdev_io->internal.split_current_offset_blocks = bdev_io->u.bdev.offset_blocks;

	bdev_io->internal.split_remaining_num_blocks = bdev_io->u.bdev.num_blocks;

	bdev_io->internal.split_outstanding = 0;

	bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;



	switch (bdev_io->type) {
@@ -6926,8 +6926,8 @@ bdev_abort_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 
		}

	}



	parent_io->u.bdev.split_outstanding--;

	if (parent_io->u.bdev.split_outstanding == 0) {

	parent_io->internal.split_outstanding--;

	if (parent_io->internal.split_outstanding == 0) {

		if (parent_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {

			bdev_abort_retry(parent_io);

		} else {
@@ -7075,7 +7075,7 @@ bdev_abort_retry(void *ctx) 
	}



	/* Use split_outstanding to manage the progress of aborting I/Os. */

	parent_io->u.bdev.split_outstanding = matched_ios;

	parent_io->internal.split_outstanding = matched_ios;

}



static void
@@ -7097,7 +7097,7 @@ bdev_abort(struct spdk_bdev_io *parent_io) 
	}



	/* Use split_outstanding to manage the progress of aborting I/Os. */

	parent_io->u.bdev.split_outstanding = matched_ios;

	parent_io->internal.split_outstanding = matched_ios;

}



int

lib/bdev/part.c

+2 −2
Original line number Diff line number Diff line
@@ -269,7 +269,7 @@ bdev_part_complete_io(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 
	}





	cb = part_io->u.bdev.stored_user_cb;

	cb = part_io->internal.stored_user_cb;

	if (cb != NULL) {

		cb(part_io, success, NULL);

	} else {
@@ -300,7 +300,7 @@ spdk_bdev_part_submit_request_ext(struct spdk_bdev_part_channel *ch, struct spdk 
	uint64_t offset, remapped_offset, remapped_src_offset;

	int rc = 0;



	bdev_io->u.bdev.stored_user_cb = cb;

	bdev_io->internal.stored_user_cb = cb;



	offset = bdev_io->u.bdev.offset_blocks;

	remapped_offset = offset + part->internal.offset_blocks;