Commit c834f9cc authored by Jacek Kalwas's avatar Jacek Kalwas Committed by Tomasz Zawadzki
Browse files

bdev: mark (un)likely to improve hot path 



Signed-off-by: default avatarJacek Kalwas <jacek.kalwas@intel.com>
Change-Id: I1903be21e913081afa51ec98ab14b8509f4eb21c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21337


Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
parent 001fe4ec

lib/bdev/bdev.c

+29 −25
Original line number Diff line number Diff line
@@ -5343,12 +5343,12 @@ bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *c 
	struct spdk_bdev_io *bdev_io;

	struct spdk_bdev_channel *channel = __io_ch_to_bdev_ch(ch);



	if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {

	if (spdk_unlikely(!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks))) {

		return -EINVAL;

	}



	bdev_io = bdev_channel_get_io(channel);

	if (!bdev_io) {

	if (spdk_unlikely(!bdev_io)) {

		return -ENOMEM;

	}
@@ -5430,16 +5430,18 @@ spdk_bdev_readv_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel * 
		if (spdk_unlikely(!_bdev_io_check_opts(opts, iov))) {

			return -EINVAL;

		}

		md = opts->metadata;

	}



	if (md && !spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {

		md = opts->metadata;

		if (md) {

			if (spdk_unlikely(!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc)))) {

				return -EINVAL;

			}



	if (md && !_is_buf_allocated(iov)) {

			if (spdk_unlikely(!_is_buf_allocated(iov))) {

				return -EINVAL;

			}

		}

	}



	return bdev_readv_blocks_with_md(desc, ch, iov, iovcnt, md, offset_blocks,

					 num_blocks,
@@ -5547,16 +5549,16 @@ bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel * 
	struct spdk_bdev_io *bdev_io;

	struct spdk_bdev_channel *channel = __io_ch_to_bdev_ch(ch);



	if (!desc->write) {

	if (spdk_unlikely(!desc->write)) {

		return -EBADF;

	}



	if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {

	if (spdk_unlikely(!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks))) {

		return -EINVAL;

	}



	bdev_io = bdev_channel_get_io(channel);

	if (!bdev_io) {

	if (spdk_unlikely(!bdev_io)) {

		return -ENOMEM;

	}
@@ -5639,16 +5641,18 @@ spdk_bdev_writev_blocks_ext(struct spdk_bdev_desc *desc, struct spdk_io_channel 
		if (spdk_unlikely(!_bdev_io_check_opts(opts, iov))) {

			return -EINVAL;

		}

		md = opts->metadata;

	}



	if (md && !spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {

		md = opts->metadata;

		if (md) {

			if (spdk_unlikely(!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc)))) {

				return -EINVAL;

			}



	if (md && !_is_buf_allocated(iov)) {

			if (spdk_unlikely(!_is_buf_allocated(iov))) {

				return -EINVAL;

			}

		}

	}



	return bdev_writev_blocks_with_md(desc, ch, iov, iovcnt, md, offset_blocks, num_blocks,

					  bdev_get_ext_io_opt(opts, memory_domain, NULL),
@@ -7252,7 +7256,7 @@ spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status sta 
	struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;

	struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;



	if (bdev_io->internal.status != SPDK_BDEV_IO_STATUS_PENDING) {

	if (spdk_unlikely(bdev_io->internal.status != SPDK_BDEV_IO_STATUS_PENDING)) {

		SPDK_ERRLOG("Unexpected completion on IO from %s module, status was %s\n",

			    spdk_bdev_get_module_name(bdev),

			    bdev_io_status_get_string(bdev_io->internal.status));
@@ -7395,7 +7399,7 @@ spdk_bdev_io_complete_nvme_status(struct spdk_bdev_io *bdev_io, uint32_t cdw0, i 
{

	enum spdk_bdev_io_status status;



	if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS) {

	if (spdk_likely(sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS)) {

		status = SPDK_BDEV_IO_STATUS_SUCCESS;

	} else if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_ABORTED_BY_REQUEST) {

		status = SPDK_BDEV_IO_STATUS_ABORTED;
@@ -7428,12 +7432,12 @@ spdk_bdev_io_get_nvme_status(const struct spdk_bdev_io *bdev_io, uint32_t *cdw0, 
		return;

	}



	if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NVME_ERROR) {

		*sct = bdev_io->internal.error.nvme.sct;

		*sc = bdev_io->internal.error.nvme.sc;

	} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {

	if (spdk_likely(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS)) {

		*sct = SPDK_NVME_SCT_GENERIC;

		*sc = SPDK_NVME_SC_SUCCESS;

	} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NVME_ERROR) {

		*sct = bdev_io->internal.error.nvme.sct;

		*sc = bdev_io->internal.error.nvme.sc;

	} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_ABORTED) {

		*sct = SPDK_NVME_SCT_GENERIC;

		*sc = SPDK_NVME_SC_ABORTED_BY_REQUEST;