Commit ecc80dfc authored by John Levon's avatar John Levon Committed by Tomasz Zawadzki
Browse files

util: add spdk_iov_xfer_*() API 



Add a new API for incremental copying in or out of an iovec, and replace
current code to use the new API.

Signed-off-by: default avatarJohn Levon <john.levon@nutanix.com>
Change-Id: I088b784aef821310699478989e61411952066c18
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16193


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 833992d0

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -131,6 +131,9 @@ names with this option to restrict allowed RPCs to only that list. 
Added `--vfio-vf-token` command line option to specify a VF token (UUID)

shared between SR-IOV PF and VFs for vfio_pci driver.



The `spdk_iov_xfer_*()` APIs were added for handling incremental copying between

an iovec and a buffer.



### json



Added API `spdk_json_write_double` and `spdk_json_write_named_double` to allow

include/spdk/util.h

+31 −0
Original line number Diff line number Diff line
@@ -169,6 +169,37 @@ size_t spdk_iovcpy(struct iovec *siov, size_t siovcnt, struct iovec *diov, size_ 
 */

size_t spdk_iovmove(struct iovec *siov, size_t siovcnt, struct iovec *diov, size_t diovcnt);



/**

 * Transfer state for iterative copying in or out of an iovec.

 */

struct spdk_iov_xfer {

	struct iovec *iovs;

	int iovcnt;

	int cur_iov_idx;

	size_t cur_iov_offset;

};



/**

 * Initialize a transfer context to point to the given iovec.

 */

void

spdk_iov_xfer_init(struct spdk_iov_xfer *ix, struct iovec *iovs, int iovcnt);



/**

 * Copy from the given buf up to buf_len bytes, into the given ix iovec

 * iterator, advancing the iterator as needed.. Returns the number of bytes

 * copied.

 */

size_t

spdk_iov_xfer_from_buf(struct spdk_iov_xfer *ix, const void *buf, size_t buf_len);



/**

 * Copy from the given ix iovec iterator into the given buf up to buf_len

 * bytes, advancing the iterator as needed. Returns the number of bytes copied.

 */

size_t

spdk_iov_xfer_to_buf(struct spdk_iov_xfer *ix, const void *buf, size_t buf_len);



/**

 * Copy iovs contents to buf through memcpy.

 */

lib/nvmf/ctrlr.c

+31 −81
Original line number Diff line number Diff line
@@ -53,56 +53,6 @@ static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME 


static void _nvmf_request_complete(void *ctx);



struct copy_iovs_ctx {

	struct iovec *iovs;

	int iovcnt;

	int cur_iov_idx;

	size_t cur_iov_offset;

};



static void

_init_copy_iovs_ctx(struct copy_iovs_ctx *copy_ctx, struct iovec *iovs, int iovcnt)

{

	copy_ctx->iovs = iovs;

	copy_ctx->iovcnt = iovcnt;

	copy_ctx->cur_iov_idx = 0;

	copy_ctx->cur_iov_offset = 0;

}



static size_t

_copy_buf_to_iovs(struct copy_iovs_ctx *copy_ctx, const void *buf, size_t buf_len)

{

	size_t len, iov_remain_len, copied_len = 0;

	struct iovec *iov;



	if (buf_len == 0) {

		return 0;

	}



	while (copy_ctx->cur_iov_idx < copy_ctx->iovcnt) {

		iov = &copy_ctx->iovs[copy_ctx->cur_iov_idx];

		iov_remain_len = iov->iov_len - copy_ctx->cur_iov_offset;

		if (iov_remain_len == 0) {

			copy_ctx->cur_iov_idx++;

			copy_ctx->cur_iov_offset = 0;

			continue;

		}



		len = spdk_min(iov_remain_len, buf_len - copied_len);

		memcpy((char *)iov->iov_base + copy_ctx->cur_iov_offset,

		       (const char *)buf + copied_len,

		       len);

		copied_len += len;

		copy_ctx->cur_iov_offset += len;



		if (buf_len == copied_len) {

			return copied_len;

		}

	}



	return copied_len;

}



static inline void

nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp,

			      uint8_t iattr, uint16_t ipo)
@@ -1712,7 +1662,7 @@ nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 
	struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;

	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	SPDK_DEBUGLOG(nvmf, "Get Features - Host Identifier\n");
@@ -1723,14 +1673,14 @@ nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req) 
		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

	}



	if (req->data == NULL || req->length < sizeof(ctrlr->hostid)) {

	if (req->iovcnt < 1 || req->length < sizeof(ctrlr->hostid)) {

		SPDK_ERRLOG("Invalid data buffer for Get Features - Host Identifier\n");

		response->status.sc = SPDK_NVME_SC_INVALID_FIELD;

		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

	}



	_init_copy_iovs_ctx(&copy_ctx, req->iov, req->iovcnt);

	_copy_buf_to_iovs(&copy_ctx, &ctrlr->hostid, sizeof(ctrlr->hostid));

	spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);

	spdk_iov_xfer_from_buf(&ix, &ctrlr->hostid, sizeof(ctrlr->hostid));



	return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

}
@@ -1858,11 +1808,11 @@ nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req) 
	struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;

	struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;

	struct spdk_nvme_host_behavior host_behavior = {};

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	SPDK_DEBUGLOG(nvmf, "Get Features - Host Behavior Support\n");



	if (req->data == NULL || req->length < sizeof(struct spdk_nvme_host_behavior)) {

	if (req->iovcnt < 1 || req->length < sizeof(struct spdk_nvme_host_behavior)) {

		SPDK_ERRLOG("invalid data buffer for Host Behavior Support\n");

		response->status.sct = SPDK_NVME_SCT_GENERIC;

		response->status.sc = SPDK_NVME_SC_INVALID_FIELD;
@@ -1871,8 +1821,8 @@ nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req) 


	host_behavior.acre = ctrlr->acre_enabled;



	_init_copy_iovs_ctx(&copy_ctx, req->iov, req->iovcnt);

	_copy_buf_to_iovs(&copy_ctx, &host_behavior, sizeof(host_behavior));

	spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);

	spdk_iov_xfer_from_buf(&ix, &host_behavior, sizeof(host_behavior));



	return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

}
@@ -2151,9 +2101,9 @@ nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, 
{

	struct spdk_nvme_firmware_page fw_page;

	size_t copy_len;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	_init_copy_iovs_ctx(&copy_ctx, iovs, iovcnt);

	spdk_iov_xfer_init(&ix, iovs, iovcnt);



	memset(&fw_page, 0, sizeof(fw_page));

	fw_page.afi.active_slot = 1;
@@ -2163,7 +2113,7 @@ nvmf_get_firmware_slot_log_page(struct iovec *iovs, int iovcnt, uint64_t offset, 
	if (offset < sizeof(fw_page)) {

		copy_len = spdk_min(sizeof(fw_page) - offset, length);

		if (copy_len > 0) {

			_copy_buf_to_iovs(&copy_ctx, (const char *)&fw_page + offset, copy_len);

			spdk_iov_xfer_from_buf(&ix, (const char *)&fw_page + offset, copy_len);

		}

	}

}
@@ -2259,9 +2209,9 @@ nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iov 
	size_t copy_len, copied_len;

	uint32_t num_anagrp = 0, anagrpid;

	struct spdk_nvmf_ns *ns;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	_init_copy_iovs_ctx(&copy_ctx, iovs, iovcnt);

	spdk_iov_xfer_init(&ix, iovs, iovcnt);



	if (length == 0) {

		goto done;
@@ -2283,7 +2233,7 @@ nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iov 
		ana_hdr.change_count = 0;



		copy_len = spdk_min(sizeof(ana_hdr) - offset, length);

		copied_len = _copy_buf_to_iovs(&copy_ctx, (const char *)&ana_hdr + offset, copy_len);

		copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_hdr + offset, copy_len);

		assert(copied_len == copy_len);

		length -= copied_len;

		offset = 0;
@@ -2308,7 +2258,7 @@ nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iov 
			ana_desc.ana_state = nvmf_ctrlr_get_ana_state(ctrlr, anagrpid);



			copy_len = spdk_min(sizeof(ana_desc) - offset, length);

			copied_len = _copy_buf_to_iovs(&copy_ctx, (const char *)&ana_desc + offset,

			copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ana_desc + offset,

							    copy_len);

			assert(copied_len == copy_len);

			length -= copied_len;
@@ -2334,7 +2284,7 @@ nvmf_get_ana_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec *iovs, int iov 
			}



			copy_len = spdk_min(sizeof(uint32_t) - offset, length);

			copied_len = _copy_buf_to_iovs(&copy_ctx, (const char *)&ns->nsid + offset,

			copied_len = spdk_iov_xfer_from_buf(&ix, (const char *)&ns->nsid + offset,

							    copy_len);

			assert(copied_len == copy_len);

			length -= copied_len;
@@ -2385,14 +2335,14 @@ nvmf_get_changed_ns_list_log_page(struct spdk_nvmf_ctrlr *ctrlr, 
				  struct iovec *iovs, int iovcnt, uint64_t offset, uint32_t length, uint32_t rae)

{

	size_t copy_length;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	_init_copy_iovs_ctx(&copy_ctx, iovs, iovcnt);

	spdk_iov_xfer_init(&ix, iovs, iovcnt);



	if (offset < sizeof(ctrlr->changed_ns_list)) {

		copy_length = spdk_min(length, sizeof(ctrlr->changed_ns_list) - offset);

		if (copy_length) {

			_copy_buf_to_iovs(&copy_ctx, (char *)&ctrlr->changed_ns_list + offset, copy_length);

			spdk_iov_xfer_from_buf(&ix, (char *)&ctrlr->changed_ns_list + offset, copy_length);

		}

	}
@@ -2450,11 +2400,11 @@ nvmf_get_cmds_and_effects_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec * 
{

	uint32_t page_size = sizeof(struct spdk_nvme_cmds_and_effect_log_page);

	size_t copy_len = 0;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_nvme_cmds_and_effect_log_page cmds_and_effect_log_page = g_cmds_and_effect_log_page;

	struct spdk_nvme_cmds_and_effect_entry csupp_and_lbcc_effect_entry = {1, 1, 0, 0, 0, 0, 0, 0};

	struct spdk_iov_xfer ix;



	_init_copy_iovs_ctx(&copy_ctx, iovs, iovcnt);

	spdk_iov_xfer_init(&ix, iovs, iovcnt);



	if (offset < page_size) {

		if (ctrlr->subsys->zone_append_supported) {
@@ -2462,7 +2412,7 @@ nvmf_get_cmds_and_effects_log_page(struct spdk_nvmf_ctrlr *ctrlr, struct iovec * 
				csupp_and_lbcc_effect_entry;

		}

		copy_len = spdk_min(page_size - offset, length);

		_copy_buf_to_iovs(&copy_ctx, (char *)(&cmds_and_effect_log_page) + offset, copy_len);

		spdk_iov_xfer_from_buf(&ix, (char *)(&cmds_and_effect_log_page) + offset, copy_len);

	}

}
@@ -2472,9 +2422,9 @@ nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 
{

	uint32_t unit_log_len, avail_log_len, next_pos, copy_len;

	struct spdk_nvmf_reservation_log *log, *log_tmp;

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	_init_copy_iovs_ctx(&copy_ctx, iovs, iovcnt);

	spdk_iov_xfer_init(&ix, iovs, iovcnt);



	unit_log_len = sizeof(struct spdk_nvme_reservation_notification_log);

	/* No available log, return zeroed log pages */
@@ -2495,7 +2445,7 @@ nvmf_get_reservation_notification_log_page(struct spdk_nvmf_ctrlr *ctrlr, 
		next_pos += unit_log_len;

		if (next_pos > offset) {

			copy_len = spdk_min(next_pos - offset, length);

			_copy_buf_to_iovs(&copy_ctx, &log->log, copy_len);

			spdk_iov_xfer_from_buf(&ix, &log->log, copy_len);

			length -= copy_len;

			offset += copy_len;

		}
@@ -3055,9 +3005,9 @@ nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 
	struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys;

	int ret = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

	char tmpbuf[SPDK_NVME_IDENTIFY_BUFLEN] = "";

	struct copy_iovs_ctx copy_ctx;

	struct spdk_iov_xfer ix;



	if (req->data == NULL || req->length < SPDK_NVME_IDENTIFY_BUFLEN) {

	if (req->iovcnt < 1 || req->length < SPDK_NVME_IDENTIFY_BUFLEN) {

		SPDK_DEBUGLOG(nvmf, "identify command with invalid buffer\n");

		rsp->status.sct = SPDK_NVME_SCT_GENERIC;

		rsp->status.sc = SPDK_NVME_SC_INVALID_FIELD;
@@ -3076,7 +3026,7 @@ nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 
	 * We must use a temporary buffer: it's entirely possible the out buffer

	 * is split across more than one IOV.

	 */

	_init_copy_iovs_ctx(&copy_ctx, req->iov, req->iovcnt);

	spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);



	SPDK_DEBUGLOG(nvmf, "Received identify command with CNS 0x%02x\n", cns);
@@ -3105,7 +3055,7 @@ nvmf_ctrlr_identify(struct spdk_nvmf_request *req) 
	}



	if (ret == SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE) {

		_copy_buf_to_iovs(&copy_ctx, tmpbuf, sizeof(tmpbuf));

		spdk_iov_xfer_from_buf(&ix, tmpbuf, sizeof(tmpbuf));

	}



	return ret;

lib/nvmf/ctrlr_bdev.c

+19 −8
Original line number Diff line number Diff line
@@ -581,7 +581,7 @@ nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 
	uint16_t nr, i;

	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;

	struct spdk_nvme_dsm_range *dsm_range;

	struct spdk_iov_xfer ix;

	uint64_t lba;

	uint32_t lba_count;

	int rc;
@@ -611,10 +611,15 @@ nvmf_bdev_ctrlr_unmap(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 
		unmap_ctx->count--;	/* dequeued */

	}



	dsm_range = (struct spdk_nvme_dsm_range *)req->data;

	spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);



	for (i = unmap_ctx->range_index; i < nr; i++) {

		lba = dsm_range[i].starting_lba;

		lba_count = dsm_range[i].length;

		struct spdk_nvme_dsm_range dsm_range = { 0 };



		spdk_iov_xfer_to_buf(&ix, &dsm_range, sizeof(dsm_range));



		lba = dsm_range.starting_lba;

		lba_count = dsm_range.length;



		unmap_ctx->count++;
@@ -667,7 +672,8 @@ nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 
	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;

	uint64_t sdlba = ((uint64_t)cmd->cdw11 << 32) + cmd->cdw10;

	struct spdk_nvme_scc_source_range *range;

	struct spdk_nvme_scc_source_range range = { 0 };

	struct spdk_iov_xfer ix;

	int rc;



	SPDK_DEBUGLOG(nvmf, "Copy command: SDLBA %lu, NR %u, desc format %u, PRINFOR %u, "
@@ -696,7 +702,10 @@ nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 
		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

	}



	/* We support only one source range */

	/*

	 * We support only one source range, and rely on this with the xfer

	 * below.

	 */

	if (cmd->cdw12_bits.copy.nr > 0) {

		response->status.sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;

		response->status.sc = SPDK_NVME_SC_CMD_SIZE_LIMIT_SIZE_EXCEEDED;
@@ -709,8 +718,10 @@ nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc, 
		return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE;

	}



	range = req->data;

	rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range->slba, range->nlb + 1,

	spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);

	spdk_iov_xfer_to_buf(&ix, &range, sizeof(range));



	rc = spdk_bdev_copy_blocks(desc, ch, sdlba, range.slba, range.nlb + 1,

				   nvmf_bdev_ctrlr_complete_cmd, req);

	if (spdk_unlikely(rc)) {

		if (rc == -ENOMEM) {

lib/util/iov.c

+68 −20
Original line number Diff line number Diff line
@@ -144,31 +144,79 @@ spdk_iovmove(struct iovec *siov, size_t siovcnt, struct iovec *diov, size_t diov 
}



void

spdk_copy_iovs_to_buf(void *buf, size_t buf_len, struct iovec *iovs, int iovcnt)

spdk_iov_xfer_init(struct spdk_iov_xfer *ix, struct iovec *iovs, int iovcnt)

{

	ix->iovs = iovs;

	ix->iovcnt = iovcnt;

	ix->cur_iov_idx = 0;

	ix->cur_iov_offset = 0;

}



static size_t

iov_xfer(struct spdk_iov_xfer *ix, const void *buf, size_t buf_len, bool to_buf)

{

	int i;

	size_t len;

	size_t len, iov_remain_len, copied_len = 0;

	struct iovec *iov;



	if (buf_len == 0) {

		return 0;

	}



	for (i = 0; i < iovcnt; i++) {

		len = spdk_min(iovs[i].iov_len, buf_len);

		memcpy(buf, iovs[i].iov_base, len);

		buf += len;

		assert(buf_len >= len);

		buf_len -= len;

	while (ix->cur_iov_idx < ix->iovcnt) {

		iov = &ix->iovs[ix->cur_iov_idx];

		iov_remain_len = iov->iov_len - ix->cur_iov_offset;

		if (iov_remain_len == 0) {

			ix->cur_iov_idx++;

			ix->cur_iov_offset = 0;

			continue;

		}



		len = spdk_min(iov_remain_len, buf_len - copied_len);



		if (to_buf) {

			memcpy((char *)buf + copied_len,

			       iov->iov_base + ix->cur_iov_offset, len);

		} else {

			memcpy((char *)iov->iov_base + ix->cur_iov_offset,

			       (const char *)buf + copied_len, len);

		}

		copied_len += len;

		ix->cur_iov_offset += len;



		if (buf_len == copied_len) {

			return copied_len;

		}

	}



	return copied_len;

}



size_t

spdk_iov_xfer_from_buf(struct spdk_iov_xfer *ix, const void *buf, size_t buf_len)

{

	return iov_xfer(ix, buf, buf_len, false);

}



size_t

spdk_iov_xfer_to_buf(struct spdk_iov_xfer *ix, const void *buf, size_t buf_len)

{

	return iov_xfer(ix, buf, buf_len, true);

}



void

spdk_copy_buf_to_iovs(struct iovec *iovs, int iovcnt, void *buf, size_t buf_len)

spdk_copy_iovs_to_buf(void *buf, size_t buf_len, struct iovec *iovs, int iovcnt)

{

	int i;

	size_t len;

	struct spdk_iov_xfer ix;



	for (i = 0; i < iovcnt; i++) {

		len = spdk_min(iovs[i].iov_len, buf_len);

		memcpy(iovs[i].iov_base, buf, len);

		buf += len;

		assert(buf_len >= len);

		buf_len -= len;

	spdk_iov_xfer_init(&ix, iovs, iovcnt);

	spdk_iov_xfer_to_buf(&ix, buf, buf_len);

}



void

spdk_copy_buf_to_iovs(struct iovec *iovs, int iovcnt, void *buf, size_t buf_len)

{

	struct spdk_iov_xfer ix;



	spdk_iov_xfer_init(&ix, iovs, iovcnt);

	spdk_iov_xfer_from_buf(&ix, buf, buf_len);

}