Commit d3d6d19b authored by Changpeng Liu's avatar Changpeng Liu Committed by Daniel Verkamp
Browse files

spdk: add scattered payloads support to NVMe driver 



For the purpose to support different types of input scattered payloads,
such as iovs or scattered list, we define common method in the NVMe
driver, users should implement their own functions to iterate each
segment memory.

Change-Id: Id2765747296a66997518281af0db04888ffc4b53
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent b8d31e24

include/spdk/nvme.h

+64 −0
Original line number Diff line number Diff line
@@ -364,6 +364,23 @@ enum nvme_namespace_flags { 
 */

uint32_t nvme_ns_get_flags(struct nvme_namespace *ns);



/**

 * Restart the SGL walk to the specified offset when the command has scattered payloads.

 *

 * The cb_arg parameter is the value passed to readv/writev.

 */

typedef void (*nvme_req_reset_sgl_fn_t)(void *cb_arg, uint32_t offset);



/**

 * Fill out *address and *length with the current SGL entry and advance to the next

 * entry for the next time the callback is invoked.

 *

 * The cb_arg parameter is the value passed to readv/writev.

 * The address parameter contains the physical address of this segment.

 * The length parameter contains the length of this physical segment.

 */

typedef int (*nvme_req_next_sge_fn_t)(void *cb_arg, uint64_t *address, uint32_t *length);



/**

 * \brief Submits a write I/O to the specified NVMe namespace.

 *
@@ -384,6 +401,29 @@ int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, 
		      uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,

		      void *cb_arg);



/**

 * \brief Submits a write I/O to the specified NVMe namespace.

 *

 * \param ns NVMe namespace to submit the write I/O

 * \param lba starting LBA to write the data

 * \param lba_count length (in sectors) for the write operation

 * \param cb_fn callback function to invoke when the I/O is completed

 * \param cb_arg argument to pass to the callback function

 * \param reset_sgl_fn callback function to reset scattered payload

 * \param next_sge_fn callback function to iterate each scattered

 * payload memory segment

 *

 * \return 0 if successfully submitted, ENOMEM if an nvme_request

 *	     structure cannot be allocated for the I/O request

 *

 * This function is thread safe and can be called at any point after

 * nvme_register_io_thread().

 */

int nvme_ns_cmd_writev(struct nvme_namespace *ns, uint64_t lba, uint32_t lba_count,

		       nvme_cb_fn_t cb_fn, void *cb_arg,

		       nvme_req_reset_sgl_fn_t reset_sgl_fn,

		       nvme_req_next_sge_fn_t next_sge_fn);



/**

 * \brief Submits a read I/O to the specified NVMe namespace.

 *
@@ -404,6 +444,30 @@ int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, 
		     uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,

		     void *cb_arg);



/**

 * \brief Submits a read I/O to the specified NVMe namespace.

 *

 * \param ns NVMe namespace to submit the read I/O

 * \param lba starting LBA to read the data

 * \param lba_count length (in sectors) for the read operation

 * \param cb_fn callback function to invoke when the I/O is completed

 * \param cb_arg argument to pass to the callback function

 * \param reset_sgl_fn callback function to reset scattered payload

 * \param next_sge_fn callback function to iterate each scattered

 * payload memory segment

 *

 * \return 0 if successfully submitted, ENOMEM if an nvme_request

 *	     structure cannot be allocated for the I/O request

 *

 * This function is thread safe and can be called at any point after

 * nvme_register_io_thread().

 */

int nvme_ns_cmd_readv(struct nvme_namespace *ns, uint64_t lba, uint32_t lba_count,

		      nvme_cb_fn_t cb_fn, void *cb_arg,

		      nvme_req_reset_sgl_fn_t reset_sgl_fn,

		      nvme_req_next_sge_fn_t next_sge_fn);





/**

 * \brief Submits a deallocation request to the specified NVMe namespace.

 *

lib/nvme/nvme.c

+5 −10
Original line number Diff line number Diff line
@@ -145,16 +145,11 @@ nvme_allocate_request(void *payload, uint32_t payload_size, 
	req->cb_fn = cb_fn;

	req->cb_arg = cb_arg;

	req->timeout = true;

	nvme_assert((payload == NULL && payload_size == 0) ||

		    (payload != NULL && payload_size != 0),

		    ("Invalid argument combination of payload and payload_size\n"));

	if (payload == NULL || payload_size == 0) {

		req->u.payload = NULL;

		req->payload_size = 0;

	} else {

	req->sgl_offset = 0;

	req->parent = NULL;



	req->u.payload = payload;

	req->payload_size = payload_size;

	}



	return req;

}

lib/nvme/nvme_internal.h

+7 −0
Original line number Diff line number Diff line
@@ -159,6 +159,13 @@ struct nvme_request { 
	 *  status once all child requests are completed.

	 */

	struct nvme_completion		parent_status;



	/**

	 * Functions for retrieving physical addresses for scattered payloads.

	 */

	nvme_req_reset_sgl_fn_t reset_sgl_fn;

	nvme_req_next_sge_fn_t next_sge_fn;

	uint32_t sgl_offset;

};



struct nvme_completion_poll_status {

lib/nvme/nvme_ns_cmd.c

+59 −9
Original line number Diff line number Diff line
@@ -41,7 +41,8 @@ 
static struct nvme_request *

_nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba,

		uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg,

		uint32_t opc);

		uint32_t opc, nvme_req_reset_sgl_fn_t reset_sgl_fn,

		nvme_req_next_sge_fn_t next_sge_fn);



static void

nvme_cb_complete_child(void *child_arg, const struct nvme_completion *cpl)
@@ -91,10 +92,13 @@ _nvme_ns_cmd_split_request(struct nvme_namespace *ns, void *payload, 
			   uint64_t lba, uint32_t lba_count,

			   nvme_cb_fn_t cb_fn, void *cb_arg, uint32_t opc,

			   struct nvme_request *req,

			   uint32_t sectors_per_max_io, uint32_t sector_mask)

			   uint32_t sectors_per_max_io, uint32_t sector_mask,

			   nvme_req_reset_sgl_fn_t reset_sgl_fn,

			   nvme_req_next_sge_fn_t next_sge_fn)

{

	uint32_t		sector_size = ns->sector_size;

	uint32_t		remaining_lba_count = lba_count;

	uint32_t		offset = 0;

	struct nvme_request	*child;



	while (remaining_lba_count > 0) {
@@ -102,7 +106,7 @@ _nvme_ns_cmd_split_request(struct nvme_namespace *ns, void *payload, 
		lba_count = nvme_min(remaining_lba_count, lba_count);



		child = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn,

					cb_arg, opc);

					cb_arg, opc, reset_sgl_fn, next_sge_fn);

		if (child == NULL) {

			nvme_free_request(req);

			return NULL;
@@ -110,6 +114,10 @@ _nvme_ns_cmd_split_request(struct nvme_namespace *ns, void *payload, 
		nvme_request_add_child(req, child);

		remaining_lba_count -= lba_count;

		lba += lba_count;

		if (req->u.payload == NULL) {

			child->sgl_offset = offset;

			offset += lba_count * ns->sector_size;

		} else

			payload = (void *)((uintptr_t)payload + (lba_count * sector_size));

	}
@@ -119,7 +127,8 @@ _nvme_ns_cmd_split_request(struct nvme_namespace *ns, void *payload, 
static struct nvme_request *

_nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba,

		uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg,

		uint32_t opc)

		uint32_t opc, nvme_req_reset_sgl_fn_t reset_sgl_fn,

		nvme_req_next_sge_fn_t next_sge_fn)

{

	struct nvme_request	*req;

	struct nvme_command	*cmd;
@@ -137,6 +146,9 @@ _nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba, 
		return NULL;

	}



	req->reset_sgl_fn = reset_sgl_fn;

	req->next_sge_fn = next_sge_fn;



	/*

	 * Intel DC P3*00 NVMe controllers benefit from driver-assisted striping.

	 * If this controller defines a stripe boundary and this I/O spans a stripe
@@ -147,10 +159,12 @@ _nvme_ns_cmd_rw(struct nvme_namespace *ns, void *payload, uint64_t lba, 
	    (((lba & (sectors_per_stripe - 1)) + lba_count) > sectors_per_stripe)) {



		return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc,

						  req, sectors_per_stripe, sectors_per_stripe - 1);

						  req, sectors_per_stripe, sectors_per_stripe - 1,

						  reset_sgl_fn, next_sge_fn);

	} else if (lba_count > sectors_per_max_io) {

		return _nvme_ns_cmd_split_request(ns, payload, lba, lba_count, cb_fn, cb_arg, opc,

						  req, sectors_per_max_io, 0);

						  req, sectors_per_max_io, 0,

						  reset_sgl_fn, next_sge_fn);

	} else {

		cmd = &req->cmd;

		cmd->opc = opc;
@@ -170,7 +184,25 @@ nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, 
{

	struct nvme_request *req;



	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_READ);

	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_READ, NULL, NULL);

	if (req != NULL) {

		nvme_ctrlr_submit_io_request(ns->ctrlr, req);

		return 0;

	} else {

		return ENOMEM;

	}

}



int

nvme_ns_cmd_readv(struct nvme_namespace *ns, uint64_t lba, uint32_t lba_count,

		  nvme_cb_fn_t cb_fn, void *cb_arg,

		  nvme_req_reset_sgl_fn_t reset_sgl_fn,

		  nvme_req_next_sge_fn_t next_sge_fn)

{

	struct nvme_request *req;



	req = _nvme_ns_cmd_rw(ns, NULL, lba, lba_count, cb_fn, cb_arg, NVME_OPC_READ, reset_sgl_fn,

			      next_sge_fn);

	if (req != NULL) {

		nvme_ctrlr_submit_io_request(ns->ctrlr, req);

		return 0;
@@ -185,7 +217,25 @@ nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, 
{

	struct nvme_request *req;



	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_WRITE);

	req = _nvme_ns_cmd_rw(ns, payload, lba, lba_count, cb_fn, cb_arg, NVME_OPC_WRITE, NULL, NULL);

	if (req != NULL) {

		nvme_ctrlr_submit_io_request(ns->ctrlr, req);

		return 0;

	} else {

		return ENOMEM;

	}

}



int

nvme_ns_cmd_writev(struct nvme_namespace *ns, uint64_t lba, uint32_t lba_count,

		   nvme_cb_fn_t cb_fn, void *cb_arg,

		   nvme_req_reset_sgl_fn_t reset_sgl_fn,

		   nvme_req_next_sge_fn_t next_sge_fn)

{

	struct nvme_request *req;



	req = _nvme_ns_cmd_rw(ns, NULL, lba, lba_count, cb_fn, cb_arg, NVME_OPC_WRITE, reset_sgl_fn,

			      next_sge_fn);

	if (req != NULL) {

		nvme_ctrlr_submit_io_request(ns->ctrlr, req);

		return 0;

lib/nvme/nvme_qpair.c

+92 −1
Original line number Diff line number Diff line
@@ -668,9 +668,94 @@ _nvme_fail_request_ctrlr_failed(struct nvme_qpair *qpair, struct nvme_request *r 
					   NVME_SC_ABORTED_BY_REQUEST, true);

}



static int

_nvme_qpair_build_sgl_request(struct nvme_qpair *qpair, struct nvme_request *req,

			      struct nvme_tracker *tr)

{

	int rc;

	uint64_t phys_addr;

	uint32_t data_transfered, remaining_transfer_len, length;

	uint32_t nseg, cur_nseg, total_nseg, last_nseg, modulo, unaligned;

	uint32_t sge_count = 0;

	uint64_t prp2 = 0;

	struct nvme_request *parent;



	/*

	 * Build scattered payloads.

	 */



	parent = req->parent ? req->parent : req;

	nvme_assert(req->reset_sgl_fn != NULL, ("sgl reset callback required\n"));

	req->reset_sgl_fn(parent->cb_arg, req->sgl_offset);



	remaining_transfer_len = req->payload_size;

	total_nseg = 0;

	last_nseg = 0;



	while (remaining_transfer_len > 0) {

		nvme_assert(req->next_sge_fn != NULL, ("sgl callback required\n"));

		rc = req->next_sge_fn(parent->cb_arg, &phys_addr, &length);

		if (rc)

			return -1;



		data_transfered = nvme_min(remaining_transfer_len, length);



		nseg = data_transfered >> nvme_u32log2(PAGE_SIZE);

		modulo = data_transfered & (PAGE_SIZE - 1);

		unaligned = phys_addr & (PAGE_SIZE - 1);

		if (modulo || unaligned) {

			nseg += 1 + ((modulo + unaligned - 1) >> nvme_u32log2(PAGE_SIZE));

		}



		if (total_nseg == 0) {

			req->cmd.psdt = NVME_PSDT_PRP;

			req->cmd.dptr.prp.prp1 = phys_addr;

		}



		total_nseg += nseg;

		sge_count++;

		remaining_transfer_len -= data_transfered;



		if (total_nseg == 2) {

			if (sge_count == 1)

				tr->req->cmd.dptr.prp.prp2 = phys_addr + PAGE_SIZE - unaligned;

			else if (sge_count == 2)

				tr->req->cmd.dptr.prp.prp2 = phys_addr;

			/* save prp2 value */

			prp2 = tr->req->cmd.dptr.prp.prp2;

		} else if (total_nseg > 2) {

			if (sge_count == 1)

				cur_nseg = 1;

			else

				cur_nseg = 0;



			tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;

			while (cur_nseg < nseg) {

				if (prp2) {

					tr->prp[0] = prp2;

					tr->prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;

				} else

					tr->prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;



				last_nseg++;

				cur_nseg++;



				/* physical address and length check */

				if (remaining_transfer_len || (!remaining_transfer_len && (cur_nseg < nseg))) {

					if ((length & (PAGE_SIZE - 1)) || unaligned)

						return -1;

				}

			}

		}

	}



	return 0;

}



void

nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)

{

	int			rc;

	struct nvme_tracker	*tr;

	struct nvme_request	*child_req;

	uint64_t phys_addr;
@@ -718,7 +803,7 @@ nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 
	tr->req = req;

	req->cmd.cid = tr->cid;



	if (req->payload_size) {

	if (req->u.payload) {

		/*

		 * Build PRP list describing payload buffer.

		 */
@@ -754,6 +839,12 @@ nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) 
				cur_nseg++;

			}

		}

	} else if (req->u.payload == NULL && req->payload_size != 0) {

		rc = _nvme_qpair_build_sgl_request(qpair, req, tr);

		if (rc < 0) {

			_nvme_fail_request_bad_vtophys(qpair, tr);

			return;

		}

	}



	nvme_qpair_submit_tracker(qpair, tr);