Commit eb9ef5cc authored by Changpeng Liu's avatar Changpeng Liu Committed by Daniel Verkamp
Browse files

nvme: Add SGL support in NVMe driver 



For those NVMe controllers which can support SGL feature in
firmware, we will use SGL for scattered payloads.

Change-Id: If688e6494ed62e8cba1d55fc6372c6e162cc09c3
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent 98416108

lib/nvme/nvme_ctrlr.c

+6 −0
Original line number Diff line number Diff line
@@ -842,6 +842,11 @@ nvme_ctrlr_start(struct spdk_nvme_ctrlr *ctrlr) 


	nvme_ctrlr_set_supported_log_pages(ctrlr);

	nvme_ctrlr_set_supported_features(ctrlr);



	if (ctrlr->cdata.sgls.supported) {

		ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED;

	}



	return 0;

}
@@ -908,6 +913,7 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) 


	ctrlr->is_resetting = false;

	ctrlr->is_failed = false;

	ctrlr->flags = 0;



	nvme_mutex_init_recursive(&ctrlr->ctrlr_lock);

lib/nvme/nvme_internal.h

+21 −2
Original line number Diff line number Diff line
@@ -98,6 +98,12 @@ 
#define NVME_MIN_IO_TRACKERS	(4)

#define NVME_MAX_IO_TRACKERS	(1024)



/*

 * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL

 *  segment.

 */

#define NVME_MAX_SGL_DESCRIPTORS	(256)



/*

 * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES

 *  for each controller.
@@ -128,6 +134,13 @@ enum nvme_payload_type { 
	NVME_PAYLOAD_TYPE_SGL,

};



/*

 * Controller support flags.

 */

enum spdk_nvme_ctrlr_flags {

	SPDK_NVME_CTRLR_SGL_SUPPORTED	= 0x1, /**< The SGL is supported */

};



/**

 * Descriptor for a request data payload.

 *
@@ -233,8 +246,11 @@ struct nvme_tracker { 
	struct nvme_request		*req;

	uint16_t			cid;



	uint64_t			prp_bus_addr;

	uint64_t			prp_sgl_bus_addr;

	union {

		uint64_t			prp[NVME_MAX_PRP_LIST_ENTRIES];

		struct spdk_nvme_sgl_descriptor	sgl[NVME_MAX_SGL_DESCRIPTORS];

	} u;

};



struct spdk_nvme_qpair {
@@ -342,6 +358,9 @@ struct spdk_nvme_ctrlr { 


	bool				is_failed;



	/** Controller support flags */

	uint64_t			flags;



	/* Cold data (not accessed in normal I/O path) is after this point. */



	enum nvme_ctrlr_state		state;

lib/nvme/nvme_qpair.c

+84 −10
Original line number Diff line number Diff line
@@ -289,7 +289,7 @@ nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl) 
static void

nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phys_addr)

{

	tr->prp_bus_addr = phys_addr + offsetof(struct nvme_tracker, prp);

	tr->prp_sgl_bus_addr = phys_addr + offsetof(struct nvme_tracker, u.prp);

	tr->cid = cid;

}
@@ -702,7 +702,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ 
		tr->req->cmd.dptr.prp.prp2 = nvme_vtophys(seg_addr);

	} else if (nseg > 2) {

		cur_nseg = 1;

		tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;

		tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;

		while (cur_nseg < nseg) {

			seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;

			phys_addr = nvme_vtophys(seg_addr);
@@ -710,7 +710,7 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ 
				_nvme_fail_request_bad_vtophys(qpair, tr);

				return -1;

			}

			tr->prp[cur_nseg - 1] = phys_addr;

			tr->u.prp[cur_nseg - 1] = phys_addr;

			cur_nseg++;

		}

	}
@@ -718,8 +718,78 @@ _nvme_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_requ 
	return 0;

}



/**

 * Build SGL list describing scattered payload buffer.

 */

static int

_nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,

_nvme_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,

				 struct nvme_tracker *tr)

{

	int rc;

	uint64_t phys_addr;

	uint32_t remaining_transfer_len, length;

	struct spdk_nvme_sgl_descriptor *sgl;

	uint32_t nseg = 0;



	/*

	 * Build scattered payloads.

	 */

	nvme_assert(req->payload.type == NVME_PAYLOAD_TYPE_SGL, ("sgl payload type required\n"));

	nvme_assert(req->payload.u.sgl.reset_sgl_fn != NULL, ("sgl reset callback required\n"));

	req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);



	sgl = (struct spdk_nvme_sgl_descriptor *)tr->u.sgl;

	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_SGL;

	req->cmd.dptr.sgl1.type_specific = 0;



	remaining_transfer_len = req->payload_size;



	while (remaining_transfer_len > 0) {

		nvme_assert(req->payload.u.sgl.next_sge_fn != NULL, ("sgl callback required\n"));

		rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &phys_addr, &length);

		if (rc) {

			_nvme_fail_request_bad_vtophys(qpair, tr);

			return -1;

		}



		remaining_transfer_len -= length;



		sgl->type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;

		sgl->length = length;

		sgl->address = phys_addr;

		sgl->type_specific = 0;



		sgl++;

		nseg++;



		if (nseg >= NVME_MAX_SGL_DESCRIPTORS) {

			_nvme_fail_request_bad_vtophys(qpair, tr);

			return -1;

		}

	}



	if (nseg == 1) {

		req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;

		req->cmd.dptr.sgl1.address = phys_addr;

		req->cmd.dptr.sgl1.length = length;

	} else if (nseg > 1) {

		/* For now we can only support 1 SGL segment in NVMe controller */

		req->cmd.dptr.sgl1.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;

		req->cmd.dptr.sgl1.address = tr->prp_sgl_bus_addr;

		req->cmd.dptr.sgl1.length = nseg * sizeof(struct spdk_nvme_sgl_descriptor);

	} else {

		_nvme_fail_request_bad_vtophys(qpair, tr);

		return -1;

	}



	return 0;

}



/**

 * Build PRP list describing scattered payload buffer.

 */

static int

_nvme_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,

				   struct nvme_tracker *tr)

{

	int rc;
@@ -779,13 +849,13 @@ _nvme_qpair_build_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request 
			else

				cur_nseg = 0;



			tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_bus_addr;

			tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;

			while (cur_nseg < nseg) {

				if (prp2) {

					tr->prp[0] = prp2;

					tr->prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;

					tr->u.prp[0] = prp2;

					tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;

				} else

					tr->prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;

					tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE - unaligned;



				last_nseg++;

				cur_nseg++;
@@ -810,6 +880,7 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re 
	int			rc;

	struct nvme_tracker	*tr;

	struct nvme_request	*child_req;

	struct spdk_nvme_ctrlr	*ctrlr = qpair->ctrlr;



	nvme_qpair_check_enabled(qpair);
@@ -860,7 +931,10 @@ nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *re 
			return;

		}

	} else if (req->payload.type == NVME_PAYLOAD_TYPE_SGL) {

		rc = _nvme_qpair_build_sgl_request(qpair, req, tr);

		if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED)

			rc = _nvme_qpair_build_hw_sgl_request(qpair, req, tr);

		else

			rc = _nvme_qpair_build_prps_sgl_request(qpair, req, tr);

		if (rc < 0) {

			return;

		}

test/lib/nvme/sgl/nvme_sgl.c

+18 −1
Original line number Diff line number Diff line
@@ -129,6 +129,22 @@ io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 
		io_complete_flag = 1;

}



static uint32_t build_io_request_0(struct io_request *req)

{

	int i;

	uint32_t len = 0;



	req->nseg = 1;



	req->iovs[0].iov_base = rte_zmalloc(NULL, 0x800, 4);

	req->iovs[0].iov_len = 0x800;



	for (i = 0; i < req->nseg; i++)

		len += req->iovs[i].iov_len;



	return len;

}



static uint32_t build_io_request_1(struct io_request *req)

{

	int i, found = 0;
@@ -456,7 +472,8 @@ int main(int argc, char **argv) 
	}



	foreach_dev(iter) {

		if (writev_readv_tests(iter, build_io_request_1)

		if (writev_readv_tests(iter, build_io_request_0)

		    || writev_readv_tests(iter, build_io_request_1)

		    || writev_readv_tests(iter, build_io_request_2)

		    || writev_readv_tests(iter, build_io_request_3)

		    || writev_readv_tests(iter, build_io_request_4)

test/lib/nvme/unit/nvme_qpair_c/nvme_qpair_ut.c

+7 −6
Original line number Diff line number Diff line
@@ -389,13 +389,14 @@ test_sgl_req(void) 
	CU_ASSERT(req->cmd.dptr.prp.prp1 == 0);

	CU_ASSERT(qpair.sq_tail == 1);

	sgl_tr = LIST_FIRST(&qpair.outstanding_tr);

	CU_ASSERT(sgl_tr != NULL);

	if (sgl_tr != NULL) {

		for (i = 0; i < 32; i++) {

		CU_ASSERT(sgl_tr->prp[i] == (PAGE_SIZE * (i + 1)));

			CU_ASSERT(sgl_tr->u.prp[i] == (PAGE_SIZE * (i + 1)));

		}



		LIST_REMOVE(sgl_tr, list);

		free(sgl_tr);

	}

	cleanup_submit_request_test(&qpair);

	nvme_free_request(req);

}