Commit 269910c0 authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvme: refactor PRP building code 



This also changes the SGL -> PRP case to translate each 4K page from
virtual to physical, in case the buffer is not physically contiguous.

Change-Id: If027f9d656c52c56504f0c64cd4464e16440df63
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-on: https://review.gerrithub.io/371616


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 1d304bc5

lib/nvme/nvme_pcie.c

+91 −101
Original line number Diff line number Diff line
@@ -1518,58 +1518,97 @@ nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tr 
						1 /* do not retry */, true);

}



/**

 * Build PRP list describing physically contiguous payload buffer.

/*

 * Append PRP list entries to describe a virtually contiguous buffer starting at virt_addr of len bytes.

 *

 * *prp_index will be updated to account for the number of PRP entries used.

 */

static int

nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,

				     struct nvme_tracker *tr)

nvme_pcie_prp_list_append(struct nvme_tracker *tr, uint32_t *prp_index, void *virt_addr, size_t len)

{

	struct spdk_nvme_cmd *cmd = &tr->req->cmd;

	uint64_t phys_addr;

	void *seg_addr;

	uint32_t nseg, cur_nseg, modulo, unaligned;

	void *md_payload;

	void *payload = req->payload.u.contig + req->payload_offset;

	uint32_t i;



	phys_addr = spdk_vtophys(payload);

	if (phys_addr == SPDK_VTOPHYS_ERROR) {

		nvme_pcie_fail_request_bad_vtophys(qpair, tr);

		return -1;

	SPDK_TRACELOG(SPDK_TRACE_NVME, "prp_index:%u virt_addr:%p len:%u\n",

		      *prp_index, virt_addr, (uint32_t)len);



	if (spdk_unlikely(((uintptr_t)virt_addr & 3) != 0)) {

		SPDK_TRACELOG(SPDK_TRACE_NVME, "virt_addr %p not dword aligned\n", virt_addr);

		return -EINVAL;

	}

	nseg = req->payload_size >> spdk_u32log2(PAGE_SIZE);

	modulo = req->payload_size & (PAGE_SIZE - 1);

	unaligned = phys_addr & (PAGE_SIZE - 1);

	if (modulo || unaligned) {

		nseg += 1 + ((modulo + unaligned - 1) >> spdk_u32log2(PAGE_SIZE));



	i = *prp_index;

	while (len) {

		uint32_t seg_len;



		/*

		 * prp_index 0 is stored in prp1, and the rest are stored in the prp[] array,

		 * so prp_index == count is valid.

		 */

		if (spdk_unlikely(i > SPDK_COUNTOF(tr->u.prp))) {

			SPDK_TRACELOG(SPDK_TRACE_NVME, "out of PRP entries\n");

			return -EINVAL;

		}



	if (req->payload.md) {

		md_payload = req->payload.md + req->md_offset;

		tr->req->cmd.mptr = spdk_vtophys(md_payload);

		if (tr->req->cmd.mptr == SPDK_VTOPHYS_ERROR) {

			nvme_pcie_fail_request_bad_vtophys(qpair, tr);

			return -1;

		phys_addr = spdk_vtophys(virt_addr);

		if (spdk_unlikely(phys_addr == SPDK_VTOPHYS_ERROR)) {

			SPDK_TRACELOG(SPDK_TRACE_NVME, "vtophys(%p) failed\n", virt_addr);

			return -EINVAL;

		}



		if (i == 0) {

			SPDK_TRACELOG(SPDK_TRACE_NVME, "prp1 = %p\n", (void *)phys_addr);

			cmd->dptr.prp.prp1 = phys_addr;

			seg_len = PAGE_SIZE - ((uintptr_t)virt_addr & (PAGE_SIZE - 1));

		} else {

			if ((phys_addr & (PAGE_SIZE - 1)) != 0) {

				SPDK_TRACELOG(SPDK_TRACE_NVME, "PRP %u not page aligned (%p)\n",

					      i, virt_addr);

				return -EINVAL;

			}



	tr->req->cmd.psdt = SPDK_NVME_PSDT_PRP;

	tr->req->cmd.dptr.prp.prp1 = phys_addr;

	if (nseg == 2) {

		seg_addr = payload + PAGE_SIZE - unaligned;

		tr->req->cmd.dptr.prp.prp2 = spdk_vtophys(seg_addr);

	} else if (nseg > 2) {

		cur_nseg = 1;

		tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;

		while (cur_nseg < nseg) {

			seg_addr = payload + cur_nseg * PAGE_SIZE - unaligned;

			phys_addr = spdk_vtophys(seg_addr);

			if (phys_addr == SPDK_VTOPHYS_ERROR) {

				nvme_pcie_fail_request_bad_vtophys(qpair, tr);

				return -1;

			SPDK_TRACELOG(SPDK_TRACE_NVME, "prp[%u] = %p\n", i - 1, (void *)phys_addr);

			tr->u.prp[i - 1] = phys_addr;

			seg_len = PAGE_SIZE;

		}

			tr->u.prp[cur_nseg - 1] = phys_addr;

			cur_nseg++;



		seg_len = spdk_min(seg_len, len);

		virt_addr += seg_len;

		len -= seg_len;

		i++;

	}



	cmd->psdt = SPDK_NVME_PSDT_PRP;

	if (i <= 1) {

		cmd->dptr.prp.prp2 = 0;

	} else if (i == 2) {

		cmd->dptr.prp.prp2 = tr->u.prp[0];

		SPDK_TRACELOG(SPDK_TRACE_NVME, "prp2 = %p\n", (void *)cmd->dptr.prp.prp2);

	} else {

		cmd->dptr.prp.prp2 = tr->prp_sgl_bus_addr;

		SPDK_TRACELOG(SPDK_TRACE_NVME, "prp2 = %p (PRP list)\n", (void *)cmd->dptr.prp.prp2);

	}



	*prp_index = i;

	return 0;

}



/**

 * Build PRP list describing physically contiguous payload buffer.

 */

static int

nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,

				     struct nvme_tracker *tr)

{

	uint32_t prp_index = 0;

	int rc;



	rc = nvme_pcie_prp_list_append(tr, &prp_index, req->payload.u.contig + req->payload_offset,

				       req->payload_size);

	if (rc) {

		nvme_pcie_fail_request_bad_vtophys(qpair, tr);

		return rc;

	}



	return 0;
@@ -1663,11 +1702,8 @@ nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvm 
{

	int rc;

	void *virt_addr;

	uint64_t phys_addr;

	uint32_t data_transferred, remaining_transfer_len, length;

	uint32_t nseg, cur_nseg, total_nseg, last_nseg, modulo, unaligned;

	uint32_t sge_count = 0;

	uint64_t prp2 = 0;

	uint32_t remaining_transfer_len, length;

	uint32_t prp_index = 0;



	/*

	 * Build scattered payloads.
@@ -1677,9 +1713,6 @@ nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvm 
	req->payload.u.sgl.reset_sgl_fn(req->payload.u.sgl.cb_arg, req->payload_offset);



	remaining_transfer_len = req->payload_size;

	total_nseg = 0;

	last_nseg = 0;



	while (remaining_transfer_len > 0) {

		assert(req->payload.u.sgl.next_sge_fn != NULL);

		rc = req->payload.u.sgl.next_sge_fn(req->payload.u.sgl.cb_arg, &virt_addr, &length);
@@ -1688,66 +1721,23 @@ nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvm 
			return -1;

		}



		phys_addr = spdk_vtophys(virt_addr);

		if (phys_addr == SPDK_VTOPHYS_ERROR) {

			nvme_pcie_fail_request_bad_vtophys(qpair, tr);

			return -1;

		}

		length = spdk_min(remaining_transfer_len, length);



		/*

		 * Any incompatible sges should have been handled up in the splitting routine,

		 *  but assert here as an additional check.

		 *

		 * All SGEs except last must end on a page boundary.

		 */

		assert((phys_addr & 0x3) == 0); /* Address must be dword aligned. */

		/* All SGEs except last must end on a page boundary. */

		assert((length >= remaining_transfer_len) || _is_page_aligned(phys_addr + length));

		/* All SGe except first must start on a page boundary. */

		assert((sge_count == 0) || _is_page_aligned(phys_addr));



		data_transferred = spdk_min(remaining_transfer_len, length);



		nseg = data_transferred >> spdk_u32log2(PAGE_SIZE);

		modulo = data_transferred & (PAGE_SIZE - 1);

		unaligned = phys_addr & (PAGE_SIZE - 1);

		if (modulo || unaligned) {

			nseg += 1 + ((modulo + unaligned - 1) >> spdk_u32log2(PAGE_SIZE));

		}



		if (total_nseg == 0) {

			req->cmd.psdt = SPDK_NVME_PSDT_PRP;

			req->cmd.dptr.prp.prp1 = phys_addr;

			phys_addr -= unaligned;

		}



		total_nseg += nseg;

		sge_count++;

		remaining_transfer_len -= data_transferred;



		if (total_nseg == 2) {

			if (sge_count == 1)

				tr->req->cmd.dptr.prp.prp2 = phys_addr + PAGE_SIZE;

			else if (sge_count == 2)

				tr->req->cmd.dptr.prp.prp2 = phys_addr;

			/* save prp2 value */

			prp2 = tr->req->cmd.dptr.prp.prp2;

		} else if (total_nseg > 2) {

			if (sge_count == 1)

				cur_nseg = 1;

			else

				cur_nseg = 0;



			tr->req->cmd.dptr.prp.prp2 = (uint64_t)tr->prp_sgl_bus_addr;

			while (cur_nseg < nseg) {

				if (prp2) {

					tr->u.prp[0] = prp2;

					tr->u.prp[last_nseg + 1] = phys_addr + cur_nseg * PAGE_SIZE;

				} else

					tr->u.prp[last_nseg] = phys_addr + cur_nseg * PAGE_SIZE;



				last_nseg++;

				cur_nseg++;

			}

		assert((length == remaining_transfer_len) || _is_page_aligned((uintptr_t)virt_addr + length));



		rc = nvme_pcie_prp_list_append(tr, &prp_index, virt_addr, length);

		if (rc) {

			nvme_pcie_fail_request_bad_vtophys(qpair, tr);

			return rc;

		}



		remaining_transfer_len -= length;

	}



	return 0;

test/unit/lib/nvme/nvme_pcie.c/nvme_pcie_ut.c

+138 −4
Original line number Diff line number Diff line
@@ -642,6 +642,143 @@ static void test_nvme_qpair_destroy(void) 
}

#endif



static void

prp_list_prep(struct nvme_tracker *tr, struct nvme_request *req, uint32_t *prp_index)

{

	memset(req, 0, sizeof(*req));

	memset(tr, 0, sizeof(*tr));

	tr->req = req;

	tr->prp_sgl_bus_addr = 0xDEADBEEF;

	*prp_index = 0;

}



static void

test_prp_list_append(void)

{

	struct nvme_request req;

	struct nvme_tracker tr;

	uint32_t prp_index;



	/* Non-DWORD-aligned buffer (invalid) */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100001, 0x1000) == -EINVAL);



	/* 512-byte buffer, 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x200) == 0);

	CU_ASSERT(prp_index == 1);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100000);



	/* 512-byte buffer, non-4K-aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x108000, 0x200) == 0);

	CU_ASSERT(prp_index == 1);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x108000);



	/* 4K buffer, 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x1000) == 0);

	CU_ASSERT(prp_index == 1);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100000);



	/* 4K buffer, non-4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800, 0x1000) == 0);

	CU_ASSERT(prp_index == 2);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100800);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == 0x101000);



	/* 8K buffer, 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x2000) == 0);

	CU_ASSERT(prp_index == 2);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100000);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == 0x101000);



	/* 8K buffer, non-4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800, 0x2000) == 0);

	CU_ASSERT(prp_index == 3);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100800);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == tr.prp_sgl_bus_addr);

	CU_ASSERT(tr.u.prp[0] == 0x101000);

	CU_ASSERT(tr.u.prp[1] == 0x102000);



	/* 12K buffer, 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x3000) == 0);

	CU_ASSERT(prp_index == 3);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100000);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == tr.prp_sgl_bus_addr);

	CU_ASSERT(tr.u.prp[0] == 0x101000);

	CU_ASSERT(tr.u.prp[1] == 0x102000);



	/* 12K buffer, non-4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800, 0x3000) == 0);

	CU_ASSERT(prp_index == 4);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100800);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == tr.prp_sgl_bus_addr);

	CU_ASSERT(tr.u.prp[0] == 0x101000);

	CU_ASSERT(tr.u.prp[1] == 0x102000);

	CU_ASSERT(tr.u.prp[2] == 0x103000);



	/* Two 4K buffers, both 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x1000) == 0);

	CU_ASSERT(prp_index == 1);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x900000, 0x1000) == 0);

	CU_ASSERT(prp_index == 2);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100000);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == 0x900000);



	/* Two 4K buffers, first non-4K aligned, second 4K aligned */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800, 0x1000) == 0);

	CU_ASSERT(prp_index == 2);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x900000, 0x1000) == 0);

	CU_ASSERT(prp_index == 3);

	CU_ASSERT(req.cmd.dptr.prp.prp1 == 0x100800);

	CU_ASSERT(req.cmd.dptr.prp.prp2 == tr.prp_sgl_bus_addr);

	CU_ASSERT(tr.u.prp[0] == 0x101000);

	CU_ASSERT(tr.u.prp[1] == 0x900000);



	/* Two 4K buffers, both non-4K aligned (invalid) */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800, 0x1000) == 0);

	CU_ASSERT(prp_index == 2);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x900800, 0x1000) == -EINVAL);

	CU_ASSERT(prp_index == 2);



	/* 4K buffer, 4K aligned, but vtophys fails */

	ut_fail_vtophys = true;

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000, 0x1000) == -EINVAL);

	ut_fail_vtophys = false;



	/* Largest aligned buffer that can be described in NVME_MAX_PRP_LIST_ENTRIES (plus PRP1) */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000,

					    (NVME_MAX_PRP_LIST_ENTRIES + 1) * 0x1000) == 0);

	CU_ASSERT(prp_index == NVME_MAX_PRP_LIST_ENTRIES + 1);



	/* Largest non-4K-aligned buffer that can be described in NVME_MAX_PRP_LIST_ENTRIES (plus PRP1) */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800,

					    NVME_MAX_PRP_LIST_ENTRIES * 0x1000) == 0);

	CU_ASSERT(prp_index == NVME_MAX_PRP_LIST_ENTRIES + 1);



	/* Buffer too large to be described in NVME_MAX_PRP_LIST_ENTRIES */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100000,

					    (NVME_MAX_PRP_LIST_ENTRIES + 2) * 0x1000) == -EINVAL);



	/* Non-4K-aligned buffer too large to be described in NVME_MAX_PRP_LIST_ENTRIES */

	prp_list_prep(&tr, &req, &prp_index);

	CU_ASSERT(nvme_pcie_prp_list_append(&tr, &prp_index, (void *)0x100800,

					    (NVME_MAX_PRP_LIST_ENTRIES + 1) * 0x1000) == -EINVAL);

}



int main(int argc, char **argv)

{

	CU_pSuite	suite = NULL;
@@ -657,14 +794,11 @@ int main(int argc, char **argv) 
		return CU_get_error();

	}



#if 0

	if (CU_add_test(suite, "test3", test3) == NULL

	    || CU_add_test(suite, "test4", test4) == NULL

	if (CU_add_test(suite, "prp_list_append", test_prp_list_append) == NULL

	   ) {

		CU_cleanup_registry();

		return CU_get_error();

	}

#endif



	CU_basic_set_mode(CU_BRM_VERBOSE);

	CU_basic_run_tests();