Commit b9913608 authored by Ben Walker's avatar Ben Walker Committed by Jim Harris
Browse files

nvme: Add clarifying comments when building RDMA SGLs 



Change-Id: I9e0fc92e422de3fc65c5048a63f4c7dcc46f7324
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/428727


Reviewed-by: default avatarSeth Howell <seth.howell5141@gmail.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
parent c10f8e16

lib/nvme/nvme_rdma.c

+39 −6
Original line number Diff line number Diff line
@@ -414,13 +414,17 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair) 
	TAILQ_INIT(&rqpair->outstanding_reqs);

	for (i = 0; i < rqpair->num_entries; i++) {

		struct spdk_nvme_rdma_req	*rdma_req;

		struct spdk_nvme_cmd		*cmd;

		struct spdk_nvmf_cmd		*cmd;



		rdma_req = &rqpair->rdma_reqs[i];

		cmd = &rqpair->cmds[i].cmd;

		cmd = &rqpair->cmds[i];



		rdma_req->id = i;



		/* The first RDMA sgl element will always point

		 * at this data structure. Depending on whether

		 * an NVMe-oF SGL is required, the length of

		 * this element may change. */

		rdma_req->send_sgl[0].addr = (uint64_t)cmd;

		rdma_req->send_sgl[0].lkey = rqpair->cmd_mr->lkey;
@@ -428,7 +432,7 @@ nvme_rdma_alloc_reqs(struct nvme_rdma_qpair *rqpair) 
		rdma_req->send_wr.next = NULL;

		rdma_req->send_wr.opcode = IBV_WR_SEND;

		rdma_req->send_wr.send_flags = IBV_SEND_SIGNALED;

		rdma_req->send_wr.sg_list = &rdma_req->send_sgl[0];

		rdma_req->send_wr.sg_list = rdma_req->send_sgl;

		rdma_req->send_wr.num_sge = 1; /* Need to increment if inline */

		rdma_req->send_wr.imm_data = 0;
@@ -831,7 +835,12 @@ nvme_rdma_build_null_request(struct spdk_nvme_rdma_req *rdma_req) 


	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;



	/* The first element of this SGL is pointing at an

	 * spdk_nvmf_cmd object. For this particular command,

	 * we only need the first 64 bytes corresponding to

	 * the NVMe command. */

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	nvme_sgl = &req->cmd.dptr.sgl1;

	nvme_sgl->keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;

	nvme_sgl->keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
@@ -867,12 +876,18 @@ nvme_rdma_build_contig_inline_request(struct nvme_rdma_qpair *rqpair, 
		return -EINVAL;

	}



	/* The first element of this SGL is pointing at an

	 * spdk_nvmf_cmd object. For this particular command,

	 * we only need the first 64 bytes corresponding to

	 * the NVMe command. */

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	sge_inline->addr = (uint64_t)payload;

	sge_inline->length = (uint32_t)req->payload_size;

	sge_inline->lkey = mr->lkey;



	rdma_req->send_wr.num_sge = 2;

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;

	req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;

	req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;
@@ -906,8 +921,14 @@ nvme_rdma_build_contig_request(struct nvme_rdma_qpair *rqpair, 
		return -1;

	}



	rdma_req->send_wr.num_sge = 1;

	/* The first element of this SGL is pointing at an

	 * spdk_nvmf_cmd object. For this particular command,

	 * we only need the first 64 bytes corresponding to

	 * the NVMe command. */

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	rdma_req->send_wr.num_sge = 1;



	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;

	req->cmd.dptr.sgl1.keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;

	req->cmd.dptr.sgl1.keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;
@@ -983,7 +1004,12 @@ nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair, 
	 * as a data block descriptor.

	 */

	if (num_sgl_desc == 1) {

		/* The first element of this SGL is pointing at an

		 * spdk_nvmf_cmd object. For this particular command,

		 * we only need the first 64 bytes corresponding to

		 * the NVMe command. */

		rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



		req->cmd.dptr.sgl1.keyed.type = SPDK_NVME_SGL_TYPE_KEYED_DATA_BLOCK;

		req->cmd.dptr.sgl1.keyed.subtype = SPDK_NVME_SGL_SUBTYPE_ADDRESS;

		req->cmd.dptr.sgl1.keyed.length = req->payload_size;
@@ -996,6 +1022,7 @@ nvme_rdma_build_sgl_request(struct nvme_rdma_qpair *rqpair, 
		 */

		rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd) + sizeof(struct

					       spdk_nvme_sgl_descriptor) * num_sgl_desc;



		req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_LAST_SEGMENT;

		req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;

		req->cmd.dptr.sgl1.unkeyed.length = num_sgl_desc * sizeof(struct spdk_nvme_sgl_descriptor);
@@ -1044,12 +1071,18 @@ nvme_rdma_build_sgl_inline_request(struct nvme_rdma_qpair *rqpair, 
		return -1;

	}



	/* The first element of this SGL is pointing at an

	 * spdk_nvmf_cmd object. For this particular command,

	 * we only need the first 64 bytes corresponding to

	 * the NVMe command. */

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	sge_inline->addr = (uint64_t)virt_addr;

	sge_inline->length = (uint32_t)req->payload_size;

	sge_inline->lkey = mr->lkey;



	rdma_req->send_wr.num_sge = 2;

	rdma_req->send_sgl[0].length = sizeof(struct spdk_nvme_cmd);



	req->cmd.psdt = SPDK_NVME_PSDT_SGL_MPTR_CONTIG;

	req->cmd.dptr.sgl1.unkeyed.type = SPDK_NVME_SGL_TYPE_DATA_BLOCK;

	req->cmd.dptr.sgl1.unkeyed.subtype = SPDK_NVME_SGL_SUBTYPE_OFFSET;