Commit e688d1cc authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

nvme_rdma: properly configure and store max_sges 



The max_send_sge and max_recv_sge values can be set to any value from
0...dev_attr->max_sge. WHen we actually set the attributes, we will
receive a qpair with values for max_sge greater than or equal to what we
initially set. We need to store the maximum number of SGEs for later use
when constructing work requests.
Previously we have not relied on these values since we assumed that we
would always be able to have more sges than we asked for initially. This
may change as we try to allocate more SGEs to handle splitting buffers
across memory regions.

Change-Id: Ibbeae1908b86baa3a96d9c6cd2051401aaa2197b
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/433307


Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 1180bf83

lib/nvme/nvme_rdma.c

+23 −5
Original line number Diff line number Diff line
@@ -92,14 +92,16 @@ struct nvme_rdma_ctrlr { 
struct nvme_rdma_qpair {

	struct spdk_nvme_qpair			qpair;



	struct rdma_event_channel		*cm_channel;



	struct rdma_cm_id			*cm_id;



	struct ibv_cq				*cq;



	struct	spdk_nvme_rdma_req		*rdma_reqs;



	uint32_t				max_send_sge;



	uint32_t				max_recv_sge;



	uint16_t				num_entries;



	/* Parallel arrays of response buffers + response SGLs of size num_entries */
@@ -124,6 +126,9 @@ struct nvme_rdma_qpair { 


	TAILQ_HEAD(, spdk_nvme_rdma_req)	free_reqs;

	TAILQ_HEAD(, spdk_nvme_rdma_req)	outstanding_reqs;



	/* Placed at the end of the struct since it is not used frequently */

	struct rdma_event_channel		*cm_channel;

};



struct spdk_nvme_rdma_req {
@@ -245,8 +250,15 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
{

	int			rc;

	struct ibv_qp_init_attr	attr;

	struct ibv_device_attr	dev_attr;

	struct nvme_rdma_ctrlr	*rctrlr;



	rc = ibv_query_device(rqpair->cm_id->verbs, &dev_attr);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to query RDMA device attributes.\n");

		return -1;

	}



	rqpair->cq = ibv_create_cq(rqpair->cm_id->verbs, rqpair->num_entries * 2, rqpair, NULL, 0);

	if (!rqpair->cq) {

		SPDK_ERRLOG("Unable to create completion queue: errno %d: %s\n", errno, spdk_strerror(errno));
@@ -266,14 +278,20 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair) 
	attr.recv_cq		= rqpair->cq;

	attr.cap.max_send_wr	= rqpair->num_entries; /* SEND operations */

	attr.cap.max_recv_wr	= rqpair->num_entries; /* RECV operations */

	attr.cap.max_send_sge	= NVME_RDMA_DEFAULT_TX_SGE;

	attr.cap.max_recv_sge	= NVME_RDMA_DEFAULT_RX_SGE;

	attr.cap.max_send_sge	= spdk_min(NVME_RDMA_DEFAULT_TX_SGE, dev_attr.max_sge);

	attr.cap.max_recv_sge	= spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);



	rc = rdma_create_qp(rqpair->cm_id, rctrlr->pd, &attr);



	if (rc) {

		SPDK_ERRLOG("rdma_create_qp failed\n");

		return -1;

	}



	/* ibv_create_qp will change the values in attr.cap. Make sure we store the proper value. */

	rqpair->max_send_sge = spdk_min(NVME_RDMA_DEFAULT_TX_SGE, attr.cap.max_send_sge);

	rqpair->max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, attr.cap.max_recv_sge);



	rctrlr->pd = rqpair->cm_id->qp->pd;



	rqpair->cm_id->context = &rqpair->qpair;