Commit 97a43680 authored by Seth Howell's avatar Seth Howell Committed by Jim Harris
Browse files

rdma: move cq_resize to its own function. 



Change-Id: I07aef399320fd4a014f63760670ea765d2e18b4b
Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/448190


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent fa79f64a

lib/nvmf/rdma.c

+43 −34
Original line number Diff line number Diff line
@@ -839,11 +839,46 @@ spdk_nvmf_rdma_qpair_destroy(struct spdk_nvmf_rdma_qpair *rqpair) 
}



static int

spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)

nvmf_rdma_resize_cq(struct spdk_nvmf_rdma_qpair *rqpair, struct spdk_nvmf_rdma_device *device)

{

	struct spdk_nvmf_rdma_qpair		*rqpair;

	struct spdk_nvmf_rdma_poller	*rpoller;

	int				rc, num_cqe, required_num_wr;



	/* Enlarge CQ size dynamically */

	rpoller = rqpair->poller;

	required_num_wr = rpoller->required_num_wr + MAX_WR_PER_QP(rqpair->max_queue_depth);

	num_cqe = rpoller->num_cqe;

	if (num_cqe < required_num_wr) {

		num_cqe = spdk_max(num_cqe * 2, required_num_wr);

		num_cqe = spdk_min(num_cqe, device->attr.max_cqe);

	}



	if (rpoller->num_cqe != num_cqe) {

		if (required_num_wr > device->attr.max_cqe) {

			SPDK_ERRLOG("RDMA CQE requirement (%d) exceeds device max_cqe limitation (%d)\n",

				    required_num_wr, device->attr.max_cqe);

			return -1;

		}



		SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Resize RDMA CQ from %d to %d\n", rpoller->num_cqe, num_cqe);

		rc = ibv_resize_cq(rpoller->cq, num_cqe);

		if (rc) {

			SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));

			return -1;

		}



		rpoller->num_cqe = num_cqe;

	}



	rpoller->required_num_wr = required_num_wr;

	return 0;

}



static int

spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)

{

	struct spdk_nvmf_rdma_qpair		*rqpair;

	int					rc;

	struct spdk_nvmf_rdma_transport		*rtransport;

	struct spdk_nvmf_transport		*transport;

	struct spdk_nvmf_rdma_resource_opts	opts;
@@ -871,40 +906,14 @@ spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	ibv_init_attr.cap.max_send_sge	= spdk_min(device->attr.max_sge, NVMF_DEFAULT_TX_SGE);

	ibv_init_attr.cap.max_recv_sge	= spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE);



	/* Enlarge CQ size dynamically */

	rpoller = rqpair->poller;

	required_num_wr = rpoller->required_num_wr + MAX_WR_PER_QP(rqpair->max_queue_depth);

	num_cqe = rpoller->num_cqe;

	if (num_cqe < required_num_wr) {

		num_cqe = spdk_max(num_cqe * 2, required_num_wr);

		num_cqe = spdk_min(num_cqe, device->attr.max_cqe);

	}



	if (rpoller->num_cqe != num_cqe) {

		if (required_num_wr > device->attr.max_cqe) {

			SPDK_ERRLOG("RDMA CQE requirement (%d) exceeds device max_cqe limitation (%d)\n",

				    required_num_wr, device->attr.max_cqe);

			rdma_destroy_id(rqpair->cm_id);

			rqpair->cm_id = NULL;

			spdk_nvmf_rdma_qpair_destroy(rqpair);

			return -1;

		}



		SPDK_DEBUGLOG(SPDK_LOG_RDMA, "Resize RDMA CQ from %d to %d\n", rpoller->num_cqe, num_cqe);

		rc = ibv_resize_cq(rpoller->cq, num_cqe);

		if (rc) {

			SPDK_ERRLOG("RDMA CQ resize failed: errno %d: %s\n", errno, spdk_strerror(errno));

	if (nvmf_rdma_resize_cq(rqpair, device) < 0) {

		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");

		rdma_destroy_id(rqpair->cm_id);

		rqpair->cm_id = NULL;

		spdk_nvmf_rdma_qpair_destroy(rqpair);

		return -1;

	}



		rpoller->num_cqe = num_cqe;

	}



	rpoller->required_num_wr = required_num_wr;



	rc = rdma_create_qp(rqpair->cm_id, rqpair->port->device->pd, &ibv_init_attr);

	if (rc) {

		SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno));