Commit ee06b4c2 authored by Jim Harris's avatar Jim Harris
Browse files

nvme: restore nvme_complete_request optimization for pcie 



Make nvme_free_request a helper named _nvme_free_request that takes an
explicit qpair rather than loading it from req->qpair. This restores the
optimization in the pcie path which avoids touching the request to reduce
unnecessary load dependencies.

Signed-off-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: I2b07536b1c78c3db16d5a795dfdafaa34f0f917c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/20649


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarMichael Haeuptle <michaelhaeuptle@gmail.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarDeepak Abraham Tom <deepak-abraham.tom@hpe.com>
Community-CI: Mellanox Build Bot
parent 06955fd3

lib/nvme/nvme_internal.h

+18 −7
Original line number Diff line number Diff line
@@ -1333,23 +1333,29 @@ struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpa 
		spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);



static inline void

nvme_free_request(struct nvme_request *req)

_nvme_free_request(struct nvme_request *req, struct spdk_nvme_qpair *qpair)

{

	assert(req != NULL);

	assert(req->num_children == 0);

	assert(req->qpair != NULL);

	assert(qpair != NULL);



	/* The reserved_req does not go in the free_req STAILQ - it is

	 * saved only for use with a FABRICS/CONNECT command.

	 */

	if (spdk_likely(req->qpair->reserved_req != req)) {

		STAILQ_INSERT_HEAD(&req->qpair->free_req, req, stailq);

	if (spdk_likely(qpair->reserved_req != req)) {

		STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq);



		assert(req->qpair->num_outstanding_reqs > 0);

		req->qpair->num_outstanding_reqs--;

		assert(qpair->num_outstanding_reqs > 0);

		qpair->num_outstanding_reqs--;

	}

}



static inline void

nvme_free_request(struct nvme_request *req)

{

	_nvme_free_request(req, req->qpair);

}



static inline void

nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpair *qpair,

		      struct nvme_request *req, struct spdk_nvme_cpl *cpl)
@@ -1391,7 +1397,12 @@ nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpa 
		}

	}



	nvme_free_request(req);

	/* For PCIe completions, we want to avoid touching the req itself to avoid

	 * dependencies on loading those cachelines. So call the internal helper

	 * function instead using the qpair that was passed by the caller, instead

	 * of getting it from the req.

	 */

	_nvme_free_request(req, qpair);



	if (cb_fn) {

		cb_fn(cb_arg, cpl);