Commit d02950e6 authored by Ben Walker's avatar Ben Walker Committed by Jim Harris
Browse files

nvme: Cache the cb_fn and cb_arg in the tracker 



This avoids a data dependent load to find which
callback to call in the completion path.

Change-Id: Ifa20790a7af3332a74bc45037e589668744af797
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/450558


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent f7d3dd9b

lib/nvme/nvme_ctrlr_cmd.c

+1 −1
Original line number Diff line number Diff line
@@ -529,7 +529,7 @@ spdk_nvme_ctrlr_cmd_abort_cpl(void *ctx, const struct spdk_nvme_cpl *cpl) 
			next->cpl.status.sct = SPDK_NVME_SCT_GENERIC;

			next->cpl.status.sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			next->cpl.status.dnr = 1;

			nvme_complete_request(next, &req->cpl);

			nvme_complete_request(next->cb_fn, next->cb_arg, next, &req->cpl);

			nvme_free_request(next);

		} else {

			/* If the first abort succeeds, stop iterating. */

lib/nvme/nvme_internal.h

+4 −3
Original line number Diff line number Diff line
@@ -908,7 +908,8 @@ struct nvme_request *nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpa 
		spdk_nvme_cmd_cb cb_fn, void *cb_arg, bool host_to_controller);



static inline void

nvme_complete_request(struct nvme_request *req, struct spdk_nvme_cpl *cpl)

nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg,

		      struct nvme_request *req, struct spdk_nvme_cpl *cpl)

{

	struct spdk_nvme_qpair          *qpair = req->qpair;

	struct spdk_nvme_cpl            err_cpl;
@@ -938,8 +939,8 @@ nvme_complete_request(struct nvme_request *req, struct spdk_nvme_cpl *cpl) 
		}

	}



	if (req->cb_fn) {

		req->cb_fn(req->cb_arg, cpl);

	if (cb_fn) {

		cb_fn(cb_arg, cpl);

	}

}

lib/nvme/nvme_ns_cmd.c

+1 −1
Original line number Diff line number Diff line
@@ -70,7 +70,7 @@ nvme_cb_complete_child(void *child_arg, const struct spdk_nvme_cpl *cpl) 
	}



	if (parent->num_children == 0) {

		nvme_complete_request(parent, &parent->parent_status);

		nvme_complete_request(parent->cb_fn, parent->cb_arg, parent, &parent->parent_status);

		nvme_free_request(parent);

	}

}

lib/nvme/nvme_pcie.c

+10 −8
Original line number Diff line number Diff line
@@ -55,9 +55,9 @@ 
 * NVME_MAX_SGL_DESCRIPTORS defines the maximum number of descriptors in one SGL

 *  segment.

 */

#define NVME_MAX_SGL_DESCRIPTORS	(253)

#define NVME_MAX_SGL_DESCRIPTORS	(251)



#define NVME_MAX_PRP_LIST_ENTRIES	(506)

#define NVME_MAX_PRP_LIST_ENTRIES	(505)



struct nvme_pcie_enum_ctx {

	struct spdk_nvme_probe_ctx *probe_ctx;
@@ -114,11 +114,11 @@ struct nvme_tracker { 
	struct nvme_request		*req;

	uint16_t			cid;



	uint16_t			rsvd1;

	uint16_t			rsvd0;

	uint32_t			rsvd1;



	uint32_t			rsvd2;



	uint64_t			rsvd3;

	spdk_nvme_cmd_cb		cb_fn;

	void				*cb_arg;



	uint64_t			prp_sgl_bus_addr;
@@ -1153,7 +1153,7 @@ nvme_pcie_qpair_complete_pending_admin_request(struct spdk_nvme_qpair *qpair) 


		assert(req->pid == pid);



		nvme_complete_request(req, &req->cpl);

		nvme_complete_request(req->cb_fn, req->cb_arg, req, &req->cpl);

		nvme_free_request(req);

	}

}
@@ -1286,7 +1286,7 @@ nvme_pcie_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_trac 
			req_from_current_proc = false;

			nvme_pcie_qpair_insert_pending_admin_request(qpair, req, cpl);

		} else {

			nvme_complete_request(req, cpl);

			nvme_complete_request(tr->cb_fn, tr->cb_arg, req, cpl);

		}



		if (req_from_current_proc == true) {
@@ -2003,6 +2003,8 @@ nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_reques 
	TAILQ_REMOVE(&pqpair->free_tr, tr, tq_list); /* remove tr from free_tr */

	TAILQ_INSERT_TAIL(&pqpair->outstanding_tr, tr, tq_list);

	tr->req = req;

	tr->cb_fn = req->cb_fn;

	tr->cb_arg = req->cb_arg;

	req->cmd.cid = tr->cid;



	if (req->payload_size && req->payload.md) {

lib/nvme/nvme_qpair.c

+1 −1
Original line number Diff line number Diff line
@@ -382,7 +382,7 @@ nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair, 
		nvme_qpair_print_completion(qpair, &cpl);

	}



	nvme_complete_request(req, &cpl);

	nvme_complete_request(req->cb_fn, req->cb_arg, req, &cpl);

	nvme_free_request(req);

}