Commit 47154c69 authored by Jacek Kalwas's avatar Jacek Kalwas Committed by Tomasz Zawadzki
Browse files

nvme: mark (un)likely to improve hot path 



Signed-off-by: default avatarJacek Kalwas <jacek.kalwas@intel.com>
Change-Id: I1ba5ac3d016e326e18d52c88033a71fa5ff3c406
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21339


Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
parent a6289d57

lib/nvme/nvme_internal.h

+1 −1
Original line number Diff line number Diff line
@@ -1405,7 +1405,7 @@ nvme_complete_request(spdk_nvme_cmd_cb cb_fn, void *cb_arg, struct spdk_nvme_qpa 
	 */

	_nvme_free_request(req, qpair);



	if (cb_fn) {

	if (spdk_likely(cb_fn)) {

		cb_fn(cb_arg, cpl);

	}

}

lib/nvme/nvme_qpair.c

+4 −3
Original line number Diff line number Diff line
@@ -644,7 +644,8 @@ nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair) 
	 * from the old transport connection and encourage the application to retry them. We also need

	 * to submit any queued requests that built up while we were in the connected or enabling state.

	 */

	if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTED && !qpair->ctrlr->is_resetting) {

	if (spdk_unlikely(nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTED &&

			  !qpair->ctrlr->is_resetting)) {

		nvme_qpair_set_state(qpair, NVME_QPAIR_ENABLING);

		/*

		 * PCIe is special, for fabrics transports, we can abort requests before disconnect during reset
@@ -674,8 +675,8 @@ nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair) 
	 * controller thread and we can't disconnect I/O qpairs from the controller

	 * thread.

	 */

	if (qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE &&

	    nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED) {

	if (spdk_unlikely(qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE &&

			  nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED)) {

		/* Don't disconnect PCIe qpairs. They are a special case for reset. */

		if (qpair->ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) {

			nvme_ctrlr_disconnect_qpair(qpair);

lib/nvme/nvme_tcp.c

+13 −11
Original line number Diff line number Diff line
@@ -818,7 +818,7 @@ nvme_tcp_req_init(struct nvme_tcp_qpair *tqpair, struct nvme_request *req, 
		return rc;

	}



	if (req->cmd.opc == SPDK_NVME_OPC_FABRIC) {

	if (spdk_unlikely(req->cmd.opc == SPDK_NVME_OPC_FABRIC)) {

		struct spdk_nvmf_capsule_cmd *nvmf_cmd = (struct spdk_nvmf_capsule_cmd *)&req->cmd;



		xfer = spdk_nvme_opc_get_data_transfer(nvmf_cmd->fctype);
@@ -827,7 +827,8 @@ nvme_tcp_req_init(struct nvme_tcp_qpair *tqpair, struct nvme_request *req, 
	}

	if (xfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {

		max_in_capsule_data_size = ctrlr->ioccsz_bytes;

		if ((req->cmd.opc == SPDK_NVME_OPC_FABRIC) || nvme_qpair_is_admin_queue(&tqpair->qpair)) {

		if (spdk_unlikely((req->cmd.opc == SPDK_NVME_OPC_FABRIC) ||

				  nvme_qpair_is_admin_queue(&tqpair->qpair))) {

			max_in_capsule_data_size = SPDK_NVME_TCP_IN_CAPSULE_DATA_MAX_SIZE;

		}
@@ -956,7 +957,7 @@ nvme_tcp_qpair_submit_request(struct spdk_nvme_qpair *qpair, 
		return -EAGAIN;

	}



	if (nvme_tcp_req_init(tqpair, req, tcp_req)) {

	if (spdk_unlikely(nvme_tcp_req_init(tqpair, req, tcp_req))) {

		SPDK_ERRLOG("nvme_tcp_req_init() failed\n");

		nvme_tcp_req_put(tqpair, tcp_req);

		return -1;
@@ -984,7 +985,7 @@ nvme_tcp_req_complete(struct nvme_tcp_req *tcp_req, 
	struct spdk_nvme_cpl	cpl;

	struct spdk_nvme_qpair	*qpair;

	struct nvme_request	*req;

	bool			error, print_error;

	bool			print_error;



	assert(tcp_req->req != NULL);

	req = tcp_req->req;
@@ -993,8 +994,8 @@ nvme_tcp_req_complete(struct nvme_tcp_req *tcp_req, 
	/* Cache arguments to be passed to nvme_complete_request since tcp_req can be zeroed when released */

	memcpy(&cpl, rsp, sizeof(cpl));



	error = spdk_nvme_cpl_is_error(rsp);

	print_error = error && print_on_error && !qpair->ctrlr->opts.disable_error_logging;

	if (spdk_unlikely(spdk_nvme_cpl_is_error(rsp))) {

		print_error = print_on_error && !qpair->ctrlr->opts.disable_error_logging;



		if (print_error) {

			spdk_nvme_qpair_print_command(qpair, &req->cmd);
@@ -1003,6 +1004,7 @@ nvme_tcp_req_complete(struct nvme_tcp_req *tcp_req, 
		if (print_error || SPDK_DEBUGLOG_FLAG_ENABLED("nvme")) {

			spdk_nvme_qpair_print_completion(qpair, rsp);

		}

	}



	spdk_trace_record(TRACE_NVME_TCP_COMPLETE, qpair->id, 0, (uintptr_t)req, req->cb_arg,

			  (uint32_t)req->cmd.cid, (uint32_t)cpl.status_raw);