Commit cfc716fe authored by Naresh Gottumukkala's avatar Naresh Gottumukkala Committed by Tomasz Zawadzki
Browse files

nvmf/fc: Handle out of order fused commands. 



For FC, Fused commands ordering is not gauranteed but nvmf layer expects strict ordering.
FC LLD layer can reorder this out of order commands based on FC header CSN.
Introduce a new FUSED_WAITING state that can be used by LLD to keep waiting the out of order
data.

Signed-off-by: default avatarNaresh Gottumukkala <raju.gottumukkala@broadcom.com>
Change-Id: I85eba0059f75ff3775f664e85ed2e4dc258d27a6
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5811


Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent e3c78a64

lib/nvmf/fc.c

+13 −1
Original line number Diff line number Diff line
@@ -84,7 +84,8 @@ static char *fc_req_state_strs[] = { 
	"SPDK_NVMF_FC_REQ_FAILED",

	"SPDK_NVMF_FC_REQ_ABORTED",

	"SPDK_NVMF_FC_REQ_BDEV_ABORTED",

	"SPDK_NVMF_FC_REQ_PENDING"

	"SPDK_NVMF_FC_REQ_PENDING",

	"SPDK_NVMF_FC_REQ_FUSED_WAITING"

};



#define OBJECT_NVMF_FC_IO				0xA0
@@ -105,6 +106,7 @@ static char *fc_req_state_strs[] = { 
#define TRACE_FC_REQ_ABORTED                    SPDK_TPOINT_ID(TRACE_GROUP_NVMF_FC, 0x0D)

#define TRACE_FC_REQ_BDEV_ABORTED               SPDK_TPOINT_ID(TRACE_GROUP_NVMF_FC, 0x0E)

#define TRACE_FC_REQ_PENDING                    SPDK_TPOINT_ID(TRACE_GROUP_NVMF_FC, 0x0F)

#define TRACE_FC_REQ_FUSED_WAITING		SPDK_TPOINT_ID(TRACE_GROUP_NVMF_FC, 0x10)



#define HWQP_CONN_TABLE_SIZE			8192

#define HWQP_RPI_TABLE_SIZE			4096
@@ -157,6 +159,9 @@ SPDK_TRACE_REGISTER_FN(nvmf_fc_trace, "nvmf_fc", TRACE_GROUP_NVMF_FC) 
	spdk_trace_register_description("FC_REQ_PENDING",

					TRACE_FC_REQ_PENDING,

					OWNER_NONE, OBJECT_NONE, 0, 1, "");

	spdk_trace_register_description("FC_REQ_FUSED_WAITING",

					TRACE_FC_REQ_FUSED_WAITING,

					OWNER_NONE, OBJECT_NONE, 0, 1, "");

}



/**
@@ -299,6 +304,9 @@ nvmf_fc_record_req_trace_point(struct spdk_nvmf_fc_request *fc_req, 
	case SPDK_NVMF_FC_REQ_PENDING:

		tpoint_id = TRACE_FC_REQ_PENDING;

		break;

	case SPDK_NVMF_FC_REQ_FUSED_WAITING:

		tpoint_id = TRACE_FC_REQ_FUSED_WAITING;

		break;

	default:

		assert(0);

		break;
@@ -1351,6 +1359,9 @@ nvmf_fc_request_abort(struct spdk_nvmf_fc_request *fc_req, bool send_abts, 
		/* Remove from pending */

		nvmf_fc_request_remove_from_pending(fc_req);

		goto complete;

	case SPDK_NVMF_FC_REQ_FUSED_WAITING:

		TAILQ_REMOVE(&fc_req->fc_conn->fused_waiting_queue, fc_req, fused_link);

		goto complete;

	default:

		SPDK_ERRLOG("Request in invalid state.\n");

		goto complete;
@@ -1532,6 +1543,7 @@ nvmf_fc_hwqp_handle_request(struct spdk_nvmf_fc_hwqp *hwqp, struct spdk_nvmf_fc_ 
	fc_req->req.xfer = xfer;

	fc_req->s_id = s_id;

	fc_req->d_id = d_id;

	fc_req->csn  = from_be32(&cmd_iu->cmnd_seq_num);



	nvmf_fc_record_req_trace_point(fc_req, SPDK_NVMF_FC_REQ_INIT);

lib/nvmf/fc_ls.c

+1 −0
Original line number Diff line number Diff line
@@ -355,6 +355,7 @@ nvmf_fc_ls_new_connection(struct spdk_nvmf_fc_association *assoc, uint16_t qid, 
	fc_conn->max_queue_depth = sq_size + 1;

	fc_conn->conn_state = SPDK_NVMF_FC_OBJECT_CREATED;

	TAILQ_INIT(&fc_conn->in_use_reqs);

	TAILQ_INIT(&fc_conn->fused_waiting_queue);



	/* save target port trid in connection (for subsystem

	 * listener validation in fabric connect command)

lib/nvmf/nvmf_fc.h

+5 −0
Original line number Diff line number Diff line
@@ -104,6 +104,7 @@ enum spdk_nvmf_fc_request_state { 
	SPDK_NVMF_FC_REQ_ABORTED,

	SPDK_NVMF_FC_REQ_BDEV_ABORTED,

	SPDK_NVMF_FC_REQ_PENDING,

	SPDK_NVMF_FC_REQ_FUSED_WAITING,

	SPDK_NVMF_FC_REQ_MAX_STATE,

};
@@ -240,6 +241,8 @@ struct spdk_nvmf_fc_conn { 
	/* number of read/write requests that are outstanding */

	uint16_t cur_fc_rw_depth;



	TAILQ_HEAD(, spdk_nvmf_fc_request) fused_waiting_queue;



	struct spdk_nvmf_fc_association *fc_assoc;



	uint16_t rpi;
@@ -376,8 +379,10 @@ struct spdk_nvmf_fc_request { 
	uint32_t magic;

	uint32_t s_id;

	uint32_t d_id;

	uint32_t csn;

	TAILQ_ENTRY(spdk_nvmf_fc_request) link;

	TAILQ_ENTRY(spdk_nvmf_fc_request) conn_link;

	TAILQ_ENTRY(spdk_nvmf_fc_request) fused_link;

	TAILQ_HEAD(, spdk_nvmf_fc_caller_ctx) abort_cbs;

};