Commit e89d9af0 authored by Naresh Gottumukkala's avatar Naresh Gottumukkala Committed by Jim Harris
Browse files

nvmf/fc: Cleanup FC qpair_fini code. 



As NVMF qpair maps to a FC level connection, Ideally a qpair fini
should cleanup fc level connection irrespective of what type of qpair
it is i.e admin or IO. But today IO qpair cleanup is actually a dummy.

Also FC nvmf_transport_qpair_fini is just triggering fc level connection
cleanup but not properly synchronized. Use latest async nature of
nvmf_transport_qpair_fini enhancements to fix this.

Signed-off-by: default avatarNaresh Gottumukkala <raju.gottumukkala@broadcom.com>
Change-Id: I883ed774769e2d4a2575b6d90bb3348981cd1e0d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5700


Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent b50c6bc2

lib/nvmf/fc.c

+54 −37
Original line number Diff line number Diff line
@@ -321,30 +321,6 @@ nvmf_fc_create_hash_table(const char *name, size_t num_entries, size_t key_len) 
	return rte_hash_create(&hash_params);

}



static void

nvmf_fc_handle_connection_failure(void *arg)

{

	struct spdk_nvmf_fc_conn *fc_conn = arg;

	struct spdk_nvmf_fc_ls_add_conn_api_data *api_data = NULL;



	if (!fc_conn->create_opd) {

		return;

	}

	api_data = &fc_conn->create_opd->u.add_conn;



	nvmf_fc_ls_add_conn_failure(api_data->assoc, api_data->ls_rqst,

				    api_data->args.fc_conn, api_data->aq_conn);

}



static void

nvmf_fc_handle_assoc_deletion(void *arg)

{

	struct spdk_nvmf_fc_conn *fc_conn = arg;



	nvmf_fc_delete_association(fc_conn->fc_assoc->tgtport,

				   fc_conn->fc_assoc->assoc_id, false, true, NULL, NULL);

}



void

nvmf_fc_free_conn_reqpool(struct spdk_nvmf_fc_conn *fc_conn)

{
@@ -2063,27 +2039,68 @@ nvmf_fc_request_free(struct spdk_nvmf_request *req) 
}



static void

nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair,

		    spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)

nvmf_fc_connection_delete_done_cb(void *arg)

{

	struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx = arg;



	if (fc_ctx->cb_fn) {

		spdk_thread_send_msg(fc_ctx->qpair_thread, fc_ctx->cb_fn, fc_ctx->cb_ctx);

	}

	free(fc_ctx);

}



static void

_nvmf_fc_close_qpair(void *arg)

{

	struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx = arg;

	struct spdk_nvmf_qpair *qpair = fc_ctx->qpair;

	struct spdk_nvmf_fc_conn *fc_conn;

	int rc;



	fc_conn = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_fc_conn, qpair);



	if (fc_conn->conn_id == NVMF_FC_INVALID_CONN_ID) {

		/* QP creation failure in FC tranport. Cleanup. */

		spdk_thread_send_msg(nvmf_fc_get_main_thread(),

				     nvmf_fc_handle_connection_failure, fc_conn);

	} else if (fc_conn->fc_assoc->assoc_id == fc_conn->conn_id &&

		   fc_conn->fc_assoc->assoc_state != SPDK_NVMF_FC_OBJECT_TO_BE_DELETED) {

		/* Admin connection */

		spdk_thread_send_msg(nvmf_fc_get_main_thread(),

				     nvmf_fc_handle_assoc_deletion, fc_conn);

		struct spdk_nvmf_fc_ls_add_conn_api_data *api_data = NULL;



		if (fc_conn->create_opd) {

			api_data = &fc_conn->create_opd->u.add_conn;



			nvmf_fc_ls_add_conn_failure(api_data->assoc, api_data->ls_rqst,

						    api_data->args.fc_conn, api_data->aq_conn);

		}

	} else if (fc_conn->conn_state == SPDK_NVMF_FC_OBJECT_CREATED) {

		rc = nvmf_fc_delete_connection(fc_conn, false, true,

					       nvmf_fc_connection_delete_done_cb, fc_ctx);

		if (!rc) {

			/* Wait for transport to complete its work. */

			return;

		}



		SPDK_ERRLOG("%s: Delete FC connection failed.\n", __func__);

	}



	nvmf_fc_connection_delete_done_cb(fc_ctx);

}



static void

nvmf_fc_close_qpair(struct spdk_nvmf_qpair *qpair,

		    spdk_nvmf_transport_qpair_fini_cb cb_fn, void *cb_arg)

{

	struct spdk_nvmf_fc_qpair_remove_ctx *fc_ctx;



	fc_ctx = calloc(1, sizeof(struct spdk_nvmf_fc_qpair_remove_ctx));

	if (!fc_ctx) {

		SPDK_ERRLOG("Unable to allocate close_qpair ctx.");

		if (cb_fn) {

			cb_fn(cb_arg);

		}

		return;

	}

	fc_ctx->qpair = qpair;

	fc_ctx->cb_fn = cb_fn;

	fc_ctx->cb_ctx = cb_arg;

	fc_ctx->qpair_thread = spdk_get_thread();



	spdk_thread_send_msg(nvmf_fc_get_main_thread(), _nvmf_fc_close_qpair, fc_ctx);

}



static int

lib/nvmf/fc_ls.c

+19 −3
Original line number Diff line number Diff line
@@ -419,6 +419,9 @@ nvmf_fc_do_del_conn_cbs(struct nvmf_fc_ls_op_ctx *opd, 
				SPDK_ERRLOG("Send LS response for delete connection failed\n");

			}

		}

		if (dp->del_conn_cb) {

			dp->del_conn_cb(dp->del_conn_cb_data);

		}

		free(opd);

		opd = nxt;

	}
@@ -444,7 +447,8 @@ nvmf_fc_ls_poller_delete_conn_cb(void *cb_data, enum spdk_nvmf_fc_poller_api_ret 


static int

nvmf_fc_ls_poller_delete_conn(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,

			      struct spdk_nvmf_fc_ls_rqst *ls_rqst, bool backend_initiated)

			      struct spdk_nvmf_fc_ls_rqst *ls_rqst, bool backend_initiated,

			      spdk_nvmf_fc_del_conn_cb cb_fn, void *cb_data)

{

	struct spdk_nvmf_fc_association *assoc = fc_conn->fc_assoc;

	struct spdk_nvmf_fc_ls_del_conn_api_data *api_data;
@@ -464,6 +468,8 @@ nvmf_fc_ls_poller_delete_conn(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts, 
	api_data = &opd->u.del_conn;

	api_data->assoc = assoc;

	api_data->ls_rqst = ls_rqst;

	api_data->del_conn_cb = cb_fn;

	api_data->del_conn_cb_data = cb_data;

	api_data->aq_conn = (assoc->aq_conn == fc_conn ? true : false);

	api_data->args.fc_conn = fc_conn;

	api_data->args.send_abts = send_abts;
@@ -527,7 +533,7 @@ nvmf_fc_ls_add_conn_cb(void *cb_data, enum spdk_nvmf_fc_poller_api_ret ret) 
	if (nvmf_fc_xmt_ls_rsp(tgtport, ls_rqst) != 0) {

		SPDK_ERRLOG("Send LS response for %s failed - cleaning up\n",

			    dp->aq_conn ? "association" : "connection");

		nvmf_fc_ls_poller_delete_conn(fc_conn, false, NULL, false);

		nvmf_fc_ls_poller_delete_conn(fc_conn, false, NULL, false, NULL, NULL);

	} else {

		SPDK_DEBUGLOG(nvmf_fc_ls,

			      "LS response (conn_id 0x%lx) sent\n", fc_conn->conn_id);
@@ -767,7 +773,7 @@ _nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport, 


	/* delete all of the association's connections */

	TAILQ_FOREACH(fc_conn, &assoc->fc_conns, assoc_link) {

		rc = nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL, backend_initiated);

		rc = nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL, backend_initiated, NULL, NULL);

		if (rc) {

			SPDK_ERRLOG("Delete connection failed for assoc_id 0x%lx conn_id 0x%lx\n",

				    assoc->assoc_id, fc_conn->conn_id);
@@ -1278,6 +1284,16 @@ nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport, 
					   del_assoc_cb, cb_data, false);

}



int

nvmf_fc_delete_connection(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,

			  bool backend_initiated, spdk_nvmf_fc_del_conn_cb cb_fn,

			  void *cb_data)

{

	return nvmf_fc_ls_poller_delete_conn(fc_conn, send_abts, NULL,

					     backend_initiated, cb_fn, cb_data);

}





static void

nvmf_fc_poller_api_cb_event(void *arg)

{

lib/nvmf/nvmf_fc.h

+14 −0
Original line number Diff line number Diff line
@@ -176,6 +176,13 @@ struct spdk_nvmf_fc_srsr_bufs { 
	uint16_t rpi;

};



struct spdk_nvmf_fc_qpair_remove_ctx {

	struct spdk_nvmf_qpair *qpair;

	spdk_nvmf_transport_qpair_fini_cb cb_fn;

	void *cb_ctx;

	struct spdk_thread *qpair_thread;

};



/*

 * Struct representing a nport

 */
@@ -552,6 +559,7 @@ SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_fc_rq_buf_ls_request) == 


/* Poller API structures (arguments and callback data */

typedef void (*spdk_nvmf_fc_del_assoc_cb)(void *arg, uint32_t err);

typedef void (*spdk_nvmf_fc_del_conn_cb)(void *arg);



struct spdk_nvmf_fc_ls_add_conn_api_data {

	struct spdk_nvmf_fc_poller_api_add_connection_args args;
@@ -566,6 +574,8 @@ struct spdk_nvmf_fc_ls_del_conn_api_data { 
	struct spdk_nvmf_fc_ls_rqst *ls_rqst;

	struct spdk_nvmf_fc_association *assoc;

	bool aq_conn; /* true if deleting AQ connection */

	spdk_nvmf_fc_del_conn_cb del_conn_cb;

	void *del_conn_cb_data;

};



/* used by LS disconnect association cmd handling */
@@ -922,6 +932,10 @@ int nvmf_fc_delete_association(struct spdk_nvmf_fc_nport *tgtport, 
			       spdk_nvmf_fc_del_assoc_cb del_assoc_cb,

			       void *cb_data);



int nvmf_fc_delete_connection(struct spdk_nvmf_fc_conn *fc_conn, bool send_abts,

			      bool backend_initiated, spdk_nvmf_fc_del_conn_cb cb_fn,

			      void *cb_data);



bool nvmf_ctrlr_is_on_nport(uint8_t port_hdl, uint16_t nport_hdl,

			    struct spdk_nvmf_ctrlr *ctrlr);