Commit 72800826 authored by Ben Walker's avatar Ben Walker Committed by Jim Harris
Browse files

nvmf: Quiesce I/O before closing spdk_nvmf_qpairs 



Change-Id: Iff58943f3f9db39b6fce1437a0cfe7380991daa6
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/412078


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent ed97638c

lib/nvmf/ctrlr.c

+20 −7
Original line number Diff line number Diff line
@@ -455,27 +455,40 @@ _spdk_nvmf_ctrlr_remove_qpair(void *ctx) 
	}

}



static void

_spdk_nvmf_qpair_quiesced(void *cb_arg, int status)

{

	struct spdk_nvmf_qpair *qpair = cb_arg;



	/* Send a message to the controller thread to remove the qpair from its internal

	 * list. */

	spdk_thread_send_msg(qpair->ctrlr->admin_qpair->group->thread, _spdk_nvmf_ctrlr_remove_qpair,

			     qpair);

}



static void

_spdk_nvmf_qpair_deactivate(void *ctx)

{

	struct spdk_nvmf_qpair *qpair = ctx;

	struct spdk_nvmf_ctrlr *ctrlr;



	assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE);

	qpair->state = SPDK_NVMF_QPAIR_DEACTIVATING;



	ctrlr = qpair->ctrlr;



	if (ctrlr == NULL) {

	if (qpair->ctrlr == NULL) {

		/* This qpair was never added to a controller. Skip a step

		 * and destroy it immediately. */

		_spdk_nvmf_qpair_destroy(qpair);

		return;

	}



	/* Send a message to the controller thread to remove the qpair from its internal

	 * list. */

	spdk_thread_send_msg(ctrlr->admin_qpair->group->thread, _spdk_nvmf_ctrlr_remove_qpair, qpair);

	/* Check for outstanding I/O */

	if (!TAILQ_EMPTY(&qpair->outstanding)) {

		qpair->state_cb = _spdk_nvmf_qpair_quiesced;

		qpair->state_cb_arg = qpair;

		return;

	}



	_spdk_nvmf_qpair_quiesced(qpair, 0);

}



void

lib/nvmf/nvmf_internal.h

+4 −0
Original line number Diff line number Diff line
@@ -62,6 +62,8 @@ enum spdk_nvmf_qpair_state { 
	SPDK_NVMF_QPAIR_DEACTIVATING,

};



typedef void (*spdk_nvmf_state_change_done)(void *cb_arg, int status);



struct spdk_nvmf_tgt {

	struct spdk_nvmf_tgt_opts		opts;
@@ -155,6 +157,8 @@ struct spdk_nvmf_ns { 


struct spdk_nvmf_qpair {

	enum spdk_nvmf_qpair_state		state;

	spdk_nvmf_state_change_done		state_cb;

	void					*state_cb_arg;



	struct spdk_nvmf_transport		*transport;

	struct spdk_nvmf_ctrlr			*ctrlr;

lib/nvmf/request.c

+14 −1
Original line number Diff line number Diff line
@@ -49,21 +49,34 @@ int 
spdk_nvmf_request_complete(struct spdk_nvmf_request *req)

{

	struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;

	struct spdk_nvmf_qpair *qpair;



	rsp->sqid = 0;

	rsp->status.p = 0;

	rsp->cid = req->cmd->nvme_cmd.cid;



	qpair = req->qpair;



	SPDK_DEBUGLOG(SPDK_LOG_NVMF,

		      "cpl: cid=%u cdw0=0x%08x rsvd1=%u status=0x%04x\n",

		      rsp->cid, rsp->cdw0, rsp->rsvd1,

		      *(uint16_t *)&rsp->status);



	TAILQ_REMOVE(&req->qpair->outstanding, req, link);

	TAILQ_REMOVE(&qpair->outstanding, req, link);

	if (spdk_nvmf_transport_req_complete(req)) {

		SPDK_ERRLOG("Transport request completion error!\n");

	}



	if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING) {

		assert(qpair->state_cb != NULL);



		if (TAILQ_EMPTY(&qpair->outstanding)) {

			qpair->state_cb(qpair->state_cb_arg, 0);

		}

	} else {

		assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE);

	}



	return 0;

}