Commit aa9e0ac6 authored by Changpeng Liu's avatar Changpeng Liu
Browse files

nvmf/vfio-user: support delete/create SQ 



Previously vfio-user will hit an error when creating a deleted IO SQ.
For lookup_io_q() function, actually we should not use the queue's
address to check the queue is exist or not, as when there is a memory
removal, we will also set the queue's address to NULL and reset it again,
so here we use the queue state to indicate the queue is exist or not.

Fix issue #2174.

Change-Id: I56bdd63947c2b30a598c427e64ff2dafe1cc1d2e
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9605


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent e4913b3c

lib/nvmf/vfio_user.c

+29 −19
Original line number Diff line number Diff line
@@ -130,7 +130,7 @@ struct nvme_q { 
enum nvmf_vfio_user_qpair_state {

	VFIO_USER_QPAIR_UNINITIALIZED = 0,

	VFIO_USER_QPAIR_ACTIVE,

	VFIO_USER_QPAIR_DELETED,

	VFIO_USER_QPAIR_SQ_DELETED,

	VFIO_USER_QPAIR_INACTIVE,

	VFIO_USER_QPAIR_ERROR,

};
@@ -915,14 +915,16 @@ lookup_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, const uint16_t qid, const bool i 
	}



	if (is_cq) {

		/* CQ is always exist if the queue pair wasn't null */

		q = &ctrlr->qp[qid]->cq;

		return q;

	} else {

		q = &ctrlr->qp[qid]->sq;

	}



	if (q->addr == NULL) {

		if (ctrlr->qp[qid]->state == VFIO_USER_QPAIR_SQ_DELETED ||

		    ctrlr->qp[qid]->state == VFIO_USER_QPAIR_UNINITIALIZED) {

			return NULL;

		}

		q = &ctrlr->qp[qid]->sq;

	}



	return q;

}
@@ -1207,6 +1209,13 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 


	if (is_cq) {

		*hdbl(ctrlr, io_q) = 0;

	} else {

		vu_qpair = ctrlr->qp[qid];

		*tdbl(ctrlr, io_q) = 0;

		vu_qpair->sq.head = 0;



		if (vu_qpair->state == VFIO_USER_QPAIR_SQ_DELETED) {

			vu_qpair->state = VFIO_USER_QPAIR_ACTIVE;

		} else {

			/*

			 * Create our new I/O qpair. This asynchronously invokes, on a
@@ -1215,13 +1224,12 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
			 * with a generated fabrics connect command. This command is

			 * then eventually completed via handle_queue_connect_rsp().

			 */

		vu_qpair = ctrlr->qp[qid];

			vu_qpair->create_io_sq_cmd = *cmd;

			spdk_nvmf_tgt_new_qpair(ctrlr->transport->transport.tgt,

						&vu_qpair->qpair);

		*tdbl(ctrlr, io_q) = 0;

			return 0;

		}

	}



out:

	return post_completion(ctrlr, &ctrlr->qp[0]->cq, 0, 0, cmd->cid, sc, sct);
@@ -1273,7 +1281,7 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
	vu_qpair = ctrlr->qp[cmd->cdw10_bits.delete_io_q.qid];

	if (is_cq) {

		/* SQ must have been deleted first */

		if (vu_qpair->state != VFIO_USER_QPAIR_DELETED) {

		if (vu_qpair->state != VFIO_USER_QPAIR_SQ_DELETED) {

			SPDK_ERRLOG("%s: the associated SQ must be deleted first\n", ctrlr_id(ctrlr));

			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;

			sc = SPDK_NVME_SC_INVALID_QUEUE_DELETION;
@@ -1290,7 +1298,7 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
		spdk_nvmf_qpair_disconnect(&vu_qpair->qpair, vfio_user_qpair_delete_cb, ctx);

		return 0;

	} else {

		if (vu_qpair->state == VFIO_USER_QPAIR_DELETED) {

		if (vu_qpair->state == VFIO_USER_QPAIR_SQ_DELETED) {

			SPDK_DEBUGLOG(nvmf_vfio, "%s: SQ%u is already deleted\n", ctrlr_id(ctrlr),

				      cmd->cdw10_bits.delete_io_q.qid);

			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
@@ -1303,7 +1311,9 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
		 * function to skip checking this SQ.  The queue pair will be disconnected in Delete

		 * IO CQ command.

		 */

		vu_qpair->state = VFIO_USER_QPAIR_DELETED;

		vu_qpair->state = VFIO_USER_QPAIR_SQ_DELETED;

		vfu_unmap_sg(ctrlr->endpoint->vfu_ctx, vu_qpair->sq.sg, &vu_qpair->sq.iov, 1);

		vu_qpair->sq.addr = NULL;

	}



out: