Commit 0537f914 authored by Changpeng Liu's avatar Changpeng Liu Committed by Jim Harris
Browse files

nvmf/vfio-user: support shared CQ finally 



For the purpose to support shared IO CQ feature, we will construct
the queue pair data structure at the beginning, and setup SQ/CQ
separately in CREATE IO SQ/CQ routine.

Previously we will disconnect queue pair when got a DELETE IO CQ
command, now we disconnect queue pair when got a DELETE IO SQ command,
and in the disconnect completion callback, we will release the IO SQ
resources, there is a case that the VM will just RESET/SHUTDOWN
controller when IO queue pairs are connected, for this case, we
will also try to release CQ resources in the disconnect completion
callback.

`free_qp` function now is only called when destroying a controller.

Change-Id: I45ec679ddb63bdf1feeba5dc2bd39cae3ba4aa89
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10532


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarThanos Makatos <thanos.makatos@nutanix.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 56fc3576

lib/nvmf/vfio_user.c

+188 −166
Original line number Diff line number Diff line
@@ -191,6 +191,8 @@ struct nvmf_vfio_user_ctrlr { 
	struct spdk_thread			*thread;

	struct spdk_poller			*vfu_ctx_poller;



	bool					reset_shn;



	uint16_t				cntlid;

	struct spdk_nvmf_ctrlr			*ctrlr;
@@ -718,6 +720,15 @@ map_q(struct nvmf_vfio_user_ctrlr *vu_ctrlr, struct nvme_q *q, bool is_cq, bool 
	return 0;

}



static inline void

unmap_q(struct nvmf_vfio_user_ctrlr *vu_ctrlr, struct nvme_q *q)

{

	if (q->addr) {

		vfu_unmap_sg(vu_ctrlr->endpoint->vfu_ctx, q->sg, &q->iov, 1);

		q->addr = NULL;

	}

}



static int

asq_setup(struct nvmf_vfio_user_ctrlr *ctrlr)

{
@@ -963,71 +974,65 @@ io_q_exists(struct nvmf_vfio_user_ctrlr *vu_ctrlr, const uint16_t qid, const boo 
		return false;

	}



	if (!is_cq) {

		if (vu_ctrlr->qp[qid]->sq_state == VFIO_USER_SQ_DELETED ||

		    vu_ctrlr->qp[qid]->sq_state == VFIO_USER_SQ_UNUSED) {

			return false;

		}

	if (is_cq) {

		return (vu_ctrlr->qp[qid]->cq_state != VFIO_USER_CQ_DELETED &&

			vu_ctrlr->qp[qid]->cq_state != VFIO_USER_CQ_UNUSED);

	}



	return true;

	return (vu_ctrlr->qp[qid]->sq_state != VFIO_USER_SQ_DELETED &&

		vu_ctrlr->qp[qid]->sq_state != VFIO_USER_SQ_UNUSED);

}



/* Deletes a SQ, if this SQ is the last user of the associated CQ

 * and the controller is being shut down or reset, then the CQ is

 * also deleted.

 */

static void

unmap_qp(struct nvmf_vfio_user_qpair *qp)

delete_sq_done(struct nvmf_vfio_user_ctrlr *vu_ctrlr, struct nvmf_vfio_user_qpair *vu_qpair)

{

	struct nvmf_vfio_user_ctrlr *ctrlr;

	struct nvmf_vfio_user_qpair *vu_cqpair;

	struct nvmf_vfio_user_req *vu_req;

	uint16_t cqid;

	uint32_t i;



	if (qp->ctrlr == NULL) {

		return;

	}

	ctrlr = qp->ctrlr;

	SPDK_DEBUGLOG(nvmf_vfio, "%s: delete SQ%d=%p done\n", ctrlr_id(vu_ctrlr),

		      vu_qpair->qpair.qid, vu_qpair);



	SPDK_DEBUGLOG(nvmf_vfio, "%s: unmap QP%d\n",

		      ctrlr_id(ctrlr), qp->qpair.qid);

	/* Free SQ resources */

	unmap_q(vu_ctrlr, &vu_qpair->sq);



	if (qp->sq.addr != NULL) {

		vfu_unmap_sg(ctrlr->endpoint->vfu_ctx, qp->sq.sg, &qp->sq.iov, 1);

		qp->sq.addr = NULL;

	for (i = 0; i < vu_qpair->qsize; i++) {

		vu_req = &vu_qpair->reqs_internal[i];

		free(vu_req->sg);

	}



	if (qp->cq.addr != NULL) {

		vfu_unmap_sg(ctrlr->endpoint->vfu_ctx, qp->cq.sg, &qp->cq.iov, 1);

		qp->cq.addr = NULL;

	}

	if (vu_qpair->qsize) {

		vu_qpair->qsize = 0;

		free(vu_qpair->reqs_internal);

	}

	vu_qpair->sq.size = 0;

	vu_qpair->sq_state = VFIO_USER_SQ_DELETED;



static int

remap_qp(struct nvmf_vfio_user_qpair *vu_qpair)

{

	struct nvme_q *sq, *cq;

	struct nvmf_vfio_user_ctrlr *vu_ctrlr;

	int ret;

	/* Controller RESET and SHUTDOWN are special cases,

	 * VM may not send DELETE IO SQ/CQ commands, NVMf library

	 * will disconnect IO queue pairs.

	 */

	if (vu_ctrlr->reset_shn) {

		cqid = vu_qpair->sq.cqid;

		vu_cqpair = vu_ctrlr->qp[cqid];



	vu_ctrlr = vu_qpair->ctrlr;

	sq = &vu_qpair->sq;

	cq = &vu_qpair->cq;

		SPDK_DEBUGLOG(nvmf_vfio, "%s: try to delete CQ%d=%p\n", ctrlr_id(vu_ctrlr),

			      vu_cqpair->qpair.qid, vu_cqpair);



	if (sq->size) {

		ret = map_q(vu_ctrlr, sq, false, false);

		if (ret) {

			SPDK_DEBUGLOG(nvmf_vfio, "Memory isn't ready to remap SQID %d %#lx-%#lx\n",

				      io_q_id(sq), sq->prp1, sq->prp1 + sq->size * sizeof(struct spdk_nvme_cmd));

			return -EFAULT;

		if (vu_cqpair->cq_ref) {

			vu_cqpair->cq_ref--;

		}

		if (vu_cqpair->cq_ref == 0) {

			unmap_q(vu_ctrlr, &vu_cqpair->cq);

			vu_cqpair->cq.size = 0;

			vu_cqpair->cq_state = VFIO_USER_CQ_DELETED;

		}



	if (cq->size) {

		ret = map_q(vu_ctrlr, cq, true, false);

		if (ret) {

			SPDK_DEBUGLOG(nvmf_vfio, "Memory isn't ready to remap CQID %d %#lx-%#lx\n",

				      io_q_id(cq), cq->prp1, cq->prp1 + cq->size * sizeof(struct spdk_nvme_cpl));

			return -EFAULT;

		}



	}



	return 0;

}



static void
@@ -1046,16 +1051,19 @@ free_qp(struct nvmf_vfio_user_ctrlr *ctrlr, uint16_t qid) 
		return;

	}



	SPDK_DEBUGLOG(nvmf_vfio, "%s: destroy QP%d=%p\n", ctrlr_id(ctrlr),

	SPDK_DEBUGLOG(nvmf_vfio, "%s: Free QP%d=%p\n", ctrlr_id(ctrlr),

		      qid, qpair);



	unmap_qp(qpair);

	unmap_q(ctrlr, &qpair->sq);

	unmap_q(ctrlr, &qpair->cq);



	for (i = 0; i < qpair->qsize; i++) {

		vu_req = &qpair->reqs_internal[i];

		free(vu_req->sg);

	}

	if (qpair->qsize) {

		free(qpair->reqs_internal);

	}



	free(qpair->sq.sg);

	free(qpair->cq.sg);
@@ -1064,15 +1072,11 @@ free_qp(struct nvmf_vfio_user_ctrlr *ctrlr, uint16_t qid) 
	ctrlr->qp[qid] = NULL;

}



/* This function can only fail because of memory allocation errors. */

static int

init_qp(struct nvmf_vfio_user_ctrlr *ctrlr, struct spdk_nvmf_transport *transport,

	const uint32_t qsize, const uint16_t id)

	const uint16_t id)

{

	uint32_t i;

	struct nvmf_vfio_user_qpair *qpair;

	struct nvmf_vfio_user_req *vu_req, *tmp;

	struct spdk_nvmf_request *req;



	assert(ctrlr != NULL);

	assert(transport != NULL);
@@ -1096,44 +1100,51 @@ init_qp(struct nvmf_vfio_user_ctrlr *ctrlr, struct spdk_nvmf_transport *transpor 
	qpair->qpair.qid = id;

	qpair->qpair.transport = transport;

	qpair->ctrlr = ctrlr;

	qpair->qsize = qsize;



	TAILQ_INIT(&qpair->reqs);

	ctrlr->qp[id] = qpair;



	return 0;

}



static int

alloc_sq_reqs(struct nvmf_vfio_user_ctrlr *vu_ctrlr, struct nvmf_vfio_user_qpair *vu_qpair,

	      const uint32_t qsize)

{

	uint32_t i;

	struct nvmf_vfio_user_req *vu_req, *tmp;

	struct spdk_nvmf_request *req;



	TAILQ_INIT(&vu_qpair->reqs);



	qpair->reqs_internal = calloc(qsize, sizeof(struct nvmf_vfio_user_req));

	if (qpair->reqs_internal == NULL) {

		SPDK_ERRLOG("%s: error allocating reqs: %m\n", ctrlr_id(ctrlr));

		goto reqs_err;

	vu_qpair->reqs_internal = calloc(qsize, sizeof(struct nvmf_vfio_user_req));

	if (vu_qpair->reqs_internal == NULL) {

		SPDK_ERRLOG("%s: error allocating reqs: %m\n", ctrlr_id(vu_ctrlr));

		return -ENOMEM;

	}



	for (i = 0; i < qsize; i++) {

		vu_req = &qpair->reqs_internal[i];

		vu_req = &vu_qpair->reqs_internal[i];

		vu_req->sg = calloc(NVMF_VFIO_USER_MAX_IOVECS, dma_sg_size());

		if (vu_req->sg == NULL) {

			goto sg_err;

		}



		req = &vu_req->req;

		req->qpair = &qpair->qpair;

		req->qpair = &vu_qpair->qpair;

		req->rsp = (union nvmf_c2h_msg *)&vu_req->rsp;

		req->cmd = (union nvmf_h2c_msg *)&vu_req->cmd;



		TAILQ_INSERT_TAIL(&qpair->reqs, vu_req, link);

		TAILQ_INSERT_TAIL(&vu_qpair->reqs, vu_req, link);

	}



	ctrlr->qp[id] = qpair;

	vu_qpair->qsize = qsize;

	return 0;



sg_err:

	TAILQ_FOREACH_SAFE(vu_req, &qpair->reqs, link, tmp) {

	TAILQ_FOREACH_SAFE(vu_req, &vu_qpair->reqs, link, tmp) {

		free(vu_req->sg);

	}

	free(qpair->reqs_internal);



reqs_err:

	free(qpair->sq.sg);

	free(qpair->cq.sg);

	free(qpair);

	free(vu_qpair->reqs_internal);

	return -ENOMEM;

}
@@ -1186,6 +1197,15 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
		      "%s: create I/O %cQ%d: QSIZE=%#x\n", ctrlr_id(ctrlr),

		      is_cq ? 'C' : 'S', qid, qsize);



	if (ctrlr->qp[qid] == NULL) {

		err = init_qp(ctrlr, ctrlr->qp[0]->qpair.transport, qid);

		if (err != 0) {

			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			goto out;

		}

	}

	vu_qpair = ctrlr->qp[qid];



	if (is_cq) {

		if (cmd->cdw11_bits.create_io_cq.pc != 0x1) {

			SPDK_ERRLOG("%s: non-PC CQ not supporred\n", ctrlr_id(ctrlr));
@@ -1199,16 +1219,7 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
			goto out;

		}



		err = init_qp(ctrlr, ctrlr->qp[0]->qpair.transport, qsize, qid);

		if (err != 0) {

			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			goto out;

		}



		io_q = &ctrlr->qp[qid]->cq;

		io_q->ien = cmd->cdw11_bits.create_io_cq.ien;

		io_q->iv = cmd->cdw11_bits.create_io_cq.iv;

		io_q->phase = true;

		io_q = &vu_qpair->cq;

	} else {

		cqid = cmd->cdw11_bits.create_io_sq.cqid;

		if (cqid == 0 || cqid >= vu_transport->transport.opts.max_qpairs_per_ctrlr) {
@@ -1231,18 +1242,10 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
			sc = SPDK_NVME_SC_INVALID_FIELD;

			goto out;

		}

		/* TODO: support shared IO CQ */

		if (qid != cqid) {

			SPDK_ERRLOG("%s: doesn't support shared CQ now\n", ctrlr_id(ctrlr));

			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;

			sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;

		}



		io_q = &ctrlr->qp[qid]->sq;

		io_q->cqid = cqid;

		ctrlr->qp[io_q->cqid]->cq_ref++;

		io_q = &vu_qpair->sq;

		SPDK_DEBUGLOG(nvmf_vfio, "%s: SQ%d CQID=%d\n", ctrlr_id(ctrlr),

			      qid, io_q->cqid);

			      qid, cqid);

	}



	io_q->is_cq = is_cq;
@@ -1261,15 +1264,25 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
		      qid, cmd->dptr.prp.prp1, (unsigned long long)io_q->addr);



	if (is_cq) {

		io_q->ien = cmd->cdw11_bits.create_io_cq.ien;

		io_q->iv = cmd->cdw11_bits.create_io_cq.iv;

		io_q->phase = true;

		io_q->tail = 0;

		vu_qpair->cq_state = VFIO_USER_CQ_CREATED;

		*hdbl(ctrlr, io_q) = 0;

	} else {

		vu_qpair = ctrlr->qp[qid];

		err = alloc_sq_reqs(ctrlr, vu_qpair, qsize);

		if (err < 0) {

			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			SPDK_ERRLOG("%s: failed to allocate SQ requests: %m\n", ctrlr_id(ctrlr));

			goto out;

		}

		io_q->cqid = cqid;

		io_q->head = 0;

		ctrlr->qp[io_q->cqid]->cq_ref++;

		vu_qpair->sq_state = VFIO_USER_SQ_CREATED;

		*tdbl(ctrlr, io_q) = 0;

		vu_qpair->sq.head = 0;



		if (vu_qpair->sq_state == VFIO_USER_SQ_DELETED) {

			vu_qpair->sq_state = VFIO_USER_SQ_ACTIVE;

		} else {

		/*

		 * Create our new I/O qpair. This asynchronously invokes, on a

		 * suitable poll group, the nvmf_vfio_user_poll_group_add()
@@ -1282,27 +1295,26 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
					&vu_qpair->qpair);

		return 0;

	}

	}



out:

	return post_completion(ctrlr, &ctrlr->qp[0]->cq, 0, 0, cmd->cid, sc, sct);

}



/* For ADMIN I/O DELETE COMPLETION QUEUE the NVMf library will disconnect and free

/* For ADMIN I/O DELETE SUBMISSION QUEUE the NVMf library will disconnect and free

 * queue pair, so save the command in a context.

 */

struct vfio_user_delete_cq_ctx {

struct vfio_user_delete_sq_ctx {

	struct nvmf_vfio_user_ctrlr *vu_ctrlr;

	struct spdk_nvme_cmd delete_io_cq_cmd;

	struct spdk_nvme_cmd delete_io_sq_cmd;

};



static void

vfio_user_qpair_delete_cb(void *cb_arg)

{

	struct vfio_user_delete_cq_ctx *ctx = cb_arg;

	struct vfio_user_delete_sq_ctx *ctx = cb_arg;

	struct nvmf_vfio_user_ctrlr *vu_ctrlr = ctx->vu_ctrlr;



	post_completion(vu_ctrlr, &vu_ctrlr->qp[0]->cq, 0, 0, ctx->delete_io_cq_cmd.cid,

	post_completion(vu_ctrlr, &vu_ctrlr->qp[0]->cq, 0, 0, ctx->delete_io_sq_cmd.cid,

			SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC);

	free(ctx);

}
@@ -1317,7 +1329,7 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
	uint16_t sct = SPDK_NVME_SCT_GENERIC;

	uint16_t sc = SPDK_NVME_SC_SUCCESS;

	struct nvmf_vfio_user_qpair *vu_qpair;

	struct vfio_user_delete_cq_ctx *ctx;

	struct vfio_user_delete_sq_ctx *ctx;



	SPDK_DEBUGLOG(nvmf_vfio, "%s: delete I/O %cQ: QID=%d\n",

		      ctrlr_id(ctrlr), is_cq ? 'C' : 'S',
@@ -1333,11 +1345,6 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 


	vu_qpair = ctrlr->qp[cmd->cdw10_bits.delete_io_q.qid];

	if (is_cq) {

		if (vu_qpair->sq_state == VFIO_USER_SQ_UNUSED) {

			free_qp(ctrlr, cmd->cdw10_bits.delete_io_q.qid);

			goto out;

		}



		if (vu_qpair->cq_ref) {

			SPDK_ERRLOG("%s: the associated SQ must be deleted first\n", ctrlr_id(ctrlr));

			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
@@ -1345,6 +1352,10 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
			goto out;

		}



		unmap_q(ctrlr, &vu_qpair->cq);

		vu_qpair->cq.size = 0;

		vu_qpair->cq_state = VFIO_USER_CQ_DELETED;

	} else {

		ctx = calloc(1, sizeof(*ctx));

		if (!ctx) {

			sct = SPDK_NVME_SCT_GENERIC;
@@ -1352,29 +1363,14 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
			goto out;

		}

		ctx->vu_ctrlr = ctrlr;

		ctx->delete_io_cq_cmd = *cmd;

		spdk_nvmf_qpair_disconnect(&vu_qpair->qpair, vfio_user_qpair_delete_cb, ctx);

		return 0;

	} else {

		if (vu_qpair->sq_state == VFIO_USER_SQ_DELETED) {

			SPDK_DEBUGLOG(nvmf_vfio, "%s: SQ%u is already deleted\n", ctrlr_id(ctrlr),

				      cmd->cdw10_bits.delete_io_q.qid);

			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;

			sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;

			goto out;

		}

		ctx->delete_io_sq_cmd = *cmd;



		/*

		 * This doesn't actually delete the SQ, We're merely telling the poll_group_poll

		 * function to skip checking this SQ.  The queue pair will be disconnected in Delete

		 * IO CQ command.

		 */

		vu_qpair->sq_state = VFIO_USER_SQ_DELETED;

		vfu_unmap_sg(ctrlr->endpoint->vfu_ctx, vu_qpair->sq.sg, &vu_qpair->sq.iov, 1);

		vu_qpair->sq.addr = NULL;



		assert(ctrlr->qp[vu_qpair->sq.cqid]->cq_ref);

		ctrlr->qp[vu_qpair->sq.cqid]->cq_ref--;



		spdk_nvmf_qpair_disconnect(&vu_qpair->qpair, vfio_user_qpair_delete_cb, ctx);

		return 0;

	}



out:
@@ -1506,7 +1502,9 @@ disable_admin_queue(struct nvmf_vfio_user_ctrlr *ctrlr) 
{

	assert(ctrlr->qp[0] != NULL);



	unmap_qp(ctrlr->qp[0]);

	unmap_q(ctrlr, &ctrlr->qp[0]->sq);

	unmap_q(ctrlr, &ctrlr->qp[0]->cq);



	ctrlr->qp[0]->sq.size = 0;

	ctrlr->qp[0]->sq.head = 0;

	ctrlr->qp[0]->cq.size = 0;
@@ -1518,7 +1516,8 @@ memory_region_add_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 
{

	struct nvmf_vfio_user_endpoint *endpoint = vfu_get_private(vfu_ctx);

	struct nvmf_vfio_user_ctrlr *ctrlr;

	struct nvmf_vfio_user_qpair *qpair;

	struct nvmf_vfio_user_qpair *qpair, *cqpair;

	struct nvme_q *sq, *cq;

	int ret;



	/*
@@ -1566,10 +1565,28 @@ memory_region_add_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 
			continue;

		}



		ret = remap_qp(qpair);

		cqpair = ctrlr->qp[qpair->sq.cqid];

		cq = &cqpair->cq;

		sq = &qpair->sq;



		/* For shared CQ case, we will use cq->addr to avoid mapping CQ multiple times */

		if (cq->size && !cq->addr) {

			ret = map_q(ctrlr, cq, true, false);

			if (ret) {

				SPDK_DEBUGLOG(nvmf_vfio, "Memory isn't ready to remap CQID %d %#lx-%#lx\n",

					      cq->cqid, cq->prp1, cq->prp1 + cq->size * sizeof(struct spdk_nvme_cpl));

				continue;

			}

		}



		if (sq->size) {

			ret = map_q(ctrlr, sq, false, false);

			if (ret) {

				SPDK_DEBUGLOG(nvmf_vfio, "Memory isn't ready to remap SQID %d %#lx-%#lx\n",

					      qpair->qpair.qid, sq->prp1, sq->prp1 + sq->size * sizeof(struct spdk_nvme_cmd));

				continue;

			}

		}

		qpair->sq_state = VFIO_USER_SQ_ACTIVE;

		SPDK_DEBUGLOG(nvmf_vfio, "Remap QP %u successfully\n", qpair->qpair.qid);

	}
@@ -1581,7 +1598,7 @@ memory_region_remove_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 
{

	struct nvmf_vfio_user_endpoint *endpoint = vfu_get_private(vfu_ctx);

	struct nvmf_vfio_user_ctrlr *ctrlr;

	struct nvmf_vfio_user_qpair *qpair;

	struct nvmf_vfio_user_qpair *qpair, *cqpair;

	void *map_start, *map_end;

	int ret = 0;
@@ -1612,14 +1629,15 @@ memory_region_remove_cb(vfu_ctx_t *vfu_ctx, vfu_dma_info_t *info) 


	pthread_mutex_lock(&endpoint->lock);

	TAILQ_FOREACH(qpair, &ctrlr->connected_qps, tailq) {

		if ((qpair->cq.addr >= map_start && qpair->cq.addr <= map_end) ||

		    (qpair->sq.addr >= map_start && qpair->sq.addr <= map_end)) {

			/* TODO: Ideally we should disconnect this queue pair

			 * before returning to caller.

			 */

			unmap_qp(qpair);

		if (qpair->sq.addr >= map_start && qpair->sq.addr <= map_end) {

			unmap_q(ctrlr, &qpair->sq);

			qpair->sq_state = VFIO_USER_SQ_INACTIVE;

		}



		cqpair = ctrlr->qp[qpair->sq.cqid];

		if (cqpair->cq.addr >= map_start && cqpair->cq.addr <= map_end) {

			unmap_q(ctrlr, &cqpair->cq);

		}

	}

	pthread_mutex_unlock(&endpoint->lock);
@@ -1641,7 +1659,6 @@ nvmf_vfio_user_prop_req_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) 
{

	struct nvmf_vfio_user_qpair *vu_qpair = cb_arg;

	struct nvmf_vfio_user_ctrlr *vu_ctrlr;

	bool disable_admin = false;

	int ret;



	assert(vu_qpair != NULL);
@@ -1674,18 +1691,19 @@ nvmf_vfio_user_prop_req_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) 
						return ret;

					}

					vu_qpair->sq_state = VFIO_USER_SQ_ACTIVE;

					vu_ctrlr->reset_shn = false;

				} else {

					disable_admin = true;

					vu_ctrlr->reset_shn = true;

				}

			}



			if (diff.bits.shn) {

				if (cc.bits.shn == SPDK_NVME_SHN_NORMAL || cc.bits.shn == SPDK_NVME_SHN_ABRUPT) {

					disable_admin = true;

					vu_ctrlr->reset_shn = true;

				}

			}



			if (disable_admin) {

			if (vu_ctrlr->reset_shn) {

				SPDK_DEBUGLOG(nvmf_vfio,

					      "%s: UNMAP Admin queue\n",

					      ctrlr_id(vu_ctrlr));
@@ -2034,18 +2052,16 @@ _free_ctrlr(void *ctx) 
}



static void

free_ctrlr(struct nvmf_vfio_user_ctrlr *ctrlr, bool free_qps)

free_ctrlr(struct nvmf_vfio_user_ctrlr *ctrlr)

{

	int i;

	assert(ctrlr != NULL);



	SPDK_DEBUGLOG(nvmf_vfio, "free %s\n", ctrlr_id(ctrlr));



	if (free_qps) {

	for (i = 0; i < NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR; i++) {

		free_qp(ctrlr, i);

	}

	}



	if (ctrlr->thread == spdk_get_thread()) {

		_free_ctrlr(ctrlr);
@@ -2075,7 +2091,13 @@ nvmf_vfio_user_create_ctrlr(struct nvmf_vfio_user_transport *transport, 
	TAILQ_INIT(&ctrlr->connected_qps);



	/* Then, construct an admin queue pair */

	err = init_qp(ctrlr, &transport->transport, NVMF_VFIO_USER_DEFAULT_AQ_DEPTH, 0);

	err = init_qp(ctrlr, &transport->transport, 0);

	if (err != 0) {

		free(ctrlr);

		goto out;

	}



	err = alloc_sq_reqs(ctrlr, ctrlr->qp[0], NVMF_VFIO_USER_DEFAULT_AQ_DEPTH);

	if (err != 0) {

		free(ctrlr);

		goto out;
@@ -2212,7 +2234,7 @@ nvmf_vfio_user_stop_listen(struct spdk_nvmf_transport *transport, 
				/* Users may kill NVMeoF target while VM

				 * is connected, free all resources.

				 */

				free_ctrlr(endpoint->ctrlr, true);

				free_ctrlr(endpoint->ctrlr);

			}

			nvmf_vfio_user_destroy_endpoint(endpoint);

			pthread_mutex_unlock(&vu_transport->lock);
@@ -2421,7 +2443,7 @@ vfio_user_qpair_disconnect_cb(void *ctx) 


	if (TAILQ_EMPTY(&ctrlr->connected_qps)) {

		endpoint->ctrlr = NULL;

		free_ctrlr(ctrlr, false);

		free_ctrlr(ctrlr);

	}

	pthread_mutex_unlock(&endpoint->lock);

}
@@ -2453,7 +2475,7 @@ vfio_user_destroy_ctrlr(struct nvmf_vfio_user_ctrlr *ctrlr) 
	pthread_mutex_lock(&endpoint->lock);

	if (TAILQ_EMPTY(&ctrlr->connected_qps)) {

		endpoint->ctrlr = NULL;

		free_ctrlr(ctrlr, false);

		free_ctrlr(ctrlr);

		pthread_mutex_unlock(&endpoint->lock);

		return 0;

	}
@@ -2519,7 +2541,7 @@ handle_queue_connect_rsp(struct nvmf_vfio_user_req *req, void *cb_arg) 
	if (spdk_nvme_cpl_is_error(&req->req.rsp->nvme_cpl)) {

		SPDK_ERRLOG("SC %u, SCT %u\n", req->req.rsp->nvme_cpl.status.sc, req->req.rsp->nvme_cpl.status.sct);

		endpoint->ctrlr = NULL;

		free_ctrlr(vu_ctrlr, true);

		free_ctrlr(vu_ctrlr);

		return -1;

	}
@@ -2696,7 +2718,7 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair, 
	TAILQ_REMOVE(&vu_ctrlr->connected_qps, vu_qpair, tailq);

	pthread_mutex_unlock(&vu_ctrlr->endpoint->lock);



	free_qp(vu_ctrlr, qpair->qid);

	delete_sq_done(vu_ctrlr, vu_qpair);



	if (cb_fn) {

		cb_fn(cb_arg);