Commit bcd0ea8c authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Ben Walker
Browse files

nvmf/vfio_user: Post SQ delete cpl when qpair is destroyed 



This patch removes usage of cb_fn argument of
spdk_nvmf_qpair_disconnect API. Instead of relying
on the callback, post a completion on delete SQ
command when transport qpair_fini is called.

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: I68dec97ea94e89f48a8667da82f88b5e24fc0d88
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17168


Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent 6693862f

lib/nvmf/vfio_user.c

+19 −12
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (C) 2020 Intel Corporation.

 *   Copyright (c) 2019-2022, Nutanix Inc. All rights reserved.

 *   Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 *   Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 */



/*
@@ -318,6 +318,8 @@ struct nvmf_vfio_user_sq { 
	 */

	struct spdk_nvme_cmd			create_io_sq_cmd;



	struct vfio_user_delete_sq_ctx		*delete_ctx;



	/* Currently unallocated reqs. */

	TAILQ_HEAD(, nvmf_vfio_user_req)	free_reqs;

	/* Poll group entry */
@@ -2240,11 +2242,11 @@ out: 
}



/* For ADMIN I/O DELETE SUBMISSION QUEUE the NVMf library will disconnect and free

 * queue pair, so save the command in a context.

 * queue pair, so save the command id and controller in a context.

 */

struct vfio_user_delete_sq_ctx {

	struct nvmf_vfio_user_ctrlr *vu_ctrlr;

	struct spdk_nvme_cmd delete_io_sq_cmd;

	uint16_t cid;

};



static void
@@ -2263,7 +2265,7 @@ vfio_user_qpair_delete_cb(void *cb_arg) 
				     cb_arg);

	} else {

		post_completion(vu_ctrlr, admin_cq, 0, 0,

				ctx->delete_io_sq_cmd.cid,

				ctx->cid,

				SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC);

		free(ctx);

	}
@@ -2280,7 +2282,6 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
	uint16_t sc = SPDK_NVME_SC_SUCCESS;

	struct nvmf_vfio_user_sq *sq;

	struct nvmf_vfio_user_cq *cq;

	struct vfio_user_delete_sq_ctx *ctx;



	SPDK_DEBUGLOG(nvmf_vfio, "%s: delete I/O %cqid:%d\n",

		      ctrlr_id(ctrlr), is_cq ? 'c' : 's',
@@ -2309,21 +2310,20 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, 
		 * VM reboot or CC.EN change, so we have to delete it in all

		 * other cases.

		 */

		ctx = calloc(1, sizeof(*ctx));

		if (!ctx) {

		sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid];

		sq->delete_ctx = calloc(1, sizeof(*sq->delete_ctx));

		if (!sq->delete_ctx) {

			sct = SPDK_NVME_SCT_GENERIC;

			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

			goto out;

		}

		ctx->vu_ctrlr = ctrlr;

		ctx->delete_io_sq_cmd = *cmd;



		sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid];

		sq->delete_ctx->vu_ctrlr = ctrlr;

		sq->delete_ctx->cid = cmd->cid;

		sq->sq_state = VFIO_USER_SQ_DELETED;

		assert(ctrlr->cqs[sq->cqid]->cq_ref);

		ctrlr->cqs[sq->cqid]->cq_ref--;



		spdk_nvmf_qpair_disconnect(&sq->qpair, vfio_user_qpair_delete_cb, ctx);

		spdk_nvmf_qpair_disconnect(&sq->qpair, NULL, NULL);

		return 0;

	}
@@ -5325,11 +5325,14 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair, 
	struct nvmf_vfio_user_sq *sq;

	struct nvmf_vfio_user_ctrlr *vu_ctrlr;

	struct nvmf_vfio_user_endpoint *endpoint;

	struct vfio_user_delete_sq_ctx *del_ctx;



	assert(qpair != NULL);

	sq = SPDK_CONTAINEROF(qpair, struct nvmf_vfio_user_sq, qpair);

	vu_ctrlr = sq->ctrlr;

	endpoint = vu_ctrlr->endpoint;

	del_ctx = sq->delete_ctx;

	sq->delete_ctx = NULL;



	pthread_mutex_lock(&endpoint->lock);

	TAILQ_REMOVE(&vu_ctrlr->connected_sqs, sq, tailq);
@@ -5348,6 +5351,10 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair, 
	}

	pthread_mutex_unlock(&endpoint->lock);



	if (del_ctx) {

		vfio_user_qpair_delete_cb(del_ctx);

	}



	if (cb_fn) {

		cb_fn(cb_arg);

	}