Commit 4bdfe13d authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

lib/nvmf: make spdk_nvmf_qpair_disconnect thread safe. 



This function should be the synchronization point for all
disconnects regardless of whether they begin on the transport,
from an RPC, or in response to application termination.

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: If3553ab3a9e265b0938c84832cb9f774852d7565
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3674


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarZiye Yang <ziye.yang@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent a70d01d6

include/spdk/nvmf_transport.h

+1 −0
Original line number Diff line number Diff line
@@ -123,6 +123,7 @@ struct spdk_nvmf_qpair { 
	uint16_t				qid;

	uint16_t				sq_head;

	uint16_t				sq_head_max;

	bool					disconnect_started;



	struct spdk_nvmf_request		*first_fused_req;

lib/nvmf/Makefile

+2 −2
Original line number Diff line number Diff line
@@ -34,8 +34,8 @@ 
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)

include $(SPDK_ROOT_DIR)/mk/spdk.common.mk



SO_VER := 5

SO_MINOR := 1

SO_VER := 6

SO_MINOR := 0



C_SRCS = ctrlr.c ctrlr_discovery.c ctrlr_bdev.c \

	 subsystem.c nvmf.c nvmf_rpc.c transport.c tcp.c

lib/nvmf/nvmf.c

+30 −13
Original line number Diff line number Diff line
@@ -940,11 +940,27 @@ _nvmf_qpair_destroy(void *ctx, int status) 
	spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx);

}



static void

_nvmf_qpair_disconnect_msg(void *ctx)

{

	struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx;



	spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx);

	free(ctx);

}



int

spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx)

{

	struct nvmf_qpair_disconnect_ctx *qpair_ctx;



	if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) {

		if (cb_fn) {

			cb_fn(ctx);

		}

		return 0;

	}



	/* If we get a qpair in the uninitialized state, we can just destroy it immediately */

	if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) {

		nvmf_transport_qpair_fini(qpair);
@@ -954,19 +970,20 @@ spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_ 
		return 0;

	}



	/* The queue pair must be disconnected from the thread that owns it */

	assert(qpair->group->thread == spdk_get_thread());



	if (qpair->state != SPDK_NVMF_QPAIR_ACTIVE) {

		/* This can occur if the connection is killed by the target,

		 * which results in a notification that the connection

		 * died. Send a message to defer the processing of this

		 * callback. This allows the stack to unwind in the case

		 * where a bunch of connections are disconnected in

		 * a loop. */

		if (cb_fn) {

			spdk_thread_send_msg(qpair->group->thread, cb_fn, ctx);

	assert(qpair->group != NULL);

	if (spdk_get_thread() != qpair->group->thread) {

		/* clear the atomic so we can set it on the next call on the proper thread. */

		__atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED);

		qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx));

		if (!qpair_ctx) {

			SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n");

			return -ENOMEM;

		}

		qpair_ctx->qpair = qpair;

		qpair_ctx->cb_fn = cb_fn;

		qpair_ctx->thread = qpair->group->thread;

		qpair_ctx->ctx = ctx;

		spdk_thread_send_msg(qpair->group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx);

		return 0;

	}
@@ -1337,7 +1354,7 @@ nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 
		_nvmf_poll_group_remove_subsystem_cb(ctx, 0);

	}



	if (rc != 0) {

	if (rc != 0 && rc != -EINPROGRESS) {

		free(ctx);

		goto fini;

	}