Commit 81fc34df authored by Krzysztof Goreczny's avatar Krzysztof Goreczny Committed by Konrad Sztyber
Browse files

nvmf/tcp: Add support for the interrupt mode in NVMe-of TCP 



Add one dedicated test interrupt.sh that checks if all pollers are
indeed working in the interrupt mode.
Also, run entire nvmf_target_core.sh scope in both polling and
interrupt mode to check if from functional perspective everything works
in both cases.
Most tests will (for now) still have busy pollers for the backend
storage as only aio supports the interrupt mode but that's ok.

Change-Id: I0a61e82cd6372fe1bf97ae4c912f6f3266207f99
Signed-off-by: default avatarKrzysztof Goreczny <krzysztof.goreczny@dell.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/22746


Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
parent a6696d7f

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -4,6 +4,8 @@ 


### nvmf



Added support for interrupt mode in the NVMe-of TCP transport.



Enable iobuf based queuing for nvmf requests when there is not enough free buffers available.

Perspective from the user of the spdk_nvmf_request_get_buffers() API is that whenever all iovecs

are allocated immediately then nothing changes compared to the previous implementation.

lib/nvmf/tcp.c

+68 −8
Original line number Diff line number Diff line
@@ -683,6 +683,7 @@ nvmf_tcp_destroy(struct spdk_nvmf_transport *transport, 
	}



	spdk_poller_unregister(&ttransport->accept_poller);

	spdk_sock_group_unregister_interrupt(ttransport->listen_sock_group);

	spdk_sock_group_close(&ttransport->listen_sock_group);

	free(ttransport);
@@ -696,12 +697,20 @@ static int nvmf_tcp_accept(void *ctx); 


static void nvmf_tcp_accept_cb(void *ctx, struct spdk_sock_group *group, struct spdk_sock *sock);



static void

nvmf_tcp_poller_set_interrupt_mode_nop(struct spdk_poller *poller, void *cb_arg,

				       bool interrupt_mode)

{

}



static struct spdk_nvmf_transport *

nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts)

{

	struct spdk_nvmf_tcp_transport *ttransport;

	uint32_t sge_count;

	uint32_t min_shared_buffers;

	int rc;

	uint64_t period;



	ttransport = calloc(1, sizeof(*ttransport));

	if (!ttransport) {
@@ -817,13 +826,16 @@ nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 
		}

	}



	ttransport->accept_poller = SPDK_POLLER_REGISTER(nvmf_tcp_accept, &ttransport->transport,

				    opts->acceptor_poll_rate);

	period = spdk_interrupt_mode_is_enabled() ? 0 : opts->acceptor_poll_rate;

	ttransport->accept_poller = SPDK_POLLER_REGISTER(nvmf_tcp_accept, &ttransport->transport, period);

	if (!ttransport->accept_poller) {

		free(ttransport);

		return NULL;

	}



	spdk_poller_register_interrupt(ttransport->accept_poller, nvmf_tcp_poller_set_interrupt_mode_nop,

				       NULL);



	ttransport->listen_sock_group = spdk_sock_group_create(NULL);

	if (ttransport->listen_sock_group == NULL) {

		SPDK_ERRLOG("Failed to create socket group for listen sockets\n");
@@ -832,6 +844,18 @@ nvmf_tcp_create(struct spdk_nvmf_transport_opts *opts) 
		return NULL;

	}



	if (spdk_interrupt_mode_is_enabled()) {

		rc = SPDK_SOCK_GROUP_REGISTER_INTERRUPT(ttransport->listen_sock_group,

							SPDK_INTERRUPT_EVENT_IN | SPDK_INTERRUPT_EVENT_OUT, nvmf_tcp_accept, &ttransport->transport);

		if (rc != 0) {

			SPDK_ERRLOG("Failed to register interrupt for listen socker sock group\n");

			spdk_sock_group_close(&ttransport->listen_sock_group);

			spdk_poller_unregister(&ttransport->accept_poller);

			free(ttransport);

			return NULL;

		}

	}



	return &ttransport->transport;

}
@@ -1569,17 +1593,26 @@ nvmf_tcp_control_msg_list_free(struct spdk_nvmf_tcp_control_msg_list *list) 
	free(list);

}



static int nvmf_tcp_poll_group_poll(struct spdk_nvmf_transport_poll_group *group);



static int

nvmf_tcp_poll_group_intr(void *ctx)

{

	struct spdk_nvmf_transport_poll_group *group = ctx;

	int ret = 0;



	ret = nvmf_tcp_poll_group_poll(group);



	return ret != 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE;

}



static struct spdk_nvmf_transport_poll_group *

nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport,

			   struct spdk_nvmf_poll_group *group)

{

	struct spdk_nvmf_tcp_transport	*ttransport;

	struct spdk_nvmf_tcp_poll_group *tgroup;



	if (spdk_interrupt_mode_is_enabled()) {

		SPDK_ERRLOG("TCP transport does not support interrupt mode\n");

		return NULL;

	}

	int rc;



	tgroup = calloc(1, sizeof(*tgroup));

	if (!tgroup) {
@@ -1617,6 +1650,15 @@ nvmf_tcp_poll_group_create(struct spdk_nvmf_transport *transport, 
		ttransport->next_pg = tgroup;

	}



	if (spdk_interrupt_mode_is_enabled()) {

		rc = SPDK_SOCK_GROUP_REGISTER_INTERRUPT(tgroup->sock_group,

							SPDK_INTERRUPT_EVENT_IN | SPDK_INTERRUPT_EVENT_OUT, nvmf_tcp_poll_group_intr, &tgroup->group);

		if (rc != 0) {

			SPDK_ERRLOG("Failed to register interrupt for sock group\n");

			goto cleanup;

		}

	}



	return &tgroup->group;



cleanup:
@@ -1668,6 +1710,7 @@ nvmf_tcp_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) 
	struct spdk_nvmf_tcp_transport *ttransport;



	tgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_tcp_poll_group, group);

	spdk_sock_group_unregister_interrupt(tgroup->sock_group);

	spdk_sock_group_close(&tgroup->sock_group);

	if (tgroup->control_msg_list) {

		nvmf_tcp_control_msg_list_free(tgroup->control_msg_list);
@@ -3325,7 +3368,7 @@ nvmf_tcp_req_process(struct spdk_nvmf_tcp_transport *ttransport, 
}



static void

nvmf_tcp_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock)

tcp_sock_cb(void *arg)

{

	struct spdk_nvmf_tcp_qpair *tqpair = arg;

	int rc;
@@ -3339,6 +3382,12 @@ nvmf_tcp_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *soc 
	}

}



static void

nvmf_tcp_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock)

{

	tcp_sock_cb(arg);

}



static int

nvmf_tcp_poll_group_add(struct spdk_nvmf_transport_poll_group *group,

			struct spdk_nvmf_qpair *qpair)
@@ -3421,9 +3470,11 @@ nvmf_tcp_req_complete(struct spdk_nvmf_request *req) 
{

	struct spdk_nvmf_tcp_transport *ttransport;

	struct spdk_nvmf_tcp_req *tcp_req;

	struct spdk_nvmf_tcp_qpair *tqpair;



	ttransport = SPDK_CONTAINEROF(req->qpair->transport, struct spdk_nvmf_tcp_transport, transport);

	tcp_req = SPDK_CONTAINEROF(req, struct spdk_nvmf_tcp_req, req);

	tqpair = SPDK_CONTAINEROF(req->qpair, struct spdk_nvmf_tcp_qpair, qpair);



	switch (tcp_req->state) {

	case TCP_REQUEST_STATE_EXECUTING:
@@ -3435,6 +3486,15 @@ nvmf_tcp_req_complete(struct spdk_nvmf_request *req) 
		break;

	case TCP_REQUEST_STATE_AWAITING_ZCOPY_RELEASE:

		nvmf_tcp_req_set_state(tcp_req, TCP_REQUEST_STATE_COMPLETED);

		/* In the interrupt mode it's possible that all responses are already written out over

		 * the socket but zero-copy buffers are still not released. In that case there won't be

		 * any event to trigger further socket processing. Send msg to a thread to avoid deadlock.

		 */

		if (spdk_unlikely(spdk_interrupt_mode_is_enabled() &&

				  tqpair->recv_state == NVME_TCP_PDU_RECV_STATE_AWAIT_REQ &&

				  spdk_nvmf_qpair_is_active(&tqpair->qpair))) {

			spdk_thread_send_msg(spdk_get_thread(), tcp_sock_cb, tqpair);

		}

		break;

	default:

		SPDK_ERRLOG("Unexpected request state %d (cntlid:%d, qid:%d)\n",

test/common/autotest_common.sh

+3 −0
Original line number Diff line number Diff line
@@ -309,6 +309,9 @@ for i in "$@"; do 
		--no-hugepages)

			NO_HUGE=(--no-huge -s 1024)

			;;

		--interrupt-mode)

			TEST_INTERRUPT_MODE=1

			;;

	esac

done

test/nvmf/common.sh

+4 −0
Original line number Diff line number Diff line
@@ -30,6 +30,10 @@ function build_nvmf_app_args() { 


	NVMF_APP+=("${NO_HUGE[@]}")



	if [ "$TEST_INTERRUPT_MODE" -eq 1 ]; then

		NVMF_APP+=(--interrupt-mode)

	fi



	if [ -n "$SPDK_HUGE_DIR" ]; then

		NVMF_APP+=(--huge-dir "$SPDK_HUGE_DIR")

	elif [ $SPDK_RUN_NON_ROOT -eq 1 ]; then

test/nvmf/nvmf.sh

+6 −0
Original line number Diff line number Diff line
@@ -14,3 +14,9 @@ fi 
run_test "nvmf_target_core" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT

run_test "nvmf_target_extra" $rootdir/test/nvmf/nvmf_target_extra.sh --transport=$SPDK_TEST_NVMF_TRANSPORT

run_test "nvmf_host" $rootdir/test/nvmf/nvmf_host.sh --transport=$SPDK_TEST_NVMF_TRANSPORT



# Interrupt mode for now is supported only on the target, with the TCP transport and posix or ssl socket implementations.

if [[ "$SPDK_TEST_NVMF_TRANSPORT" = "tcp" && $SPDK_TEST_URING -eq 0 ]]; then

	run_test "nvmf_target_core_interrupt_mode" $rootdir/test/nvmf/nvmf_target_core.sh --transport=$SPDK_TEST_NVMF_TRANSPORT --interrupt-mode

	run_test "nvmf_interrupt" $rootdir/test/nvmf/target/interrupt.sh --transport=$SPDK_TEST_NVMF_TRANSPORT --interrupt-mode

fi