Commit 2dff9736 authored by Jacek Kalwas's avatar Jacek Kalwas Committed by Tomasz Zawadzki
Browse files

sock: use nonblock connect 



With previous implementation connect was blocking, time was not
deterministic and depending on global networking settings.

Checked VM to VM connect blocking time just to have some reference.
1) with allowed traffic; few ms (4-6)
2) not allowed traffic from 4420/tcp; as above; EHOSTUNREACH errno
3) no listener; as above, ECONNREFUSED errno
4) drop rule on target; 135s; ETIMEDOUT errno
5) link down on target; 70s; EHOSTUNREACH errno
6) link down on target + arp table flush; 3s; EHOSTUNREACH errno
7) non existing ip; as above

Blocking behavior is maintained with this change however it is done on
posix poll() and not connect(). Following patch will introduce timeout
opt. 1-3 scenarios results are the same with the change, however 4-7
can be significantly mitigated.

This is a step in the middle of transitioning to async connect.

Change-Id: Ia2f347a3cfdcd427b8302e0220448b0f34d19dd4
Signed-off-by: default avatarJacek Kalwas <jacek.kalwas@nutanix.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/25794


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz@tzawadzki.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
parent e7cf1014

include/spdk_internal/sock_module.h

+2 −0
Original line number Diff line number Diff line
@@ -31,6 +31,8 @@ extern "C" { 
#define MIN_SO_SNDBUF_SIZE (4 * 1024)

#define IOV_BATCH_SIZE 64



#define SPDK_SOCK_DEFAULT_CONNECT_TIMEOUT -1



struct spdk_sock {

	struct spdk_net_impl		*net_impl;

	struct spdk_sock_opts		opts;

lib/sock/sock.c

+48 −2
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include "spdk/util.h"

#include "spdk/trace.h"

#include "spdk/thread.h"

#include "spdk/string.h"

#include "spdk_internal/trace_defs.h"



#define SPDK_SOCK_DEFAULT_PRIORITY 0
@@ -457,8 +458,18 @@ spdk_sock_posix_fd_connect(int fd, struct addrinfo *res, struct spdk_sock_opts * 
	const char *src_addr;

	uint16_t src_port;

	struct addrinfo hints, *src_ai;

	int rc;

	int rc, err, flag;

	struct pollfd pfd = {.fd = fd, .events = POLLOUT};

	socklen_t len = sizeof(err);

	bool nonblock;



	flag = fcntl(fd, F_GETFL);

	if (flag < 0) {

		SPDK_ERRLOG("fcntl can't get file status flag, fd: %d (%s)\n", fd, spdk_strerror(errno));

		return -1;

	}



	nonblock = flag & O_NONBLOCK;

	src_addr = SPDK_GET_FIELD(opts, src_addr, NULL, opts->opts_size);

	src_port = SPDK_GET_FIELD(opts, src_port, 0, opts->opts_size);

	if (src_addr != NULL || src_port != 0) {
@@ -483,12 +494,47 @@ spdk_sock_posix_fd_connect(int fd, struct addrinfo *res, struct spdk_sock_opts * 
		freeaddrinfo(src_ai);

	}



	if (!nonblock && spdk_fd_set_nonblock(fd)) {

		return -1;

	}



	rc = connect(fd, res->ai_addr, res->ai_addrlen);

	if (rc != 0) {

	if (rc != 0 && errno != EINPROGRESS) {

		SPDK_ERRLOG("connect() failed, errno = %d\n", errno);

		return 1;

	}



	rc = poll(&pfd, 1, SPDK_SOCK_DEFAULT_CONNECT_TIMEOUT);

	if (rc < 0) {

		SPDK_ERRLOG("poll() failed, errno = %d\n", errno);

		return -1;

	}



	if (rc == 0) {

		SPDK_ERRLOG("poll() timeout after %d ms\n", SPDK_SOCK_DEFAULT_CONNECT_TIMEOUT);

		return -1;

	}



	if (!(pfd.revents & POLLOUT)) {

		SPDK_ERRLOG("poll() returned %hx events without POLLOUT\n", pfd.revents);

		return -1;

	}



	rc = getsockopt(fd, SOL_SOCKET, SO_ERROR, &err, &len);

	if (rc != 0) {

		SPDK_ERRLOG("getsockopt() failed, errno = %d\n", errno);

		return -1;

	}



	if (err) {

		SPDK_ERRLOG("connect() failed, err = %d\n", err);

		return 1;

	}



	if (!nonblock && spdk_fd_clear_nonblock(fd)) {

		return -1;

	}



	return 0;

}

mk/spdk.lib_deps.mk

+1 −1
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ DEPDIRS-env_dpdk := log util 


DEPDIRS-ioat := log

DEPDIRS-idxd := log util

DEPDIRS-sock := log $(JSON_LIBS) trace

DEPDIRS-sock := log $(JSON_LIBS) trace util

DEPDIRS-util := log

DEPDIRS-vmd := log util

DEPDIRS-dma := log