Commit 7706450f authored by Michael Haeuptle's avatar Michael Haeuptle Committed by Jim Harris
Browse files

nvme_rdma: Support TOS for RDMA initiator 



The spdk_nvme_ctrlr_opts now supports a transport_tos option
that allows setting of the 'type of service' value in the IPv4 header.

This is needed to support lossless RoCE setups.

Note: Only RDMA is supported at this point.

Change-Id: I21825fc197c60f539a7d2d651a970ea380d8b56d
Signed-off-by: default avatarMichael Haeuptle <michael.haeuptle@hpe.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15908


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent ce92d919

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -110,6 +110,11 @@ tags will appear in SPDK's log at the warn level. As the SPDK application exits, 
will log a summary of how many times `SPDK_LOG_DEPRECATED()` was called for each

tag that was logged at least once.



### nvme



Added `transport_tos` to `spdk_nvme_ctrlr_opts` to support setting of the "type of service"

value in the IPv4 header. Only RDMA is supported at this time.



## v22.09



### accel

CONFIG

+1 −0
Original line number Diff line number Diff line
@@ -92,6 +92,7 @@ CONFIG_FIO_SOURCE_DIR=/usr/src/fio 
CONFIG_RDMA=n

CONFIG_RDMA_SEND_WITH_INVAL=n

CONFIG_RDMA_SET_ACK_TIMEOUT=n

CONFIG_RDMA_SET_TOS=n

CONFIG_RDMA_PROV=verbs



# Enable NVMe Character Devices.

configure

+9 −0
Original line number Diff line number Diff line
@@ -846,6 +846,15 @@ than or equal to 4.14 will see significantly reduced performance. 
		echo "RDMA_OPTION_ID_ACK_TIMEOUT is not supported"

	fi



	if echo -e '#include <rdma/rdma_cma.h>\n' \

		'int main(void) { return !!RDMA_OPTION_ID_TOS; }\n' \

		| "${BUILD_CMD[@]}" -c - 2> /dev/null; then

		CONFIG[RDMA_SET_TOS]="y"

	else

		CONFIG[RDMA_SET_TOS]="n"

		echo "RDMA_OPTION_ID_TOS is not supported"

	fi



	if [ "${CONFIG[RDMA_PROV]}" == "mlx5_dv" ]; then

		if ! echo -e '#include <spdk/stdinc.h>\n' \

			'#include <infiniband/mlx5dv.h>\n' \

include/spdk/nvme.h

+8 −1
Original line number Diff line number Diff line
@@ -275,8 +275,15 @@ struct spdk_nvme_ctrlr_opts { 
	 * 12B (header) + 2B (hash) + 176B (base64 for 1024b + crc32) + 3B (colons) + 1B (NULL) + 6B (extra space for future)

	 */

	char psk[200];



	/**

	 * It is used for RDMA transport.

	 *

	 * Set the IP protocol type of service value for RDMA transport. Default is 0, which means that the TOS will not be set.

	 */

	uint8_t transport_tos;

} __attribute__((packed));

SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_ctrlr_opts) == 817, "Incorrect size");

SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_ctrlr_opts) == 818, "Incorrect size");



/**

 * NVMe acceleration operation callback.

lib/nvme/nvme_internal.h

+2 −0
Original line number Diff line number Diff line
@@ -171,6 +171,8 @@ extern pid_t g_spdk_nvme_pid; 
#define SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED	(0)

#define SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT	SPDK_NVME_TRANSPORT_ACK_TIMEOUT_DISABLED



#define SPDK_NVME_TRANSPORT_TOS_DISABLED	(0)



#define MIN_KEEP_ALIVE_TIMEOUT_IN_MS	(10000)



/* We want to fit submission and completion rings each in a single 2MB