Commit 8ffbc77d authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Konrad Sztyber
Browse files

lib/mlx5: API to query device capabilities 



Only crypto caps are fetched for now, the API will
be extended later

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: I0fafe6e4de3a116ed68226feb28548af8645a722
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23106


Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Community-CI: Mellanox Build Bot
parent b8b8feb8

include/spdk_internal/mlx5.h

+24 −0
Original line number Diff line number Diff line
@@ -144,6 +144,30 @@ struct spdk_mlx5_umr_crypto_attr { 
	uint64_t keytag; /* Must match DEK's keytag or 0 */

};



struct spdk_mlx5_device_crypto_caps {

	bool wrapped_crypto_operational;

	bool wrapped_crypto_going_to_commissioning;

	bool wrapped_import_method_aes_xts;

	bool single_block_le_tweak;

	bool multi_block_be_tweak;

	bool multi_block_le_tweak;

};



struct spdk_mlx5_device_caps {

	/* Content of this structure is valid only if crypto_supported is true */

	struct spdk_mlx5_device_crypto_caps crypto;

	bool crypto_supported;

};



/**

 * Query device capabilities

 *

 * \param context Context of a device to query

 * \param caps Device capabilities

 * \return 0 on success, negated errno on failure.

 */

int spdk_mlx5_device_query_caps(struct ibv_context *context, struct spdk_mlx5_device_caps *caps);



/**

 * Create Completion Queue

 *

lib/mlx5/mlx5_crypto.c

+93 −15
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#include "spdk/util.h"

#include "spdk_internal/mlx5.h"

#include "spdk_internal/rdma_utils.h"

#include "mlx5_ifc.h"



#define MLX5_VENDOR_ID_MELLANOX 0x2c9
@@ -111,7 +112,11 @@ spdk_mlx5_crypto_devs_get(int *dev_num) 
{

	struct ibv_context **rdma_devs, **rdma_devs_out = NULL, *dev;

	struct ibv_device_attr dev_attr;

	struct mlx5dv_context dv_dev_attr;

	struct ibv_port_attr port_attr;

	struct spdk_mlx5_device_caps dev_caps;

	uint8_t in[DEVX_ST_SZ_BYTES(query_nic_vport_context_in)];

	uint8_t out[DEVX_ST_SZ_BYTES(query_nic_vport_context_out)];

	uint8_t devx_v;

	int num_rdma_devs = 0, i, rc;

	int num_crypto_devs = 0;
@@ -130,7 +135,6 @@ spdk_mlx5_crypto_devs_get(int *dev_num) 


	for (i = 0; i < num_rdma_devs; i++) {

		dev = rdma_devs[i];



		rc = ibv_query_device(dev, &dev_attr);

		if (rc) {

			SPDK_ERRLOG("Failed to query dev %s, skipping\n", dev->device->name);
@@ -145,30 +149,52 @@ spdk_mlx5_crypto_devs_get(int *dev_num) 
			continue;

		}



		memset(&dv_dev_attr, 0, sizeof(dv_dev_attr));

		dv_dev_attr.comp_mask |= MLX5DV_CONTEXT_MASK_CRYPTO_OFFLOAD;

		rc = mlx5dv_query_device(dev, &dv_dev_attr);

		rc = ibv_query_port(dev, 1, &port_attr);

		if (rc) {

			SPDK_ERRLOG("Failed to query port attributes for device %s, rc %d\n", dev->device->name, rc);

			continue;

		}



		if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {

			/* Port may be ethernet but roce is still disabled */

			memset(in, 0, sizeof(in));

			memset(out, 0, sizeof(out));

			DEVX_SET(query_nic_vport_context_in, in, opcode, MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);

			rc = mlx5dv_devx_general_cmd(dev, in, sizeof(in), out, sizeof(out));

			if (rc) {

				SPDK_ERRLOG("Failed to get VPORT context for device %s. Assuming ROCE is disabled\n",

					    dev->device->name);

				continue;

			}



			devx_v = DEVX_GET(query_nic_vport_context_out, out, nic_vport_context.roce_en);

			if (!devx_v) {

				SPDK_ERRLOG("Device %s, RoCE disabled\n", dev->device->name);

				continue;

			}

		}



		memset(&dev_caps, 0, sizeof(dev_caps));

		rc = spdk_mlx5_device_query_caps(dev, &dev_caps);

		if (rc) {

			SPDK_ERRLOG("Failed to query mlx5 dev %s, skipping\n", dev->device->name);

			continue;

		}

		if (!(dv_dev_attr.crypto_caps.flags & MLX5DV_CRYPTO_CAPS_CRYPTO)) {

			SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support crypto, skipping\n", dev->device->name);

		if (!dev_caps.crypto_supported) {

			SPDK_WARNLOG("dev %s crypto engine doesn't support crypto\n", dev->device->name);

			continue;

		}

		if (!(dv_dev_attr.crypto_caps.crypto_engines & (MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS |

				MLX5DV_CRYPTO_ENGINES_CAP_AES_XTS_SINGLE_BLOCK))) {

			SPDK_DEBUGLOG(mlx5, "dev %s crypto engine doesn't support AES_XTS, skipping\n", dev->device->name);

		if (!(dev_caps.crypto.single_block_le_tweak || dev_caps.crypto.multi_block_le_tweak ||

		      dev_caps.crypto.multi_block_be_tweak)) {

			SPDK_WARNLOG("dev %s crypto engine doesn't support AES_XTS\n", dev->device->name);

			continue;

		}

		if (dv_dev_attr.crypto_caps.wrapped_import_method &

		    MLX5DV_CRYPTO_WRAPPED_IMPORT_METHOD_CAP_AES_XTS) {

			SPDK_WARNLOG("dev %s uses wrapped import method (0x%x) which is not supported by mlx5 accel module\n",

				     dev->device->name, dv_dev_attr.crypto_caps.wrapped_import_method);

		if (dev_caps.crypto.wrapped_import_method_aes_xts) {

			SPDK_WARNLOG("dev %s uses wrapped import method which is not supported by mlx5 lib\n",

				     dev->device->name);

			continue;

		}



		SPDK_NOTICELOG("Crypto dev %s\n", dev->device->name);

		rdma_devs_out[num_crypto_devs++] = dev;

	}
@@ -197,6 +223,58 @@ spdk_mlx5_crypto_devs_release(struct ibv_context **rdma_devs) 
	}

}



int

spdk_mlx5_device_query_caps(struct ibv_context *context, struct spdk_mlx5_device_caps *caps)

{

	uint16_t opmod = MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE |

			 HCA_CAP_OPMOD_GET_CUR;

	uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};

	uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {};

	int rc;



	DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);

	DEVX_SET(query_hca_cap_in, in, op_mod, opmod);



	rc = mlx5dv_devx_general_cmd(context, in, sizeof(in), out, sizeof(out));

	if (rc) {

		return rc;

	}



	caps->crypto_supported = DEVX_GET(query_hca_cap_out, out, capability.cmd_hca_cap.crypto);

	if (!caps->crypto_supported) {

		return 0;

	}



	caps->crypto.single_block_le_tweak = DEVX_GET(query_hca_cap_out,

					     out, capability.cmd_hca_cap.aes_xts_single_block_le_tweak);

	caps->crypto.multi_block_be_tweak = DEVX_GET(query_hca_cap_out, out,

					    capability.cmd_hca_cap.aes_xts_multi_block_be_tweak);

	caps->crypto.multi_block_le_tweak = DEVX_GET(query_hca_cap_out, out,

					    capability.cmd_hca_cap.aes_xts_multi_block_le_tweak);



	opmod = MLX5_SET_HCA_CAP_OP_MOD_CRYPTO | HCA_CAP_OPMOD_GET_CUR;

	memset(&out, 0, sizeof(out));

	memset(&in, 0, sizeof(in));



	DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);

	DEVX_SET(query_hca_cap_in, in, op_mod, opmod);



	rc = mlx5dv_devx_general_cmd(context, in, sizeof(in), out, sizeof(out));

	if (rc) {

		return rc;

	}



	caps->crypto.wrapped_crypto_operational = DEVX_GET(query_hca_cap_out, out,

			capability.crypto_caps.wrapped_crypto_operational);

	caps->crypto.wrapped_crypto_going_to_commissioning = DEVX_GET(query_hca_cap_out, out,

			capability.crypto_caps .wrapped_crypto_going_to_commissioning);

	caps->crypto.wrapped_import_method_aes_xts = (DEVX_GET(query_hca_cap_out, out,

			capability.crypto_caps.wrapped_import_method) &

			MLX5_CRYPTO_CAPS_WRAPPED_IMPORT_METHOD_AES) != 0;



	return 0;

}



void

spdk_mlx5_crypto_keytag_destroy(struct spdk_mlx5_crypto_keytag *keytag)

{

lib/mlx5/mlx5_ifc.h

+2 −1
Original line number Diff line number Diff line
@@ -1221,7 +1221,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { 
	u8         steering_format_version[0x4];

	u8         create_qp_start_hint[0x18];



	u8         reserved_at_460[0x9];

	u8         reserved_at_460[0x8];

	u8	   aes_xts_multi_block_le_tweak[0x1];

	u8         crypto[0x1];

	u8         reserved_at_46a[0x6];

	u8         max_num_eqs[0x10];

lib/mlx5/spdk_mlx5.map

+1 −0
Original line number Diff line number Diff line 
{

	global:



    spdk_mlx5_device_query_caps;

    spdk_mlx5_cq_create;

    spdk_mlx5_cq_destroy;

    spdk_mlx5_qp_create;