Commit b8b8feb8 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Konrad Sztyber
Browse files

lib/mlx5: Add API to configure crypto UMR 



Add API to configure UMR with crypto capabilities.
UMR configuration work on top of an mkey obtained from
the spdk_mlx5_mkey_pool. Crypto UMR requires the pool
to be created with crypto capabilities, that means that
device backed by PD supports crypto capabilities

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: Ic3c346a4b838804a52d43cb7e0d7e3ffa86bacf4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23105


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent 29e074df

include/spdk_internal/mlx5.h

+60 −2
Original line number Diff line number Diff line
@@ -21,6 +21,8 @@ 
 * WQEBB - Work Queue Element Build Block (64 bytes)

 * CQE - Completion Queue Entry

 * BSF - Byte Stream Format - part of UMR WQ which describes specific data properties such as encryption or signature

 * UMR - User Memory Region

 * DEK - Data Encryption Key

 */



struct spdk_mlx5_crypto_dek;
@@ -105,6 +107,43 @@ struct spdk_mlx5_mkey_pool_obj { 
	} sig;

};



struct spdk_mlx5_umr_attr {

	struct ibv_sge *sge;

	uint32_t mkey; /* User Memory Region key to configure */

	uint32_t umr_len;

	uint16_t sge_count;

};



enum spdk_mlx5_encryption_order {

	SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_WIRE_SIGNATURE    = 0x0,

	SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_MEMORY_SIGNATURE  = 0x1,

	SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_WIRE          = 0x2,

	SPDK_MLX5_ENCRYPTION_ORDER_ENCRYPTED_RAW_MEMORY        = 0x3,

};



enum spdk_mlx5_block_size_selector {

	SPDK_MLX5_BLOCK_SIZE_SELECTOR_RESERVED	= 0,

	SPDK_MLX5_BLOCK_SIZE_SELECTOR_512	= 1,

	SPDK_MLX5_BLOCK_SIZE_SELECTOR_520	= 2,

	SPDK_MLX5_BLOCK_SIZE_SELECTOR_4096	= 3,

	SPDK_MLX5_BLOCK_SIZE_SELECTOR_4160	= 4,

};



enum spdk_mlx5_crypto_key_tweak_mode {

	SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_BE	= 0,

	SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_LE	= 1,

};



struct spdk_mlx5_umr_crypto_attr {

	uint8_t enc_order; /* see \ref enum spdk_mlx5_encryption_order */

	uint8_t bs_selector; /* see \ref enum spdk_mlx5_block_size_selector */

	uint8_t tweak_mode; /* see \ref enum spdk_mlx5_crypto_key_tweak_mode */

	/* Low level ID of the Data Encryption Key */

	uint32_t dek_obj_id;

	uint64_t xts_iv;

	uint64_t keytag; /* Must match DEK's keytag or 0 */

};



/**

 * Create Completion Queue

 *
@@ -181,7 +220,7 @@ void spdk_mlx5_qp_complete_send(struct spdk_mlx5_qp *qp); 
 * \param dstaddr Remote address to write \b sge to

 * \param rkey Remote memory key

 * \param wrid wrid which is returned in the CQE

 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion or 0

 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0

 * \return 0 on success, negated errno on failure

 */

int spdk_mlx5_qp_rdma_write(struct spdk_mlx5_qp *qp, struct ibv_sge *sge, uint32_t sge_count,
@@ -196,12 +235,31 @@ int spdk_mlx5_qp_rdma_write(struct spdk_mlx5_qp *qp, struct ibv_sge *sge, uint32 
 * \param dstaddr Remote address to read into \b sge

 * \param rkey Remote memory key

 * \param wrid wrid which is returned in the CQE

 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion or 0

 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0

 * \return 0 on success, negated errno on failure

 */

int spdk_mlx5_qp_rdma_read(struct spdk_mlx5_qp *qp, struct ibv_sge *sge, uint32_t sge_count,

			   uint64_t dstaddr, uint32_t rkey, uint64_t wrid, uint32_t flags);



/**

 * Configure User Memory Region obtained using \ref spdk_mlx5_mkey_pool_get_bulk with crypto capabilities.

 *

 * Besides crypto capabilities, it allows to gather memory chunks into virtually contig (from the NIC point of view)

 * memory space with start address 0. The user must ensure that \b qp's capacity is enough to perform this operation.

 * It only works if the UMR pool was created with crypto capabilities.

 *

 * \param qp Qpair to be used for UMR configuration. If RDMA operation which references this UMR is used on the same \b qp

 * then it is not necessary to wait for the UMR configuration to complete. Instead, first RDMA operation after UMR

 * configuration must have flag SPDK_MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE set to 1

 * \param umr_attr Common UMR attributes, describe memory layout

 * \param crypto_attr Crypto UMR attributes

 * \param wr_id wrid which is returned in the CQE

 * \param flags SPDK_MLX5_WQE_CTRL_CE_CQ_UPDATE to have a signaled completion; Any of SPDK_MLX5_WQE_CTRL_FENCE* or 0

 * \return 0 on success, negated errno on failure

 */

int spdk_mlx5_umr_configure_crypto(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr,

				   struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, uint32_t flags);



/**

 * Return a NULL terminated array of devices which support crypto operation on Nvidia NICs

 *

lib/mlx5/mlx5_priv.h

+14 −0
Original line number Diff line number Diff line
@@ -107,7 +107,21 @@ static uint8_t g_mlx5_ce_map[3][4] = { 
		[2] = SPDK_MLX5_WQE_CTRL_CE_CQ_NO_FLUSH_ERROR,

		[3] = SPDK_MLX5_WQE_CTRL_CE_CQ_ECE

	}

};



struct mlx5_crypto_bsf_seg {

	uint8_t		size_type;

	uint8_t		enc_order;

	uint8_t		rsvd0;

	uint8_t		enc_standard;

	__be32		raw_data_size;

	uint8_t		crypto_block_size_pointer;

	uint8_t		rsvd1[7];

	uint8_t		xts_initial_tweak[16];

	__be32		dek_pointer;

	uint8_t		rsvd2[4];

	uint8_t		keytag[8];

	uint8_t		rsvd3[16];

};



static inline uint8_t

lib/mlx5/mlx5_umr.c

+282 −0
Original line number Diff line number Diff line
@@ -15,6 +15,8 @@ 
#include "mlx5_ifc.h"



#define MLX5_UMR_POOL_VALID_FLAGS_MASK (~(SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO))

#define MLX5_CRYPTO_BSF_P_TYPE_CRYPTO (0x1)

#define MLX5_CRYPTO_BSF_SIZE_64B (0x2)



RB_HEAD(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj);
@@ -471,3 +473,283 @@ spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool, 


	spdk_mempool_put_bulk(pool->mpool, (void **)mkeys, mkeys_count);

}



static inline void

_mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords,

			   uint64_t mkey_mask)

{

	ctrl->flags |= MLX5_WQE_UMR_CTRL_FLAG_INLINE;

	ctrl->klm_octowords = htobe16(klms_octowords);

	/*

	 * Going to modify two properties of KLM mkey:

	 *  1. 'free' field: change this mkey from in free to in use

	 *  2. 'len' field: to include the total bytes in iovec

	 */

	mkey_mask |= MLX5_WQE_UMR_CTRL_MKEY_MASK_FREE | MLX5_WQE_UMR_CTRL_MKEY_MASK_LEN;



	ctrl->mkey_mask |= htobe64(mkey_mask);

}



static inline void

mlx5_set_umr_ctrl_seg_mtt(struct mlx5_wqe_umr_ctrl_seg *ctrl, uint32_t klms_octowords)

{

	_mlx5_set_umr_ctrl_seg_mtt(ctrl, klms_octowords, 0);

}



static inline void

mlx5_set_umr_ctrl_seg_bsf_size(struct mlx5_wqe_umr_ctrl_seg *ctrl, int bsf_size)

{

	ctrl->bsf_octowords = htobe16(SPDK_ALIGN_CEIL(SPDK_CEIL_DIV(bsf_size, 16), 4));

}



static inline void

mlx5_set_umr_mkey_seg_mtt(struct mlx5_wqe_mkey_context_seg *mkey,

			  struct spdk_mlx5_umr_attr *umr_attr)

{

	mkey->len = htobe64(umr_attr->umr_len);

}



static void

mlx5_set_umr_mkey_seg(struct mlx5_wqe_mkey_context_seg *mkey,

		      struct spdk_mlx5_umr_attr *umr_attr)

{

	memset(mkey, 0, 64);

	mlx5_set_umr_mkey_seg_mtt(mkey, umr_attr);

}



static inline void

mlx5_set_umr_inline_klm_seg(struct mlx5_wqe_umr_klm_seg *klm, struct ibv_sge *sge)

{

	klm->byte_count = htobe32(sge->length);

	klm->mkey = htobe32(sge->lkey);

	klm->address = htobe64(sge->addr);

}



static void *

mlx5_build_inline_mtt(struct mlx5_hw_qp *qp, uint32_t *to_end, struct mlx5_wqe_umr_klm_seg *dst_klm,

		      struct spdk_mlx5_umr_attr *umr_attr)

{

	struct ibv_sge *src_sge = umr_attr->sge;

	int num_wqebbs = umr_attr->sge_count / 4;

	int tail = umr_attr->sge_count & 0x3;

	int i;



	for (i = 0; i < num_wqebbs; i++) {

		mlx5_set_umr_inline_klm_seg(&dst_klm[0], src_sge++);

		mlx5_set_umr_inline_klm_seg(&dst_klm[1], src_sge++);

		mlx5_set_umr_inline_klm_seg(&dst_klm[2], src_sge++);

		mlx5_set_umr_inline_klm_seg(&dst_klm[3], src_sge++);

		/* sizeof(*dst_klm) * 4 == MLX5_SEND_WQE_BB */

		dst_klm = mlx5_qp_get_next_wqebb(qp, to_end, dst_klm);

	}



	if (!tail) {

		return dst_klm;

	}



	for (i = 0; i < tail; i++) {

		mlx5_set_umr_inline_klm_seg(&dst_klm[i], src_sge++);

	}



	/* Fill PAD entries to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB) */

	memset(&dst_klm[i], 0, MLX5_SEND_WQE_BB - sizeof(struct mlx5_wqe_umr_klm_seg) * tail);



	return mlx5_qp_get_next_wqebb(qp, to_end, dst_klm);

}



static inline void

mlx5_set_umr_crypto_bsf_seg(struct mlx5_crypto_bsf_seg *bsf, struct spdk_mlx5_umr_crypto_attr *attr,

			    uint32_t raw_data_size, uint8_t bsf_size)

{

	uint64_t *iv = (void *)bsf->xts_initial_tweak;



	memset(bsf, 0, sizeof(*bsf));

	switch (attr->tweak_mode) {

	case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_LE:

		iv[0] = htole64(attr->xts_iv);

		iv[1] = 0;

		break;

	case SPDK_MLX5_CRYPTO_KEY_TWEAK_MODE_SIMPLE_LBA_BE:

		iv[0] = 0;

		iv[1] = htobe64(attr->xts_iv);

		break;

	default:

		assert(false && "unsupported tweak mode");

		break;

	}



	bsf->size_type = (bsf_size << 6) | MLX5_CRYPTO_BSF_P_TYPE_CRYPTO;

	bsf->enc_order = attr->enc_order;

	bsf->raw_data_size = htobe32(raw_data_size);

	bsf->crypto_block_size_pointer = attr->bs_selector;

	bsf->dek_pointer = htobe32(attr->dek_obj_id);

	*((uint64_t *)bsf->keytag) = attr->keytag;

}



static inline void

mlx5_umr_configure_with_wrap_around_crypto(struct spdk_mlx5_qp *qp,

		struct spdk_mlx5_umr_attr *umr_attr, struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id,

		uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb, uint32_t mtt_size)

{

	struct mlx5_hw_qp *hw = &qp->hw;

	struct mlx5_wqe_ctrl_seg *ctrl;

	struct mlx5_wqe_ctrl_seg *gen_ctrl;

	struct mlx5_wqe_umr_ctrl_seg *umr_ctrl;

	struct mlx5_wqe_mkey_context_seg *mkey;

	struct mlx5_wqe_umr_klm_seg *klm;

	struct mlx5_crypto_bsf_seg *bsf;

	uint8_t fm_ce_se;

	uint32_t pi, to_end;



	fm_ce_se = mlx5_qp_fm_ce_se_update(qp, (uint8_t)flags);



	ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw);

	pi = hw->sq_pi & (hw->sq_wqe_cnt - 1);

	to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB;



	/*

	 * sizeof(gen_ctrl) + sizeof(umr_ctrl) == MLX5_SEND_WQE_BB,

	 * so do not need to worry about wqe buffer wrap around.

	 *

	 * build genenal ctrl segment

	 */

	gen_ctrl = ctrl;

	mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0,

			  hw->qp_num, fm_ce_se,

			  SPDK_CEIL_DIV(wqe_size, 16), 0,

			  htobe32(umr_attr->mkey));



	/* build umr ctrl segment */

	umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1);

	memset(umr_ctrl, 0, sizeof(*umr_ctrl));

	mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size);

	mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg));



	/* build mkey context segment */

	mkey = mlx5_qp_get_next_wqebb(hw, &to_end, ctrl);

	mlx5_set_umr_mkey_seg(mkey, umr_attr);



	klm = mlx5_qp_get_next_wqebb(hw, &to_end, mkey);

	bsf = mlx5_build_inline_mtt(hw, &to_end, klm, umr_attr);



	mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B);



	mlx5_qp_wqe_submit(qp, ctrl, umr_wqe_n_bb, pi);



	mlx5_qp_set_comp(qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb);

	assert(qp->tx_available >= umr_wqe_n_bb);

	qp->tx_available -= umr_wqe_n_bb;

}



static inline void

mlx5_umr_configure_full_crypto(struct spdk_mlx5_qp *dv_qp, struct spdk_mlx5_umr_attr *umr_attr,

			       struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id,

			       uint32_t flags, uint32_t wqe_size, uint32_t umr_wqe_n_bb,

			       uint32_t mtt_size)

{

	struct mlx5_hw_qp *hw = &dv_qp->hw;

	struct mlx5_wqe_ctrl_seg *ctrl;

	struct mlx5_wqe_ctrl_seg *gen_ctrl;

	struct mlx5_wqe_umr_ctrl_seg *umr_ctrl;

	struct mlx5_wqe_mkey_context_seg *mkey;

	struct mlx5_wqe_umr_klm_seg *klm;

	struct mlx5_crypto_bsf_seg *bsf;

	uint8_t fm_ce_se;

	uint32_t pi;

	uint32_t i;



	fm_ce_se = mlx5_qp_fm_ce_se_update(dv_qp, (uint8_t)flags);



	ctrl = (struct mlx5_wqe_ctrl_seg *)mlx5_qp_get_wqe_bb(hw);

	pi = hw->sq_pi & (hw->sq_wqe_cnt - 1);

	gen_ctrl = ctrl;

	mlx5_set_ctrl_seg(gen_ctrl, hw->sq_pi, MLX5_OPCODE_UMR, 0,

			  hw->qp_num, fm_ce_se,

			  SPDK_CEIL_DIV(wqe_size, 16), 0,

			  htobe32(umr_attr->mkey));



	/* build umr ctrl segment */

	umr_ctrl = (struct mlx5_wqe_umr_ctrl_seg *)(gen_ctrl + 1);

	memset(umr_ctrl, 0, sizeof(*umr_ctrl));

	mlx5_set_umr_ctrl_seg_mtt(umr_ctrl, mtt_size);

	mlx5_set_umr_ctrl_seg_bsf_size(umr_ctrl, sizeof(struct mlx5_crypto_bsf_seg));



	/* build mkey context segment */

	mkey = (struct mlx5_wqe_mkey_context_seg *)(umr_ctrl + 1);

	mlx5_set_umr_mkey_seg(mkey, umr_attr);



	klm = (struct mlx5_wqe_umr_klm_seg *)(mkey + 1);

	for (i = 0; i < umr_attr->sge_count; i++) {

		mlx5_set_umr_inline_klm_seg(klm, &umr_attr->sge[i]);

		/* sizeof(*klm) * 4 == MLX5_SEND_WQE_BB */

		klm = klm + 1;

	}

	/* fill PAD if existing */

	/* PAD entries is to make whole mtt aligned to 64B(MLX5_SEND_WQE_BB),

	 * So it will not happen wrap around during fill PAD entries. */

	for (; i < mtt_size; i++) {

		memset(klm, 0, sizeof(*klm));

		klm = klm + 1;

	}



	bsf = (struct mlx5_crypto_bsf_seg *)klm;

	mlx5_set_umr_crypto_bsf_seg(bsf, crypto_attr, umr_attr->umr_len, MLX5_CRYPTO_BSF_SIZE_64B);



	mlx5_qp_wqe_submit(dv_qp, ctrl, umr_wqe_n_bb, pi);



	mlx5_qp_set_comp(dv_qp, pi, wr_id, fm_ce_se, umr_wqe_n_bb);

	assert(dv_qp->tx_available >= umr_wqe_n_bb);

	dv_qp->tx_available -= umr_wqe_n_bb;

}



int

spdk_mlx5_umr_configure_crypto(struct spdk_mlx5_qp *qp, struct spdk_mlx5_umr_attr *umr_attr,

			       struct spdk_mlx5_umr_crypto_attr *crypto_attr, uint64_t wr_id, uint32_t flags)

{

	struct mlx5_hw_qp *hw = &qp->hw;

	uint32_t pi, to_end, umr_wqe_n_bb;

	uint32_t wqe_size, mtt_size;

	uint32_t inline_klm_size;



	if (!spdk_unlikely(umr_attr->sge_count)) {

		return -EINVAL;

	}



	pi = hw->sq_pi & (hw->sq_wqe_cnt - 1);

	to_end = (hw->sq_wqe_cnt - pi) * MLX5_SEND_WQE_BB;



	/*

	 * UMR WQE LAYOUT:

	 * -----------------------------------------------------------------------

	 * | gen_ctrl | umr_ctrl | mkey_ctx | inline klm mtt | inline crypto bsf |

	 * -----------------------------------------------------------------------

	 *   16bytes    48bytes    64bytes   sge_count*16 bytes      64 bytes

	 *

	 * Note: size of inline klm mtt should be aligned to 64 bytes.

	 */

	wqe_size = sizeof(struct mlx5_wqe_ctrl_seg) + sizeof(struct mlx5_wqe_umr_ctrl_seg) +

		   sizeof(struct mlx5_wqe_mkey_context_seg);

	mtt_size = SPDK_ALIGN_CEIL(umr_attr->sge_count, 4);

	inline_klm_size = mtt_size * sizeof(struct mlx5_wqe_umr_klm_seg);

	wqe_size += inline_klm_size;

	wqe_size += sizeof(struct mlx5_crypto_bsf_seg);



	umr_wqe_n_bb = SPDK_CEIL_DIV(wqe_size, MLX5_SEND_WQE_BB);

	if (spdk_unlikely(umr_wqe_n_bb > qp->tx_available)) {

		return -ENOMEM;

	}

	if (spdk_unlikely(umr_attr->sge_count > qp->max_send_sge)) {

		return -E2BIG;

	}



	if (spdk_unlikely(to_end < wqe_size)) {

		mlx5_umr_configure_with_wrap_around_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size,

				umr_wqe_n_bb,

				mtt_size);

	} else {

		mlx5_umr_configure_full_crypto(qp, umr_attr, crypto_attr, wr_id, flags, wqe_size, umr_wqe_n_bb,

					       mtt_size);

	}



	return 0;

}

lib/mlx5/spdk_mlx5.map

+2 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ 
    spdk_mlx5_qp_rdma_write;

    spdk_mlx5_qp_rdma_read;



    spdk_mlx5_umr_configure_crypto;



    spdk_mlx5_crypto_devs_get;

    spdk_mlx5_crypto_devs_release;

    spdk_mlx5_crypto_keytag_create;