lib/mlx5: Add API to create/destroy and use mkey pools (29e074df) · Commits · Public Repositories / spdk

include/spdk_internal/mlx5.h

+91 −0

Original line number	Diff line number	Diff line
		@@ -8,6 +8,8 @@

		#include <infiniband/mlx5dv.h>

		#include "spdk/tree.h"

		#define SPDK_MLX5_DEV_MAX_NAME_LEN 64

		/* API for low level PRM based mlx5 driver implementation. Some terminology:
		@@ -18,6 +20,7 @@
		* WQE - Work Queue Element
		* WQEBB - Work Queue Element Build Block (64 bytes)
		* CQE - Completion Queue Entry
		* BSF - Byte Stream Format - part of UMR WQ which describes specific data properties such as encryption or signature
		*/

		struct spdk_mlx5_crypto_dek;
		@@ -76,6 +79,32 @@ struct spdk_mlx5_cq_completion {
		int status;
		};

		struct spdk_mlx5_mkey_pool;

		enum spdk_mlx5_mkey_pool_flags {
		SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO = 1 << 0,
		/* Max number of pools of different types */
		SPDK_MLX5_MKEY_POOL_FLAG_COUNT = 1,
		};

		struct spdk_mlx5_mkey_pool_param {
		uint32_t mkey_count;
		uint32_t cache_per_thread;
		/* enum spdk_mlx5_mkey_pool_flags */
		uint32_t flags;
		};

		struct spdk_mlx5_mkey_pool_obj {
		uint32_t mkey;
		/* Determines which pool the mkey belongs to. See \ref spdk_mlx5_mkey_pool_flags */
		uint8_t pool_flag;
		RB_ENTRY(spdk_mlx5_mkey_pool_obj) node;
		struct {
		uint32_t sigerr_count;
		bool sigerr;
		} sig;
		};

		/**
		* Create Completion Queue
		*
		@@ -238,4 +267,66 @@ int spdk_mlx5_crypto_set_attr(struct mlx5dv_crypto_attr *attr_out,
		*/
		int spdk_mlx5_crypto_devs_allow(const char *const dev_names[], size_t devs_count);

		/**
		* Creates a pool of memory keys for a given \b PD. If params::flags has SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO enabled,
		* then a device associated with PD must support crypto operations.
		*
		* Can be called several times for different PDs. Has no effect if a pool for \b PD with the same \b flags already exists
		*
		* \param params Parameter of the memory pool
		* \param pd Protection Domain
		* \return 0 on success, errno on failure
		*/
		int spdk_mlx5_mkey_pool_init(struct spdk_mlx5_mkey_pool_param params, struct ibv_pd pd);

		/**
		* Destroy mkey pools with the given \b flags and \b pd which was created by \ref spdk_mlx5_mkey_pool_init.
		*
		* The pool reference must be released by \ref spdk_mlx5_mkey_pool_put_ref before calling this function.
		*
		* \param pd Protection Domain
		* \param flags Specifies type of the pool to delete.
		* \return 0 on success, negated errno on failure
		*/
		int spdk_mlx5_mkey_pool_destroy(uint32_t flags, struct ibv_pd *pd);

		/**
		* Get a reference to mkey pool specified by PD, increment internal reference counter.
		*
		* \param pd PD to get a mkey pool for
		* \param flags Required mkey pool flags, see \ref enum spdk_mlx5_mkey_pool_flags
		* \return Pointer to the mkey pool on success or NULL on error
		*/
		struct spdk_mlx5_mkey_pool spdk_mlx5_mkey_pool_get_ref(struct ibv_pd pd, uint32_t flags);

		/**
		* Put the mkey pool reference.
		*
		* The pool is NOT destroyed if even reference counter reaches 0
		*
		* \param pool Mkey pool pointer
		*/
		void spdk_mlx5_mkey_pool_put_ref(struct spdk_mlx5_mkey_pool *pool);

		/**
		* Get several mkeys from the pool
		*
		* \param pool mkey pool
		* \param mkeys array of mkey pointers to be filled by this function
		* \param mkeys_count number of mkeys to get from the pool
		* \return 0 on success, errno on failure
		*/
		int spdk_mlx5_mkey_pool_get_bulk(struct spdk_mlx5_mkey_pool *pool,
		struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count);

		/**
		* Return mkeys to the pool
		*
		* \param pool mkey pool
		* \param mkeys array of mkey pointers to be returned to the pool
		* \param mkeys_count number of mkeys to be returned to the pool
		*/
		void spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool,
		struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count);

		#endif /* SPDK_MLX5_H */

lib/mlx5/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
		SO_VER := 3
		SO_MINOR := 0

		C_SRCS = mlx5_crypto.c mlx5_qp.c mlx5_dma.c
		C_SRCS = mlx5_crypto.c mlx5_qp.c mlx5_dma.c mlx5_umr.c
		LIBNAME = mlx5

		LOCAL_SYS_LIBS += -lmlx5 -libverbs -lrdmacm

lib/mlx5/mlx5_ifc.h

+13 −3

Original line number	Diff line number	Diff line
		@@ -888,7 +888,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
		u8 qp_data_in_order[0x1];
		u8 reserved_at_63[0x8];
		u8 log_dma_mmo_max_size[0x5];
		u8 reserved_at_70[0x10];
		u8 relaxed_ordering_write_pci_enabled[0x1];
		u8 reserved_at_77[0xf];

		u8 log_max_srq_sz[0x8];
		u8 log_max_qp_sz[0x8];
		@@ -905,12 +906,14 @@ struct mlx5_ifc_cmd_hca_cap_bits {

		u8 reserved_at_c0[0x8];
		u8 log_max_cq_sz[0x8];
		u8 reserved_at_d0[0xb];
		u8 relaxed_ordering_write_umr[0x1];
		u8 relaxed_ordering_read_umr[0x1];
		u8 reserved_at_d2[0x9];
		u8 log_max_cq[0x5];

		u8 log_max_eq_sz[0x8];
		u8 relaxed_ordering_write[0x1];
		u8 reserved_at_e9[0x1];
		u8 relaxed_ordering_read[0x1];
		u8 log_max_mkey[0x6];
		u8 tunneled_atomic[0x1];
		u8 as_notify[0x1];
		@@ -1611,6 +1614,13 @@ enum {
		enum {
		MLX5_MKC_ACCESS_MODE_MTT = 0x1,
		MLX5_MKC_ACCESS_MODE_KLMS = 0x2,
		MLX5_MKC_ACCESS_MODE_KLMFBS = 0x3,
		};

		struct mlx5_ifc_klm_bits {
		u8 byte_count[0x20];
		u8 mkey[0x20];
		u8 address[0x40];
		};

		struct mlx5_ifc_mkc_bits {

lib/mlx5/mlx5_priv.h

+21 −0

Original line number	Diff line number	Diff line
		@@ -252,3 +252,24 @@ mlx5_cq_find_qp(struct spdk_mlx5_cq *cq, uint32_t qp_num)
		}
		return cq->qps[qpn_upper].table[qpn_mask];
		}

		static inline int
		mlx5_get_pd_id(struct ibv_pd pd, uint32_t pd_id)
		{
		struct mlx5dv_pd pd_info;
		struct mlx5dv_obj obj;
		int rc;

		if (!pd) {
		return -EINVAL;
		}
		obj.pd.in = pd;
		obj.pd.out = &pd_info;
		rc = mlx5dv_init_obj(&obj, MLX5DV_OBJ_PD);
		if (rc) {
		return rc;
		}
		*pd_id = pd_info.pdn;

		return 0;
		}

lib/mlx5/mlx5_umr.c

0 → 100644

+473 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: BSD-3-Clause
		* Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
		*/

		#include <infiniband/verbs.h>

		#include "spdk/log.h"
		#include "spdk/util.h"
		#include "spdk/likely.h"
		#include "spdk/thread.h"
		#include "spdk/tree.h"

		#include "spdk_internal/rdma_utils.h"
		#include "mlx5_priv.h"
		#include "mlx5_ifc.h"

		#define MLX5_UMR_POOL_VALID_FLAGS_MASK (~(SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO))

		RB_HEAD(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj);

		struct mlx5_relaxed_ordering_caps {
		bool relaxed_ordering_write_pci_enabled;
		bool relaxed_ordering_write;
		bool relaxed_ordering_read;
		bool relaxed_ordering_write_umr;
		bool relaxed_ordering_read_umr;
		};

		struct mlx5_mkey_attr {
		uint64_t addr;
		uint64_t size;
		uint32_t log_entity_size;
		struct mlx5_wqe_data_seg *klm;
		uint32_t klm_count;
		/* Size of bsf in octowords. If 0 then bsf is disabled */
		uint32_t bsf_octowords;
		bool crypto_en;
		bool relaxed_ordering_write;
		bool relaxed_ordering_read;
		};

		struct mlx5_mkey {
		struct mlx5dv_devx_obj *devx_obj;
		uint32_t mkey;
		uint64_t addr;
		};

		struct spdk_mlx5_mkey_pool {
		struct ibv_pd *pd;
		struct spdk_mempool *mpool;
		struct mlx5_mkeys_tree tree;
		struct mlx5_mkey **mkeys;
		uint32_t num_mkeys;
		uint32_t refcnt;
		uint32_t flags;
		TAILQ_ENTRY(spdk_mlx5_mkey_pool) link;
		};

		static int
		mlx5_key_obj_compare(struct spdk_mlx5_mkey_pool_obj key1, struct spdk_mlx5_mkey_pool_obj key2)
		{
		return key1->mkey < key2->mkey ? -1 : key1->mkey > key2->mkey;
		}

		RB_GENERATE_STATIC(mlx5_mkeys_tree, spdk_mlx5_mkey_pool_obj, node, mlx5_key_obj_compare);

		static TAILQ_HEAD(mlx5_mkey_pool_head,
		spdk_mlx5_mkey_pool) g_mkey_pools = TAILQ_HEAD_INITIALIZER(g_mkey_pools);
		static pthread_mutex_t g_mkey_pool_lock = PTHREAD_MUTEX_INITIALIZER;

		#define SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM 128

		static struct mlx5_mkey *
		mlx5_mkey_create(struct ibv_pd pd, struct mlx5_mkey_attr attr)
		{
		struct mlx5_wqe_data_seg *klms = attr->klm;
		uint32_t klm_count = attr->klm_count;
		int in_size_dw = DEVX_ST_SZ_DW(create_mkey_in) +
		(klm_count ? SPDK_ALIGN_CEIL(klm_count, 4) : 0) * DEVX_ST_SZ_DW(klm);
		uint32_t in[in_size_dw];
		uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {0};
		void *mkc;
		uint32_t translation_size;
		struct mlx5_mkey *cmkey;
		struct ibv_context *ctx = pd->context;
		uint32_t pd_id = 0;
		uint32_t i;
		uint8_t *klm;

		cmkey = calloc(1, sizeof(*cmkey));
		if (!cmkey) {
		SPDK_ERRLOG("failed to alloc cross_mkey\n");
		return NULL;
		}

		memset(in, 0, in_size_dw * 4);
		DEVX_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
		mkc = DEVX_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

		if (klm_count > 0) {
		klm = (uint8_t *)DEVX_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
		translation_size = SPDK_ALIGN_CEIL(klm_count, 4);

		for (i = 0; i < klm_count; i++) {
		DEVX_SET(klm, klm, byte_count, klms[i].byte_count);
		DEVX_SET(klm, klm, mkey, klms[i].lkey);
		DEVX_SET64(klm, klm, address, klms[i].addr);
		klms += DEVX_ST_SZ_BYTES(klm);
		}

		for (; i < translation_size; i++) {
		DEVX_SET(klm, klms, byte_count, 0x0);
		DEVX_SET(klm, klms, mkey, 0x0);
		DEVX_SET64(klm, klms, address, 0x0);
		klm += DEVX_ST_SZ_BYTES(klm);
		}
		}

		DEVX_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ?
		MLX5_MKC_ACCESS_MODE_KLMFBS :
		MLX5_MKC_ACCESS_MODE_KLMS);
		DEVX_SET(mkc, mkc, log_page_size, attr->log_entity_size);

		mlx5_get_pd_id(pd, &pd_id);
		DEVX_SET(create_mkey_in, in, translations_octword_actual_size, klm_count);
		if (klm_count == 0) {
		DEVX_SET(mkc, mkc, free, 0x1);
		}
		DEVX_SET(mkc, mkc, lw, 0x1);
		DEVX_SET(mkc, mkc, lr, 0x1);
		DEVX_SET(mkc, mkc, rw, 0x1);
		DEVX_SET(mkc, mkc, rr, 0x1);
		DEVX_SET(mkc, mkc, umr_en, 1);
		DEVX_SET(mkc, mkc, qpn, 0xffffff);
		DEVX_SET(mkc, mkc, pd, pd_id);
		DEVX_SET(mkc, mkc, translations_octword_size,
		SPDK_KLM_MAX_TRANSLATION_ENTRIES_NUM);
		DEVX_SET(mkc, mkc, relaxed_ordering_write,
		attr->relaxed_ordering_write);
		DEVX_SET(mkc, mkc, relaxed_ordering_read,
		attr->relaxed_ordering_read);
		DEVX_SET64(mkc, mkc, start_addr, attr->addr);
		DEVX_SET64(mkc, mkc, len, attr->size);
		DEVX_SET(mkc, mkc, mkey_7_0, 0x42);
		if (attr->crypto_en) {
		DEVX_SET(mkc, mkc, crypto_en, 1);
		}
		if (attr->bsf_octowords) {
		DEVX_SET(mkc, mkc, bsf_en, 1);
		DEVX_SET(mkc, mkc, bsf_octword_size, attr->bsf_octowords);
		}

		cmkey->devx_obj = mlx5dv_devx_obj_create(ctx, in, sizeof(in), out,
		sizeof(out));
		if (!cmkey->devx_obj) {
		SPDK_ERRLOG("mlx5dv_devx_obj_create() failed to create mkey, errno:%d\n", errno);
		goto out_err;
		}

		cmkey->mkey = DEVX_GET(create_mkey_out, out, mkey_index) << 8 \| 0x42;
		return cmkey;

		out_err:
		free(cmkey);
		return NULL;
		}

		static int
		mlx5_mkey_destroy(struct mlx5_mkey *mkey)
		{
		int ret = 0;

		if (mkey->devx_obj) {
		ret = mlx5dv_devx_obj_destroy(mkey->devx_obj);
		}

		free(mkey);

		return ret;
		}

		static int
		mlx5_query_relaxed_ordering_caps(struct ibv_context *context,
		struct mlx5_relaxed_ordering_caps *caps)
		{
		uint8_t in[DEVX_ST_SZ_BYTES(query_hca_cap_in)] = {};
		uint8_t out[DEVX_ST_SZ_BYTES(query_hca_cap_out)] = {};
		int ret;

		DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
		DEVX_SET(query_hca_cap_in, in, op_mod,
		MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE_CAP_2);
		ret = mlx5dv_devx_general_cmd(context, in, sizeof(in),
		out, sizeof(out));
		if (ret) {
		return ret;
		}

		caps->relaxed_ordering_write_pci_enabled = DEVX_GET(query_hca_cap_out,
		out, capability.cmd_hca_cap.relaxed_ordering_write_pci_enabled);
		caps->relaxed_ordering_write = DEVX_GET(query_hca_cap_out, out,
		capability.cmd_hca_cap.relaxed_ordering_write);
		caps->relaxed_ordering_read = DEVX_GET(query_hca_cap_out, out,
		capability.cmd_hca_cap.relaxed_ordering_read);
		caps->relaxed_ordering_write_umr = DEVX_GET(query_hca_cap_out,
		out, capability.cmd_hca_cap.relaxed_ordering_write_umr);
		caps->relaxed_ordering_read_umr = DEVX_GET(query_hca_cap_out,
		out, capability.cmd_hca_cap.relaxed_ordering_read_umr);
		return 0;
		}

		static int
		mlx5_mkey_pool_create_mkey(struct mlx5_mkey *_mkey, struct ibv_pd pd,
		struct mlx5_relaxed_ordering_caps *caps, uint32_t flags)
		{
		struct mlx5_mkey *mkey;
		struct mlx5_mkey_attr mkey_attr = {};
		uint32_t bsf_size = 0;

		mkey_attr.addr = 0;
		mkey_attr.size = 0;
		mkey_attr.log_entity_size = 0;
		mkey_attr.relaxed_ordering_write = caps->relaxed_ordering_write;
		mkey_attr.relaxed_ordering_read = caps->relaxed_ordering_read;
		mkey_attr.klm_count = 0;
		mkey_attr.klm = NULL;
		if (flags & SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO) {
		mkey_attr.crypto_en = true;
		bsf_size += 64;
		}
		mkey_attr.bsf_octowords = bsf_size / 16;

		mkey = mlx5_mkey_create(pd, &mkey_attr);
		if (!mkey) {
		SPDK_ERRLOG("Failed to create mkey on dev %s\n", pd->context->device->name);
		return -EINVAL;
		}
		*_mkey = mkey;

		return 0;
		}

		static void
		mlx5_set_mkey_in_pool(struct spdk_mempool mp, void cb_arg, void *_mkey, unsigned obj_idx)
		{
		struct spdk_mlx5_mkey_pool_obj *mkey = _mkey;
		struct spdk_mlx5_mkey_pool *pool = cb_arg;

		assert(obj_idx < pool->num_mkeys);
		assert(pool->mkeys[obj_idx] != NULL);
		mkey->mkey = pool->mkeys[obj_idx]->mkey;
		mkey->pool_flag = pool->flags & 0xf;
		mkey->sig.sigerr_count = 1;
		mkey->sig.sigerr = false;

		RB_INSERT(mlx5_mkeys_tree, &pool->tree, mkey);
		}

		static const char *g_mkey_pool_names[] = {
		[SPDK_MLX5_MKEY_POOL_FLAG_CRYPTO] = "crypto",
		};

		static void
		mlx5_mkey_pool_destroy(struct spdk_mlx5_mkey_pool *pool)
		{
		uint32_t i;

		if (pool->mpool) {
		spdk_mempool_free(pool->mpool);
		}
		if (pool->mkeys) {
		for (i = 0; i < pool->num_mkeys; i++) {
		if (pool->mkeys[i]) {
		mlx5_mkey_destroy(pool->mkeys[i]);
		pool->mkeys[i] = NULL;
		}
		}
		free(pool->mkeys);
		}
		TAILQ_REMOVE(&g_mkey_pools, pool, link);
		free(pool);
		}

		static int
		mlx5_mkey_pools_init(struct spdk_mlx5_mkey_pool_param params, struct ibv_pd pd)
		{
		struct spdk_mlx5_mkey_pool *new_pool;
		struct mlx5_mkey **mkeys;
		struct mlx5_relaxed_ordering_caps caps;
		uint32_t j, pdn;
		int rc;
		char pool_name[32];

		new_pool = calloc(1, sizeof(*new_pool));
		if (!new_pool) {
		rc = -ENOMEM;
		goto err;
		}
		TAILQ_INSERT_TAIL(&g_mkey_pools, new_pool, link);
		rc = mlx5_query_relaxed_ordering_caps(pd->context, &caps);
		if (rc) {
		SPDK_ERRLOG("Failed to get relaxed ordering capabilities, dev %s\n",
		pd->context->device->dev_name);
		goto err;
		}
		mkeys = calloc(params->mkey_count, sizeof(struct mlx5_mkey *));
		if (!mkeys) {
		rc = -ENOMEM;
		goto err;
		}
		new_pool->mkeys = mkeys;
		new_pool->num_mkeys = params->mkey_count;
		new_pool->pd = pd;
		new_pool->flags = params->flags;
		for (j = 0; j < params->mkey_count; j++) {
		rc = mlx5_mkey_pool_create_mkey(&mkeys[j], pd, &caps, params->flags);
		if (rc) {
		goto err;
		}
		}
		rc = mlx5_get_pd_id(pd, &pdn);
		if (rc) {
		SPDK_ERRLOG("Failed to get pdn, pd %p\n", pd);
		goto err;
		}
		rc = snprintf(pool_name, 32, "%s_%s_%04u", pd->context->device->name,
		g_mkey_pool_names[new_pool->flags], pdn);
		if (rc < 0) {
		goto err;
		}
		RB_INIT(&new_pool->tree);
		new_pool->mpool = spdk_mempool_create_ctor(pool_name, params->mkey_count,
		sizeof(struct spdk_mlx5_mkey_pool_obj),
		params->cache_per_thread, SPDK_ENV_SOCKET_ID_ANY,
		mlx5_set_mkey_in_pool, new_pool);
		if (!new_pool->mpool) {
		SPDK_ERRLOG("Failed to create mempool\n");
		rc = -ENOMEM;
		goto err;
		}

		return 0;

		err:
		mlx5_mkey_pool_destroy(new_pool);

		return rc;
		}

		static struct spdk_mlx5_mkey_pool *
		mlx5_mkey_pool_get(struct ibv_pd *pd, uint32_t flags)
		{
		struct spdk_mlx5_mkey_pool *pool;

		TAILQ_FOREACH(pool, &g_mkey_pools, link) {
		if (pool->pd == pd && pool->flags == flags) {
		return pool;
		}
		}

		return NULL;
		}

		int
		spdk_mlx5_mkey_pool_init(struct spdk_mlx5_mkey_pool_param params, struct ibv_pd pd)
		{
		int rc;

		if (!pd) {
		return -EINVAL;
		}

		if (!params \|\| !params->mkey_count) {
		return -EINVAL;
		}
		if ((params->flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) {
		SPDK_ERRLOG("Invalid flags %x\n", params->flags);
		return -EINVAL;
		}
		if (params->cache_per_thread > params->mkey_count \|\| !params->cache_per_thread) {
		params->cache_per_thread = params->mkey_count * 3 / 4 / spdk_env_get_core_count();
		}

		pthread_mutex_lock(&g_mkey_pool_lock);
		if (mlx5_mkey_pool_get(pd, params->flags) != NULL) {
		pthread_mutex_unlock(&g_mkey_pool_lock);
		return -EEXIST;
		}

		rc = mlx5_mkey_pools_init(params, pd);
		pthread_mutex_unlock(&g_mkey_pool_lock);

		return rc;
		}

		int
		spdk_mlx5_mkey_pool_destroy(uint32_t flags, struct ibv_pd *pd)
		{
		struct spdk_mlx5_mkey_pool *pool;
		int rc = 0;

		if (!pd) {
		return -EINVAL;
		}

		if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) {
		SPDK_ERRLOG("Invalid flags %x\n", flags);
		return -EINVAL;
		}

		pthread_mutex_lock(&g_mkey_pool_lock);
		pool = mlx5_mkey_pool_get(pd, flags);
		if (!pool) {
		SPDK_ERRLOG("Cant find a pool for PD %p, flags %x\n", pd, flags);
		pthread_mutex_unlock(&g_mkey_pool_lock);
		return -ENODEV;
		}
		if (pool->refcnt) {
		SPDK_WARNLOG("Can't delete pool pd %p, dev %s\n", pool->pd, pool->pd->context->device->dev_name);
		rc = -EAGAIN;
		} else {
		mlx5_mkey_pool_destroy(pool);
		}
		pthread_mutex_unlock(&g_mkey_pool_lock);

		return rc;
		}

		struct spdk_mlx5_mkey_pool *
		spdk_mlx5_mkey_pool_get_ref(struct ibv_pd *pd, uint32_t flags)
		{
		struct spdk_mlx5_mkey_pool *pool;

		if ((flags & MLX5_UMR_POOL_VALID_FLAGS_MASK) != 0) {
		SPDK_ERRLOG("Invalid flags %x\n", flags);
		return NULL;
		}

		pthread_mutex_lock(&g_mkey_pool_lock);
		pool = mlx5_mkey_pool_get(pd, flags);
		if (pool) {
		pool->refcnt++;
		}
		pthread_mutex_unlock(&g_mkey_pool_lock);

		return pool;
		}

		void
		spdk_mlx5_mkey_pool_put_ref(struct spdk_mlx5_mkey_pool *pool)
		{
		pthread_mutex_lock(&g_mkey_pool_lock);
		pool->refcnt--;
		pthread_mutex_unlock(&g_mkey_pool_lock);
		}

		int
		spdk_mlx5_mkey_pool_get_bulk(struct spdk_mlx5_mkey_pool *pool,
		struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count)
		{
		assert(pool->mpool);

		return spdk_mempool_get_bulk(pool->mpool, (void **)mkeys, mkeys_count);
		}

		void
		spdk_mlx5_mkey_pool_put_bulk(struct spdk_mlx5_mkey_pool *pool,
		struct spdk_mlx5_mkey_pool_obj **mkeys, uint32_t mkeys_count)
		{
		assert(pool->mpool);

		spdk_mempool_put_bulk(pool->mpool, (void **)mkeys, mkeys_count);
		}

Admin message