Commit 31d2a116 authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvmf: remove rx_desc and tx_desc mempools 



The RDMA tx_desc and rx_desc pools were only used at startup; all
descriptors are immediately allocated and put into a queue, and the
mempool functionality was never used at runtime.

Change-Id: I2882274962550191a555c8483b8f7be2854b32ec
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent e7dce1cf

lib/nvmf/nvmf.c

+0 −39
Original line number Diff line number Diff line
@@ -65,9 +65,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) 
 *

 * SPDK_NVMF_DESC_POOL_SIZE: The total number of RDMA descriptors

 * needed for all possible admin and I/O queue requests.

 *

 * SPDK_NVMF_TX_DESC_POOL_SIZE: The total number of RDMA descriptors

 * needed for all possible admin and I/O queue tx requests.

 */

#define SPDK_NVMF_ADMINQ_POOL_SIZE(spdk)	(MAX_SUBSYSTEMS * \

						 (spdk->MaxSessionsPerSubsystem) * \
@@ -81,10 +78,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF) 
#define SPDK_NVMF_DESC_POOL_SIZE(spdk)	(SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \

					 SPDK_NVMF_IOQ_POOL_SIZE(spdk))



/* Per each connection, cq depth is double of sq depth. Double the memory accordingly. */

#define SPDK_NVMF_TX_DESC_POOL_SIZE(spdk)	((SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \

						 SPDK_NVMF_IOQ_POOL_SIZE(spdk)) * 2)



#define SPDK_NVMF_MAX_CONNECTIONS(spdk)	(MAX_SUBSYSTEMS * \

					 ((spdk)->MaxSessionsPerSubsystem) * \

					 ((spdk)->MaxConnectionsPerSession))
@@ -145,36 +138,6 @@ spdk_nvmf_initialize_pools(struct spdk_nvmf_globals *spdk_nvmf) 
		      spdk_nvmf->bb_large_pool,

		      (SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf) * LARGE_BB_MAX_SIZE));



	spdk_nvmf->rx_desc_pool =

		rte_mempool_create("RDMA RX Desc Pool",

				   SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf),

				   sizeof(struct nvme_qp_rx_desc),

				   0, 0,

				   NULL, NULL, NULL, NULL,

				   SOCKET_ID_ANY, 0);

	if (!spdk_nvmf->rx_desc_pool) {

		SPDK_ERRLOG("create RX Desc pool failed\n");

		return -1;

	}

	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Rx descriptor pool %p, size 0x%lx bytes\n",

		      spdk_nvmf->rx_desc_pool,

		      (SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_rx_desc)));



	spdk_nvmf->tx_desc_pool =

		rte_mempool_create("RDMA TX Desc Pool",

				   SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf),

				   sizeof(struct nvme_qp_tx_desc),

				   0, 0,

				   NULL, NULL, NULL, NULL,

				   SOCKET_ID_ANY, 0);

	if (!spdk_nvmf->tx_desc_pool) {

		SPDK_ERRLOG("create TX Desc pool failed\n");

		return -1;

	}

	SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Tx descriptor pool %p, size 0x%lx bytes\n",

		      spdk_nvmf->tx_desc_pool,

		      (SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_tx_desc)));



	return 0;

}
@@ -196,8 +159,6 @@ spdk_nvmf_check_pools(void) 
	int rc = 0;



	rc += spdk_nvmf_check_pool(spdk_nvmf->nvme_request_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf));

	rc += spdk_nvmf_check_pool(spdk_nvmf->rx_desc_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf));

	rc += spdk_nvmf_check_pool(spdk_nvmf->tx_desc_pool, SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf));

	rc += spdk_nvmf_check_pool(spdk_nvmf->bb_small_pool, SPDK_NVMF_ADMINQ_POOL_SIZE(spdk_nvmf));

	rc += spdk_nvmf_check_pool(spdk_nvmf->bb_large_pool, SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf));

lib/nvmf/nvmf_internal.h

+0 −2
Original line number Diff line number Diff line
@@ -113,8 +113,6 @@ struct spdk_nvmf_globals { 
	int MaxConnectionsPerSession;

	int MaxRecvDataSegmentLength;



	struct rte_mempool *rx_desc_pool;

	struct rte_mempool *tx_desc_pool;

	struct rte_mempool *nvme_request_pool;

	struct rte_mempool *bb_small_pool;

	struct rte_mempool *bb_large_pool;

lib/nvmf/rdma.c

+11 −12
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ 
#include <rte_debug.h>

#include <rte_cycles.h>

#include <rte_timer.h>

#include <rte_malloc.h>

#include <rte_mempool.h>



#include "conn.h"
@@ -186,7 +187,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn) 
			SPDK_ERRLOG("Unable to de-register rx mr\n");

		}



		rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp_rx);

		rte_free(tmp_rx);

	}



	STAILQ_FOREACH(tmp_tx, &conn->rdma.qp_tx_desc, link) {
@@ -197,7 +198,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn) 
			SPDK_ERRLOG("Unable to de-register tx mr\n");

		}



		rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp_tx);

		rte_free(tmp_tx);

	}

}
@@ -910,9 +911,8 @@ alloc_qp_rx_desc(struct spdk_nvmf_conn *conn) 


	/* Allocate buffer for rx descriptors (RX WQE + Msg Buffer) */

	for (i = 0; i < conn->sq_depth; i++) {

		rx_desc = NULL;

		rc = rte_mempool_get(g_nvmf_tgt.rx_desc_pool, (void **)&rx_desc);

		if ((rc < 0) || !rx_desc) {

		rx_desc = rte_zmalloc("nvmf_rx_desc", sizeof(*rx_desc), 0);

		if (!rx_desc) {

			SPDK_ERRLOG("Unable to get rx desc object\n");

			goto fail;

		}
@@ -998,7 +998,7 @@ fail: 
			}

		}



		rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)rx_desc);

		rte_free(rx_desc);

	}



	STAILQ_FOREACH(tmp, &conn->rdma.qp_rx_desc, link) {
@@ -1020,7 +1020,7 @@ fail: 
			SPDK_ERRLOG("Unable to de-register rx mr\n");

		}



		rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp);

		rte_free(tmp);

	}



	return -ENOMEM;
@@ -1036,9 +1036,8 @@ alloc_qp_tx_desc(struct spdk_nvmf_conn *conn) 


	/* Initialize the tx descriptors */

	for (i = 0; i < conn->cq_depth; i++) {

		tx_desc = NULL;

		rc = rte_mempool_get(g_nvmf_tgt.tx_desc_pool, (void **)&tx_desc);

		if ((rc < 0) || !tx_desc) {

		tx_desc = rte_zmalloc("nvmf_tx_desc", sizeof(*tx_desc), 0);

		if (!tx_desc) {

			SPDK_ERRLOG("Unable to get tx desc object\n");

			goto fail;

		}
@@ -1079,7 +1078,7 @@ fail: 
			}

		}



		rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tx_desc);

		rte_free(tx_desc);

	}



	STAILQ_FOREACH(tmp, &conn->rdma.qp_tx_desc, link) {
@@ -1090,7 +1089,7 @@ fail: 
			SPDK_ERRLOG("Unable to de-register tx mr\n");

		}



		rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp);

		rte_free(tmp);

	}



	return -ENOMEM;