Commit 503cddac authored by Jim Harris's avatar Jim Harris
Browse files

bdev: add global structure for bdev_io options 



Currently this just includes the overall size of the
spdk_bdev_io pool and the size of the per-thread
spdk_bdev_io caches.

Later patches will allow configuring these via
INI config file and JSON-RPC.

Signed-off-by: default avatarJim Harris <james.r.harris@intel.com>
Change-Id: I7e235ee6d2d7123d8460eeacde999c7b51017c43

Reviewed-on: https://review.gerrithub.io/414710


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarDariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent e30535fe

include/spdk/bdev.h

+9 −0
Original line number Diff line number Diff line
@@ -128,6 +128,15 @@ struct spdk_bdev_io_stat { 
	uint64_t ticks_rate;

};



struct spdk_bdev_opts {

	uint32_t bdev_io_pool_size;

	uint32_t bdev_io_cache_size;

};



void spdk_bdev_get_opts(struct spdk_bdev_opts *opts);



int spdk_bdev_set_opts(struct spdk_bdev_opts *opts);



/**

 * Block device initialization callback.

 *

lib/bdev/bdev.c

+32 −4
Original line number Diff line number Diff line
@@ -104,6 +104,11 @@ static struct spdk_bdev_mgr g_bdev_mgr = { 
	.module_init_complete = false,

};



static struct spdk_bdev_opts	g_bdev_opts = {

	.bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE,

	.bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE,

};



static spdk_bdev_init_cb	g_init_cb_fn = NULL;

static void			*g_init_cb_arg = NULL;
@@ -158,6 +163,7 @@ struct spdk_bdev_mgmt_channel { 
	 */

	bdev_io_stailq_t per_thread_cache;

	uint32_t	per_thread_cache_count;

	uint32_t	bdev_io_cache_size;



	TAILQ_HEAD(, spdk_bdev_shared_resource) shared_resources;

};
@@ -249,6 +255,26 @@ struct spdk_bdev_iostat_ctx { 


static void spdk_bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);



void

spdk_bdev_get_opts(struct spdk_bdev_opts *opts)

{

	*opts = g_bdev_opts;

}



int

spdk_bdev_set_opts(struct spdk_bdev_opts *opts)

{

	if (opts->bdev_io_pool_size < opts->bdev_io_cache_size * spdk_thread_get_count()) {

		SPDK_ERRLOG("bdev_io_pool_size %" PRIu32 " is not compatible with bdev_io_cache_size %" PRIu32

			    " and %" PRIu32 " threads\n", opts->bdev_io_pool_size, opts->bdev_io_cache_size,

			    spdk_thread_get_count());

		return -1;

	}



	g_bdev_opts = *opts;

	return 0;

}



struct spdk_bdev *

spdk_bdev_first(void)

{
@@ -483,6 +509,7 @@ spdk_bdev_mgmt_channel_create(void *io_device, void *ctx_buf) 


	STAILQ_INIT(&ch->per_thread_cache);

	ch->per_thread_cache_count = 0;

	ch->bdev_io_cache_size = g_bdev_opts.bdev_io_cache_size;



	TAILQ_INIT(&ch->shared_resources);
@@ -606,6 +633,7 @@ spdk_bdev_modules_init(void) 
	g_bdev_mgr.module_init_complete = true;

	return rc;

}



void

spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)

{
@@ -621,7 +649,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg) 
	snprintf(mempool_name, sizeof(mempool_name), "bdev_io_%d", getpid());



	g_bdev_mgr.bdev_io_pool = spdk_mempool_create(mempool_name,

				  SPDK_BDEV_IO_POOL_SIZE,

				  g_bdev_opts.bdev_io_pool_size,

				  sizeof(struct spdk_bdev_io) +

				  spdk_bdev_module_get_max_ctx_size(),

				  0,
@@ -697,10 +725,10 @@ spdk_bdev_mgr_unregister_cb(void *io_device) 
{

	spdk_bdev_fini_cb cb_fn = g_fini_cb_fn;



	if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != SPDK_BDEV_IO_POOL_SIZE) {

	if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != g_bdev_opts.bdev_io_pool_size) {

		SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n",

			    spdk_mempool_count(g_bdev_mgr.bdev_io_pool),

			    SPDK_BDEV_IO_POOL_SIZE);

			    g_bdev_opts.bdev_io_pool_size);

	}



	if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) {
@@ -862,7 +890,7 @@ spdk_bdev_put_io(struct spdk_bdev_io *bdev_io) 
		spdk_bdev_io_put_buf(bdev_io);

	}



	if (ch->per_thread_cache_count < SPDK_BDEV_IO_CACHE_SIZE) {

	if (ch->per_thread_cache_count < ch->bdev_io_cache_size) {

		ch->per_thread_cache_count++;

		STAILQ_INSERT_TAIL(&ch->per_thread_cache, bdev_io, internal.buf_link);

	} else {