Commit f420b9ef authored by Ziye Yang's avatar Ziye Yang Committed by Tomasz Zawadzki
Browse files

bdev: Make the small and large buffer pool configurable. 



This patch includes the following work:

1 Add two fields in spdk_bdev_opts structure and make the
two fields configurable.
2 Update the unit test in bdev_ut.c
3 Revise the set_bdev_options rpc call and add the support
to use the two new fields.

Change-Id: Idd6073b3a3ffaba8161e3ffa9444a9e533e67f6d
Signed-off-by: default avatarZiye Yang <ziye.yang@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5664


Community-CI: Broadcom CI
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 590b6e75

CHANGELOG.md

+6 −1
Original line number Diff line number Diff line
@@ -6,7 +6,9 @@ 


An `opts_size`element was added in the `spdk_bdev_opts` structure to solve the

ABI compatiblity issue between different SPDK version. And also add `opts_size`

parameter in spdk_bdev_get_opts function.

parameter in spdk_bdev_get_opts function. Two fields `small_buf_pool_size` and

`large_buf_pool_size` were added into spdk_bdev_opts, which were used to determine

the small and large buffer pool size of the whole bdev module.



### event
@@ -66,6 +68,9 @@ listening on portals for a portal group until all associated target nodes are cr 
at startup, otherwise some iSCSI initiators may fail to re-login when SPDK iSCSI

target application restarts.



Two optional parameter `--small-buf-pool-size` and `--large-buf-pool-size` were added

into `bdev_set_options` function.



## ioat



The PCI BDF whitelist option has been removed from the ioat_scan_accel_engine RPC.

include/spdk/bdev.h

+3 −0
Original line number Diff line number Diff line
@@ -194,6 +194,9 @@ struct spdk_bdev_opts { 
	 * New added fields should be put at the end of the struct.

	 */

	size_t opts_size;



	uint32_t small_buf_pool_size;

	uint32_t large_buf_pool_size;

};



/**

lib/bdev/bdev.c

+23 −7
Original line number Diff line number Diff line
@@ -134,6 +134,8 @@ static struct spdk_bdev_opts g_bdev_opts = { 
	.bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE,

	.bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE,

	.bdev_auto_examine = SPDK_BDEV_AUTO_EXAMINE,

	.small_buf_pool_size = BUF_SMALL_POOL_SIZE,

	.large_buf_pool_size = BUF_LARGE_POOL_SIZE,

};



static spdk_bdev_init_cb	g_init_cb_fn = NULL;
@@ -393,10 +395,12 @@ spdk_bdev_get_opts(struct spdk_bdev_opts *opts, size_t opts_size) 
	SET_FIELD(bdev_io_pool_size);

	SET_FIELD(bdev_io_cache_size);

	SET_FIELD(bdev_auto_examine);

	SET_FIELD(small_buf_pool_size);

	SET_FIELD(large_buf_pool_size);



	/* Do not remove this statement, you should always update this statement when you adding a new field,

	 * and do not forget to add the SET_FIELD statement for your added field. */

	SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_opts) == 24, "Incorrect size");

	SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_opts) == 32, "Incorrect size");



#undef SET_FIELD

}
@@ -430,6 +434,16 @@ spdk_bdev_set_opts(struct spdk_bdev_opts *opts) 
		return -1;

	}



	if (opts->small_buf_pool_size < BUF_SMALL_POOL_SIZE) {

		SPDK_ERRLOG("small_buf_pool_size must be at least %" PRIu32 "\n", BUF_SMALL_POOL_SIZE);

		return -1;

	}



	if (opts->large_buf_pool_size < BUF_LARGE_POOL_SIZE) {

		SPDK_ERRLOG("large_buf_pool_size must be at least %" PRIu32 "\n", BUF_LARGE_POOL_SIZE);

		return -1;

	}



#define SET_FIELD(field) \

        if (offsetof(struct spdk_bdev_opts, field) + sizeof(opts->field) <= opts->opts_size) { \

                g_bdev_opts.field = opts->field; \
@@ -438,6 +452,8 @@ spdk_bdev_set_opts(struct spdk_bdev_opts *opts) 
	SET_FIELD(bdev_io_pool_size);

	SET_FIELD(bdev_io_cache_size);

	SET_FIELD(bdev_auto_examine);

	SET_FIELD(small_buf_pool_size);

	SET_FIELD(large_buf_pool_size);



	g_bdev_opts.opts_size = opts->opts_size;
@@ -1305,7 +1321,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg) 
	snprintf(mempool_name, sizeof(mempool_name), "buf_small_pool_%d", getpid());



	g_bdev_mgr.buf_small_pool = spdk_mempool_create(mempool_name,

				    BUF_SMALL_POOL_SIZE,

				    g_bdev_opts.small_buf_pool_size,

				    SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +

				    SPDK_BDEV_POOL_ALIGNMENT,

				    cache_size,
@@ -1320,7 +1336,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg) 
	snprintf(mempool_name, sizeof(mempool_name), "buf_large_pool_%d", getpid());



	g_bdev_mgr.buf_large_pool = spdk_mempool_create(mempool_name,

				    BUF_LARGE_POOL_SIZE,

				    g_bdev_opts.large_buf_pool_size,

				    SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +

				    SPDK_BDEV_POOL_ALIGNMENT,

				    cache_size,
@@ -1374,10 +1390,10 @@ bdev_mgr_unregister_cb(void *io_device) 
	}



	if (g_bdev_mgr.buf_small_pool) {

		if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) {

		if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != g_bdev_opts.small_buf_pool_size) {

			SPDK_ERRLOG("Small buffer pool count is %zu but should be %u\n",

				    spdk_mempool_count(g_bdev_mgr.buf_small_pool),

				    BUF_SMALL_POOL_SIZE);

				    g_bdev_opts.small_buf_pool_size);

			assert(false);

		}
@@ -1385,10 +1401,10 @@ bdev_mgr_unregister_cb(void *io_device) 
	}



	if (g_bdev_mgr.buf_large_pool) {

		if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != BUF_LARGE_POOL_SIZE) {

		if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != g_bdev_opts.large_buf_pool_size) {

			SPDK_ERRLOG("Large buffer pool count is %zu but should be %u\n",

				    spdk_mempool_count(g_bdev_mgr.buf_large_pool),

				    BUF_LARGE_POOL_SIZE);

				    g_bdev_opts.large_buf_pool_size);

			assert(false);

		}

lib/bdev/bdev_rpc.c

+13 −0
Original line number Diff line number Diff line
@@ -46,12 +46,16 @@ struct spdk_rpc_set_bdev_opts { 
	uint32_t bdev_io_pool_size;

	uint32_t bdev_io_cache_size;

	bool bdev_auto_examine;

	uint32_t small_buf_pool_size;

	uint32_t large_buf_pool_size;

};



static const struct spdk_json_object_decoder rpc_set_bdev_opts_decoders[] = {

	{"bdev_io_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, bdev_io_pool_size), spdk_json_decode_uint32, true},

	{"bdev_io_cache_size", offsetof(struct spdk_rpc_set_bdev_opts, bdev_io_cache_size), spdk_json_decode_uint32, true},

	{"bdev_auto_examine", offsetof(struct spdk_rpc_set_bdev_opts, bdev_auto_examine), spdk_json_decode_bool, true},

	{"small_buf_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, small_buf_pool_size), spdk_json_decode_uint32, true},

	{"large_buf_pool_size", offsetof(struct spdk_rpc_set_bdev_opts, large_buf_pool_size), spdk_json_decode_uint32, true},

};



static void
@@ -63,6 +67,8 @@ rpc_bdev_set_options(struct spdk_jsonrpc_request *request, const struct spdk_jso 


	rpc_opts.bdev_io_pool_size = UINT32_MAX;

	rpc_opts.bdev_io_cache_size = UINT32_MAX;

	rpc_opts.small_buf_pool_size = UINT32_MAX;

	rpc_opts.large_buf_pool_size = UINT32_MAX;

	rpc_opts.bdev_auto_examine = true;



	if (params != NULL) {
@@ -83,6 +89,13 @@ rpc_bdev_set_options(struct spdk_jsonrpc_request *request, const struct spdk_jso 
		bdev_opts.bdev_io_cache_size = rpc_opts.bdev_io_cache_size;

	}

	bdev_opts.bdev_auto_examine = rpc_opts.bdev_auto_examine;

	if (rpc_opts.small_buf_pool_size != UINT32_MAX) {

		bdev_opts.small_buf_pool_size = rpc_opts.small_buf_pool_size;

	}

	if (rpc_opts.large_buf_pool_size != UINT32_MAX) {

		bdev_opts.large_buf_pool_size = rpc_opts.large_buf_pool_size;

	}



	rc = spdk_bdev_set_opts(&bdev_opts);



	if (rc != 0) {

scripts/rpc.py

+5 −1
Original line number Diff line number Diff line
@@ -171,12 +171,16 @@ if __name__ == "__main__": 
        rpc.bdev.bdev_set_options(args.client,

                                  bdev_io_pool_size=args.bdev_io_pool_size,

                                  bdev_io_cache_size=args.bdev_io_cache_size,

                                  bdev_auto_examine=args.bdev_auto_examine)

                                  bdev_auto_examine=args.bdev_auto_examine,

                                  small_buf_pool_size=args.small_buf_pool_size,

                                  large_buf_pool_size=args.large_buf_pool_size)



    p = subparsers.add_parser('bdev_set_options', aliases=['set_bdev_options'],

                              help="""Set options of bdev subsystem""")

    p.add_argument('-p', '--bdev-io-pool-size', help='Number of bdev_io structures in shared buffer pool', type=int)

    p.add_argument('-c', '--bdev-io-cache-size', help='Maximum number of bdev_io structures cached per thread', type=int)

    p.add_argument('-s', '--small-buf-pool-size', help='Maximum number of small buf (i.e., 8KB) pool size', type=int)

    p.add_argument('-l', '--large-buf-pool-size', help='Maximum number of large buf (i.e., 64KB) pool size', type=int)

    group = p.add_mutually_exclusive_group()

    group.add_argument('-e', '--enable-auto-examine', dest='bdev_auto_examine', help='Allow to auto examine', action='store_true')

    group.add_argument('-d', '--disable-auto-examine', dest='bdev_auto_examine', help='Not allow to auto examine', action='store_false')