Commit f1900e4d authored by Jim Harris's avatar Jim Harris
Browse files

thread: convert iobuf nodes to 1-sized arrays 



This is just a step towards supporting a bigger array, 1 element per
NUMA node.

Signed-off-by: default avatarJim Harris <jim.harris@samsung.com>
Change-Id: If235aa9120965921cda6291043169a5c55ceb144
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24520


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Community CI Samsung <spdk.community.ci.samsung@gmail.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 904d7e5f

include/spdk/thread.h

+4 −2
Original line number Diff line number Diff line
@@ -1108,14 +1108,16 @@ struct spdk_iobuf_node_cache { 
	struct spdk_iobuf_pool_cache	large;

};



#define SPDK_IOBUF_MAX_NUMA_NODES	1



/** iobuf channel */

struct spdk_iobuf_channel {

	/* Buffer cache */

	struct spdk_iobuf_node_cache	cache;

	/** Module pointer */

	const void			*module;

	/** Parent IO channel */

	struct spdk_io_channel		*parent;

	/* Buffer cache */

	struct spdk_iobuf_node_cache	cache[SPDK_IOBUF_MAX_NUMA_NODES];

};



/**

lib/bdev/bdev.c

+1 −1
Original line number Diff line number Diff line
@@ -1821,7 +1821,7 @@ bdev_io_get_buf(struct spdk_bdev_io *bdev_io, uint64_t len) 
	mgmt_ch = bdev_io->internal.ch->shared_resource->mgmt_ch;

	max_len = bdev_io_get_max_buf_len(bdev_io, len);



	if (spdk_unlikely(max_len > mgmt_ch->iobuf.cache.large.bufsize)) {

	if (spdk_unlikely(max_len > mgmt_ch->iobuf.cache[0].large.bufsize)) {

		SPDK_ERRLOG("Length %" PRIu64 " is larger than allowed\n", max_len);

		bdev_io_get_buf_complete(bdev_io, false);

		return;

lib/thread/iobuf.c

+17 −17
Original line number Diff line number Diff line
@@ -35,7 +35,7 @@ struct iobuf_channel_node { 
};



struct iobuf_channel {

	struct iobuf_channel_node	node;

	struct iobuf_channel_node	node[SPDK_IOBUF_MAX_NUMA_NODES];

	struct spdk_iobuf_channel	*channels[IOBUF_MAX_CHANNELS];

};
@@ -52,11 +52,11 @@ struct iobuf_node { 
};



struct iobuf {

	struct iobuf_node		node;

	struct spdk_iobuf_opts		opts;

	TAILQ_HEAD(, iobuf_module)	modules;

	spdk_iobuf_finish_cb		finish_cb;

	void				*finish_arg;

	struct iobuf_node		node[SPDK_IOBUF_MAX_NUMA_NODES];

};



static struct iobuf g_iobuf = {
@@ -83,7 +83,7 @@ iobuf_channel_create_cb(void *io_device, void *ctx) 
	struct iobuf_channel *ch = ctx;

	struct iobuf_channel_node *node;



	node = &ch->node;

	node = &ch->node[0];

	STAILQ_INIT(&node->small_queue);

	STAILQ_INIT(&node->large_queue);
@@ -96,7 +96,7 @@ iobuf_channel_destroy_cb(void *io_device, void *ctx) 
	struct iobuf_channel *ch = ctx;

	struct iobuf_channel_node *node __attribute__((unused));



	node = &ch->node;

	node = &ch->node[0];

	assert(STAILQ_EMPTY(&node->small_queue));

	assert(STAILQ_EMPTY(&node->large_queue));

}
@@ -197,7 +197,7 @@ spdk_iobuf_initialize(void) 
	opts->small_bufsize = SPDK_ALIGN_CEIL(opts->small_bufsize, IOBUF_ALIGNMENT);

	opts->large_bufsize = SPDK_ALIGN_CEIL(opts->large_bufsize, IOBUF_ALIGNMENT);



	node = &g_iobuf.node;

	node = &g_iobuf.node[0];

	rc = iobuf_node_initialize(node);

	if (rc) {

		return rc;
@@ -223,7 +223,7 @@ iobuf_unregister_cb(void *io_device) 
		free(module);

	}



	node = &g_iobuf.node;

	node = &g_iobuf.node[0];

	iobuf_node_free(node);



	if (g_iobuf.finish_cb != NULL) {
@@ -341,7 +341,7 @@ spdk_iobuf_channel_init(struct spdk_iobuf_channel *ch, const char *name, 
	struct iobuf_module *module;

	struct spdk_iobuf_buffer *buf;

	struct spdk_iobuf_node_cache *cache;

	struct iobuf_node *node = &g_iobuf.node;

	struct iobuf_node *node = &g_iobuf.node[0];

	struct iobuf_channel_node *ch_node;

	uint32_t i;
@@ -378,8 +378,8 @@ spdk_iobuf_channel_init(struct spdk_iobuf_channel *ch, const char *name, 


	ch->parent = ioch;

	ch->module = module;

	cache = &ch->cache;

	ch_node = &iobuf_ch->node;

	cache = &ch->cache[0];

	ch_node = &iobuf_ch->node[0];



	cache->small.queue = &ch_node->small_queue;

	cache->large.queue = &ch_node->large_queue;
@@ -434,10 +434,10 @@ spdk_iobuf_channel_fini(struct spdk_iobuf_channel *ch) 
	struct spdk_iobuf_buffer *buf;

	struct iobuf_channel *iobuf_ch;

	struct spdk_iobuf_node_cache *cache;

	struct iobuf_node *node = &g_iobuf.node;

	struct iobuf_node *node = &g_iobuf.node[0];

	uint32_t i;



	cache = &ch->cache;

	cache = &ch->cache[0];



	/* Make sure none of the wait queue entries are coming from this module */

	STAILQ_FOREACH(entry, cache->small.queue, stailq) {
@@ -549,7 +549,7 @@ spdk_iobuf_for_each_entry(struct spdk_iobuf_channel *ch, 
	struct spdk_iobuf_node_cache *cache;

	int rc;



	cache = &ch->cache;

	cache = &ch->cache[0];



	rc = iobuf_pool_for_each_entry(ch, &cache->small, cb_fn, cb_ctx);

	if (rc != 0) {
@@ -565,7 +565,7 @@ spdk_iobuf_entry_abort(struct spdk_iobuf_channel *ch, struct spdk_iobuf_entry *e 
	struct spdk_iobuf_node_cache *cache;

	struct spdk_iobuf_pool_cache *pool;



	cache = &ch->cache;

	cache = &ch->cache[0];



	if (len <= cache->small.bufsize) {

		pool = &cache->small;
@@ -587,7 +587,7 @@ spdk_iobuf_get(struct spdk_iobuf_channel *ch, uint64_t len, 
	struct spdk_iobuf_pool_cache *pool;

	void *buf;



	cache = &ch->cache;

	cache = &ch->cache[0];



	assert(spdk_io_channel_get_thread(ch->parent) == spdk_get_thread());

	if (len <= cache->small.bufsize) {
@@ -643,7 +643,7 @@ spdk_iobuf_put(struct spdk_iobuf_channel *ch, void *buf, uint64_t len) 
	struct spdk_iobuf_pool_cache *pool;

	size_t sz;



	cache = &ch->cache;

	cache = &ch->cache[0];



	assert(spdk_io_channel_get_thread(ch->parent) == spdk_get_thread());

	if (len <= cache->small.bufsize) {
@@ -724,12 +724,12 @@ iobuf_get_channel_stats(struct spdk_io_channel_iter *iter) 
			if (strcmp(it->module, module->name) == 0) {

				struct spdk_iobuf_pool_cache *cache;



				cache = &channel->cache.small;

				cache = &channel->cache[0].small;

				it->small_pool.cache += cache->stats.cache;

				it->small_pool.main += cache->stats.main;

				it->small_pool.retry += cache->stats.retry;



				cache = &channel->cache.large;

				cache = &channel->cache[0].large;

				it->large_pool.cache += cache->stats.cache;

				it->large_pool.main += cache->stats.main;

				it->large_pool.retry += cache->stats.retry;

test/common/lib/test_iobuf.c

+3 −3
Original line number Diff line number Diff line
@@ -56,7 +56,7 @@ spdk_iobuf_channel_init(struct spdk_iobuf_channel *ch, const char *name, 
{

	struct spdk_iobuf_node_cache *cache;



	cache = &ch->cache;

	cache = &ch->cache[0];



	STAILQ_INIT(&g_iobuf_entries);

	cache->small.cache_count = small_cache_size;
@@ -78,7 +78,7 @@ spdk_iobuf_get(struct spdk_iobuf_channel *ch, uint64_t len, 


	HANDLE_RETURN_MOCK(spdk_iobuf_get);



	cache = &ch->cache;

	cache = &ch->cache[0];



	if (len > g_iobuf.opts.small_bufsize) {

		pool = &cache->large;
@@ -118,7 +118,7 @@ spdk_iobuf_put(struct spdk_iobuf_channel *ch, void *buf, uint64_t len) 
	struct spdk_iobuf_pool_cache *pool;

	uint32_t *count;



	cache = &ch->cache;

	cache = &ch->cache[0];



	if (len > g_iobuf.opts.small_bufsize) {

		pool = &cache->large;

test/unit/lib/accel/accel.c/accel_ut.c

+2 −2
Original line number Diff line number Diff line
@@ -2328,7 +2328,7 @@ test_sequence_accel_buffers(void) 
	 * available

	 */

	accel_ch = spdk_io_channel_get_ctx(ioch);

	small = &accel_ch->iobuf.cache.small;

	small = &accel_ch->iobuf.cache[0].small;

	small_cache_count = small->cache_count;

	STAILQ_INIT(&small_cache);

	STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer);
@@ -2677,7 +2677,7 @@ test_sequence_memory_domain(void) 
	seq = NULL;

	/* Make sure the buffer pool is empty */

	accel_ch = spdk_io_channel_get_ctx(ioch);

	small = &accel_ch->iobuf.cache.small;

	small = &accel_ch->iobuf.cache[0].small;

	small_cache_count = small->cache_count;

	STAILQ_INIT(&small_cache);

	STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer);