Commit df559ab6 authored by Jiewei Ke's avatar Jiewei Ke Committed by Tomasz Zawadzki
Browse files

thread: speed up io_channel lookup by using rbtree 



Use the macros for red black tree provided by Free BSD to speed up
io_channel lookup.

Signed-off-by: default avatarJiewei Ke <jiewei@smartx.com>
Change-Id: Icfd87a8a2f60c082a17b8c501a03faba83edb762
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7895


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarGangCao <gang.cao@intel.com>
parent e45450d2

lib/thread/thread.c

+58 −49
Original line number Diff line number Diff line
@@ -39,7 +39,6 @@ 
#include "spdk/string.h"

#include "spdk/thread.h"

#include "spdk/trace.h"

#include "spdk/tree.h"

#include "spdk/util.h"

#include "spdk/fd_group.h"
@@ -143,7 +142,7 @@ struct spdk_thread { 
	enum spdk_thread_state		state;

	int				pending_unregister_count;



	TAILQ_HEAD(, spdk_io_channel)	io_channels;

	RB_HEAD(io_channel_tree, spdk_io_channel)	io_channels;

	TAILQ_ENTRY(spdk_thread)			tailq;



	char				name[SPDK_MAX_THREAD_NAME_LEN + 1];
@@ -196,6 +195,14 @@ io_device_cmp(struct io_device *dev1, struct io_device *dev2) 


RB_GENERATE_STATIC(io_device_tree, io_device, node, io_device_cmp);



static int

io_channel_cmp(struct spdk_io_channel *ch1, struct spdk_io_channel *ch2)

{

	return (ch1->dev < ch2->dev ? -1 : ch1->dev > ch2->dev);

}



RB_GENERATE_STATIC(io_channel_tree, spdk_io_channel, node, io_channel_cmp);



struct spdk_msg {

	spdk_msg_fn		fn;

	void			*arg;
@@ -347,7 +354,7 @@ _free_thread(struct spdk_thread *thread) 
	struct spdk_msg *msg;

	struct spdk_poller *poller, *ptmp;



	TAILQ_FOREACH(ch, &thread->io_channels, tailq) {

	RB_FOREACH(ch, io_channel_tree, &thread->io_channels) {

		SPDK_ERRLOG("thread %s still has channel for io_device %s\n",

			    thread->name, ch->dev->name);

	}
@@ -422,7 +429,7 @@ spdk_thread_create(const char *name, struct spdk_cpuset *cpumask) 
		spdk_cpuset_negate(&thread->cpumask);

	}



	TAILQ_INIT(&thread->io_channels);

	RB_INIT(&thread->io_channels);

	TAILQ_INIT(&thread->active_pollers);

	RB_INIT(&thread->timed_pollers);

	TAILQ_INIT(&thread->paused_pollers);
@@ -538,7 +545,7 @@ thread_exit(struct spdk_thread *thread, uint64_t now) 
		return;

	}



	TAILQ_FOREACH(ch, &thread->io_channels, tailq) {

	RB_FOREACH(ch, io_channel_tree, &thread->io_channels) {

		SPDK_INFOLOG(thread,

			     "thread %s still has channel for io_device %s\n",

			     thread->name, ch->dev->name);
@@ -1741,13 +1748,13 @@ spdk_thread_get_next_paused_poller(struct spdk_poller *prev) 
struct spdk_io_channel *

spdk_thread_get_first_io_channel(struct spdk_thread *thread)

{

	return TAILQ_FIRST(&thread->io_channels);

	return RB_MIN(io_channel_tree, &thread->io_channels);

}



struct spdk_io_channel *

spdk_thread_get_next_io_channel(struct spdk_io_channel *prev)

{

	return TAILQ_NEXT(prev, tailq);

	return RB_NEXT(io_channel_tree, &thread->io_channels, prev);

}



struct call_thread {
@@ -2024,6 +2031,15 @@ spdk_io_device_get_name(struct io_device *dev) 
	return dev->name;

}



static struct spdk_io_channel *

thread_get_io_channel(struct spdk_thread *thread, struct io_device *dev)

{

	struct spdk_io_channel find = {};



	find.dev = dev;

	return RB_FIND(io_channel_tree, &thread->io_channels, &find);

}



struct spdk_io_channel *

spdk_get_io_channel(void *io_device)

{
@@ -2053,8 +2069,8 @@ spdk_get_io_channel(void *io_device) 
		return NULL;

	}



	TAILQ_FOREACH(ch, &thread->io_channels, tailq) {

		if (ch->dev == dev) {

	ch = thread_get_io_channel(thread, dev);

	if (ch != NULL) {

		ch->ref++;



		SPDK_DEBUGLOG(thread, "Get io_channel %p for io_device %s (%p) on thread %s refcnt %u\n",
@@ -2069,7 +2085,6 @@ spdk_get_io_channel(void *io_device) 
				  (uint64_t)spdk_io_channel_get_ctx(ch), ch->ref);

		return ch;

	}

	}



	ch = calloc(1, sizeof(*ch) + dev->ctx_size);

	if (ch == NULL) {
@@ -2083,7 +2098,7 @@ spdk_get_io_channel(void *io_device) 
	ch->thread = thread;

	ch->ref = 1;

	ch->destroy_ref = 0;

	TAILQ_INSERT_TAIL(&thread->io_channels, ch, tailq);

	RB_INSERT(io_channel_tree, &thread->io_channels, ch);



	SPDK_DEBUGLOG(thread, "Get io_channel %p for io_device %s (%p) on thread %s refcnt %u\n",

		      ch, dev->name, dev->io_device, thread->name, ch->ref);
@@ -2095,7 +2110,7 @@ spdk_get_io_channel(void *io_device) 
	rc = dev->create_cb(io_device, (uint8_t *)ch + sizeof(*ch));

	if (rc != 0) {

		pthread_mutex_lock(&g_devlist_mutex);

		TAILQ_REMOVE(&ch->thread->io_channels, ch, tailq);

		RB_REMOVE(io_channel_tree, &ch->thread->io_channels, ch);

		dev->refcnt--;

		free(ch);

		pthread_mutex_unlock(&g_devlist_mutex);
@@ -2138,7 +2153,7 @@ put_io_channel(void *arg) 
	}



	pthread_mutex_lock(&g_devlist_mutex);

	TAILQ_REMOVE(&ch->thread->io_channels, ch, tailq);

	RB_REMOVE(io_channel_tree, &ch->thread->io_channels, ch);

	pthread_mutex_unlock(&g_devlist_mutex);



	/* Don't hold the devlist mutex while the destroy_cb is called. */
@@ -2283,11 +2298,7 @@ _call_channel(void *ctx) 
	 *  the fn() on this thread.

	 */

	pthread_mutex_lock(&g_devlist_mutex);

	TAILQ_FOREACH(ch, &i->cur_thread->io_channels, tailq) {

		if (ch->dev == i->dev) {

			break;

		}

	}

	ch = thread_get_io_channel(i->cur_thread, i->dev);

	pthread_mutex_unlock(&g_devlist_mutex);



	if (ch) {
@@ -2327,8 +2338,8 @@ spdk_for_each_channel(void *io_device, spdk_channel_msg fn, void *ctx, 
	}



	TAILQ_FOREACH(thread, &g_threads, tailq) {

		TAILQ_FOREACH(ch, &thread->io_channels, tailq) {

			if (ch->dev == i->dev) {

		ch = thread_get_io_channel(thread, i->dev);

		if (ch != NULL) {

			ch->dev->for_each_count++;

			i->cur_thread = thread;

			i->ch = ch;
@@ -2338,7 +2349,6 @@ spdk_for_each_channel(void *io_device, spdk_channel_msg fn, void *ctx, 
			return;

		}

	}

	}



end:

	pthread_mutex_unlock(&g_devlist_mutex);
@@ -2364,8 +2374,8 @@ spdk_for_each_channel_continue(struct spdk_io_channel_iter *i, int status) 
	}

	thread = TAILQ_NEXT(i->cur_thread, tailq);

	while (thread) {

		TAILQ_FOREACH(ch, &thread->io_channels, tailq) {

			if (ch->dev == i->dev) {

		ch = thread_get_io_channel(thread, i->dev);

		if (ch != NULL) {

			i->cur_thread = thread;

			i->ch = ch;

			pthread_mutex_unlock(&g_devlist_mutex);
@@ -2373,7 +2383,6 @@ spdk_for_each_channel_continue(struct spdk_io_channel_iter *i, int status) 
			assert(rc == 0);

			return;

		}

		}

		thread = TAILQ_NEXT(thread, tailq);

	}

lib/thread/thread_internal.h

+3 −3
Original line number Diff line number Diff line
@@ -34,8 +34,8 @@ 
#define SPDK_THREAD_INTERNAL_H_



#include "spdk/assert.h"

#include "spdk/queue.h"

#include "spdk/thread.h"

#include "spdk/tree.h"



/**

 * \brief Represents a per-thread channel for accessing an I/O device.
@@ -51,10 +51,10 @@ struct spdk_io_channel { 
	struct io_device		*dev;

	uint32_t			ref;

	uint32_t			destroy_ref;

	TAILQ_ENTRY(spdk_io_channel)	tailq;

	RB_ENTRY(spdk_io_channel)	node;

	spdk_io_channel_destroy_cb	destroy_cb;



	uint8_t				_padding[48];

	uint8_t				_padding[40];

	/*

	 * Modules will allocate extra memory off the end of this structure

	 *  to store references to hardware-specific references (i.e. NVMe queue