Commit 78accbf4 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

lib/event: Count reactor CPU stats (idle/busy tsc) 



Following the idea of thread CPU stats, add reactor CPU stats.

Reactor CPU stats accumulates run time of spdk_thread_poll() calls
to idle TSC or busy TSC according to their return codes.

Add necessary unit tests.

Signed-off-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Signed-off-by: default avatarMaciej Szwed <maciej.szwed@intel.com>
Change-Id: I1a1391e79d74387c68f1651a61c8900e4c6faf66
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1501


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 49826b36

doc/jsonrpc.md

+2 −0
Original line number Diff line number Diff line
@@ -518,6 +518,8 @@ Example response: 
    "reactors": [

      {

        "lcore": 0,

        "busy": 41289723495,

        "idle": 3624832946,

        "lw_threads": [

          {

            "name": "app_thread",

include/spdk_internal/event.h

+3 −0
Original line number Diff line number Diff line
@@ -86,6 +86,9 @@ struct spdk_reactor { 
	/* The last known rusage values */

	struct rusage					rusage;

	uint64_t					last_rusage;



	uint64_t					busy_tsc;

	uint64_t					idle_tsc;

} __attribute__((aligned(SPDK_CACHE_LINE_SIZE)));



int spdk_reactors_init(void);

lib/event/reactor.c

+11 −2
Original line number Diff line number Diff line
@@ -315,13 +315,22 @@ reactor_run(struct spdk_reactor *reactor) 
{

	struct spdk_thread	*thread;

	struct spdk_lw_thread	*lw_thread, *tmp;

	uint64_t		now;

	int			rc;



	_spdk_event_queue_run_batch(reactor);



	TAILQ_FOREACH_SAFE(lw_thread, &reactor->threads, link, tmp) {

		thread = spdk_thread_get_from_ctx(lw_thread);

		spdk_thread_poll(thread, 0, reactor->tsc_last);

		reactor->tsc_last = spdk_thread_get_last_tsc(thread);

		rc = spdk_thread_poll(thread, 0, reactor->tsc_last);



		now = spdk_thread_get_last_tsc(thread);

		if (rc == 0) {

			reactor->idle_tsc += now - reactor->tsc_last;

		} else if (rc > 0) {

			reactor->busy_tsc += now - reactor->tsc_last;

		}

		reactor->tsc_last = now;



		if (spdk_unlikely(lw_thread->resched)) {

			lw_thread->resched = false;

module/event/rpc/app_rpc.c

+2 −0
Original line number Diff line number Diff line
@@ -374,6 +374,8 @@ rpc_framework_get_reactors(void *arg1, void *arg2) 


	spdk_json_write_object_begin(ctx->w);

	spdk_json_write_named_uint32(ctx->w, "lcore", current_core);

	spdk_json_write_named_uint64(ctx->w, "busy", reactor->busy_tsc);

	spdk_json_write_named_uint64(ctx->w, "idle", reactor->idle_tsc);



	spdk_json_write_named_array_begin(ctx->w, "lw_threads");

	TAILQ_FOREACH(lw_thread, &reactor->threads, link) {

test/unit/lib/event/reactor.c/reactor_ut.c

+123 −0
Original line number Diff line number Diff line
@@ -299,6 +299,128 @@ test_for_each_reactor(void) 
	free_cores();

}



static int

poller_run_idle(void *ctx)

{

	uint64_t delay_us = (uint64_t)ctx;



	spdk_delay_us(delay_us);



	return 0;

}



static int

poller_run_busy(void *ctx)

{

	uint64_t delay_us = (uint64_t)ctx;



	spdk_delay_us(delay_us);



	return 1;

}



static void

test_reactor_stats(void)

{

	struct spdk_cpuset cpuset = {};

	struct spdk_thread *thread1, *thread2;

	struct spdk_reactor *reactor;

	struct spdk_poller *busy1, *idle1, *busy2, *idle2;

	int rc __attribute__((unused));



	/* Test case is the following:

	 * Create a reactor on CPU core0.

	 * Create thread1 and thread2 simultaneously on reactor0 at TSC = 100.

	 * Reactor runs

	 * - thread1 for 100 with busy

	 * - thread2 for 200 with idle

	 * - thread1 for 300 with idle

	 * - thread2 for 400 with busy.

	 * Then,

	 * - both elapsed TSC of thread1 and thread2 should be 1000 (= 100 + 900).

	 * - busy TSC of reactor should be 500 (= 100 + 400).

	 * - idle TSC of reactor should be 500 (= 200 + 300).

	 */



	allocate_cores(1);



	CU_ASSERT(spdk_reactors_init() == 0);



	spdk_cpuset_set_cpu(&cpuset, 0, true);



	MOCK_SET(spdk_env_get_current_core, 0);

	MOCK_SET(spdk_get_ticks, 100);



	thread1 = spdk_thread_create(NULL, &cpuset);

	SPDK_CU_ASSERT_FATAL(thread1 != NULL);



	thread2 = spdk_thread_create(NULL, &cpuset);

	SPDK_CU_ASSERT_FATAL(thread2 != NULL);



	reactor = spdk_reactor_get(0);

	SPDK_CU_ASSERT_FATAL(reactor != NULL);



	reactor->tsc_last = 100;



	spdk_set_thread(thread1);

	busy1 = spdk_poller_register(poller_run_busy, (void *)100, 0);

	CU_ASSERT(busy1 != NULL);



	spdk_set_thread(thread2);

	idle2 = spdk_poller_register(poller_run_idle, (void *)300, 0);

	CU_ASSERT(idle2 != NULL);



	reactor_run(reactor);



	CU_ASSERT(thread1->tsc_last == 200);

	CU_ASSERT(thread1->stats.busy_tsc == 100);

	CU_ASSERT(thread1->stats.idle_tsc == 0);

	CU_ASSERT(thread2->tsc_last == 500);

	CU_ASSERT(thread2->stats.busy_tsc == 0);

	CU_ASSERT(thread2->stats.idle_tsc == 300);



	CU_ASSERT(reactor->busy_tsc == 100);

	CU_ASSERT(reactor->idle_tsc == 300);



	spdk_set_thread(thread1);

	spdk_poller_unregister(&busy1);

	idle1 = spdk_poller_register(poller_run_idle, (void *)200, 0);

	CU_ASSERT(idle1 != NULL);



	spdk_set_thread(thread2);

	spdk_poller_unregister(&idle2);

	busy2 = spdk_poller_register(poller_run_busy, (void *)400, 0);

	CU_ASSERT(busy2 != NULL);



	reactor_run(reactor);



	CU_ASSERT(thread1->tsc_last == 700);

	CU_ASSERT(thread1->stats.busy_tsc == 100);

	CU_ASSERT(thread1->stats.idle_tsc == 200);

	CU_ASSERT(thread2->tsc_last == 1100);

	CU_ASSERT(thread2->stats.busy_tsc == 400);

	CU_ASSERT(thread2->stats.idle_tsc == 300);



	CU_ASSERT(reactor->busy_tsc == 500);

	CU_ASSERT(reactor->idle_tsc == 500);



	spdk_set_thread(thread1);

	spdk_poller_unregister(&idle1);

	spdk_thread_exit(thread1);



	spdk_set_thread(thread2);

	spdk_poller_unregister(&busy2);

	spdk_thread_exit(thread2);



	reactor_run(reactor);



	CU_ASSERT(TAILQ_EMPTY(&reactor->threads));



	spdk_reactors_fini();



	free_cores();

}



int

main(int argc, char **argv)

{
@@ -316,6 +438,7 @@ main(int argc, char **argv) 
	CU_ADD_TEST(suite, test_schedule_thread);

	CU_ADD_TEST(suite, test_reschedule_thread);

	CU_ADD_TEST(suite, test_for_each_reactor);

	CU_ADD_TEST(suite, test_reactor_stats);



	CU_basic_set_mode(CU_BRM_VERBOSE);

	CU_basic_run_tests();