Commit c09bfe89 authored by Jim Harris's avatar Jim Harris
Browse files

lib/env: add spdk_unaffinitize_thread 



This can be used by threads within SPDK to undo the
affinitization done either by DPDK (for DPDK lcores)
or by inheriting the parent's thread when using
pthread_create().

This will be used by the stub app to unaffinitize
the reactor core to allow the scheduler to flexibly
move it to an idle core.

Signed-off-by: default avatarJim Harris <james.r.harris@intel.com>
Change-Id: I7e550953cd9dcd7fd9d98bfbb70660994f2eefcd

Reviewed-on: https://review.gerrithub.io/366680


Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent c588baab

examples/nvme/fio_plugin/fio_plugin.c

+1 −18
Original line number Diff line number Diff line
@@ -177,23 +177,6 @@ attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 
	fio_file_set_size_known(f);

}



static void

cpu_core_unaffinitized(void)

{

	cpu_set_t mask;

	int i;

	int num = sysconf(_SC_NPROCESSORS_CONF);



	CPU_ZERO(&mask);

	for (i = 0; i < num; i++) {

		CPU_SET(i, &mask);

	}



	if (pthread_setaffinity_np(pthread_self(), sizeof(mask), &mask) < 0) {

		SPDK_ERRLOG("set thread affinity failed\n");

	}

}



/* Called once at initialization. This is responsible for gathering the size of

 * each "file", which in our case are in the form

 * 'key=value [key=value] ... ns=value'
@@ -234,7 +217,7 @@ static int spdk_fio_setup(struct thread_data *td) 
		opts.mem_size = 512;

		spdk_env_init(&opts);

		spdk_env_initialized = true;

		cpu_core_unaffinitized();

		spdk_unaffinitize_thread();

	}



	for_each_file(td, f, i) {

include/spdk/env.h

+5 −0
Original line number Diff line number Diff line
@@ -376,6 +376,11 @@ int spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf); 
 */

int spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr);



/**

 * Removes any CPU affinitization from the current thread.

 */

void spdk_unaffinitize_thread(void);



/**

 * Call a function with CPU affinity unset.

 *

lib/env_dpdk/env.c

+18 −10
Original line number Diff line number Diff line
@@ -228,19 +228,12 @@ void spdk_delay_us(unsigned int us) 
	rte_delay_us(us);

}



void *

spdk_call_unaffinitized(void *cb(void *arg), void *arg)

void

spdk_unaffinitize_thread(void)

{

	rte_cpuset_t orig_cpuset, new_cpuset;

	void *ret;

	rte_cpuset_t new_cpuset;

	long num_cores, i;



	if (cb == NULL) {

		return NULL;

	}



	rte_thread_get_affinity(&orig_cpuset);



	CPU_ZERO(&new_cpuset);



	num_cores = sysconf(_SC_NPROCESSORS_CONF);
@@ -251,6 +244,21 @@ spdk_call_unaffinitized(void *cb(void *arg), void *arg) 
	}



	rte_thread_set_affinity(&new_cpuset);

}



void *

spdk_call_unaffinitized(void *cb(void *arg), void *arg)

{

	rte_cpuset_t orig_cpuset;

	void *ret;



	if (cb == NULL) {

		return NULL;

	}



	rte_thread_get_affinity(&orig_cpuset);



	spdk_unaffinitize_thread();



	ret = cb(arg);