Commit b618f056 authored by Yue-Zhu's avatar Yue-Zhu Committed by Jim Harris
Browse files

rbd_bdev: add core mask for librbd/librados threads 



Add core_mask option to allow specify core mask for librbd/librados
threads (non-spdk threads), when registering rbd cluster via bdev_rbd_register_cluster.
Also, update bdev_rbd_get_clusters_info to get librbd/librados core mask.

Signed-off-by: default avatarYue-Zhu <yue.zhu@ibm.com>
Change-Id: Iedeb508bbcf2ec97c3fdd81a6564bbddb4fc660a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/16451


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Community-CI: Mellanox Build Bot
parent 6e540558

include/spdk/stdinc.h

+1 −0
Original line number Diff line number Diff line
@@ -70,6 +70,7 @@ extern "C" { 
/* Linux */

#ifdef __linux__

#include <sys/eventfd.h>

#include <sched.h>

#endif



#ifdef __cplusplus

module/bdev/rbd/bdev_rbd.c

+63 −2
Original line number Diff line number Diff line
@@ -64,6 +64,7 @@ struct bdev_rbd_cluster { 
	char **config_param;

	char *config_file;

	char *key_file;

	char *core_mask;

	rados_t cluster;

	uint32_t ref;

	STAILQ_ENTRY(bdev_rbd_cluster) link;
@@ -83,6 +84,7 @@ bdev_rbd_cluster_free(struct bdev_rbd_cluster *entry) 
	free(entry->key_file);

	free(entry->user_id);

	free(entry->name);

	free(entry->core_mask);

	free(entry);

}
@@ -848,6 +850,10 @@ dump_single_cluster_entry(struct bdev_rbd_cluster *entry, struct spdk_json_write 
		spdk_json_write_named_string(w, "key_file", entry->key_file);

	}



	if (entry->core_mask) {

		spdk_json_write_named_string(w, "core_mask", entry->core_mask);

	}



	spdk_json_write_object_end(w);

}
@@ -902,11 +908,43 @@ static const struct spdk_bdev_fn_table rbd_fn_table = { 
	.write_config_json	= bdev_rbd_write_config_json,

};



static int

rbd_thread_set_cpumask(struct spdk_cpuset *set)

{

#ifdef __linux__

	uint32_t lcore;

	cpu_set_t mask;



	assert(set != NULL);

	CPU_ZERO(&mask);



	/* get the core id on current spdk_cpuset and set to cpu_set_t */

	for (lcore = 0; lcore < SPDK_CPUSET_SIZE; lcore++) {

		if (spdk_cpuset_get_cpu(set, lcore)) {

			CPU_SET(lcore, &mask);

		}

	}



	/* change current thread core mask */

	if (sched_setaffinity(0, sizeof(mask), &mask) < 0) {

		SPDK_ERRLOG("Set non SPDK thread cpu mask error (errno=%d)\n", errno);

		return -1;

	}



	return 0;

#else

	SPDK_ERRLOG("SPDK non spdk thread cpumask setup supports only Linux platform now.\n");

	return -ENOTSUP;

#endif

}





static int

rbd_register_cluster(const char *name, const char *user_id, const char *const *config_param,

		     const char *config_file, const char *key_file)

		     const char *config_file, const char *key_file, const char *core_mask)

{

	struct bdev_rbd_cluster *entry;

	struct spdk_cpuset rbd_core_mask = {};

	int rc;



	pthread_mutex_lock(&g_map_bdev_rbd_cluster_mutex);
@@ -964,6 +1002,29 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c 
		}

	}



	if (core_mask) {

		entry->core_mask = strdup(core_mask);

		if (entry->core_mask == NULL) {

			SPDK_ERRLOG("Core_mask=%s allocation failed on entry = %p\n", core_mask, entry);

			goto err_handle;

		}



		if (spdk_cpuset_parse(&rbd_core_mask, entry->core_mask) < 0) {

			SPDK_ERRLOG("Invalid cpumask=%s on entry = %p\n", entry->core_mask, entry);

			goto err_handle;

		}



		if (rbd_thread_set_cpumask(&rbd_core_mask) < 0) {

			SPDK_ERRLOG("Failed to change rbd threads to core_mask %s on entry = %p\n", core_mask, entry);

			goto err_handle;

		}

	}





	/* If rbd thread core mask is given, rados_create() must execute with

	 * the affinity set by rbd_thread_set_cpumask(). The affinity set

	 * by rbd_thread_set_cpumask() will be reverted once rbd_register_cluster() returns

	 * and when we leave the spdk_call_unaffinitized context. */

	rc = rados_create(&entry->cluster, user_id);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to create rados_t struct\n");
@@ -1062,7 +1123,7 @@ _bdev_rbd_register_cluster(void *arg) 


	rc = rbd_register_cluster((const char *)info->name, (const char *)info->user_id,

				  (const char *const *)info->config_param, (const char *)info->config_file,

				  (const char *)info->key_file);

				  (const char *)info->key_file, info->core_mask);

	if (rc) {

		ret = NULL;

	}

module/bdev/rbd/bdev_rbd.h

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ struct cluster_register_info { 
	char **config_param;

	char *config_file;

	char *key_file;

	char *core_mask;

};



void bdev_rbd_free_config(char **config);

module/bdev/rbd/bdev_rbd_rpc.c

+3 −1
Original line number Diff line number Diff line
@@ -230,6 +230,7 @@ free_rpc_register_cluster(struct cluster_register_info *req) 
	bdev_rbd_free_config(req->config_param);

	free(req->config_file);

	free(req->key_file);

	free(req->core_mask);

}



static const struct spdk_json_object_decoder rpc_register_cluster_decoders[] = {
@@ -237,7 +238,8 @@ static const struct spdk_json_object_decoder rpc_register_cluster_decoders[] = { 
	{"user_id", offsetof(struct cluster_register_info, user_id), spdk_json_decode_string, true},

	{"config_param", offsetof(struct cluster_register_info, config_param), bdev_rbd_decode_config, true},

	{"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true},

	{"key_file", offsetof(struct cluster_register_info, key_file), spdk_json_decode_string, true}

	{"key_file", offsetof(struct cluster_register_info, key_file), spdk_json_decode_string, true},

	{"core_mask", offsetof(struct cluster_register_info, core_mask), spdk_json_decode_string, true}

};



static void

python/spdk/rpc/bdev.py

+4 −1
Original line number Diff line number Diff line
@@ -1060,7 +1060,7 @@ def bdev_zone_block_delete(client, name): 
    return client.call('bdev_zone_block_delete', params)





def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None, key_file=None):

def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config_file=None, key_file=None, core_mask=None):

    """Create a Rados Cluster object of the Ceph RBD backend.



    Args:
@@ -1069,6 +1069,7 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config 
        config_param: map of config keys to values (optional)

        config_file: file path of Ceph configuration file (optional)

        key_file: file path of Ceph key file (optional)

        core_mask: core mask for librbd IO context threads (optional)



    Returns:

        Name of registered Rados Cluster object.
@@ -1083,6 +1084,8 @@ def bdev_rbd_register_cluster(client, name, user=None, config_param=None, config 
        params['config_file'] = config_file

    if key_file is not None:

        params['key_file'] = key_file

    if core_mask is not None:

        params['core_mask'] = core_mask



    return client.call('bdev_rbd_register_cluster', params)