Commit 20c8a3b8 authored by Tan Long's avatar Tan Long Committed by Tomasz Zawadzki
Browse files

bdev/rbd: Add key_file to the rbd_register_cluster RPC 



In project practice, config_file and key_file are often used to connect
to a rados cluster, config_file includes "mon_host" and other rados
configurations like "rbd_cache", and key_file includes the secret key
and the access authority to each pool for current user. This patch adds
key_file option, user can specify config_file and key_file or only config_param
to connect rados cluster. This will make it much more flexible for users with
his/her convenience.

Signed-off-by: default avatarTan Long <tanl12@chinatelecom.cn>
Change-Id: I6b49aad70b578bdeb3ac8ea9ca0fcbd931582025
Signed-off-by: default avatarTan Long <tanl12@chinatelecom.cn>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10485


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarGangCao <gang.cao@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarXiaodong Liu <xiaodong.liu@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@mellanox.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 93ef69ef

CHANGELOG.md

+3 −0
Original line number Diff line number Diff line
@@ -28,6 +28,9 @@ removed in SPDK 22.04, and the parameter `transport_retry_count` is added and us 


An new parameter `bdev_retry_count` is added to the RPC `bdev_nvme_set_options`.



Added 'key_file' parameter to the 'rbd_register_cluster' RPC.  It is an optional parameter to

specify a keyring file to connect to a RADOS cluster.



### nvme



New APIs, `spdk_nvme_ctrlr_disconnect`, `spdk_nvme_ctrlr_reconnect_async`, and

doc/jsonrpc.md

+19 −14
Original line number Diff line number Diff line
@@ -3606,37 +3606,42 @@ name | Required | string | Registerd Rados cluster objec 
user_id                 | Optional | string      | Ceph ID (i.e. admin, not client.admin)

config_param            | Optional | string map  | Explicit librados configuration

config_file             | Optional | string      | File path of libraodos configuration file

key_file                | Optional | string      | File path of libraodos key file



This RPC registers a Rados Cluster object handle which is only known

to rbd module, it uses user_id + config_param or user_id + config_file to

identify a Rados cluster object.



If no config_param is specified, Ceph configuration files must exist with

all relevant settings for accessing the Ceph cluster. If a config map is

passed, the configuration files are ignored and instead all key/value

pairs are passed to rados_conf_set to configure cluster access. In

practice, "mon_host" (= list of monitor address+port) and "key" (= the

secret key stored in Ceph keyrings) are enough.

to rbd module, it uses user_id + config_param or user_id + config_file +

key_file or user_id + config_param + config_file + key_file to identify

a Rados cluster object.



When accessing the Ceph cluster as some user other than "admin" (the

default), the "user_id" has to be set.



The configuration items and secret key can be specified by setting config_param,

config_file and key_file, all of them, or none of them. If only config_param is

passed, all key/value pairs are passed to rados_conf_set to configure cluster access.

In practice, "mon_host" (= list of monitor address+port) and "key" (= the secret key

stored in Ceph keyrings) are enough. If config_file and key_file are specified, they must

exist with all relevant settings for accessing the Ceph cluster. If config_param, config_file

and key_file are specified, get the key/value pairs from config_file first and set to

rados_conf_set function, then set pairs in config_param and keyring in key_file. If nothing

is specified, it will get configuration file and key file from the default location

/etc/ceph/ceph.conf and /etc/ceph/ceph.client.user_id.keyring.



#### Result



Name of newly created Rados cluster object.



#### Example



Example request with `key` from `/etc/ceph/ceph.client.admin.keyring`:

Example request:



~~

{

  "params": {

    "name": "rbd_cluster",

    "config_param": {

      "mon_host": "192.168.7.1:6789,192.168.7.2:6789",

      "key": "AQDwf8db7zR1GRAA5k7NKXjS5S5V4mntwUDnGQ==",

    }

    "user_id": cinder,

    "config_file": "/root/ceph_conf/ceph.conf",

    "key_file": "/root/ceph_conf/ceph.client.cinder.keyring"

  },

  "jsonrpc": "2.0",

  "method": "bdev_rbd_register_cluster",

module/bdev/rbd/bdev_rbd.c

+28 −2
Original line number Diff line number Diff line
@@ -93,6 +93,7 @@ struct bdev_rbd_cluster { 
	char *user_id;

	char **config_param;

	char *config_file;

	char *key_file;

	rados_t cluster;

	uint32_t ref;

	STAILQ_ENTRY(bdev_rbd_cluster) link;
@@ -109,6 +110,7 @@ bdev_rbd_cluster_free(struct bdev_rbd_cluster *entry) 


	bdev_rbd_free_config(entry->config_param);

	free(entry->config_file);

	free(entry->key_file);

	free(entry->user_id);

	free(entry->name);

	free(entry);
@@ -803,6 +805,9 @@ bdev_rbd_cluster_dump_entry(const char *cluster_name, struct spdk_json_write_ctx 
		} else if (entry->config_file) {

			spdk_json_write_named_string(w, "config_file", entry->config_file);

		}

		if (entry->key_file) {

			spdk_json_write_named_string(w, "key_file", entry->key_file);

		}



		pthread_mutex_unlock(&g_map_bdev_rbd_cluster_mutex);

		return;
@@ -910,6 +915,9 @@ dump_single_cluster_entry(struct bdev_rbd_cluster *entry, struct spdk_json_write 
	} else if (entry->config_file) {

		spdk_json_write_named_string(w, "config_file", entry->config_file);

	}

	if (entry->key_file) {

		spdk_json_write_named_string(w, "key_file", entry->key_file);

	}



	spdk_json_write_object_end(w);

}
@@ -967,7 +975,7 @@ static const struct spdk_bdev_fn_table rbd_fn_table = { 


static int

rbd_register_cluster(const char *name, const char *user_id, const char *const *config_param,

		     const char *config_file)

		     const char *config_file, const char *key_file)

{

	struct bdev_rbd_cluster *entry;

	int rc;
@@ -1017,6 +1025,14 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c 
		}

	}



	if (key_file) {

		entry->key_file = strdup(key_file);

		if (entry->key_file == NULL) {

			SPDK_ERRLOG("Failed to save the key_file=%s on entry = %p\n", key_file, entry);

			goto err_handle;

		}

	}



	rc = rados_create(&entry->cluster, user_id);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to create rados_t struct\n");
@@ -1043,6 +1059,15 @@ rbd_register_cluster(const char *name, const char *user_id, const char *const *c 
		}

	}



	if (key_file) {

		rc = rados_conf_set(entry->cluster, "keyring", key_file);

		if (rc < 0) {

			SPDK_ERRLOG("Failed to set keyring = %s\n", key_file);

			rados_shutdown(entry->cluster);

			goto err_handle;

		}

	}



	rc = rados_connect(entry->cluster);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to connect to rbd_pool on cluster=%p\n", entry->cluster);
@@ -1104,7 +1129,8 @@ _bdev_rbd_register_cluster(void *arg) 
	int rc;



	rc = rbd_register_cluster((const char *)info->name, (const char *)info->user_id,

				  (const char *const *)info->config_param, (const char *)info->config_file);

				  (const char *const *)info->config_param, (const char *)info->config_file,

				  (const char *)info->key_file);

	if (rc) {

		ret = NULL;

	}

module/bdev/rbd/bdev_rbd.h

+1 −0
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ struct cluster_register_info { 
	char *user_id;

	char **config_param;

	char *config_file;

	char *key_file;

};



void bdev_rbd_free_config(char **config);

module/bdev/rbd/bdev_rbd_rpc.c

+3 −1
Original line number Diff line number Diff line
@@ -269,13 +269,15 @@ free_rpc_register_cluster(struct cluster_register_info *req) 
	free(req->user_id);

	bdev_rbd_free_config(req->config_param);

	free(req->config_file);

	free(req->key_file);

}



static const struct spdk_json_object_decoder rpc_register_cluster_decoders[] = {

	{"name", offsetof(struct cluster_register_info, name), spdk_json_decode_string, true},

	{"user_id", offsetof(struct cluster_register_info, user_id), spdk_json_decode_string, true},

	{"config_param", offsetof(struct cluster_register_info, config_param), bdev_rbd_decode_config, true},

	{"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true}

	{"config_file", offsetof(struct cluster_register_info, config_file), spdk_json_decode_string, true},

	{"key_file", offsetof(struct cluster_register_info, key_file), spdk_json_decode_string, true}

};



static void