Commit 90520634 authored by Changpeng Liu's avatar Changpeng Liu
Browse files

bdev/nvme: configure the number of requests allocated for one NVMe I/O queue via RPC 



A single I/O may allocate more than one request, since splitting may be
necessary to conform to the device's maximum transfer size, PRP list
compatibility requirements, or driver-assisted striping.  Very big
I/O request sent from application may get error due to limited resources
in NVMe driver layer, so here we add an optional parameter to make the
parameter can be configured by users.

Fix issue #745.

Change-Id: I7824232c54865b052dcd0ec6e91484c3837fc2c4
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/461182


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent bf7460f5

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -133,6 +133,11 @@ a thinly provisioned logical volume. See documentation for complete details. 


### nvme



Added an optional parameter `--io-queue-requests` to RPC `set_bdev_nvme_options`, which

can be used to change the number of requests allocated for one NVMe I/O queue.  For

very big I/O size, e.g. 128MiB, with this option user will not get an error due to

limited requests in NVMe driver layer.



Added spdk_nvme_ctrlr_get_transport_id() to get the transport ID from a

previously attached controller.

doc/jsonrpc.md

+3 −1
Original line number Diff line number Diff line
@@ -1402,6 +1402,7 @@ timeout_us | Optional | number | Timeout for each command, 
retry_count                | Optional | number      | The number of attempts per I/O before an I/O fails

nvme_adminq_poll_period_us | Optional | number      | How often the admin queue is polled for asynchronous events in microseconds

nvme_ioq_poll_period_us    | Optional | number      | How often I/O queues are polled for completions, in microseconds. Default: 0 (as fast as possible).

io_queue_requests          | Optional | number      | The number of requests allocated for each NVMe I/O queue. Default: 512.



### Example
@@ -1414,7 +1415,8 @@ request: 
    "retry_count": 5,

    "nvme_adminq_poll_period_us": 2000,

    "timeout_us": 10000000,

    "action_on_timeout": "reset"

    "action_on_timeout": "reset",

    "io_queue_requests" : 2048,

  },

  "jsonrpc": "2.0",

  "method": "set_bdev_nvme_options",

lib/bdev/nvme/bdev_nvme.c

+5 −0
Original line number Diff line number Diff line
@@ -107,6 +107,7 @@ static struct spdk_bdev_nvme_opts g_opts = { 
	.retry_count = SPDK_NVME_DEFAULT_RETRY_COUNT,

	.nvme_adminq_poll_period_us = 1000000ULL,

	.nvme_ioq_poll_period_us = 0,

	.io_queue_requests = 0,

};



#define NVME_HOTPLUG_POLL_PERIOD_MAX			10000000ULL
@@ -537,6 +538,8 @@ bdev_nvme_create_cb(void *io_device, void *ctx_buf) 


	spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts));

	opts.delay_pcie_doorbell = true;

	opts.io_queue_requests = spdk_max(g_opts.io_queue_requests, opts.io_queue_requests);

	g_opts.io_queue_requests = opts.io_queue_requests;



	ch->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, &opts, sizeof(opts));
@@ -1098,6 +1101,7 @@ spdk_bdev_nvme_set_opts(const struct spdk_bdev_nvme_opts *opts) 


	return 0;

}



struct set_nvme_hotplug_ctx {

	uint64_t period_us;

	bool enabled;
@@ -2118,6 +2122,7 @@ bdev_nvme_config_json(struct spdk_json_write_ctx *w) 
	spdk_json_write_named_uint32(w, "retry_count", g_opts.retry_count);

	spdk_json_write_named_uint64(w, "nvme_adminq_poll_period_us", g_opts.nvme_adminq_poll_period_us);

	spdk_json_write_named_uint64(w, "nvme_ioq_poll_period_us", g_opts.nvme_ioq_poll_period_us);

	spdk_json_write_named_uint32(w, "io_queue_requests", g_opts.io_queue_requests);

	spdk_json_write_object_end(w);



	spdk_json_write_object_end(w);

lib/bdev/nvme/bdev_nvme.h

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ struct spdk_bdev_nvme_opts { 
	uint32_t retry_count;

	uint64_t nvme_adminq_poll_period_us;

	uint64_t nvme_ioq_poll_period_us;

	uint32_t io_queue_requests;

};



typedef void (*spdk_bdev_create_nvme_fn)(void *ctx, int rc);

lib/bdev/nvme/bdev_nvme_rpc.c

+1 −0
Original line number Diff line number Diff line
@@ -75,6 +75,7 @@ static const struct spdk_json_object_decoder rpc_bdev_nvme_options_decoders[] = 
	{"retry_count", offsetof(struct spdk_bdev_nvme_opts, retry_count), spdk_json_decode_uint32, true},

	{"nvme_adminq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_adminq_poll_period_us), spdk_json_decode_uint64, true},

	{"nvme_ioq_poll_period_us", offsetof(struct spdk_bdev_nvme_opts, nvme_ioq_poll_period_us), spdk_json_decode_uint64, true},

	{"io_queue_requests", offsetof(struct spdk_bdev_nvme_opts, io_queue_requests), spdk_json_decode_uint32, true},

};



static void