Commit 60241941 authored by Karl Bonde Torp's avatar Karl Bonde Torp Committed by Konrad Sztyber
Browse files

nvmf: add support for a passthrough subsystem 



The intention is to enable the use of unsupported/custom NVMe command
sets, e.g. KV, without having to implement support for a specific csi.

Only namespaces supporting NVMe passthrough for both I/O and admin
commands can be added to the passthrough subsystem.

I/O commands and admin commands with a specified NSID will be sent via
the passthrough path.

Admin commands meant for the controller will be handled by the subsystem
as usual.

Change-Id: I5c47ffcee8834187d3e0bcc1f60388c03256c46f
Signed-off-by: default avatarKarl Bonde Torp <k.torp@samsung.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24057


Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
parent c156f3e1

doc/jsonrpc.md

+1 −0
Original line number Diff line number Diff line
@@ -8272,6 +8272,7 @@ min_cntlid | Optional | number | Minimum controller ID. Def 
max_cntlid                 | Optional | number      | Maximum controller ID. Default: 0xffef

max_discard_size_kib       | Optional | number      | Maximum discard size (Kib). Default: 0

max_write_zeroes_size_kib  | Optional | number      | Maximum write_zeroes size (Kib). Default: 0

passthrough                | Optional | boolean     | Use NVMe passthrough for I/O commands and namespace-directed admin commands. Default: `false`.



#### Example

lib/nvmf/ctrlr.c

+15 −0
Original line number Diff line number Diff line
@@ -56,6 +56,7 @@ static struct spdk_nvmf_custom_admin_cmd g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME 


static void _nvmf_request_complete(void *ctx);

int nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr);

static int nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req);



static inline void

nvmf_invalid_connect_response(struct spdk_nvmf_fabric_connect_rsp *rsp,
@@ -3768,6 +3769,13 @@ nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req) 
		}

	}



	/* We only want to send passthrough admin commands to namespaces.

	 * However, we don't want to passthrough a command with intended for all namespaces.

	 */

	if (ctrlr->subsys->passthrough && cmd->nsid && cmd->nsid != SPDK_NVME_GLOBAL_NS_TAG) {

		return nvmf_passthru_admin_cmd(req);

	}



	switch (cmd->opc) {

	case SPDK_NVME_OPC_GET_LOG_PAGE:

		return nvmf_ctrlr_get_log_page(req);
@@ -4434,6 +4442,13 @@ nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 
		qpair->first_fused_req = NULL;

	}



	if (ctrlr->subsys->passthrough) {

		assert(ns->passthrough_nsid > 0);

		req->cmd->nvme_cmd.nsid = ns->passthrough_nsid;



		return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req);

	}



	if (spdk_nvmf_request_using_zcopy(req)) {

		assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT);

		return nvmf_bdev_ctrlr_zcopy_start(bdev, desc, ch, req);

lib/nvmf/nvmf_internal.h

+1 −0
Original line number Diff line number Diff line
@@ -350,6 +350,7 @@ struct spdk_nvmf_subsystem { 
	TAILQ_HEAD(, nvmf_subsystem_state_change_ctx)	state_changes;

	/* In-band authentication sequence number, protected by ->mutex */

	uint32_t					auth_seqnum;

	bool						passthrough;

};



static int

lib/nvmf/nvmf_rpc.c

+4 −0
Original line number Diff line number Diff line
@@ -332,6 +332,7 @@ struct rpc_subsystem_create { 
	uint16_t max_cntlid;

	uint64_t max_discard_size_kib;

	uint64_t max_write_zeroes_size_kib;

	bool passthrough;

};



static const struct spdk_json_object_decoder rpc_subsystem_create_decoders[] = {
@@ -346,6 +347,7 @@ static const struct spdk_json_object_decoder rpc_subsystem_create_decoders[] = { 
	{"max_cntlid", offsetof(struct rpc_subsystem_create, max_cntlid), spdk_json_decode_uint16, true},

	{"max_discard_size_kib", offsetof(struct rpc_subsystem_create, max_discard_size_kib), spdk_json_decode_uint64, true},

	{"max_write_zeroes_size_kib", offsetof(struct rpc_subsystem_create, max_write_zeroes_size_kib), spdk_json_decode_uint64, true},

	{"passthrough", offsetof(struct rpc_subsystem_create, passthrough), spdk_json_decode_bool, true},

};



static void
@@ -452,6 +454,8 @@ rpc_nvmf_create_subsystem(struct spdk_jsonrpc_request *request, 
		goto cleanup;

	}



	subsystem->passthrough = req->passthrough;



	rc = spdk_nvmf_subsystem_start(subsystem,

				       rpc_nvmf_subsystem_started,

				       request);

lib/nvmf/subsystem.c

+3 −0
Original line number Diff line number Diff line
@@ -2135,6 +2135,9 @@ spdk_nvmf_subsystem_add_ns_ext(struct spdk_nvmf_subsystem *subsystem, const char 
		     strlen(spdk_bdev_get_module_name(ns->bdev)))) {

		nvme_ns = (struct spdk_nvme_ns *) spdk_bdev_get_module_ctx(ns->desc);

		ns->passthrough_nsid = spdk_nvme_ns_get_id(nvme_ns);

	} else if (subsystem->passthrough) {

		SPDK_ERRLOG("Only bdev_nvme namespaces can be added to a passthrough subsystem.\n");

		goto err;

	}



	/* Cache the zcopy capability of the bdev device */