Commit d20b90b2 authored by Cunyin Chang's avatar Cunyin Chang Committed by Jim Harris
Browse files

nvmf: Add support for RPC interfaces. 



Change-Id: I6f0fe35bf2876df181ad11294b62d64d97dcac2c
Signed-off-by: default avatarCunyin Chang <cunyin.chang@intel.com>
parent 1be062de

app/nvmf_tgt/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ CFLAGS += $(DPDK_INC) 
# TODO: remove this once NVMf has a public API header

CFLAGS += -I$(SPDK_ROOT_DIR)/lib



C_SRCS := conf.c nvmf_tgt.c

C_SRCS := conf.c nvmf_tgt.c nvmf_rpc.c



SPDK_LIBS = \

	$(SPDK_ROOT_DIR)/lib/nvmf/libspdk_nvmf.a \

app/nvmf_tgt/conf.c

+154 −1
Original line number Diff line number Diff line
@@ -310,7 +310,7 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctr 
	SPDK_NOTICELOG("Attaching NVMe device %x:%x:%x.%x to subsystem %s\n",

		       found_domain, found_bus, found_dev, found_func, ctx->subsystem->subnqn);



	rc = nvmf_subsystem_add_ctrlr(ctx->subsystem, ctrlr);

	rc = nvmf_subsystem_add_ctrlr(ctx->subsystem, ctrlr, dev);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to add controller to subsystem\n");

	}
@@ -566,3 +566,156 @@ spdk_nvmf_parse_conf(void) 


	return 0;

}



int

spdk_nvmf_parse_subsystem_for_rpc(const char *name,

				  const char *mode, uint32_t lcore,

				  int num_listen_addresses, struct rpc_listen_address *addresses,

				  int num_hosts, char *hosts[], const char *bdf,

				  const char *sn, int num_devs, char *dev_list[])

{

	struct spdk_nvmf_subsystem *subsystem;

	struct nvmf_tgt_subsystem *app_subsys;

	int i, ret;

	uint64_t mask;

	int num = 0;



	if (name == NULL) {

		SPDK_ERRLOG("No NQN specified for Subsystem %d\n", num);

		return -1;

	}



	if (num_listen_addresses > MAX_LISTEN_ADDRESSES) {

		SPDK_ERRLOG("invalid listen adresses number\n");

		return -1;

	}



	if (num_hosts > MAX_HOSTS) {

		SPDK_ERRLOG("invalid hosts number\n");

		return -1;

	}



	app_subsys = nvmf_tgt_subsystem_first();

	while (app_subsys) {

		if (num < app_subsys->subsystem->num) {

			num = app_subsys->subsystem->num + 1;

		}

		app_subsys = nvmf_tgt_subsystem_next(app_subsys);

	}



	/* Determine which core to assign to the subsystem */

	mask = spdk_app_get_core_mask();

	lcore = spdk_nvmf_allocate_lcore(mask, lcore);



	app_subsys = nvmf_tgt_create_subsystem(num, name, SPDK_NVMF_SUBTYPE_NVME, lcore);

	if (app_subsys == NULL) {

		SPDK_ERRLOG("Subsystem creation failed\n");

		return -1;

	}

	subsystem = app_subsys->subsystem;



	if (mode == NULL) {

		SPDK_ERRLOG("No Mode specified for Subsystem %d\n", num);

		return -1;

	}



	if (strcasecmp(mode, "Direct") == 0) {

		subsystem->mode = NVMF_SUBSYSTEM_MODE_DIRECT;

	} else if (strcasecmp(mode, "Virtual") == 0) {

		subsystem->mode = NVMF_SUBSYSTEM_MODE_VIRTUAL;

	} else {

		SPDK_ERRLOG("Invalid Subsystem mode: %s\n", mode);

		return -1;

	}



	/* Parse Listen sections */

	for (i = 0; i < num_listen_addresses; i++) {

		const struct spdk_nvmf_transport *transport;



		transport = spdk_nvmf_transport_get(addresses[i].transport);

		if (transport == NULL) {

			SPDK_ERRLOG("Unknown transport type '%s'\n", addresses[i].transport);

			return -1;

		}



		spdk_nvmf_subsystem_add_listener(subsystem, transport, addresses[i].traddr, addresses[i].trsvcid);

	}



	/* Parse Host sections */

	for (i = 0; i < num_hosts; i++) {

		char *host_nqn;



		host_nqn = hosts[i];

		if (strcmp(host_nqn, "All") == 0)

			break;

		spdk_nvmf_subsystem_add_host(subsystem, host_nqn);

	}



	if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {

		struct spdk_nvmf_probe_ctx ctx = { 0 };



		if (bdf == NULL) {

			SPDK_ERRLOG("Subsystem %d: missing NVMe directive\n", num);

			return -1;

		}



		ctx.subsystem = subsystem;

		ctx.found = false;

		if (strcmp(bdf, "*") == 0) {

			ctx.any = true;

		} else {

			ret = sscanf(bdf, "%x:%x:%x.%x", &ctx.domain, &ctx.bus, &ctx.device, &ctx.function);

			if (ret != 4) {

				SPDK_ERRLOG("Invalid format for NVMe BDF: %s\n", bdf);

				return -1;

			}

			ctx.any = false;

		}



		if (spdk_nvme_probe(&ctx, probe_cb, attach_cb, NULL)) {

			SPDK_ERRLOG("One or more controllers failed in spdk_nvme_probe()\n");

		}

	} else {

		struct spdk_bdev *bdev;

		const char *namespace;



		if (sn == NULL) {

			SPDK_ERRLOG("Subsystem %d: missing serial number\n", num);

			return -1;

		}

		if (spdk_nvmf_validate_sn(sn) != 0) {

			return -1;

		}



		if (num_devs > MAX_VIRTUAL_NAMESPACE) {

			return -1;

		}



		subsystem->dev.virtual.ns_count = 0;

		snprintf(subsystem->dev.virtual.sn, MAX_SN_LEN, "%s", sn);

		subsystem->ops = &spdk_nvmf_virtual_ctrlr_ops;



		for (i = 0; i < num_devs; i++) {

			namespace = dev_list[i];

			if (!namespace) {

				SPDK_ERRLOG("Namespace %d: missing block device\n", i);

				return -1;

			}

			bdev = spdk_bdev_get_by_name(namespace);

			if (spdk_nvmf_subsystem_add_ns(subsystem, bdev)) {

				return -1;

			}



			SPDK_NOTICELOG("Attaching block device %s to subsystem %s\n",

				       bdev->name, subsystem->subnqn);



		}

	}



	ret = spdk_nvmf_acceptor_init();

	if (ret < 0) {

		SPDK_ERRLOG("spdk_nvmf_acceptor_start() failed\n");

		return -1;

	}

	return 0;

}

app/nvmf_tgt/nvmf_rpc.c

0 → 100644
+390 −0
Original line number Diff line number Diff line 
/*-

 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation.

 *   All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions

 *   are met:

 *

 *     * Redistributions of source code must retain the above copyright

 *       notice, this list of conditions and the following disclaimer.

 *     * Redistributions in binary form must reproduce the above copyright

 *       notice, this list of conditions and the following disclaimer in

 *       the documentation and/or other materials provided with the

 *       distribution.

 *     * Neither the name of Intel Corporation nor the names of its

 *       contributors may be used to endorse or promote products derived

 *       from this software without specific prior written permission.

 *

 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */



#include <stdlib.h>

#include <stdio.h>

#include <pciaccess.h>

#include "nvmf_tgt.h"

#include "nvmf/subsystem.h"

#include "nvmf/transport.h"

#include "spdk/log.h"

#include "spdk/rpc.h"

#include "spdk/pci.h"

#include "spdk/nvme.h"



static void

dump_nvmf_subsystem(struct spdk_json_write_ctx *w, struct spdk_nvmf_subsystem *subsystem)

{

	struct spdk_nvmf_listen_addr *listen_addr;

	struct spdk_nvmf_host		*host;



	spdk_json_write_object_begin(w);



	spdk_json_write_name(w, "core");

	spdk_json_write_int32(w, subsystem->lcore);



	spdk_json_write_name(w, "nqn");

	spdk_json_write_string(w, subsystem->subnqn);

	if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) {

		spdk_json_write_name(w, "mode");

		if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {

			spdk_json_write_string(w, "direct");

		} else {

			spdk_json_write_string(w, "virtual");

		}

	}

	spdk_json_write_name(w, "subtype");

	if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) {

		spdk_json_write_string(w, "NVMe");

	} else {

		spdk_json_write_string(w, "Discovery");

	}



	spdk_json_write_name(w, "listen_addrs");

	spdk_json_write_array_begin(w);



	TAILQ_FOREACH(listen_addr, &subsystem->listen_addrs, link) {

		spdk_json_write_object_begin(w);

		spdk_json_write_name(w, "transport");

		spdk_json_write_string(w, listen_addr->transport->name);

		spdk_json_write_name(w, "traddr");

		spdk_json_write_string(w, listen_addr->traddr);

		spdk_json_write_name(w, "trsvcid");

		spdk_json_write_string(w, listen_addr->trsvcid);

		spdk_json_write_object_end(w);

	}

	spdk_json_write_array_end(w);



	spdk_json_write_name(w, "hosts");

	spdk_json_write_array_begin(w);



	TAILQ_FOREACH(host, &subsystem->hosts, link) {

		spdk_json_write_object_begin(w);

		spdk_json_write_name(w, "nqn");

		spdk_json_write_string(w, host->nqn);

		spdk_json_write_object_end(w);

	}

	spdk_json_write_array_end(w);



	if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) {

		if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {

			struct spdk_pci_device *dev = subsystem->dev.direct.pci_dev;

			uint16_t found_domain = spdk_pci_device_get_domain(dev);

			uint8_t found_bus	 = spdk_pci_device_get_bus(dev);

			uint8_t found_dev	 = spdk_pci_device_get_dev(dev);

			uint8_t found_func	 = spdk_pci_device_get_func(dev);

			spdk_json_write_name(w, "pci_address");

			spdk_json_write_object_begin(w);

			spdk_json_write_name(w, "domain");

			spdk_json_write_int32(w, found_domain);

			spdk_json_write_name(w, "bus");

			spdk_json_write_int32(w, found_bus);

			spdk_json_write_name(w, "device");

			spdk_json_write_int32(w, found_dev);

			spdk_json_write_name(w, "function");

			spdk_json_write_int32(w, found_func);

			spdk_json_write_object_end(w);

		} else {

			int i;

			spdk_json_write_name(w, "serial_number");

			spdk_json_write_string(w, subsystem->dev.virtual.sn);

			spdk_json_write_name(w, "namespaces");

			spdk_json_write_array_begin(w);

			for (i = 0; i < subsystem->dev.virtual.ns_count; i++) {

				spdk_json_write_object_begin(w);

				spdk_json_write_name(w, "nsid");

				spdk_json_write_int32(w, i + 1);

				spdk_json_write_name(w, "name");

				spdk_json_write_string(w, subsystem->dev.virtual.ns_list[i]->name);

				spdk_json_write_object_end(w);

			}

			spdk_json_write_array_end(w);



		}

	}

	spdk_json_write_object_end(w);

}



static void

spdk_rpc_get_nvmf_subsystems(struct spdk_jsonrpc_server_conn *conn,

			     const struct spdk_json_val *params,

			     const struct spdk_json_val *id)

{

	struct spdk_json_write_ctx *w;

	struct nvmf_tgt_subsystem	*tgt_subsystem;



	if (params != NULL) {

		spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS,

						 "get_nvmf_subsystems requires no parameters");

		return;

	}



	if (id == NULL) {

		return;

	}



	w = spdk_jsonrpc_begin_result(conn, id);

	spdk_json_write_array_begin(w);

	tgt_subsystem = nvmf_tgt_subsystem_first();

	while (tgt_subsystem) {

		dump_nvmf_subsystem(w, tgt_subsystem->subsystem);

		tgt_subsystem = nvmf_tgt_subsystem_next(tgt_subsystem);

	}

	spdk_json_write_array_end(w);

	spdk_jsonrpc_end_result(conn, w);

}

SPDK_RPC_REGISTER("get_nvmf_subsystems", spdk_rpc_get_nvmf_subsystems)



#define RPC_MAX_LISTEN_ADDRESSES 255

#define RPC_MAX_HOSTS 255



struct rpc_listen_addresses {

	size_t num_listen_address;

	struct rpc_listen_address addresses[RPC_MAX_LISTEN_ADDRESSES];

};



static const struct spdk_json_object_decoder rpc_listen_address_decoders[] = {

	{"transport", offsetof(struct rpc_listen_address, transport), spdk_json_decode_string},

	{"traddr", offsetof(struct rpc_listen_address, traddr), spdk_json_decode_string},

	{"trsvcid", offsetof(struct rpc_listen_address, trsvcid), spdk_json_decode_string},

};



static int

decode_rpc_listen_address(const struct spdk_json_val *val, void *out)

{

	struct rpc_listen_address *req = (struct rpc_listen_address *)out;

	if (spdk_json_decode_object(val, rpc_listen_address_decoders,

				    sizeof(rpc_listen_address_decoders) / sizeof(*rpc_listen_address_decoders),

				    req)) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n");

		return -1;

	}

	return 0;

}



static int

decode_rpc_listen_addresses(const struct spdk_json_val *val, void *out)

{

	struct rpc_listen_addresses *listen_addresses = out;

	return spdk_json_decode_array(val, decode_rpc_listen_address, &listen_addresses->addresses,

				      RPC_MAX_LISTEN_ADDRESSES,

				      &listen_addresses->num_listen_address, sizeof(struct rpc_listen_address));

}



struct rpc_hosts {

	size_t num_hosts;

	char *hosts[RPC_MAX_HOSTS];

};



static int

decode_rpc_hosts(const struct spdk_json_val *val, void *out)

{

	struct rpc_hosts *rpc_hosts = out;



	return spdk_json_decode_array(val, spdk_json_decode_string, rpc_hosts->hosts, RPC_MAX_HOSTS,

				      &rpc_hosts->num_hosts, sizeof(char *));

}







struct rpc_dev_names {

	size_t num_names;

	char *names[MAX_VIRTUAL_NAMESPACE];

};



static int

decode_rpc_dev_names(const struct spdk_json_val *val, void *out)

{

	struct rpc_dev_names *dev_names = out;



	return spdk_json_decode_array(val, spdk_json_decode_string, dev_names->names,

				      MAX_VIRTUAL_NAMESPACE,

				      &dev_names->num_names, sizeof(char *));

}



static void

free_rpc_dev_names(struct rpc_dev_names *r)

{

	size_t i;



	for (i = 0; i < r->num_names; i++) {

		free(r->names[i]);

	}

}



static void

free_rpc_listen_addresses(struct rpc_listen_addresses *r)

{

	size_t i;



	for (i = 0; i < r->num_listen_address; i++) {

		free(r->addresses[i].transport);

		free(r->addresses[i].traddr);

		free(r->addresses[i].trsvcid);

	}

}



static void

free_rpc_hosts(struct rpc_hosts *r)

{

	size_t i;



	for (i = 0; i < r->num_hosts; i++) {

		free(r->hosts[i]);

	}

}



struct rpc_subsystem {

	int32_t core;

	char *mode;

	char *nqn;

	struct rpc_listen_addresses listen_addresses;

	struct rpc_hosts hosts;

	char *pci_address;

	char *serial_number;

	struct rpc_dev_names namespaces;

};



static void

free_rpc_subsystem(struct rpc_subsystem *req)

{

	free_rpc_dev_names(&req->namespaces);

	free_rpc_listen_addresses(&req->listen_addresses);

	free_rpc_hosts(&req->hosts);

}



static const struct spdk_json_object_decoder rpc_subsystem_decoders[] = {

	{"core", offsetof(struct rpc_subsystem, core), spdk_json_decode_int32, true},

	{"mode", offsetof(struct rpc_subsystem, mode), spdk_json_decode_string},

	{"nqn", offsetof(struct rpc_subsystem, nqn), spdk_json_decode_string},

	{"listen_addresses", offsetof(struct rpc_subsystem, listen_addresses), decode_rpc_listen_addresses},

	{"hosts", offsetof(struct rpc_subsystem, hosts), decode_rpc_hosts},

	{"pci_address", offsetof(struct rpc_subsystem, pci_address), spdk_json_decode_string, true},

	{"serial_number", offsetof(struct rpc_subsystem, serial_number), spdk_json_decode_string, true},

	{"namespaces", offsetof(struct rpc_subsystem, namespaces), decode_rpc_dev_names, true},

};



static void

spdk_rpc_construct_nvmf_subsystem(struct spdk_jsonrpc_server_conn *conn,

				  const struct spdk_json_val *params,

				  const struct spdk_json_val *id)

{

	struct rpc_subsystem req = {};

	struct spdk_json_write_ctx *w;

	int ret;



	if (spdk_json_decode_object(params, rpc_subsystem_decoders,

				    sizeof(rpc_subsystem_decoders) / sizeof(*rpc_subsystem_decoders),

				    &req)) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n");

		goto invalid;

	}



	ret = spdk_nvmf_parse_subsystem_for_rpc(req.nqn, req.mode, req.core,

						req.listen_addresses.num_listen_address,

						req.listen_addresses.addresses,

						req.hosts.num_hosts, req.hosts.hosts, req.pci_address,

						req.serial_number,

						req.namespaces.num_names, req.namespaces.names);

	if (ret) {

		goto invalid;

	}



	free_rpc_subsystem(&req);



	w = spdk_jsonrpc_begin_result(conn, id);

	spdk_json_write_bool(w, true);

	spdk_jsonrpc_end_result(conn, w);

	return;



invalid:

	spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");

	free_rpc_subsystem(&req);

}

SPDK_RPC_REGISTER("construct_nvmf_subsystem", spdk_rpc_construct_nvmf_subsystem)



struct rpc_delete_subsystem {

	char *nqn;

};



static void

free_rpc_delete_subsystem(struct rpc_delete_subsystem *r)

{

	free(r->nqn);

}



static const struct spdk_json_object_decoder rpc_delete_subsystem_decoders[] = {

	{"nqn", offsetof(struct rpc_delete_subsystem, nqn), spdk_json_decode_string},

};



static void

spdk_rpc_delete_nvmf_subsystem(struct spdk_jsonrpc_server_conn *conn,

			       const struct spdk_json_val *params,

			       const struct spdk_json_val *id)

{

	struct rpc_delete_subsystem req = {};

	struct spdk_json_write_ctx *w;



	if (spdk_json_decode_object(params, rpc_delete_subsystem_decoders,

				    sizeof(rpc_delete_subsystem_decoders) / sizeof(*rpc_delete_subsystem_decoders),

				    &req)) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n");

		goto invalid;

	}



	if (req.nqn == NULL) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "missing name param\n");

		goto invalid;

	}



	if (nvmf_tgt_shutdown_subsystem_by_nqn(req.nqn)) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "shutdown_subsystem failed\n");

		goto invalid;

	}



	free_rpc_delete_subsystem(&req);



	if (id == NULL) {

		return;

	}



	w = spdk_jsonrpc_begin_result(conn, id);

	spdk_json_write_bool(w, true);

	spdk_jsonrpc_end_result(conn, w);

	return;



invalid:

	spdk_jsonrpc_send_error_response(conn, id, SPDK_JSONRPC_ERROR_INVALID_PARAMS, "Invalid parameters");

	free_rpc_delete_subsystem(&req);

}

SPDK_RPC_REGISTER("delete_nvmf_subsystem", spdk_rpc_delete_nvmf_subsystem)

app/nvmf_tgt/nvmf_tgt.c

+27 −0
Original line number Diff line number Diff line
@@ -266,6 +266,33 @@ nvmf_tgt_delete_subsystems(void) 
	}

}



struct nvmf_tgt_subsystem *

nvmf_tgt_subsystem_first(void)

{

	return TAILQ_FIRST(&g_subsystems);

}



struct nvmf_tgt_subsystem *

nvmf_tgt_subsystem_next(struct nvmf_tgt_subsystem *subsystem)

{

	return TAILQ_NEXT(subsystem, tailq);

}



int

nvmf_tgt_shutdown_subsystem_by_nqn(const char *nqn)

{

	struct nvmf_tgt_subsystem *tgt_subsystem, *subsys_tmp;



	TAILQ_FOREACH_SAFE(tgt_subsystem, &g_subsystems, tailq, subsys_tmp) {

		if (strcmp(tgt_subsystem->subsystem->subnqn, nqn) == 0) {

			TAILQ_REMOVE(&g_subsystems, tgt_subsystem, tailq);

			nvmf_tgt_delete_subsystem(tgt_subsystem);

			return 0;

		}

	}

	return -1;

}



static void

usage(void)

{

app/nvmf_tgt/nvmf_tgt.h

+22 −0
Original line number Diff line number Diff line
@@ -39,6 +39,12 @@ 
#include "spdk/nvmf_spec.h"

#include "spdk/queue.h"



struct rpc_listen_address {

	char *transport;

	char *traddr;

	char *trsvcid;

};



struct spdk_nvmf_tgt_conf {

	uint32_t acceptor_lcore;

};
@@ -54,10 +60,26 @@ struct nvmf_tgt_subsystem { 


extern struct spdk_nvmf_tgt_conf g_spdk_nvmf_tgt_conf;



struct nvmf_tgt_subsystem *

nvmf_tgt_subsystem_first(void);



struct nvmf_tgt_subsystem *

nvmf_tgt_subsystem_next(struct nvmf_tgt_subsystem *subsystem);



int spdk_nvmf_parse_conf(void);



struct nvmf_tgt_subsystem *nvmf_tgt_create_subsystem(int num,

		const char *name,

		enum spdk_nvmf_subtype subtype,

		uint32_t lcore);



int

spdk_nvmf_parse_subsystem_for_rpc(const char *name,

				  const char *mode, uint32_t lcore,

				  int num_listen_addresses, struct rpc_listen_address *addresses,

				  int num_hosts, char *hosts[], const char *bdf,

				  const char *sn, int num_devs, char *dev_list[]);



int

nvmf_tgt_shutdown_subsystem_by_nqn(const char *nqn);

#endif