Commit 29342a9e authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

external_code/nvme: device attach/detach 



The NVMe devices can now be enumerated, attached, and detached.  To
simplify the driver, the probing step has been omitted - all available
controllers are attached and need to be detached later.

The driver registers itself as a PCI driver via a call to
SPDK_PCI_DRIVER_REGISTER() and then uses spdk_pci_enumerate() to
enumerate available NVMe devices and attach them.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: Id03e2f4365f4f7ca98178be70278d0c4b7b34b26
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6664


Community-CI: Broadcom CI
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 91b30e7a

test/external_code/nvme/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -33,7 +33,7 @@ 


PKG_CONFIG_PATH = $(SPDK_LIB_DIR)/pkgconfig



DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_log)

DEPLIBS := $(shell PKG_CONFIG_PATH="$(PKG_CONFIG_PATH)" pkg-config --libs spdk_env_dpdk)



shared:

	$(CC) $(COMMON_CFLAGS) -c -fPIC nvme.c -o nvme.o

test/external_code/nvme/identify.c

+24 −1
Original line number Diff line number Diff line
@@ -34,14 +34,37 @@ 
#include "spdk/stdinc.h"

#include "nvme.h"



static void

attach_cb(void *cb_ctx, const struct spdk_pci_addr *addr,

	  struct nvme_ctrlr *ctrlr)

{

	char fmtaddr[32] = {};



	(void)cb_ctx;



	spdk_pci_addr_fmt(fmtaddr, sizeof(fmtaddr), addr);

	printf("Found NVMe controller at: %s\n", fmtaddr);



	nvme_detach(ctrlr);

}



int

main(int argc, const char **argv)

{

	struct spdk_env_opts opts;

	int rc;



	(void)argc;



	rc = nvme_probe(NULL, NULL);

	spdk_env_opts_init(&opts);

	opts.name = "identify";



	if (spdk_env_init(&opts) != 0) {

		fprintf(stderr, "%s: unable to initialize SPDK env\n", argv[0]);

		return 1;

	}



	rc = nvme_probe(attach_cb, NULL);

	if (rc != 0) {

		fprintf(stderr, "%s: nvme probe failed\n", argv[0]);

		return 1;

test/external_code/nvme/nvme.c

+81 −4
Original line number Diff line number Diff line
@@ -35,19 +35,96 @@ 
#include "spdk/stdinc.h"

#include "nvme.h"



struct nvme_ctrlr {

	/* Underlying PCI device */

	struct spdk_pci_device	*pci_device;

	TAILQ_ENTRY(nvme_ctrlr)	tailq;

};



static struct spdk_pci_id nvme_pci_driver_id[] = {

	{

		.class_id = SPDK_PCI_CLASS_NVME,

		.vendor_id = SPDK_PCI_ANY_ID,

		.device_id = SPDK_PCI_ANY_ID,

		.subvendor_id = SPDK_PCI_ANY_ID,

		.subdevice_id = SPDK_PCI_ANY_ID,

	},

	{ .vendor_id = 0, /* sentinel */ },

};



SPDK_PCI_DRIVER_REGISTER(nvme_external, nvme_pci_driver_id, SPDK_PCI_DRIVER_NEED_MAPPING);

static TAILQ_HEAD(, nvme_ctrlr) g_nvme_ctrlrs = TAILQ_HEAD_INITIALIZER(g_nvme_ctrlrs);



static int

pcie_enum_cb(void *ctx, struct spdk_pci_device *pci_dev)

{

	struct nvme_ctrlr *ctrlr;

	TAILQ_HEAD(, nvme_ctrlr) *ctrlrs = ctx;

	char addr[32] = {};



	spdk_pci_addr_fmt(addr, sizeof(addr), &pci_dev->addr);



	ctrlr = calloc(1, sizeof(*ctrlr));

	if (!ctrlr) {

		SPDK_ERRLOG("Failed to allocate NVMe controller: %s\n", addr);

		return -1;

	}



	if (spdk_pci_device_claim(pci_dev)) {

		SPDK_ERRLOG("Failed to claim PCI device: %s\n", addr);

		free(ctrlr);

		return -1;

	}



	ctrlr->pci_device = pci_dev;

	TAILQ_INSERT_TAIL(ctrlrs, ctrlr, tailq);



	return 0;

}



static void

free_ctrlr(struct nvme_ctrlr *ctrlr)

{

	spdk_pci_device_unclaim(ctrlr->pci_device);

	spdk_pci_device_detach(ctrlr->pci_device);

	free(ctrlr);

}



int

nvme_probe(nvme_attach_cb attach_cb, void *cb_ctx)

{

	(void)attach_cb;

	(void)cb_ctx;

	struct nvme_ctrlr *ctrlr, *tmp;

	TAILQ_HEAD(, nvme_ctrlr) ctrlrs = TAILQ_HEAD_INITIALIZER(ctrlrs);

	int rc;



	rc = spdk_pci_enumerate(spdk_pci_get_driver("nvme_external"),

				pcie_enum_cb, &ctrlrs);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to enumerate PCI devices\n");

		while (!TAILQ_EMPTY(&ctrlrs)) {

			ctrlr = TAILQ_FIRST(&ctrlrs);

			TAILQ_REMOVE(&ctrlrs, ctrlr, tailq);

			free_ctrlr(ctrlr);

		}



		return rc;

	}



	TAILQ_FOREACH_SAFE(ctrlr, &ctrlrs, tailq, tmp) {

		TAILQ_REMOVE(&ctrlrs, ctrlr, tailq);

		TAILQ_INSERT_TAIL(&g_nvme_ctrlrs, ctrlr, tailq);



		attach_cb(cb_ctx, &ctrlr->pci_device->addr, ctrlr);

	}



	return -ENOTSUP;

	return 0;

}



void

nvme_detach(struct nvme_ctrlr *ctrlr)

{

	(void) ctrlr;

	TAILQ_REMOVE(&g_nvme_ctrlrs, ctrlr, tailq);

	free_ctrlr(ctrlr);

}



SPDK_LOG_REGISTER_COMPONENT(nvme_external)

test/external_code/nvme/nvme.h

+8 −2
Original line number Diff line number Diff line
@@ -42,8 +42,8 @@ struct nvme_ctrlr; 
 * Callback for nvme_probe() to report a device that has been attached to

 * the userspace NVMe driver.

 *

 * \param cb_ctx Opaque value passed to spdk_nvme_attach_cb().

 * \param addr NVMe PCI address.

 * \param cb_ctx Opaque value passed to nvme_attach_cb().

 * \param addr The PCI address of the NVMe controller.

 * \param ctrlr Opaque handle to NVMe controller.

 */

typedef void (*nvme_attach_cb)(void *cb_ctx, const struct spdk_pci_addr *addr,
@@ -52,6 +52,9 @@ typedef void (*nvme_attach_cb)(void *cb_ctx, const struct spdk_pci_addr *addr, 
/**

 * Enumerate PCIe bus and attach all NVMe devices found to the driver.

 *

 * This function is not thread safe and should only be called from one thread at

 * a time while no other threads are actively using any NVMe devices.

 *

 * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of

 * the callbacks.

 * \param attach_cb will be called for each NVMe device found
@@ -64,6 +67,9 @@ int nvme_probe(nvme_attach_cb attach_cb, void *ctx); 
 * Detach specified device returned by nvme_probe()'s attach_cb. After returning

 * the nvme_ctrlr handle is no longer valid.

 *

 * This function should be called from a single thread while no other threads

 * are actively using the NVMe device.

 *

 * \param ctrlr Opaque handle to NVMe controller.

 */

void nvme_detach(struct nvme_ctrlr *ctrlr);