Commit fd6ebca7 authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvmf: only save PCI address in direct ctrlr data 



The NVMe over Fabrics target was storing the PCI device pointer for each
direct-mode controller, but it only really needs the PCI address, which
is exposed via the get_nvmf_subsystems RPC.

Also update the same code path to use the new spdk_pci_device_get_addr()
function for brevity.

Change-Id: I0708b3331b7c279c1a86f0d7459b5deb40dd7c89
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent ed1e7ca2

app/nvmf_tgt/conf.c

+4 −7
Original line number Diff line number Diff line
@@ -366,19 +366,16 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctr 
	  const struct spdk_nvme_ctrlr_opts *opts)

{

	struct spdk_nvmf_probe_ctx *ctx = cb_ctx;

	uint16_t found_domain = spdk_pci_device_get_domain(dev);

	uint8_t found_bus    = spdk_pci_device_get_bus(dev);

	uint8_t found_dev    = spdk_pci_device_get_dev(dev);

	uint8_t found_func   = spdk_pci_device_get_func(dev);

	struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(dev);

	int rc;

	char path[MAX_STRING_LEN];

	int numa_node = -1;



	SPDK_NOTICELOG("Attaching NVMe device %p at %x:%x:%x.%x to subsystem %p\n",

		       ctrlr, found_domain, found_bus, found_dev, found_func, ctx->subsystem);

		       ctrlr, pci_addr.domain, pci_addr.bus, pci_addr.dev, pci_addr.func, ctx->subsystem);



	snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%1u/numa_node",

		 found_domain, found_bus, found_dev, found_func);

		 pci_addr.domain, pci_addr.bus, pci_addr.dev, pci_addr.func);



	numa_node = spdk_get_numa_node_value(path);

	if (numa_node >= 0) {
@@ -391,7 +388,7 @@ attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctr 
		}

	}



	rc = nvmf_subsystem_add_ctrlr(ctx->subsystem, ctrlr, dev);

	rc = nvmf_subsystem_add_ctrlr(ctx->subsystem, ctrlr, &pci_addr);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to add controller to subsystem\n");

	}

app/nvmf_tgt/nvmf_rpc.c

+4 −5
Original line number Diff line number Diff line
@@ -101,13 +101,12 @@ dump_nvmf_subsystem(struct spdk_json_write_ctx *w, struct nvmf_tgt_subsystem *tg 
	if (subsystem->subtype == SPDK_NVMF_SUBTYPE_NVME) {

		if (subsystem->mode == NVMF_SUBSYSTEM_MODE_DIRECT) {

			char pci_str[20];

			struct spdk_pci_device *dev = subsystem->dev.direct.pci_dev;



			snprintf(pci_str, sizeof(pci_str), "%04x:%02x:%02x.%x",

				 spdk_pci_device_get_domain(dev),

				 spdk_pci_device_get_bus(dev),

				 spdk_pci_device_get_dev(dev),

				 spdk_pci_device_get_func(dev));

				 subsystem->dev.direct.pci_addr.domain,

				 subsystem->dev.direct.pci_addr.bus,

				 subsystem->dev.direct.pci_addr.dev,

				 subsystem->dev.direct.pci_addr.func);



			spdk_json_write_name(w, "pci_address");

			spdk_json_write_string(w, pci_str);

include/spdk/nvmf.h

+3 −3
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 


#include <stdint.h>



#include "spdk/env.h"

#include "spdk/nvmf_spec.h"

#include "spdk/queue.h"
@@ -65,7 +66,6 @@ struct spdk_nvme_ctrlr; 
struct spdk_nvmf_transport;

struct spdk_nvmf_request;

struct spdk_nvmf_conn;

struct spdk_pci_device;



typedef void (*spdk_nvmf_subsystem_connect_fn)(void *cb_ctx, struct spdk_nvmf_request *req);

typedef void (*spdk_nvmf_subsystem_disconnect_fn)(void *cb_ctx, struct spdk_nvmf_conn *conn);
@@ -130,7 +130,7 @@ struct spdk_nvmf_subsystem { 
		struct {

			struct spdk_nvme_ctrlr *ctrlr;

			struct spdk_nvme_qpair *io_qpair;

			struct spdk_pci_device *pci_dev;

			struct spdk_pci_addr pci_addr;

		} direct;



		struct {
@@ -183,7 +183,7 @@ spdk_nvmf_subsystem_add_host(struct spdk_nvmf_subsystem *subsystem, 


int

nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,

			 struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *dev);

			 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_pci_addr *pci_addr);



void spdk_nvmf_subsystem_poll(struct spdk_nvmf_subsystem *subsystem);

lib/nvmf/subsystem.c

+2 −2
Original line number Diff line number Diff line
@@ -261,10 +261,10 @@ spdk_nvmf_subsystem_add_host(struct spdk_nvmf_subsystem *subsystem, char *host_n 


int

nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,

			 struct spdk_nvme_ctrlr *ctrlr, struct spdk_pci_device *dev)

			 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_pci_addr *pci_addr)

{

	subsystem->dev.direct.ctrlr = ctrlr;

	subsystem->dev.direct.pci_dev = dev;

	subsystem->dev.direct.pci_addr = *pci_addr;

	/* Assume that all I/O will be handled on one thread for now */

	subsystem->dev.direct.io_qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, 0);

	if (subsystem->dev.direct.io_qpair == NULL) {