Commit 55bdd885 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

env/pci: add detach() callback to pci_device_provider 



This makes it possible to notify other PCI device providers (VMD) that a
PCI device is no longer used.  The VMD will driver will unhook that
device and free any resources tied to it.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I42752afbb371a1d33972dac50fd679f68d05b597
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13887


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTom Nabarro <tom.nabarro@intel.com>
parent 690eebb4

include/spdk/env.h

+9 −0
Original line number Diff line number Diff line
@@ -1182,6 +1182,15 @@ struct spdk_pci_device_provider { 
	 */

	int (*attach_cb)(const struct spdk_pci_addr *addr);



	/**

	 * Callback executed to detach a given PCI device.  The provider to detach the device is

	 * selected based on the type of the device and the name of the provider (i.e. dev->type ==

	 * provider->name).

	 *

	 * \param dev PCI device to detach.

	 */

	void (*detach_cb)(struct spdk_pci_device *dev);



	TAILQ_ENTRY(spdk_pci_device_provider) tailq;

};

lib/env_dpdk/pci.c

+18 −10
Original line number Diff line number Diff line
@@ -121,6 +121,13 @@ detach_rte_cb(void *_dev) 
	remove_rte_dev(_dev);

}



/* if it's a physical device we need to deal with DPDK on

 * a different process and we can't just unset one flag

 * here. We also want to stop using any device resources

 * so that the device isn't "in use" by the userspace driver

 * once we detach it. This would allow attaching the device

 * to a different process, or to a kernel driver like nvme.

 */

static void

detach_rte(struct spdk_pci_device *dev)

{
@@ -525,24 +532,24 @@ pci_device_fini(struct rte_pci_device *_dev) 
void

spdk_pci_device_detach(struct spdk_pci_device *dev)

{

	struct spdk_pci_device_provider *provider;



	assert(dev->internal.attached);



	if (dev->internal.claim_fd >= 0) {

		spdk_pci_device_unclaim(dev);

	}



	dev->internal.attached = false;

	if (strcmp(dev->type, "pci") == 0) {

		/* if it's a physical device we need to deal with DPDK on

		 * a different process and we can't just unset one flag

		 * here. We also want to stop using any device resources

		 * so that the device isn't "in use" by the userspace driver

		 * once we detach it. This would allow attaching the device

		 * to a different process, or to a kernel driver like nvme.

		 */

		detach_rte(dev);

	TAILQ_FOREACH(provider, &g_pci_device_providers, tailq) {

		if (strcmp(dev->type, provider->name) == 0) {

			break;

		}

	}



	assert(provider != NULL);

	dev->internal.attached = false;

	provider->detach_cb(dev);



	cleanup_pci_devices();

}
@@ -633,6 +640,7 @@ pci_attach_rte(const struct spdk_pci_addr *addr) 
static struct spdk_pci_device_provider g_pci_rte_provider = {

	.name = "pci",

	.attach_cb = pci_attach_rte,

	.detach_cb = detach_rte,

};



SPDK_PCI_REGISTER_DEVICE_PROVIDER(pci, &g_pci_rte_provider);

lib/vmd/vmd.c

+12 −0
Original line number Diff line number Diff line
@@ -1478,9 +1478,21 @@ vmd_attach_device(const struct spdk_pci_addr *addr) 
	return -ENODEV;

}



static void

vmd_detach_device(struct spdk_pci_device *pci_dev)

{

	struct vmd_pci_device *dev = SPDK_CONTAINEROF(pci_dev, struct vmd_pci_device, pci);



	assert(strcmp(spdk_pci_device_get_type(pci_dev), "vmd") == 0);

	assert(vmd_find_device(&pci_dev->addr) != NULL);



	vmd_remove_device(dev);

}



static struct spdk_pci_device_provider g_vmd_device_provider = {

	.name = "vmd",

	.attach_cb = vmd_attach_device,

	.detach_cb = vmd_detach_device,

};



SPDK_PCI_REGISTER_DEVICE_PROVIDER(vmd, &g_vmd_device_provider);

test/env/pci/pci_ut.c

+19 −0
Original line number Diff line number Diff line
@@ -99,6 +99,25 @@ ut_enum_cb(void *ctx, struct spdk_pci_device *dev) 
	return 0;

}



static int

ut_attach_cb(const struct spdk_pci_addr *addr)

{

	return -ENODEV;

}



static void

ut_detach_cb(struct spdk_pci_device *dev)

{

}



static struct spdk_pci_device_provider g_ut_provider = {

	.name = "custom",

	.attach_cb = ut_attach_cb,

	.detach_cb = ut_detach_cb,

};



SPDK_PCI_REGISTER_DEVICE_PROVIDER(ut, &g_ut_provider);



static void

pci_hook_test(void)

{