Commit f0b52280 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

vmd: read domain's config space to get BAR's physical address 



Since commit a36bc251 ("env_dpdk: Automatically map PCI BARs into
VFIO") the physical address of the BAR returned by
spdk_pci_device_map_bar() can be either an actual physical address or a
virtual one (if IOMMU is enabled).  Since the VMD driver always needs an
actual physical address to fill out the base/limit registers and the
BARs of the devices behind the VMD, we read config space to get the
correct address, regardless of IOVA configuration.

Fixes #3117.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I16bc221a9621bb1a08950697ba4537ae2274277f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/20187


Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent bedc760d

lib/vmd/vmd.c

+32 −4
Original line number Diff line number Diff line
@@ -1162,19 +1162,47 @@ vmd_scan_pcibus(struct vmd_pci_bus *bus) 
	return dev_cnt;

}



static int

vmd_domain_map_bar(struct vmd_adapter *vmd, uint32_t bar,

		   void **vaddr, uint64_t *paddr, uint64_t *size)

{

	uint64_t unused;

	int rc;



	rc = spdk_pci_device_map_bar(vmd->pci, bar, vaddr, &unused, size);

	if (rc != 0) {

		return rc;

	}



	/* Depending on the IOVA configuration, the physical address of the BAR returned by

	 * spdk_pci_device_map_bar() can be either an actual physical address or a virtual one (if

	 * IOMMU is enabled).  Since we do need an actual physical address to fill out the

	 * base/limit registers and the BARs of the devices behind the VMD, read the config space to

	 * get the correct address, regardless of IOVA configuration. */

	rc = spdk_pci_device_cfg_read(vmd->pci, paddr, sizeof(*paddr),

				      PCI_BAR0_OFFSET + bar * PCI_BAR_SIZE);

	if (rc != 0) {

		return rc;

	}



	*paddr &= PCI_BAR_MEMORY_ADDR_OFFSET;



	return 0;

}



static int

vmd_domain_map_bars(struct vmd_adapter *vmd)

{

	int rc;



	rc = spdk_pci_device_map_bar(vmd->pci, 0, (void **)&vmd->cfg_vaddr,

	rc = vmd_domain_map_bar(vmd, 0, (void **)&vmd->cfg_vaddr,

				&vmd->cfgbar, &vmd->cfgbar_size);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to map config bar: %s\n", spdk_strerror(-rc));

		return rc;

	}



	rc = spdk_pci_device_map_bar(vmd->pci, 2, (void **)&vmd->mem_vaddr,

	rc = vmd_domain_map_bar(vmd, 2, (void **)&vmd->mem_vaddr,

				&vmd->membar, &vmd->membar_size);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to map memory bar: %s\n", spdk_strerror(-rc));

lib/vmd/vmd_spec.h

+4 −0
Original line number Diff line number Diff line
@@ -38,6 +38,10 @@ 
#define  PCI_BRIDGE_MEMORY_SHIFT    16

#define  PCI_CONFIG_ACCESS_DELAY    500



#define PCI_BAR0_OFFSET			0x10

#define PCI_BAR_SIZE			4

#define PCI_BAR_MEMORY_ADDR_OFFSET	(~0xfull)



#define PCI_MAX_CFG_SIZE            0x1000



#define PCI_HEADER_TYPE             0x0e