Commit 951bb3a4 authored by Darek Stojaczyk's avatar Darek Stojaczyk Committed by Jim Harris
Browse files

env/pci: move the vfio init delay to nvme/pci 



This is an NVMe-specific issue and I/OA or VirtIO devices don't
need it. Additionally, the delay is now asynchronous, meaning
that potentially multiple NVMe controllers can wait all at once.

The drawback of this change is that we're needlessly waiting
even when using uio_pci_generic. However, since the delay does
not block anymore, its impact is significantly minimized.

Change-Id: I5d16a7fd7cb66c785acb687f14690e95f6188b9e
Signed-off-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/429414


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent a084a906

lib/env_dpdk/pci.c

+0 −8
Original line number Diff line number Diff line
@@ -59,14 +59,6 @@ spdk_pci_device_init(struct rte_pci_driver *driver, 
		return 1;

	}



	if (device->kdrv == RTE_KDRV_VFIO && spdk_process_is_primary()) {

		/*

		 * TODO: This is a workaround for an issue where the device is not ready after VFIO reset.

		 * Figure out what is actually going wrong and remove this sleep.

		 */

		usleep(500 * 1000);

	}



	rc = ctx->cb_fn(ctx->cb_arg, (struct spdk_pci_device *)device);

	if (rc != 0) {

		return rc;

lib/nvme/nvme_ctrlr.c

+24 −1
Original line number Diff line number Diff line
@@ -608,6 +608,8 @@ static const char * 
nvme_ctrlr_state_string(enum nvme_ctrlr_state state)

{

	switch (state) {

	case NVME_CTRLR_STATE_INIT_DELAY:

		return "delay init";

	case NVME_CTRLR_STATE_INIT:

		return "init";

	case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1:
@@ -1869,6 +1871,22 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) 
	 * Check if the current initialization step is done or has timed out.

	 */

	switch (ctrlr->state) {

	case NVME_CTRLR_STATE_INIT_DELAY:

		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms);

		/*

		 * Controller may need some delay before it's enabled.

		 *

		 * This is a workaround for an issue where the PCIe-attached NVMe controller

		 * is not ready after VFIO reset. We delay the initialization rather than the

		 * enabling itself, because this is required only for the very first enabling

		 * - directly after a VFIO reset.

		 *

		 * TODO: Figure out what is actually going wrong.

		 */

		SPDK_DEBUGLOG(SPDK_LOG_NVME, "Adding 500 ms delay before initializing the controller\n");

		ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (500 * spdk_get_ticks_hz() / 1000);

		break;



	case NVME_CTRLR_STATE_INIT:

		/* Begin the hardware initialization by making sure the controller is disabled. */

		if (cc.bits.en) {
@@ -2113,7 +2131,12 @@ nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr) 
{

	int rc;



	if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) {

		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE);

	} else {

		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);

	}



	ctrlr->flags = 0;

	ctrlr->free_io_qids = NULL;

	ctrlr->is_resetting = false;

lib/nvme/nvme_internal.h

+5 −0
Original line number Diff line number Diff line
@@ -388,6 +388,11 @@ struct spdk_nvme_ns { 
 * State of struct spdk_nvme_ctrlr (in particular, during initialization).

 */

enum nvme_ctrlr_state {

	/**

	 * Wait before initializing the controller.

	 */

	NVME_CTRLR_STATE_INIT_DELAY,



	/**

	 * Controller has not been initialized yet.

	 */