Commit af130056 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvme: Set crtlr state to disconnected 



Add new ctrlr state `disconnected`
When the controller disconnect process completes,
admin qpair is disconnected and free_io_qids is
released. That means that IO qpairs can not be
created. Code to create Io qpairs checks that
the ctrlr must be in READY state, so it is
needed to update the ctrlr state once disconnect
completes to disallow creating IO qpairs

Since change of the state and qpair creation
migth be executed on different threads, lock
controller's mutex in spdk_nvme_ctrlr_alloc_io_qpair

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: Ib882a5da7c0706f0a271628aa4bd11e65ff21ddf
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/18784


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent 5256f0ef

lib/nvme/nvme_ctrlr.c

+22 −9
Original line number Diff line number Diff line
@@ -446,14 +446,16 @@ spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 
			       size_t opts_size)

{



	struct spdk_nvme_qpair		*qpair;

	struct spdk_nvme_qpair		*qpair = NULL;

	struct spdk_nvme_io_qpair_opts	opts;

	int				rc;



	nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);



	if (spdk_unlikely(ctrlr->state != NVME_CTRLR_STATE_READY)) {

		/* When controller is resetting or initializing, free_io_qids is deleted or not created yet.

		 * We can't create IO qpair in that case */

		return NULL;

		goto unlock;

	}



	/*
@@ -472,14 +474,14 @@ spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 
			if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) {

				NVME_CTRLR_ERRLOG(ctrlr, "sq buffer size %" PRIx64 " is too small for sq size %zx\n",

						  opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd)));

				return NULL;

				goto unlock;

			}

		}

		if (opts.cq.vaddr) {

			if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) {

				NVME_CTRLR_ERRLOG(ctrlr, "cq buffer size %" PRIx64 " is too small for cq size %zx\n",

						  opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl)));

				return NULL;

				goto unlock;

			}

		}

	}
@@ -487,21 +489,23 @@ spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 
	qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts);



	if (qpair == NULL || opts.create_only == true) {

		return qpair;

		goto unlock;

	}



	rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair);

	if (rc != 0) {

		NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_connect_io_qpair() failed\n");

		nvme_robust_mutex_lock(&ctrlr->ctrlr_lock);

		nvme_ctrlr_proc_remove_io_qpair(qpair);

		TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq);

		spdk_bit_array_set(ctrlr->free_io_qids, qpair->id);

		nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair);

		nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);

		return NULL;

		qpair = NULL;

		goto unlock;

	}



unlock:

	nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock);



	return qpair;

}
@@ -1422,6 +1426,8 @@ nvme_ctrlr_state_string(enum nvme_ctrlr_state state) 
		return "ready";

	case NVME_CTRLR_STATE_ERROR:

		return "error";

	case NVME_CTRLR_STATE_DISCONNECTED:

		return "disconnected";

	}

	return "unknown";

};
@@ -1659,6 +1665,9 @@ nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr) 
	nvme_ctrlr_free_iocs_specific_data(ctrlr);



	spdk_bit_array_free(&ctrlr->free_io_qids);



	/* Set the state back to DISCONNECTED to cause a full hardware reset. */

	nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISCONNECTED, NVME_TIMEOUT_INFINITE);

}



int
@@ -3793,7 +3802,11 @@ nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) 
		}

		break;



	case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT */

	case NVME_CTRLR_STATE_DISCONNECTED:

		nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE);

		break;



	case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT and NVME_CTRLR_STATE_DISCONNECTED */

		rc = nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq);

		if (rc == 0) {

			nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ,

lib/nvme/nvme_internal.h

+6 −1
Original line number Diff line number Diff line
@@ -804,7 +804,12 @@ enum nvme_ctrlr_state { 
	/**

	 * Controller initialization has an error.

	 */

	NVME_CTRLR_STATE_ERROR

	NVME_CTRLR_STATE_ERROR,



	/**

	 * Admin qpair was disconnected, controller needs to be re-initialized

	 */

	NVME_CTRLR_STATE_DISCONNECTED,

};



#define NVME_TIMEOUT_INFINITE		0