Commit 8e088c9b authored by Ben Walker's avatar Ben Walker Committed by Changpeng Liu
Browse files

nvme: Minimize memory accesses when checking if mmio required



Don't touch the shadow doorbells if it isn't necessary.

The flag could be combined into a bit mask with other
flags in a future patch.

Change-Id: I9ffd16468d29f0f0868cf849f7fece327eb6a294
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/447967


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent a741e341
Loading
Loading
Loading
Loading
+45 −25
Original line number Diff line number Diff line
@@ -143,18 +143,6 @@ struct nvme_pcie_qpair {
	/* Completion queue head doorbell */
	volatile uint32_t *cq_hdbl;

	/* Submission queue shadow tail doorbell */
	volatile uint32_t *sq_shadow_tdbl;

	/* Completion queue shadow head doorbell */
	volatile uint32_t *cq_shadow_hdbl;

	/* Submission queue event index */
	volatile uint32_t *sq_eventidx;

	/* Completion queue event index */
	volatile uint32_t *cq_eventidx;

	/* Submission queue */
	struct spdk_nvme_cmd *cmd;

@@ -177,10 +165,9 @@ struct nvme_pcie_qpair {
	uint16_t sq_head;

	uint8_t phase;

	bool is_enabled;

	bool delay_pcie_doorbell;
	bool has_shadow_doorbell;

	/*
	 * Base qpair structure.
@@ -189,6 +176,20 @@ struct nvme_pcie_qpair {
	 */
	struct spdk_nvme_qpair qpair;

	struct {
		/* Submission queue shadow tail doorbell */
		volatile uint32_t *sq_tdbl;

		/* Completion queue shadow head doorbell */
		volatile uint32_t *cq_hdbl;

		/* Submission queue event index */
		volatile uint32_t *sq_eventidx;

		/* Completion queue event index */
		volatile uint32_t *cq_eventidx;
	} shadow_doorbell;

	/*
	 * Fields below this point should not be touched on the normal I/O path.
	 */
@@ -1176,11 +1177,16 @@ nvme_pcie_qpair_ring_sq_doorbell(struct spdk_nvme_qpair *qpair)
{
	struct nvme_pcie_qpair	*pqpair = nvme_pcie_qpair(qpair);
	struct nvme_pcie_ctrlr	*pctrlr = nvme_pcie_ctrlr(qpair->ctrlr);
	bool need_mmio = true;

	if (spdk_likely(nvme_pcie_qpair_update_mmio_required(qpair,
	if (spdk_unlikely(pqpair->has_shadow_doorbell)) {
		need_mmio = nvme_pcie_qpair_update_mmio_required(qpair,
				pqpair->sq_tail,
			pqpair->sq_shadow_tdbl,
			pqpair->sq_eventidx))) {
				pqpair->shadow_doorbell.sq_tdbl,
				pqpair->shadow_doorbell.sq_eventidx);
	}

	if (spdk_likely(need_mmio)) {
		spdk_wmb();
		g_thread_mmio_ctrlr = pctrlr;
		spdk_mmio_write_4(pqpair->sq_tdbl, pqpair->sq_tail);
@@ -1562,10 +1568,17 @@ _nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme
	}

	if (ctrlr->shadow_doorbell) {
		pqpair->sq_shadow_tdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 0) * pctrlr->doorbell_stride_u32;
		pqpair->cq_shadow_hdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 1) * pctrlr->doorbell_stride_u32;
		pqpair->sq_eventidx = ctrlr->eventidx + (2 * qpair->id + 0) * pctrlr->doorbell_stride_u32;
		pqpair->cq_eventidx = ctrlr->eventidx + (2 * qpair->id + 1) * pctrlr->doorbell_stride_u32;
		pqpair->shadow_doorbell.sq_tdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 0) *
						  pctrlr->doorbell_stride_u32;
		pqpair->shadow_doorbell.cq_hdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 1) *
						  pctrlr->doorbell_stride_u32;
		pqpair->shadow_doorbell.sq_eventidx = ctrlr->eventidx + (2 * qpair->id + 0) *
						      pctrlr->doorbell_stride_u32;
		pqpair->shadow_doorbell.cq_eventidx = ctrlr->eventidx + (2 * qpair->id + 1) *
						      pctrlr->doorbell_stride_u32;
		pqpair->has_shadow_doorbell = true;
	} else {
		pqpair->has_shadow_doorbell = false;
	}
	nvme_pcie_qpair_reset(qpair);

@@ -2116,9 +2129,16 @@ nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_
	}

	if (num_completions > 0) {
		if (spdk_likely(nvme_pcie_qpair_update_mmio_required(qpair, pqpair->cq_head,
				pqpair->cq_shadow_hdbl,
				pqpair->cq_eventidx))) {
		bool need_mmio = true;

		if (spdk_unlikely(pqpair->has_shadow_doorbell)) {
			need_mmio = nvme_pcie_qpair_update_mmio_required(qpair,
					pqpair->cq_head,
					pqpair->shadow_doorbell.cq_hdbl,
					pqpair->shadow_doorbell.cq_eventidx);
		}

		if (spdk_likely(need_mmio)) {
			g_thread_mmio_ctrlr = pctrlr;
			spdk_mmio_write_4(pqpair->cq_hdbl, pqpair->cq_head);
			g_thread_mmio_ctrlr = NULL;