Commit 460327ea authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

ioat, nvme: factor out MMIO helper functions 



NVMe doesn't require the specific 64-bit MMIO ordering on 32-bit
platforms performed in spdk_mmio_read_8(), but it doesn't hurt.
We have to pick one of the two possible orderings, so pick the one
required by I/OAT.

Change-Id: I2b909d64d0c077b797d0f64a11d78d1ecc55eec7
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 25cad6ff

include/spdk/mmio.h

0 → 100644
+91 −0
Original line number Diff line number Diff line 
/*-

 *   BSD LICENSE

 *

 *   Copyright(c) 2015 Intel Corporation. All rights reserved.

 *   All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions

 *   are met:

 *

 *     * Redistributions of source code must retain the above copyright

 *       notice, this list of conditions and the following disclaimer.

 *     * Redistributions in binary form must reproduce the above copyright

 *       notice, this list of conditions and the following disclaimer in

 *       the documentation and/or other materials provided with the

 *       distribution.

 *     * Neither the name of Intel Corporation nor the names of its

 *       contributors may be used to endorse or promote products derived

 *       from this software without specific prior written permission.

 *

 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */



#ifndef SPDK_MMIO_H

#define SPDK_MMIO_H



#include <inttypes.h>



#ifdef __x86_64__

#define SPDK_MMIO_64BIT	1 /* Can do atomic 64-bit memory read/write (over PCIe) */

#else

#define SPDK_MMIO_64BIT	0

#endif



static inline uint32_t

spdk_mmio_read_4(const volatile uint32_t *addr)

{

	return *addr;

}



static inline void

spdk_mmio_write_4(volatile uint32_t *addr, uint32_t val)

{

	*addr = val;

}



static inline uint64_t

spdk_mmio_read_8(volatile uint64_t *addr)

{

	uint64_t val;

	volatile uint32_t *addr32 = (volatile uint32_t *)addr;



	if (SPDK_MMIO_64BIT) {

		val = *addr;

	} else {

		/*

		 * Read lower 4 bytes before upper 4 bytes.

		 * This particular order is required by I/OAT.

		 * If the other order is required, use a pair of spdk_mmio_read_4() calls.

		 */

		val = addr32[0];

		val |= (uint64_t)addr32[1] << 32;

	}



	return val;

}



static inline void

spdk_mmio_write_8(volatile uint64_t *addr, uint64_t val)

{

	volatile uint32_t *addr32 = (volatile uint32_t *)addr;



	if (SPDK_MMIO_64BIT) {

		*addr = val;

	} else {

		addr32[0] = (uint32_t)val;

		addr32[1] = (uint32_t)(val >> 32);

	}

}



#endif

lib/ioat/ioat.c

+3 −34
Original line number Diff line number Diff line
@@ -116,53 +116,22 @@ ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id) 
	return false;

}





static uint64_t

ioat_mmio_read_8(volatile uint64_t *addr)

{

	uint64_t val;

	volatile uint32_t *addr32 = (volatile uint32_t *)addr;



	if (IOAT_64BIT_IO) {

		val = *addr;

	} else {

		/* Must read lower 4 bytes before upper 4 bytes. */

		val = addr32[0];

		val |= (uint64_t)addr32[1] << 32;

	}



	return val;

}



static void

ioat_mmio_write_8(volatile uint64_t *addr, uint64_t val)

{

	volatile uint32_t *addr32 = (volatile uint32_t *)addr;



	if (IOAT_64BIT_IO) {

		*addr = val;

	} else {

		addr32[0] = (uint32_t)val;

		addr32[1] = (uint32_t)(val >> 32);

	}

}



static uint64_t

ioat_get_chansts(struct ioat_channel *ioat)

{

	return ioat_mmio_read_8(&ioat->regs->chansts);

	return spdk_mmio_read_8(&ioat->regs->chansts);

}



static void

ioat_write_chancmp(struct ioat_channel *ioat, uint64_t addr)

{

	ioat_mmio_write_8(&ioat->regs->chancmp, addr);

	spdk_mmio_write_8(&ioat->regs->chancmp, addr);

}



static void

ioat_write_chainaddr(struct ioat_channel *ioat, uint64_t addr)

{

	ioat_mmio_write_8(&ioat->regs->chainaddr, addr);

	spdk_mmio_write_8(&ioat->regs->chainaddr, addr);

}



static inline void

lib/ioat/ioat_internal.h

+1 −6
Original line number Diff line number Diff line
@@ -44,16 +44,11 @@ 
#include <inttypes.h>



#include "spdk/queue.h"

#include "spdk/mmio.h"



/* Allocate 2 << 15 (32K) descriptors per channel by default. */

#define IOAT_DEFAULT_ORDER			15



#ifdef __x86_64__

#define IOAT_64BIT_IO	1 /* Can do atomic 64-bit memory read/write (over PCIe) */

#else

#define IOAT_64BIT_IO	0

#endif



struct ioat_descriptor {

	ioat_callback_t		callback_fn;

	void			*callback_arg;

lib/nvme/nvme_internal.h

+4 −21
Original line number Diff line number Diff line
@@ -51,6 +51,7 @@ 


#include "spdk/queue.h"

#include "spdk/barrier.h"

#include "spdk/mmio.h"



#define NVME_MAX_PRP_LIST_ENTRIES	(32)
@@ -304,32 +305,14 @@ extern struct nvme_driver g_nvme_driver; 


#define INTEL_DC_P3X00_DEVID	0x09538086



static inline uint32_t

_nvme_mmio_read_4(const volatile uint32_t *addr)

{

	return *addr;

}



static inline void

_nvme_mmio_write_4(volatile uint32_t *addr, uint32_t val)

{

	*addr = val;

}



static inline void

_nvme_mmio_write_8(volatile uint64_t *addr, uint64_t val)

{

	*addr = val;

}



#define nvme_mmio_read_4(sc, reg) \

	_nvme_mmio_read_4(&(sc)->regs->reg)

	spdk_mmio_read_4(&(sc)->regs->reg)



#define nvme_mmio_write_4(sc, reg, val) \

	_nvme_mmio_write_4(&(sc)->regs->reg, val)

	spdk_mmio_write_4(&(sc)->regs->reg, val)



#define nvme_mmio_write_8(sc, reg, val) \

	_nvme_mmio_write_8(&(sc)->regs->reg, val)

	spdk_mmio_write_8(&(sc)->regs->reg, val)



#define nvme_delay		usleep

lib/nvme/nvme_qpair.c

+2 −2
Original line number Diff line number Diff line
@@ -480,7 +480,7 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair, uint32_t max_completion 
			qpair->phase = !qpair->phase;

		}



		_nvme_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);

		spdk_mmio_write_4(qpair->cq_hdbl, qpair->cq_head);



		if (max_completions > 0 && --max_completions == 0) {

			break;
@@ -635,7 +635,7 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr) 
	}



	wmb();

	_nvme_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);

	spdk_mmio_write_4(qpair->sq_tdbl, qpair->sq_tail);

}



static void