Commit 5d926c48 authored by Rui Chang's avatar Rui Chang Committed by Tomasz Zawadzki
Browse files

nvmf/vfio-user: Fix doorbell polling not working on some ARM platform 

On aarch64 platforms, doorbells update from guest VM may not be seen
on SPDK target side. This is because there is memory type mismatch
situation here. That is on guest VM side, the doorbells are treated as
device memory while on SPDK target side, it is treated as normal
memory. And this situation cause problem on ARM platform.
Refer to "https://developer.arm.com/documentation/102376/0100/


Memory-aliasing-and-mismatched-memory-types". Only using spdk_mb()
cannot fix this. Use "dc civac" to invalidate cache may solve this.

Profiling data did not show big performance degradataion.

Signed-off-by: default avatarRui Chang <rui.chang@arm.com>
Change-Id: I9a18718f8c4307b3007b18c32ab02e6796548958
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10222


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent 3911bc34

include/spdk/barrier.h

+7 −0
Original line number Diff line number Diff line
@@ -66,6 +66,9 @@ extern "C" { 
/** SMP read/write memory barrier. */

#define spdk_smp_mb()	_spdk_smp_mb()



/** Invalidate data cache, input is data pointer */

#define spdk_ivdt_dcache(pdata)	_spdk_ivdt_dcache(pdata)



#ifdef __PPC64__



#define _spdk_rmb()	__asm volatile("sync" ::: "memory")
@@ -74,6 +77,7 @@ extern "C" { 
#define _spdk_smp_rmb()	__asm volatile("lwsync" ::: "memory")

#define _spdk_smp_wmb()	__asm volatile("lwsync" ::: "memory")

#define _spdk_smp_mb()	spdk_mb()

#define _spdk_ivdt_dcache(pdata)



#elif defined(__aarch64__)
@@ -83,6 +87,7 @@ extern "C" { 
#define _spdk_smp_rmb()	__asm volatile("dmb ishld" ::: "memory")

#define _spdk_smp_wmb()	__asm volatile("dmb ishst" ::: "memory")

#define _spdk_smp_mb()	__asm volatile("dmb ish" ::: "memory")

#define _spdk_ivdt_dcache(pdata)	asm volatile("dc civac, %0" : : "r"(pdata) : "memory");



#elif defined(__i386__) || defined(__x86_64__)
@@ -96,6 +101,7 @@ extern "C" { 
#elif defined(__i386__)

#define _spdk_smp_mb()	__asm volatile("lock addl $0, -128(%%esp); " ::: "memory");

#endif

#define _spdk_ivdt_dcache(pdata)



#else
@@ -105,6 +111,7 @@ extern "C" { 
#define _spdk_smp_rmb()

#define _spdk_smp_wmb()

#define _spdk_smp_mb()

#define _spdk_ivdt_dcache(pdata)

#error Unknown architecture



#endif

lib/nvmf/vfio_user.c

+12 −0
Original line number Diff line number Diff line
@@ -2836,6 +2836,18 @@ nvmf_vfio_user_qpair_poll(struct nvmf_vfio_user_qpair *qpair) 


	ctrlr = qpair->ctrlr;



	/* On aarch64 platforms, doorbells update from guest VM may not be seen

	 * on SPDK target side. This is because there is memory type mismatch

	 * situation here. That is on guest VM side, the doorbells are treated as

	 * device memory while on SPDK target side, it is treated as normal

	 * memory. And this situation cause problem on ARM platform.

	 * Refer to "https://developer.arm.com/documentation/102376/0100/

	 * Memory-aliasing-and-mismatched-memory-types". Only using spdk_mb()

	 * cannot fix this. Use "dc civac" to invalidate cache may solve

	 * this.

	 */

	spdk_ivdt_dcache(tdbl(ctrlr, &qpair->sq));



	/* Load-Acquire. */

	new_tail = *tdbl(ctrlr, &qpair->sq);