Commit e2d3cc65 authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

nvme: replace rte_memcpy with specialized function 



Use the knowledge that both the source and destination of
nvme_copy_command() are aligned to emit the aligned variants of the
SSE2/AVX mov instructions.

Change-Id: I0a7e32a3bb10b9a1920cd85691b79fa7172eecb3
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent eb9d77a9

examples/nvme/fio_plugin/fio_plugin.c

+0 −1
Original line number Diff line number Diff line
@@ -42,7 +42,6 @@ 
#include "rte_mempool.h"

#include "rte_malloc.h"

#include "rte_eal.h"

#include "rte_memcpy.h"



#include "spdk/nvme.h"

#include "spdk/pci.h"

lib/nvme/nvme_impl.h

+0 −6
Original line number Diff line number Diff line
@@ -54,7 +54,6 @@ 
#include <rte_cycles.h>

#include <rte_malloc.h>

#include <rte_mempool.h>

#include <rte_memcpy.h>



#ifdef USE_PCIACCESS

#include <pciaccess.h>
@@ -327,9 +326,4 @@ nvme_mutex_init_recursive(nvme_mutex_t *mtx) 
	return rc;

}



/**

 * Copy a struct nvme_command from one memory location to another.

 */

#define nvme_copy_command(dst, src)	rte_memcpy((dst), (src), sizeof(struct spdk_nvme_cmd))



#endif /* __NVME_IMPL_H__ */

lib/nvme/nvme_qpair.c

+23 −0
Original line number Diff line number Diff line
@@ -290,6 +290,29 @@ nvme_qpair_construct_tracker(struct nvme_tracker *tr, uint16_t cid, uint64_t phy 
	tr->active = false;

}



static inline void

nvme_copy_command(struct spdk_nvme_cmd *dst, const struct spdk_nvme_cmd *src)

{

	/* dst and src are known to be non-overlapping and 64-byte aligned. */

#if defined(__AVX__)

	__m256i *d256 = (__m256i *)dst;

	const __m256i *s256 = (const __m256i *)src;



	_mm256_store_si256(&d256[0], _mm256_load_si256(&s256[0]));

	_mm256_store_si256(&d256[1], _mm256_load_si256(&s256[1]));

#elif defined(__SSE2__)

	__m128i *d128 = (__m128i *)dst;

	const __m128i *s128 = (const __m128i *)src;



	_mm_store_si128(&d128[0], _mm_load_si128(&s128[0]));

	_mm_store_si128(&d128[1], _mm_load_si128(&s128[1]));

	_mm_store_si128(&d128[2], _mm_load_si128(&s128[2]));

	_mm_store_si128(&d128[3], _mm_load_si128(&s128[3]));

#else

	*dst = *src;

#endif

}



static void

nvme_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)

{

test/lib/nvme/unit/nvme_impl.h

+0 −5
Original line number Diff line number Diff line
@@ -153,9 +153,4 @@ nvme_mutex_init_recursive(nvme_mutex_t *mtx) 
	return rc;

}



/**

 * Copy a struct nvme_command from one memory location to another.

 */

#define nvme_copy_command(dst, src)	memcpy((dst), (src), sizeof(struct spdk_nvme_cmd))



#endif /* __NVME_IMPL_H__ */