Commit d8bfa4f5 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

memory: support 4KB spdk_mem_map mappings 



It's now possible to use spdk_mem_map to translate addresses with 4KB
granularity.  2MB pages are still translated in the same way they were
before this patch.  Only when a translation for a non-2MB aligned
address is set up, the 4KB mapping is used.  But it's used just for the
portion that cannot be represented by a 2MB mapping, meaning that
translating a large memory region, only the beginning and end of that
region might use the 4KB mapping.

For instance, setting up a translation for 0x1ff000-0x601000 results in:
 - 4KB mapping for the 0x100000-0x200000 region, with all but the last
   4KB page set to the default translation,
 - two 2MB mappings for the 0x200000-0x600000 region,
 - 4KB mapping for the 0x600000-0x800000 region, with all but the first
   4KB page set to the default translation.

The top-level map now has two types of 1GB maps, one for 2MB
translations and one for 4KB.  If a 1GB address space contains at least
one 2MB or 4KB mapping, the appropriate 1GB map is allocated.

Obviously, more memory is needed to store these translations.  Mapping
the same amount of memory using 4KB translations results in a 512x
increase compared to 2MB translations.  However, when all mappings are
2MB aligned, there is no difference.

Change-Id: I97726f334aeab152c3356fc515988b30bcd0635a
Signed-off-by: default avatarKonrad Sztyber <ksztyber@nvidia.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26213


Community-CI: Mellanox Build Bot
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
parent 0ebdd6a7

include/spdk/memory.h

+1 −0
Original line number Diff line number Diff line
@@ -31,6 +31,7 @@ extern "C" { 
#define VALUE_4KB		(1ULL << SHIFT_4KB)

#define MASK_4KB		(VALUE_4KB - 1)



#define _4KB_OFFSET(ptr)	(((uintptr_t)(ptr)) & MASK_4KB)

#define _2MB_OFFSET(ptr)	(((uintptr_t)(ptr)) & MASK_2MB)

#define _2MB_PAGE(ptr)		FLOOR_2MB((uintptr_t)(ptr))

#define FLOOR_2MB(x)		(((uintptr_t)(x)) & ~MASK_2MB)

lib/env_dpdk/env_internal.h

+1 −0
Original line number Diff line number Diff line
@@ -25,6 +25,7 @@ 
#define MASK_256TB	((1ULL << SHIFT_256TB) - 1)



#define SHIFT_1GB	30 /* (1 << 30) == 1 GB */

#define VALUE_1GB	(1ULL << SHIFT_1GB)

#define MASK_1GB	((1ULL << SHIFT_1GB) - 1)



int pci_env_init(void);

lib/env_dpdk/memory.c

+198 −22
Original line number Diff line number Diff line
@@ -62,15 +62,18 @@ static struct vfio_cfg g_vfio = { 
#endif



#define VFN_2MB(vaddr)		((vaddr) >> SHIFT_2MB)

#define VFN_4KB(vaddr)		((vaddr) >> SHIFT_4KB)



#define FN_2MB_TO_4KB(fn)	(fn << (SHIFT_2MB - SHIFT_4KB))

#define FN_4KB_TO_2MB(fn)	(fn >> (SHIFT_2MB - SHIFT_4KB))

#define FN_2MB_TO_4KB(fn)	((fn) << (SHIFT_2MB - SHIFT_4KB))

#define FN_4KB_TO_2MB(fn)	((fn) >> (SHIFT_2MB - SHIFT_4KB))



#define MAP_256TB_IDX(vfn_2mb)	((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB))

#define MAP_1GB_IDX(vfn_2mb)	((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB)) - 1))

#define MAP_2MB_IDX(vfn_4kb)	((vfn_4kb) & ((1ULL << (SHIFT_2MB - SHIFT_4KB)) - 1))



#define MAP_256TB_SIZE		(1ULL << (SHIFT_256TB - SHIFT_1GB))

#define MAP_1GB_SIZE		(1ULL << (SHIFT_1GB - SHIFT_2MB))

#define MAP_2MB_SIZE		(1ULL << (SHIFT_2MB - SHIFT_4KB))



/* Page is registered */

#define REG_MAP_REGISTERED	(1ULL << 62)
@@ -81,6 +84,11 @@ static struct vfio_cfg g_vfio = { 
 */

#define REG_MAP_NOTIFY_START	(1ULL << 63)



/* Third-level map for 4KB translations */

struct map_2mb4kb {

	uint64_t translation_4kb[MAP_2MB_SIZE];

};



/* Second-level map table indexed by bits [21..29] of the virtual address.

 * Each entry contains the address translation for a 2MB page or an error

 * for entries that haven't been retrieved yet.
@@ -89,12 +97,18 @@ struct map_1gb2mb { 
	uint64_t translation_2mb[MAP_1GB_SIZE];

};



/* Second-level map containing 4KB translations. */

struct map_1gb4kb {

	struct map_2mb4kb *map[MAP_1GB_SIZE];

};



/* Top-level map table indexed by bits [30..47] of the virtual address.

 * Each entry points to a second-level map table or NULL.

 */

struct map_256tb {

	struct {

		struct map_1gb2mb	*map_1gb2mb;

		struct map_1gb4kb	*map_1gb4kb;

	} map[MAP_256TB_SIZE];

};
@@ -125,19 +139,39 @@ static inline uint64_t 
mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, int *page_size)

{

	const struct map_1gb2mb *map_1gb2mb;

	uint64_t vfn_2mb, idx_1gb, idx_256tb;

	const struct map_1gb4kb *map_1gb4kb;

	const struct map_2mb4kb *map_2mb4kb;

	uint64_t translation, vfn_4kb, vfn_2mb, idx_2mb, idx_1gb, idx_256tb;



	vfn_2mb = VFN_2MB(vaddr);

	idx_256tb = MAP_256TB_IDX(vfn_2mb);

	idx_1gb = MAP_1GB_IDX(vfn_2mb);



	/* Check the 2MB map first */

	map_1gb2mb = map->map_256tb.map[idx_256tb].map_1gb2mb;

	if (spdk_unlikely(!map_1gb2mb)) {

		return map->default_translation;

	if (spdk_likely(map_1gb2mb != NULL)) {

		translation = map_1gb2mb->translation_2mb[idx_1gb];

		if (spdk_likely(translation != map->default_translation)) {

			*page_size = VALUE_2MB;

			return translation;

		}

	}



	/* There's no 2MB translation for this address, check the 4KB map */

	map_1gb4kb = map->map_256tb.map[idx_256tb].map_1gb4kb;

	if (spdk_likely(map_1gb4kb != NULL)) {

		map_2mb4kb = map_1gb4kb->map[idx_1gb];

		if (spdk_likely(map_2mb4kb != NULL)) {

			vfn_4kb = VFN_4KB(vaddr);

			idx_2mb = MAP_2MB_IDX(vfn_4kb);

			*page_size = VALUE_4KB;



			return map_2mb4kb->translation_4kb[idx_2mb];

		}

	}



	*page_size = VALUE_2MB;

	idx_1gb = MAP_1GB_IDX(vfn_2mb);

	return map_1gb2mb->translation_2mb[idx_1gb];

	return map->default_translation;

}



/*
@@ -275,10 +309,19 @@ err_unregister: 
static void

mem_map_free(struct spdk_mem_map *map)

{

	size_t i;

	struct map_1gb4kb *map_1gb4kb;

	size_t i, j;



	for (i = 0; i < SPDK_COUNTOF(map->map_256tb.map); i++) {

		free(map->map_256tb.map[i].map_1gb2mb);

		map_1gb4kb = map->map_256tb.map[i].map_1gb4kb;

		if (map_1gb4kb == NULL) {

			continue;

		}

		for (j = 0; j < SPDK_COUNTOF(map_1gb4kb->map); j++) {

			free(map_1gb4kb->map[j]);

		}

		free(map_1gb4kb);

	}

	pthread_mutex_destroy(&map->mutex);

	free(map);
@@ -557,7 +600,7 @@ spdk_mem_reserve(void *vaddr, size_t len) 
}



static struct map_1gb2mb *

mem_map_get_map_1gb2mb(struct spdk_mem_map *map, uint64_t vfn_2mb)

mem_map_get_map_1gb2mb(struct spdk_mem_map *map, uint64_t vfn_2mb, bool alloc)

{

	struct map_1gb2mb *map_1gb2mb;

	uint64_t idx_256tb = MAP_256TB_IDX(vfn_2mb);
@@ -568,8 +611,7 @@ mem_map_get_map_1gb2mb(struct spdk_mem_map *map, uint64_t vfn_2mb) 
	}



	map_1gb2mb = map->map_256tb.map[idx_256tb].map_1gb2mb;



	if (!map_1gb2mb) {

	if (!map_1gb2mb && alloc) {

		pthread_mutex_lock(&map->mutex);



		/* Recheck to make sure nobody else got the mutex first. */
@@ -596,30 +638,125 @@ mem_map_get_map_1gb2mb(struct spdk_mem_map *map, uint64_t vfn_2mb) 
	return map_1gb2mb;

}



static struct map_1gb4kb *

mem_map_get_map_1gb4kb(struct spdk_mem_map *map, uint64_t vfn_4kb, bool alloc)

{

	struct map_1gb4kb *map_1gb4kb;

	uint64_t vfn_2mb, idx_256tb;



	vfn_2mb = FN_4KB_TO_2MB(vfn_4kb);

	idx_256tb = MAP_256TB_IDX(vfn_2mb);

	if (idx_256tb >= SPDK_COUNTOF(map->map_256tb.map)) {

		return NULL;

	}



	map_1gb4kb = map->map_256tb.map[idx_256tb].map_1gb4kb;

	if (map_1gb4kb == NULL && alloc) {

		pthread_mutex_lock(&map->mutex);

		/* Recheck to make sure nobody else got the mutex first. */

		map_1gb4kb = map->map_256tb.map[idx_256tb].map_1gb4kb;

		if (map_1gb4kb == NULL) {

			map_1gb4kb = calloc(1, sizeof(*map_1gb4kb));



		}

		map->map_256tb.map[idx_256tb].map_1gb4kb = map_1gb4kb;

		pthread_mutex_unlock(&map->mutex);

	}



	return map_1gb4kb;

}



static struct map_2mb4kb *

mem_map_get_map_2mb4kb(struct spdk_mem_map *map, uint64_t vfn_4kb, bool alloc)

{

	struct map_2mb4kb *map_2mb4kb;

	struct map_1gb4kb *map_1gb4kb;

	uint64_t vfn_2mb, idx_1gb, translation;

	int page_size;

	size_t i;



	map_1gb4kb = mem_map_get_map_1gb4kb(map, vfn_4kb, alloc);

	if (map_1gb4kb == NULL) {

		return NULL;

	}



	vfn_2mb = FN_4KB_TO_2MB(vfn_4kb);

	idx_1gb = MAP_1GB_IDX(vfn_2mb);

	map_2mb4kb = map_1gb4kb->map[idx_1gb];

	if (map_2mb4kb == NULL && alloc) {

		pthread_mutex_lock(&map->mutex);

		/* Recheck to make sure nobody else got the mutex first. */

		map_2mb4kb = map_1gb4kb->map[idx_1gb];

		if (map_2mb4kb == NULL) {

			map_2mb4kb = malloc(sizeof(*map_2mb4kb));

			if (map_2mb4kb != NULL) {

				/* Fill the 4kb map with the 2mb translation, if it had any */

				translation = mem_map_translate(map, vfn_4kb << SHIFT_4KB,

								&page_size);

				for (i = 0; i < SPDK_COUNTOF(map_2mb4kb->translation_4kb); i++) {

					map_2mb4kb->translation_4kb[i] = translation;

				}

				map_1gb4kb->map[idx_1gb] = map_2mb4kb;

			}

		}

		pthread_mutex_unlock(&map->mutex);

	}



	return map_2mb4kb;

}



int

spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size,

			     uint64_t translation)

{

	uint64_t vfn_2mb;

	uint64_t vfn_4kb, vfn_2mb;

	uint64_t vfn_4kb_end, vfn_2mb_end;

	uint64_t i, idx_2mb, idx_1gb;

	struct map_1gb2mb *map_1gb2mb;

	uint64_t idx_1gb;

	struct map_2mb4kb *map_2mb4kb;



	if ((uintptr_t)vaddr & ~MASK_256TB) {

		DEBUG_PRINT("invalid usermode virtual address %" PRIu64 "\n", vaddr);

		return -EINVAL;

	}



	/* For now, only 2 MB-aligned registrations are supported */

	if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) {

	if (((uintptr_t)vaddr & MASK_4KB) || (size & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%" PRIu64 " len=%" PRIu64 "\n",

			    __func__, vaddr, size);

		return -EINVAL;

	}



	vfn_2mb = VFN_2MB(vaddr);

	/* Map the initial portion of the 4kb pages */

	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = spdk_min(FN_2MB_TO_4KB(VFN_2MB(vaddr + MASK_2MB)), VFN_4KB(vaddr + size));

	while (vfn_4kb < vfn_4kb_end) {

		map_2mb4kb = mem_map_get_map_2mb4kb(map, vfn_4kb, true);

		if (!map_2mb4kb) {

			DEBUG_PRINT("could not get %p map\n", (void *)vaddr);

			return -ENOMEM;

		}



		idx_2mb = MAP_2MB_IDX(vfn_4kb);

		map_2mb4kb->translation_4kb[idx_2mb] = translation;



		/* Set 2MB map to the default translation to indicate this region has 4KB mapping */

		vfn_2mb = FN_4KB_TO_2MB(vfn_4kb);

		map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb, false);

		if (map_1gb2mb != NULL) {

			idx_1gb = MAP_1GB_IDX(vfn_2mb);

			map_1gb2mb->translation_2mb[idx_1gb] = map->default_translation;

		}



	while (size) {

		map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb);

		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	}



	/* Map the 2mb pages */

	vfn_2mb = VFN_2MB(vaddr);

	vfn_2mb_end = VFN_2MB(vaddr + size);

	while (vfn_2mb < vfn_2mb_end) {

		map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb, true);

		if (!map_1gb2mb) {

			DEBUG_PRINT("could not get %p map\n", (void *)vaddr);

			return -ENOMEM;
@@ -628,10 +765,47 @@ spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t 
		idx_1gb = MAP_1GB_IDX(vfn_2mb);

		map_1gb2mb->translation_2mb[idx_1gb] = translation;



		/* Set up 4KB translations too in case this region later uses 4KB mapping or we're

		 * setting the default translation (which is also used to indicate a 4KB mapping).

		 */

		map_2mb4kb = mem_map_get_map_2mb4kb(map, FN_2MB_TO_4KB(vfn_2mb), false);

		if (map_2mb4kb != NULL) {

			for (i = 0; i < SPDK_COUNTOF(map_2mb4kb->translation_4kb); i++) {

				map_2mb4kb->translation_4kb[i] = translation;

			}

		}



		vaddr += VALUE_2MB;

		size -= VALUE_2MB;

		vfn_2mb++;

	}



	/* Finally, map the trailing 4kb pages */

	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = VFN_4KB(vaddr + size);

	while (vfn_4kb < vfn_4kb_end) {

		map_2mb4kb = mem_map_get_map_2mb4kb(map, vfn_4kb, true);

		if (!map_2mb4kb) {

			DEBUG_PRINT("could not get %p map\n", (void *)vaddr);

			return -ENOMEM;

		}



		idx_2mb = MAP_2MB_IDX(vfn_4kb);

		map_2mb4kb->translation_4kb[idx_2mb] = translation;



		/* Set 2MB map to the default translation to indicate this region has 4KB mapping */

		vfn_2mb = FN_4KB_TO_2MB(vfn_4kb);

		map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb, false);

		if (map_1gb2mb != NULL) {

			idx_1gb = MAP_1GB_IDX(vfn_2mb);

			map_1gb2mb->translation_2mb[idx_1gb] = map->default_translation;

		}



		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	}



	return 0;

}
@@ -657,11 +831,12 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t 
	}



	vfn_2mb = VFN_2MB(vaddr);

	cur_size = VALUE_2MB - _2MB_OFFSET(vaddr);

	curr_translation = mem_map_translate(map, vaddr, &page_size);

	cur_size = page_size - (page_size == VALUE_4KB ? _4KB_OFFSET(vaddr) : _2MB_OFFSET(vaddr));



	/* For now, we only support mappings spanning across multiple pages if they're 2MB */

	if (size == NULL || map->ops.are_contiguous == NULL ||

	    curr_translation == map->default_translation) {

	    curr_translation == map->default_translation || page_size == VALUE_4KB) {

		if (size != NULL) {

			*size = spdk_min(*size, cur_size);

		}
@@ -672,7 +847,8 @@ spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t 
	while (cur_size < *size) {

		vfn_2mb++;

		curr_translation = mem_map_translate(map, vfn_2mb << SHIFT_2MB, &page_size);

		if (!map->ops.are_contiguous(prev_translation, curr_translation)) {

		if (page_size == VALUE_4KB ||

		    !map->ops.are_contiguous(prev_translation, curr_translation)) {

			break;

		}

test/env/memory/memory_ut.c

+277 −1
Original line number Diff line number Diff line
@@ -129,6 +129,13 @@ test_mem_map_notify_checklen(void *cb_ctx, struct spdk_mem_map *map, 
	return 0;

}



static int

test_mem_map_notify_nop(void *cb_ctx, struct spdk_mem_map *map,

			enum spdk_mem_map_notify_action action, void *vaddr, size_t size)

{

	return 0;

}



static int

test_check_regions_contiguous(uint64_t addr1, uint64_t addr2)

{
@@ -155,6 +162,11 @@ struct spdk_mem_map_ops test_map_ops_notify_checklen = { 
	.are_contiguous = NULL

};



struct spdk_mem_map_ops test_map_ops_notify_nop = {

	.notify_cb = test_mem_map_notify_nop,

	.are_contiguous = NULL

};



static void

test_mem_map_alloc_free(void)

{
@@ -466,6 +478,269 @@ test_mem_map_registration_adjacent(void) 
	CU_ASSERT(map == NULL);

}



static void

test_mem_map_4kb(void)

{

	struct spdk_mem_map *map;

	const uint64_t default_translation = 0xDEADBEEF0BADF00D;

	uint64_t i, addr, traddr, size;

	int rc;



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_nop, NULL);

	SPDK_CU_ASSERT_FATAL(map != NULL);



	/* Check single 4KB page translation */

	addr = 0;

	rc = spdk_mem_map_set_translation(map, addr, VALUE_4KB, 0xfeedbeeff00d0);

	CU_ASSERT_EQUAL(rc, 0);



	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d0);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Set the next 4KB page */

	addr = VALUE_4KB;

	rc = spdk_mem_map_set_translation(map, addr, VALUE_4KB, 0xfeedbeeff00d1);

	CU_ASSERT_EQUAL(rc, 0);



	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, 0, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d0);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d1);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Clear the second page */

	rc = spdk_mem_map_clear_translation(map, addr, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, 0, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d0);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Check two 4KB pages spanning across 2MB boundary */

	addr = VALUE_2MB - VALUE_4KB;

	rc = spdk_mem_map_set_translation(map, addr, 2 * VALUE_4KB, 0xfeedbeeff00d2);

	CU_ASSERT_EQUAL(rc, 0);



	traddr = spdk_mem_map_translate(map, addr - VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d2);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d2);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Check one 4KB page + full 2MB page */

	addr = 3 * VALUE_2MB - VALUE_4KB;

	rc = spdk_mem_map_set_translation(map, addr, VALUE_4KB + VALUE_2MB, 0xfeedbeeff00d3);

	CU_ASSERT_EQUAL(rc, 0);



	traddr = spdk_mem_map_translate(map, addr - VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d3);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d3);

	CU_ASSERT_EQUAL(size, VALUE_2MB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d3);

	CU_ASSERT_EQUAL(size, 2 * VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d3);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB + VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Check the same, but switch the order (i.e. 4KB + 2MB -> 2MB + 4KB) */

	addr = 5 * VALUE_2MB;

	rc = spdk_mem_map_set_translation(map, addr, VALUE_2MB + VALUE_4KB, 0xfeedbeeff00d4);

	CU_ASSERT_EQUAL(rc, 0);



	traddr = spdk_mem_map_translate(map, addr - VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d4);

	CU_ASSERT_EQUAL(size, VALUE_2MB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d4);

	CU_ASSERT_EQUAL(size, VALUE_2MB - VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d4);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d4);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB + VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Check 2 4KB pages with one full 2MB page in the middle  */

	addr = 7 * VALUE_2MB - VALUE_4KB;

	rc = spdk_mem_map_set_translation(map, addr, 2 * VALUE_4KB + VALUE_2MB, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(rc, 0);



	traddr = spdk_mem_map_translate(map, addr - VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(size, VALUE_2MB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(size, 2 * VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d5);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB + 2 * VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Check multiple pages (2x4KB + 2x2MB + 2x4KB) */

	addr = 9 * VALUE_2MB - 2 * VALUE_4KB;

	rc = spdk_mem_map_set_translation(map, addr, 4 * VALUE_4KB + 2 * VALUE_2MB, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(rc, 0);



	traddr = spdk_mem_map_translate(map, addr - VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_2MB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, 2 * VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB + 2 * VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_2MB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, 2 * VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_2MB + 2 * VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_2MB + 3 * VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d6);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	traddr = spdk_mem_map_translate(map, addr + 2 * VALUE_2MB + 4 * VALUE_4KB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Set 4KB translation in the middle of an already translated 2MB page */

	addr = 13 * VALUE_2MB;

	rc = spdk_mem_map_set_translation(map, addr, VALUE_2MB, 0xfeedbeeff00d7);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_map_set_translation(map, addr + VALUE_4KB, VALUE_4KB, 0xfeedbeeff00d8);

	CU_ASSERT_EQUAL(rc, 0);



	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d7);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d8);

	CU_ASSERT_EQUAL(size, VALUE_4KB);



	for (i = 2 * VALUE_4KB; i < VALUE_2MB; i += VALUE_4KB) {

		size = VALUE_1GB;

		traddr = spdk_mem_map_translate(map, addr + i, &size);

		CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00d7);

		CU_ASSERT_EQUAL(size, VALUE_4KB);

	}

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, NULL);

	CU_ASSERT_EQUAL(traddr, default_translation);



	/* Set 2MB translation on an area with existing 4KB translation */

	addr = 14 * VALUE_2MB;

	rc = spdk_mem_map_set_translation(map, addr + VALUE_4KB, VALUE_4KB, 0xfeedbeeff00d9);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_map_set_translation(map, addr, VALUE_2MB, 0xfeedbeeff00da);

	CU_ASSERT_EQUAL(rc, 0);

	for (i = 0; i < VALUE_2MB; i += VALUE_4KB) {

		size = VALUE_1GB;

		traddr = spdk_mem_map_translate(map, addr + i, &size);

		CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00da);

		CU_ASSERT_EQUAL(size, VALUE_2MB - i);

	}



	/* Set 4KB + 2MB translation and then clear the 2MB containing the 4KB */

	addr = 16 * VALUE_2MB;

	rc = spdk_mem_map_set_translation(map, addr + VALUE_2MB - VALUE_4KB, VALUE_2MB + VALUE_4KB,

					  0xfeedbeeff00da);

	CU_ASSERT_EQUAL(rc, 0);



	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - 2 * VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00da);

	CU_ASSERT_EQUAL(size, VALUE_4KB);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00da);

	CU_ASSERT_EQUAL(size, VALUE_2MB);



	rc = spdk_mem_map_clear_translation(map, addr, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB - VALUE_4KB, &size);

	CU_ASSERT_EQUAL(traddr, default_translation);

	size = VALUE_1GB;

	traddr = spdk_mem_map_translate(map, addr + VALUE_2MB, &size);

	CU_ASSERT_EQUAL(traddr, 0xfeedbeeff00da);

	CU_ASSERT_EQUAL(size, VALUE_2MB);



	spdk_mem_map_free(&map);

	CU_ASSERT(map == NULL);

}



int

main(int argc, char **argv)

{
@@ -498,7 +773,8 @@ main(int argc, char **argv) 
		CU_add_test(suite, "alloc_free_mem_map", test_mem_map_alloc_free) == NULL ||

		CU_add_test(suite, "mem_map_translation", test_mem_map_translation) == NULL ||

		CU_add_test(suite, "mem_map_registration", test_mem_map_registration) == NULL ||

		CU_add_test(suite, "mem_map_adjacent_registrations", test_mem_map_registration_adjacent) == NULL

		CU_add_test(suite, "mem_map_adjacent_registrations", test_mem_map_registration_adjacent) == NULL ||

		CU_add_test(suite, "mem_map_4kb", test_mem_map_4kb) == NULL

	) {

		CU_cleanup_registry();

		return CU_get_error();