Commit d6dc356f authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

memory: support 4KB memory registrations 



It is now possible to register memory regions that doesn't start and/or
end on a 2MB boundary.

Change-Id: Ida88ea51c68bb0b12c6a7b940f5a3080b538bfce
Signed-off-by: default avatarKonrad Sztyber <ksztyber@nvidia.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26223


Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
parent 263860fc

lib/env_dpdk/memory.c

+101 −12
Original line number Diff line number Diff line
@@ -178,6 +178,15 @@ mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, int *page_size 
	return map->default_translation;

}



static bool

mem_map_is_4kb_mapping(struct spdk_mem_map *map, uint64_t vaddr)

{

	int page_size;



	mem_map_translate(map, vaddr, &page_size);

	return page_size == VALUE_4KB;

}



static int

mem_map_walk_region(struct spdk_mem_map *map, uint64_t vaddr, size_t size,

		    int (*callback)(struct spdk_mem_map *map, uint64_t addr, size_t sz, void *ctx),
@@ -234,10 +243,9 @@ static int 
mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action action)

{

	size_t idx_256tb;

	uint64_t idx_1gb;

	uint64_t idx_1gb, idx_2mb;

	uint64_t contig_start = UINT64_MAX;

	uint64_t contig_end = UINT64_MAX;

	struct map_1gb2mb *map_1gb2mb;

	int rc, page_size;



	if (!g_mem_reg_map) {
@@ -248,9 +256,8 @@ mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action ac 
	pthread_mutex_lock(&g_mem_reg_map->mutex);



	for (idx_256tb = 0; idx_256tb < MAP_256TB_SIZE; idx_256tb++) {

		map_1gb2mb = g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb2mb;



		if (!map_1gb2mb) {

		if (!g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb2mb &&

		    !g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb4kb) {

			if (contig_start != UINT64_MAX) {

				/* End of of a virtually contiguous range */

				rc = map->ops.notify_cb(map->cb_ctx, map, action,
@@ -269,6 +276,40 @@ mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action ac 
			uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);

			uint64_t reg = mem_map_translate(g_mem_reg_map, vaddr, &page_size);



			if (page_size == VALUE_4KB) {

				for (idx_2mb = 0; idx_2mb < MAP_2MB_SIZE; idx_2mb++) {

					vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB) |

						(idx_2mb << SHIFT_4KB);

					reg = mem_map_translate(g_mem_reg_map, vaddr, &page_size);



					if ((reg & REG_MAP_REGISTERED) &&

					    (contig_start == UINT64_MAX || (reg & REG_MAP_NOTIFY_START) == 0)) {

						if (contig_start == UINT64_MAX) {

							contig_start = vaddr;

						}

						contig_end = vaddr + VALUE_4KB;

					} else {

						if (contig_start != UINT64_MAX) {

							/* End of of a virtually contiguous range */

							rc = map->ops.notify_cb(map->cb_ctx, map, action,

										(void *)contig_start,

										contig_end - contig_start);

							/* Don't bother handling unregister failures. It can't be any worse */

							if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) {

								goto err_unregister;

							}

							/* This page might be a part of a neighbour

							 * region, so process it again

							 */

							idx_2mb--;

						}

						contig_start = UINT64_MAX;

					}

				}



				continue;

			}



			if ((reg & REG_MAP_REGISTERED) &&

			    (contig_start == UINT64_MAX || (reg & REG_MAP_NOTIFY_START) == 0)) {

				/* Rebuild the virtual address from the indexes */
@@ -307,14 +348,14 @@ err_unregister: 
	 */

	idx_256tb = MAP_256TB_IDX(VFN_2MB(contig_start) - 1);

	idx_1gb = MAP_1GB_IDX(VFN_2MB(contig_start) - 1);

	idx_2mb = MAP_2MB_IDX(VFN_4KB(contig_start) - 1);

	contig_start = UINT64_MAX;

	contig_end = UINT64_MAX;



	/* Unregister any memory we managed to register before the failure */

	for (; idx_256tb < SIZE_MAX; idx_256tb--) {

		map_1gb2mb = g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb2mb;



		if (!map_1gb2mb) {

		if (!g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb2mb &&

		    !g_mem_reg_map->map_256tb.map[idx_256tb].map_1gb4kb) {

			if (contig_end != UINT64_MAX) {

				/* End of of a virtually contiguous range */

				map->ops.notify_cb(map->cb_ctx, map,
@@ -331,6 +372,36 @@ err_unregister: 
			uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);

			uint64_t reg = mem_map_translate(g_mem_reg_map, vaddr, &page_size);



			if (page_size == VALUE_4KB) {

				for (; idx_2mb < UINT64_MAX; idx_2mb--) {

					vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB) |

						(idx_2mb << SHIFT_4KB);

					reg = mem_map_translate(g_mem_reg_map, vaddr, &page_size);



					if ((reg & REG_MAP_REGISTERED) &&

					    (contig_end == UINT64_MAX || (reg & REG_MAP_NOTIFY_START) == 0)) {

						if (contig_end == UINT64_MAX) {

							contig_end = vaddr + VALUE_4KB;

						}

						contig_start = vaddr;

					} else {

						if (contig_end != UINT64_MAX) {

							if (reg & REG_MAP_NOTIFY_START) {

								contig_start = vaddr;

							}

							/* End of of a virtually contiguous range */

							map->ops.notify_cb(map->cb_ctx, map,

									   SPDK_MEM_MAP_NOTIFY_UNREGISTER,

									   (void *)contig_start,

									   contig_end - contig_start);

						}

						contig_end = UINT64_MAX;

					}

				}

				idx_2mb = MAP_2MB_SIZE - 1;

				continue;

			}



			if ((reg & REG_MAP_REGISTERED) &&

			    (contig_end == UINT64_MAX || (reg & REG_MAP_NOTIFY_START) == 0)) {

				if (contig_end == UINT64_MAX) {
@@ -502,7 +573,7 @@ spdk_mem_register(void *vaddr, size_t len) 
		return -EINVAL;

	}



	if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {

	if (((uintptr_t)vaddr & MASK_4KB) || (len & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",

			    __func__, vaddr, len);

		return -EINVAL;
@@ -543,9 +614,27 @@ static int 
mem_unregister_page(struct spdk_mem_map *map, uint64_t vaddr, size_t len, void *ctx)

{

	struct iovec *region = ctx;

	uint64_t reg;

	uint64_t off, reg;

	int rc;



	/* We've already checked that the whole region we're trying to unregister was actually

	 * registered at this point.  But if we're trying to unregister a 2MB region that uses 4KB

	 * translations, we need to check each 4KB page individually, because that 2MB region could

	 * consist of multiple smaller registrations, so we might need to send multiple

	 * notifications.

	 */

	if (len > VALUE_4KB && mem_map_is_4kb_mapping(map, vaddr)) {

		assert(len == VALUE_2MB);

		for (off = 0; off < len; off += VALUE_4KB) {

			rc = mem_unregister_page(map, vaddr + off, VALUE_4KB, ctx);

			if (rc != 0) {

				return rc;

			}

		}

		/* Set translation for the whole 2MB page to free the 4KB map */

		return spdk_mem_map_set_translation(map, vaddr, len, 0);

	}



	reg = spdk_mem_map_translate(map, vaddr, NULL);

	spdk_mem_map_set_translation(map, vaddr, len, 0);

	if (region->iov_len > 0 && (reg & REG_MAP_NOTIFY_START)) {
@@ -579,7 +668,7 @@ spdk_mem_unregister(void *vaddr, size_t len) 
		return -EINVAL;

	}



	if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {

	if (((uintptr_t)vaddr & MASK_4KB) || (len & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",

			    __func__, vaddr, len);

		return -EINVAL;
@@ -648,7 +737,7 @@ spdk_mem_reserve(void *vaddr, size_t len) 
		return -EINVAL;

	}



	if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {

	if (((uintptr_t)vaddr & MASK_4KB) || (len & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",

			    __func__, vaddr, len);

		return -EINVAL;

test/env/memory/memory_ut.c

+384 −1
Original line number Diff line number Diff line
@@ -136,6 +136,81 @@ test_mem_map_notify_nop(void *cb_ctx, struct spdk_mem_map *map, 
	return 0;

}



struct ut_memreg {

	uint64_t		start;

	size_t			len;

	TAILQ_ENTRY(ut_memreg)	tailq;

};



TAILQ_HEAD(ut_memreg_tailq, ut_memreg);



static int

ut_memreg_count(struct ut_memreg_tailq *regions)

{

	struct ut_memreg *memreg;

	int count = 0;



	TAILQ_FOREACH(memreg, regions, tailq) {

		count++;

	}



	return count;

}



static struct ut_memreg *

ut_memreg_find(struct ut_memreg_tailq *regions, uint64_t vaddr, size_t len)

{

	struct ut_memreg *memreg;



	TAILQ_FOREACH(memreg, regions, tailq) {

		if (memreg->start == vaddr && memreg->len == len) {

			return memreg;

		}

	}



	return NULL;

}



static int

test_mem_map_notify_memreg(void *cb_ctx, struct spdk_mem_map *map,

			   enum spdk_mem_map_notify_action action,

			   void *vaddr, size_t len)

{

	struct ut_memreg_tailq *tailq = cb_ctx;

	struct ut_memreg *memreg;



	switch (action) {

	case SPDK_MEM_MAP_NOTIFY_REGISTER:

		if (vaddr == g_vaddr_to_fail) {

			return -1;

		}

		TAILQ_FOREACH(memreg, tailq, tailq) {

			CU_ASSERT((uint64_t)vaddr + len <= memreg->start ||

				  (uint64_t)vaddr >= memreg->start + memreg->len);

		}



		memreg = calloc(1, sizeof(*memreg));

		SPDK_CU_ASSERT_FATAL(memreg != NULL);



		memreg->start = (uint64_t)vaddr;

		memreg->len = len;

		TAILQ_INSERT_TAIL(tailq, memreg, tailq);

		break;

	case SPDK_MEM_MAP_NOTIFY_UNREGISTER:

		TAILQ_FOREACH(memreg, tailq, tailq) {

			if (memreg->start == (uint64_t)vaddr && memreg->len == len) {

				break;

			}

		}

		SPDK_CU_ASSERT_FATAL(memreg != NULL);

		TAILQ_REMOVE(tailq, memreg, tailq);

		free(memreg);

		break;

	}



	return 0;

}



static int

test_check_regions_contiguous(uint64_t addr1, uint64_t addr2)

{
@@ -172,6 +247,11 @@ struct spdk_mem_map_ops test_map_ops_notify_nop_no_contig = { 
	.are_contiguous = NULL

};



struct spdk_mem_map_ops test_map_ops_notify_memreg = {

	.notify_cb = test_mem_map_notify_memreg,

	.are_contiguous = NULL

};



static void

test_mem_map_alloc_free(void)

{
@@ -215,6 +295,7 @@ test_mem_map_alloc_free(void) 


	spdk_mem_map_free(&map);

	CU_ASSERT(map == NULL);

	g_vaddr_to_fail = (void *)UINT64_MAX;

}



static void
@@ -857,6 +938,306 @@ test_mem_map_4kb_contig_pages(void) 
	spdk_mem_map_free(&map);

}



static void

test_mem_4kb_register_notify(void)

{

	struct spdk_mem_map *map;

	uint64_t off, default_translation = 0xDEADBEEF0BADF00D;

	struct ut_memreg_tailq regions = TAILQ_HEAD_INITIALIZER(regions);

	int rc;



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);



	/* Register a single 4KB page */

	rc = spdk_mem_register((void *)0, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, 0, VALUE_4KB));

	rc = spdk_mem_unregister((void *)0, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Register two 4KB pages spanning across 2MB boundary */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, VALUE_2MB - VALUE_4KB, 2 * VALUE_4KB));

	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Register a region consisting of one 4KB page and one 2MB page */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, VALUE_2MB - VALUE_4KB, VALUE_2MB + VALUE_4KB));

	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB +  VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Register a region consisting of: 4KB page, 2MB page, 4KB page */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB + 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, VALUE_2MB - VALUE_4KB, VALUE_2MB + 2 * VALUE_4KB));

	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB +  2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Check that registration fails when it includes a registered 4KB page */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	/* Try to register a range consisting of two 4KB pages including the already registered one */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EBUSY);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	/* Try to register a 2MB page including the registered 4KB page */

	rc = spdk_mem_register((void *)0, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, -EBUSY);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	/* Try to register a range consisting of a 4KB page and 2MB page including the already

	 * registered 4KB page

	 */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EBUSY);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Try to unregister a region including unregistered pages */

	rc = spdk_mem_register((void *)0, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);



	rc = spdk_mem_unregister((void *)0, 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	rc = spdk_mem_unregister((void *)0, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	rc = spdk_mem_unregister((void *)0, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);



	rc = spdk_mem_unregister((void *)0, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Do the same but change the 4KB page's offset to the end of the 2MB page */

	rc = spdk_mem_register((void *)(VALUE_2MB - VALUE_4KB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);



	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	rc = spdk_mem_unregister((void *)0, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, -EINVAL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);



	rc = spdk_mem_unregister((void *)(VALUE_2MB - VALUE_4KB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Register two 4KB pages indivdually and unregister them both at once */

	rc = spdk_mem_register((void *)0, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)VALUE_4KB, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_unregister((void *)0, 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Register a full 2MB page via multiple 4KB registrations and unregister it all at once */

	for (off = 0; off < VALUE_2MB; off += VALUE_4KB) {

		rc = spdk_mem_register((void *)off, VALUE_4KB);

		CU_ASSERT_EQUAL(rc, 0);

	}

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), VALUE_2MB / VALUE_4KB);

	rc = spdk_mem_unregister((void *)0, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	spdk_mem_map_free(&map);

}



static void

test_mem_4kb_register_create(void)

{

	struct spdk_mem_map *map;

	uint64_t offset, default_translation = 0xDEADBEEF0BADF00D;

	struct ut_memreg_tailq regions = TAILQ_HEAD_INITIALIZER(regions);

	int rc;



	/* Register a single 4KB page, create a map, and verify the map is correctly notified  */

	offset = 0 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset, VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Register a page at the end of a 2MB region */

	offset = 1 * VALUE_2MB;

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)offset - VALUE_4KB, VALUE_4KB);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset - VALUE_4KB, VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Register two 4KB pages spanning across 2MB boundary */

	offset = 1 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset - VALUE_4KB, 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset - VALUE_4KB, 2 * VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Register the same region (4KB + 4KB), but register the pages separately */

	offset = 1 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset - VALUE_4KB, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)offset, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 2);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset - VALUE_4KB, VALUE_4KB));

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset, VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, 2 * VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Check a region of 2MB + 4KB */

	offset = 3 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset, VALUE_2MB + VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Check the same region (2MB + 4KB), but register the pages separately */

	offset = 5 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)(offset + VALUE_2MB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 2);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset, VALUE_2MB));

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset + VALUE_2MB, VALUE_4KB));



	rc = spdk_mem_unregister((void *)offset, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Do the same as above, but change the order of the pages (i.e. 2MB + 4KB -> 4KB + 2MB) */

	offset = 8 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset - VALUE_4KB, VALUE_4KB + VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 1);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset - VALUE_4KB, VALUE_4KB + VALUE_2MB));



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	offset = 10 * VALUE_2MB;

	rc = spdk_mem_register((void *)offset - VALUE_4KB, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)offset, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map != NULL);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 2);

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset - VALUE_4KB, VALUE_4KB));

	CU_ASSERT_PTR_NOT_NULL(ut_memreg_find(&regions, offset, VALUE_2MB));



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);

	spdk_mem_map_free(&map);



	/* Check failure from notify_cb() */

	offset = 11 * VALUE_2MB;

	g_vaddr_to_fail = (void *)offset;



	rc = spdk_mem_register((void *)offset - VALUE_4KB, VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)offset, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map == NULL);



	rc = spdk_mem_unregister((void *)offset - VALUE_4KB, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	/* Check the same, but choose a different region (4K + 2MB -> 2MB + 4KB) */

	offset = 13 * VALUE_2MB;

	g_vaddr_to_fail = (void *)(offset + VALUE_2MB);



	rc = spdk_mem_register((void *)offset, VALUE_2MB);

	CU_ASSERT_EQUAL(rc, 0);

	rc = spdk_mem_register((void *)(offset + VALUE_2MB), VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);



	map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_memreg, &regions);

	SPDK_CU_ASSERT_FATAL(map == NULL);



	rc = spdk_mem_unregister((void *)offset, VALUE_2MB + VALUE_4KB);

	CU_ASSERT_EQUAL(rc, 0);

	CU_ASSERT_EQUAL(ut_memreg_count(&regions), 0);



	g_vaddr_to_fail = (void *)UINT64_MAX;

}



int

main(int argc, char **argv)

{
@@ -891,7 +1272,9 @@ main(int argc, char **argv) 
		CU_add_test(suite, "mem_map_registration", test_mem_map_registration) == NULL ||

		CU_add_test(suite, "mem_map_adjacent_registrations", test_mem_map_registration_adjacent) == NULL ||

		CU_add_test(suite, "mem_map_4kb", test_mem_map_4kb) == NULL ||

		CU_add_test(suite, "mem_map_4kb_contig_pages", test_mem_map_4kb_contig_pages) == NULL

		CU_add_test(suite, "mem_map_4kb_contig_pages", test_mem_map_4kb_contig_pages) == NULL ||

		CU_add_test(suite, "mem_4kb_register_notify", test_mem_4kb_register_notify) == NULL ||

		CU_add_test(suite, "mem_4kb_register_create", test_mem_4kb_register_create) == NULL

	) {

		CU_cleanup_registry();

		return CU_get_error();