Commit 3cb6e826 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Tomasz Zawadzki
Browse files

memory: add helper for walking over non-2MB aligned regions 



Iterating over a non-2MB aligned memory regions is going to be useful in
any place that supports 4KB mappings.  This function chooses page size
based on the alignment of the address and doesn't check the map to see
if a 2MB-aligned region actually uses 4KB translations.

Change-Id: Iddb93dc5142de10a553c660461f6085aaa6679f1
Signed-off-by: default avatarKonrad Sztyber <ksztyber@nvidia.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26215


Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Community-CI: Mellanox Build Bot
parent ed0ce0fc

lib/env_dpdk/memory.c

+119 −82
Original line number Diff line number Diff line
@@ -174,6 +174,54 @@ mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, int *page_size 
	return map->default_translation;

}



static int

mem_map_walk_region(struct spdk_mem_map *map, uint64_t vaddr, size_t size,

		    int (*callback)(struct spdk_mem_map *map, uint64_t addr, size_t sz, void *ctx),

		    void *ctx)

{

	uint64_t vfn_4kb, vfn_2mb;

	uint64_t vfn_4kb_end, vfn_2mb_end;

	int rc;



	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = spdk_min(FN_2MB_TO_4KB(VFN_2MB(vaddr + MASK_2MB)), VFN_4KB(vaddr + size));

	while (vfn_4kb < vfn_4kb_end) {

		rc = callback(map, vaddr, VALUE_4KB, ctx);

		if (rc != 0) {

			return rc;

		}

		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	}



	vfn_2mb = VFN_2MB(vaddr);

	vfn_2mb_end = VFN_2MB(vaddr + size);

	while (vfn_2mb < vfn_2mb_end) {

		rc = callback(map, vaddr, VALUE_2MB, ctx);

		if (rc != 0) {

			return rc;

		}

		vaddr += VALUE_2MB;

		size -= VALUE_2MB;

		vfn_2mb++;

	}



	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = VFN_4KB(vaddr + size);

	while (vfn_4kb < vfn_4kb_end) {

		rc = callback(map, vaddr, VALUE_4KB, ctx);

		if (rc != 0) {

			return rc;

		}

		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	}



	return 0;

}



/*

 * Walk the currently registered memory via the main memory registration map

 * and call the new map's notify callback for each virtually contiguous region.
@@ -705,31 +753,15 @@ mem_map_get_map_2mb4kb(struct spdk_mem_map *map, uint64_t vfn_4kb, bool alloc) 
	return map_2mb4kb;

}



int

spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size,

			     uint64_t translation)

static int

mem_map_set_4kb_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t translation)

{

	uint64_t vfn_4kb, vfn_2mb;

	uint64_t vfn_4kb_end, vfn_2mb_end;

	uint64_t i, idx_2mb, idx_1gb;

	struct map_1gb2mb *map_1gb2mb;

	struct map_2mb4kb *map_2mb4kb;

	struct map_1gb2mb *map_1gb2mb;

	uint64_t vfn_4kb, vfn_2mb;

	uint64_t idx_2mb, idx_1gb;



	if ((uintptr_t)vaddr & ~MASK_256TB) {

		DEBUG_PRINT("invalid usermode virtual address %" PRIu64 "\n", vaddr);

		return -EINVAL;

	}



	if (((uintptr_t)vaddr & MASK_4KB) || (size & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%" PRIu64 " len=%" PRIu64 "\n",

			    __func__, vaddr, size);

		return -EINVAL;

	}



	/* Map the initial portion of the 4kb pages */

	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = spdk_min(FN_2MB_TO_4KB(VFN_2MB(vaddr + MASK_2MB)), VFN_4KB(vaddr + size));

	while (vfn_4kb < vfn_4kb_end) {

	map_2mb4kb = mem_map_get_map_2mb4kb(map, vfn_4kb, true);

	if (!map_2mb4kb) {

		DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
@@ -747,15 +779,17 @@ spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t 
		map_1gb2mb->translation_2mb[idx_1gb] = map->default_translation;

	}



		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	return 0;

}



	/* Map the 2mb pages */

static int

mem_map_set_2mb_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t translation)

{

	struct map_2mb4kb *map_2mb4kb;

	struct map_1gb2mb *map_1gb2mb;

	uint64_t i, vfn_2mb, idx_1gb;



	vfn_2mb = VFN_2MB(vaddr);

	vfn_2mb_end = VFN_2MB(vaddr + size);

	while (vfn_2mb < vfn_2mb_end) {

	map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb, true);

	if (!map_1gb2mb) {

		DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
@@ -775,38 +809,41 @@ spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t 
		}

	}



		vaddr += VALUE_2MB;

		size -= VALUE_2MB;

		vfn_2mb++;

	return 0;

}



	/* Finally, map the trailing 4kb pages */

	vfn_4kb = VFN_4KB(vaddr);

	vfn_4kb_end = VFN_4KB(vaddr + size);

	while (vfn_4kb < vfn_4kb_end) {

		map_2mb4kb = mem_map_get_map_2mb4kb(map, vfn_4kb, true);

		if (!map_2mb4kb) {

			DEBUG_PRINT("could not get %p map\n", (void *)vaddr);

			return -ENOMEM;

static int

mem_map_set_page_translation(struct spdk_mem_map *map, uint64_t vaddr, size_t page_size,

			     void *translation)

{

	switch (page_size) {

	case VALUE_4KB:

		return mem_map_set_4kb_translation(map, vaddr, (uint64_t)translation);

	case VALUE_2MB:

		return mem_map_set_2mb_translation(map, vaddr, (uint64_t)translation);

	default:

		assert(0 && "should never happan");

		return -EINVAL;

	}

}



		idx_2mb = MAP_2MB_IDX(vfn_4kb);

		map_2mb4kb->translation_4kb[idx_2mb] = translation;



		/* Set 2MB map to the default translation to indicate this region has 4KB mapping */

		vfn_2mb = FN_4KB_TO_2MB(vfn_4kb);

		map_1gb2mb = mem_map_get_map_1gb2mb(map, vfn_2mb, false);

		if (map_1gb2mb != NULL) {

			idx_1gb = MAP_1GB_IDX(vfn_2mb);

			map_1gb2mb->translation_2mb[idx_1gb] = map->default_translation;

int

spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size,

			     uint64_t translation)

{

	if ((uintptr_t)vaddr & ~MASK_256TB) {

		DEBUG_PRINT("invalid usermode virtual address %" PRIu64 "\n", vaddr);

		return -EINVAL;

	}



		vaddr += VALUE_4KB;

		size -= VALUE_4KB;

		vfn_4kb++;

	if (((uintptr_t)vaddr & MASK_4KB) || (size & MASK_4KB)) {

		DEBUG_PRINT("invalid %s parameters, vaddr=%" PRIu64 " len=%" PRIu64 "\n",

			    __func__, vaddr, size);

		return -EINVAL;

	}



	return 0;

	return mem_map_walk_region(map, vaddr, size, mem_map_set_page_translation,

				   (void *)translation);

}



int