Commit 2c43c9bc authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

vtophys: begin refactoring into general memory map 



Add a top-level structure that can be reused for other kinds of memory
address translations.

Change-Id: I046f98b76b4e98087d90095d6e9dea5cd6ab7898
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 44ceff8c

lib/env_dpdk/vtophys.c

+130 −88
Original line number Diff line number Diff line
@@ -70,13 +70,13 @@ 
/* Max value for a 16-bit ref count. */

#define VTOPHYS_MAX_REF_COUNT (0xFFFF)



/* Physical address of a single 2MB page. */

/* Translation of a single 2MB page. */

struct map_2mb {

	uint64_t paddr_2mb;

	uint64_t translation_2mb;

};



/* Second-level map table indexed by bits [21..29] of the virtual address.

 * Each entry contains the 2MB physical address or SPDK_VTOPHYS_ERROR for entries that haven't

 * Each entry contains the address translation or SPDK_VTOPHYS_ERROR for entries that haven't

 * been retrieved yet.

 */

struct map_1gb {
@@ -91,33 +91,38 @@ struct map_128tb { 
	struct map_1gb *map[1ULL << (SHIFT_128TB - SHIFT_1GB + 1)];

};



static struct map_128tb vtophys_map_128tb = {};

static pthread_mutex_t vtophys_mutex = PTHREAD_MUTEX_INITIALIZER;

/* Page-granularity memory address translation */

struct spdk_mem_map {

	struct map_128tb map_128tb;

	pthread_mutex_t mutex;

};



static struct spdk_mem_map g_vtophys_map = {{}, PTHREAD_MUTEX_INITIALIZER};



static struct map_1gb *

vtophys_get_map_1gb(uint64_t vfn_2mb)

spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb)

{

	struct map_1gb *map_1gb;

	uint64_t idx_128tb = MAP_128TB_IDX(vfn_2mb);



	map_1gb = vtophys_map_128tb.map[idx_128tb];

	map_1gb = map->map_128tb.map[idx_128tb];



	if (!map_1gb) {

		pthread_mutex_lock(&vtophys_mutex);

		pthread_mutex_lock(&map->mutex);



		/* Recheck to make sure nobody else got the mutex first. */

		map_1gb = vtophys_map_128tb.map[idx_128tb];

		map_1gb = map->map_128tb.map[idx_128tb];

		if (!map_1gb) {

			map_1gb = malloc(sizeof(struct map_1gb));

			if (map_1gb) {

				/* initialize all entries to all 0xFF (SPDK_VTOPHYS_ERROR) */

				memset(map_1gb->map, 0xFF, sizeof(map_1gb->map));

				memset(map_1gb->ref_count, 0, sizeof(map_1gb->ref_count));

				vtophys_map_128tb.map[idx_128tb] = map_1gb;

				map->map_128tb.map[idx_128tb] = map_1gb;

			}

		}



		pthread_mutex_unlock(&vtophys_mutex);

		pthread_mutex_unlock(&map->mutex);



		if (!map_1gb) {

#ifdef DEBUG
@@ -130,6 +135,116 @@ vtophys_get_map_1gb(uint64_t vfn_2mb) 
	return map_1gb;

}



static void

spdk_mem_map_register(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size, uint64_t translation)

{

	uint64_t vfn_2mb;

	struct map_1gb *map_1gb;

	uint64_t idx_1gb;

	struct map_2mb *map_2mb;

	uint16_t *ref_count;



	/* For now, only 2 MB registrations are supported */

	assert(size == 2 * 1024 * 1024);

	assert((vaddr & MASK_2MB) == 0);



	vfn_2mb = vaddr >> SHIFT_2MB;



	map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);

	if (!map_1gb) {

#ifdef DEBUG

		fprintf(stderr, "could not get %p map\n", (void *)vaddr);

#endif

		return;

	}



	idx_1gb = MAP_1GB_IDX(vfn_2mb);

	map_2mb = &map_1gb->map[idx_1gb];

	ref_count = &map_1gb->ref_count[idx_1gb];



	if (*ref_count == VTOPHYS_MAX_REF_COUNT) {

#ifdef DEBUG

		fprintf(stderr, "ref count for %p already at %d\n",

			(void *)vaddr, VTOPHYS_MAX_REF_COUNT);

#endif

		return;

	}



	map_2mb->translation_2mb = translation;



	(*ref_count)++;

}



static void

spdk_mem_map_unregister(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size)

{

	uint64_t vfn_2mb;

	struct map_1gb *map_1gb;

	uint64_t idx_1gb;

	struct map_2mb *map_2mb;

	uint16_t *ref_count;



	/* For now, only 2 MB registrations are supported */

	assert(size == 2 * 1024 * 1024);

	assert((vaddr & MASK_2MB) == 0);



	vfn_2mb = vaddr >> SHIFT_2MB;



	map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);

	if (!map_1gb) {

#ifdef DEBUG

		fprintf(stderr, "could not get %p map\n", (void *)vaddr);

#endif

		return;

	}



	idx_1gb = MAP_1GB_IDX(vfn_2mb);

	map_2mb = &map_1gb->map[idx_1gb];

	ref_count = &map_1gb->ref_count[idx_1gb];



	if (*ref_count == 0) {

#ifdef DEBUG

		fprintf(stderr, "vaddr %p not registered\n", (void *)vaddr);

#endif

		return;

	}



	(*ref_count)--;

	if (*ref_count == 0) {

		map_2mb->translation_2mb = SPDK_VTOPHYS_ERROR;

	}

}



static uint64_t

spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr)

{

	const struct map_1gb *map_1gb;

	const struct map_2mb *map_2mb;

	uint64_t idx_128tb;

	uint64_t idx_1gb;

	uint64_t vfn_2mb;



	if (spdk_unlikely(vaddr & ~MASK_128TB)) {

#ifdef DEBUG

		printf("invalid usermode virtual address %p\n", (void *)vaddr);

#endif

		return SPDK_VTOPHYS_ERROR;

	}



	vfn_2mb = vaddr >> SHIFT_2MB;

	idx_128tb = MAP_128TB_IDX(vfn_2mb);

	idx_1gb = MAP_1GB_IDX(vfn_2mb);



	map_1gb = map->map_128tb.map[idx_128tb];

	if (spdk_unlikely(!map_1gb)) {

		return SPDK_VTOPHYS_ERROR;

	}



	map_2mb = &map_1gb->map[idx_1gb];



	return map_2mb->translation_2mb;

}



static uint64_t

vtophys_get_dpdk_paddr(void *vaddr)

{
@@ -187,11 +302,6 @@ vtophys_get_paddr(uint64_t vaddr) 
static void

_spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr)

{

	struct map_1gb *map_1gb;

	uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);

	struct map_2mb *map_2mb;

	uint16_t *ref_count;



	if (paddr & MASK_2MB) {

#ifdef DEBUG

		fprintf(stderr, "invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr);
@@ -199,60 +309,13 @@ _spdk_vtophys_register_one(uint64_t vfn_2mb, uint64_t paddr) 
		return;

	}



	map_1gb = vtophys_get_map_1gb(vfn_2mb);

	if (!map_1gb) {

#ifdef DEBUG

		fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);

#endif

		return;

	}



	map_2mb = &map_1gb->map[idx_1gb];

	ref_count = &map_1gb->ref_count[idx_1gb];



	if (*ref_count == VTOPHYS_MAX_REF_COUNT) {

#ifdef DEBUG

		fprintf(stderr, "ref count for %p already at %d\n",

			(void *)(vfn_2mb << SHIFT_2MB), VTOPHYS_MAX_REF_COUNT);

#endif

		return;

	}



	map_2mb->paddr_2mb = paddr;



	(*ref_count)++;

	spdk_mem_map_register(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024, paddr);

}



static void

_spdk_vtophys_unregister_one(uint64_t vfn_2mb)

{

	struct map_1gb *map_1gb;

	uint64_t idx_1gb = MAP_1GB_IDX(vfn_2mb);

	struct map_2mb *map_2mb;

	uint16_t *ref_count;



	map_1gb = vtophys_get_map_1gb(vfn_2mb);

	if (!map_1gb) {

#ifdef DEBUG

		fprintf(stderr, "could not get vfn_2mb %p map\n", (void *)vfn_2mb);

#endif

		return;

	}



	map_2mb = &map_1gb->map[idx_1gb];

	ref_count = &map_1gb->ref_count[idx_1gb];



	if (map_2mb->paddr_2mb == SPDK_VTOPHYS_ERROR || *ref_count == 0) {

#ifdef DEBUG

		fprintf(stderr, "vaddr %p not registered\n", (void *)(vfn_2mb << SHIFT_2MB));

#endif

		return;

	}



	(*ref_count)--;

	if (*ref_count == 0) {

		map_2mb->paddr_2mb = SPDK_VTOPHYS_ERROR;

	}

	spdk_mem_map_unregister(&g_vtophys_map, vfn_2mb << SHIFT_2MB, 2 * 1024 * 1024);

}



void
@@ -347,32 +410,11 @@ spdk_vtophys_register_dpdk_mem(void) 
uint64_t

spdk_vtophys(void *buf)

{

	struct map_1gb *map_1gb;

	struct map_2mb *map_2mb;

	uint64_t idx_128tb;

	uint64_t idx_1gb;

	uint64_t vaddr, vfn_2mb, paddr_2mb;

	uint64_t vaddr, paddr_2mb;



	vaddr = (uint64_t)buf;

	if (spdk_unlikely(vaddr & ~MASK_128TB)) {

#ifdef DEBUG

		printf("invalid usermode virtual address %p\n", buf);

#endif

		return SPDK_VTOPHYS_ERROR;

	}



	vfn_2mb = vaddr >> SHIFT_2MB;

	idx_128tb = MAP_128TB_IDX(vfn_2mb);

	idx_1gb = MAP_1GB_IDX(vfn_2mb);



	map_1gb = vtophys_map_128tb.map[idx_128tb];

	if (spdk_unlikely(!map_1gb)) {

		return SPDK_VTOPHYS_ERROR;

	}



	map_2mb = &map_1gb->map[idx_1gb];



	paddr_2mb = map_2mb->paddr_2mb;

	paddr_2mb = spdk_mem_map_translate(&g_vtophys_map, vaddr);



	/*

	 * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR,