Commit 71a17628 authored by Artur Paszkiewicz's avatar Artur Paszkiewicz Committed by Jim Harris
Browse files

ftl: mempool support for durable format objects 



Allows for using shared memory in memory pools. Adds API for
accessing such pools after dirty shutdown (claiming them, ie.
marking an entry as actively used; calling the
ftl_mempool_initialize_ext will reclaim all unused entries back
to the pool). Also introduces API for accessing objects, since
using direct pointers is not possible (as addresses may change
inbetween application startups).

Signed-off-by: default avatarArtur Paszkiewicz <artur.paszkiewicz@intel.com>
Signed-off-by: default avatarKozlowski Mateusz <mateusz.kozlowski@intel.com>
Change-Id: I5325b39d68aef7e231945cee9d92c925cab2fb2a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13346


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent f1b079b4

lib/ftl/utils/ftl_df.h

0 → 100644
+46 −0
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (c) Intel Corporation.

 *   All rights reserved.

 */



#ifndef FTL_DF_H

#define FTL_DF_H



#include "spdk/stdinc.h"



/* Durable format (df) object is an offset */

typedef uint64_t ftl_df_obj_id;



#define FTL_DF_OBJ_ID_INVALID ((ftl_df_obj_id)-1)



/**

 * @brief Convert df object ptr to df object id

 *

 * @param base		allocation base address

 * @param df_obj_ptr	df object ptr

 *

 * @return df object id

 */

static inline ftl_df_obj_id

ftl_df_get_obj_id(void *base, void *df_obj_ptr)

{

	assert(base <= df_obj_ptr);

	return ((char *)df_obj_ptr - (char *)base);

}



/**

 * @brief Convert df object id to df object ptr

 *

 * @param base		allocation base address

 * @param df_obj_id	df object id

 *

 * @return df object ptr

 */

static inline void *

ftl_df_get_obj_ptr(void *base, ftl_df_obj_id df_obj_id)

{

	assert(df_obj_id != FTL_DF_OBJ_ID_INVALID);

	return ((char *)base + df_obj_id);

}



#endif /* FTL_DF_H */

lib/ftl/utils/ftl_mempool.c

+142 −0
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ 
#include "spdk/likely.h"



#include "ftl_mempool.h"

#include "ftl_bitmap.h"



struct ftl_mempool_element {

	SLIST_ENTRY(ftl_mempool_element) entry;
@@ -23,6 +24,8 @@ struct ftl_mempool { 
	size_t count;

	size_t alignment;

	int socket_id;

	struct ftl_bitmap *inuse_bmp;

	void *inuse_buf;

};



static inline bool is_element_valid(struct ftl_mempool *mpool,
@@ -124,11 +127,18 @@ ftl_mempool_destroy(struct ftl_mempool *mpool) 
	free(mpool);

}



static inline bool

ftl_mempool_is_initialized(struct ftl_mempool *mpool)

{

	return mpool->inuse_buf == NULL;

}



void *

ftl_mempool_get(struct ftl_mempool *mpool)

{

	struct ftl_mempool_element *el;



	assert(ftl_mempool_is_initialized(mpool));

	if (spdk_unlikely(SLIST_EMPTY(&mpool->list))) {

		return NULL;

	}
@@ -144,6 +154,138 @@ ftl_mempool_put(struct ftl_mempool *mpool, void *element) 
{

	struct ftl_mempool_element *el = element;



	assert(ftl_mempool_is_initialized(mpool));

	assert(is_element_valid(mpool, element));

	SLIST_INSERT_HEAD(&mpool->list, el, entry);

}



struct ftl_mempool *

ftl_mempool_create_ext(void *buffer, size_t count, size_t size, size_t alignment)

{

	struct ftl_mempool *mp;

	size_t inuse_buf_sz;



	assert(buffer);

	assert(count > 0);

	assert(size > 0);



	mp = calloc(1, sizeof(*mp));

	if (!mp) {

		goto error;

	}



	size = spdk_max(size, sizeof(struct ftl_mempool_element));



	mp->count = count;

	mp->element_size = element_size_aligned(size, alignment);

	mp->alignment = alignment;

	SLIST_INIT(&mp->list);



	/* Calculate underlying inuse_bmp's buf size */

	inuse_buf_sz = spdk_divide_round_up(mp->count, 8);

	/* The bitmap size must be a multiple of word size (8b) - round up */

	if (inuse_buf_sz & 7UL) {

		inuse_buf_sz &= ~7UL;

		inuse_buf_sz += 8;

	}



	mp->inuse_buf = calloc(1, inuse_buf_sz);

	if (!mp->inuse_buf) {

		goto error;

	}



	mp->inuse_bmp = ftl_bitmap_create(mp->inuse_buf, inuse_buf_sz);

	if (!mp->inuse_bmp) {

		goto error;

	}



	/* Map the buffer */

	mp->buffer_size = mp->element_size * mp->count;

	mp->buffer = buffer;



	return mp;



error:

	ftl_mempool_destroy_ext(mp);

	return NULL;

}



void

ftl_mempool_destroy_ext(struct ftl_mempool *mpool)

{

	if (!mpool) {

		return;

	}



	if (mpool->inuse_bmp) {

		ftl_bitmap_destroy(mpool->inuse_bmp);

	}

	free(mpool->inuse_buf);

	free(mpool);

}



void

ftl_mempool_initialize_ext(struct ftl_mempool *mpool)

{

	struct ftl_mempool_element *el;

	void *buffer = mpool->buffer;

	size_t i;



	assert(!ftl_mempool_is_initialized(mpool));



	for (i = 0; i < mpool->count; i++, buffer += mpool->element_size) {

		if (ftl_bitmap_get(mpool->inuse_bmp, i)) {

			continue;

		}

		el = buffer;

		assert(is_element_valid(mpool, el));

		SLIST_INSERT_HEAD(&mpool->list, el, entry);

	}



	ftl_bitmap_destroy(mpool->inuse_bmp);

	mpool->inuse_bmp = NULL;



	free(mpool->inuse_buf);

	mpool->inuse_buf = NULL;

}



ftl_df_obj_id

ftl_mempool_get_df_obj_id(struct ftl_mempool *mpool, void *df_obj_ptr)

{

	return ftl_df_get_obj_id(mpool->buffer, df_obj_ptr);

}



size_t

ftl_mempool_get_df_obj_index(struct ftl_mempool *mpool, void *df_obj_ptr)

{

	return ftl_mempool_get_df_obj_id(mpool, df_obj_ptr) / mpool->element_size;

}



void *

ftl_mempool_get_df_ptr(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id)

{

	return ftl_df_get_obj_ptr(mpool->buffer, df_obj_id);

}



void *

ftl_mempool_claim_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id)

{

	struct ftl_mempool_element *el = ftl_df_get_obj_ptr(mpool->buffer, df_obj_id);



	assert(!ftl_mempool_is_initialized(mpool));

	assert(df_obj_id % mpool->element_size == 0);

	assert(df_obj_id / mpool->element_size < mpool->count);



	ftl_bitmap_set(mpool->inuse_bmp, df_obj_id / mpool->element_size);

	return el;

}



void

ftl_mempool_release_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id)

{

	assert(!ftl_mempool_is_initialized(mpool));

	assert(df_obj_id % mpool->element_size == 0);

	assert(df_obj_id / mpool->element_size < mpool->count);



	ftl_bitmap_clear(mpool->inuse_bmp, df_obj_id / mpool->element_size);

}

lib/ftl/utils/ftl_mempool.h

+112 −3
Original line number Diff line number Diff line
@@ -8,12 +8,12 @@ 


#include "spdk/stdinc.h"



#include "ftl_df.h"



/* TODO: Consider porting this mempool to general SPDK utils */



/**

 * @brief Creates custom FTL memory pool using DMA kind memory

 *

 * The pool is being initialized.

 * @brief Creates and initializes custom FTL memory pool using DMA kind memory

 *

 * @param count Count of element in the memory pool

 * @param size Size of elements in the memory pool
@@ -36,6 +36,8 @@ void ftl_mempool_destroy(struct ftl_mempool *mpool); 
/**

 * @brief Gets (allocates) an element from the memory pool

 *

 * Allowed only for the initialized memory pool.

 *

 * @param mpool The memory pool

 *

 * @return Element from memory pool. If memory pool empty it returns NULL.
@@ -45,9 +47,116 @@ void *ftl_mempool_get(struct ftl_mempool *mpool); 
/**

 * @brief Puts (releases) the element to the memory pool

 *

 * Allowed only for the initialized memory pool.

 *

 * @param mpool The memory pool

 * @param element The element to be released

 */

void ftl_mempool_put(struct ftl_mempool *mpool, void *element);



/**

 * @brief Creates custom FTL memory pool using memory allocated externally

 *

 * The pool is uninitialized.

 * The uninitialized pool is accessible via ftl_mempool_claim_df() and

 * ftl_mempool_release_df() APIs. The pool's free buffer list is initialized

 * to contain only elements that were not claimed using ftl_mempool_claim_df()

 * after the call to ftl_mempool_initialize_ext.

 * See ftl_mempool_initialize_ext().

 *

 * @param buffer Externally allocated underlying memory buffer

 * @param count Count of element in the memory pool

 * @param size Size of elements in the memory pool

 * @param alignment Memory alignment of element in the memory pool

 *

 * @return Pointer to the memory pool

 */

struct ftl_mempool *ftl_mempool_create_ext(void *buffer, size_t count, size_t size,

		size_t alignment);



/**

 * @brief Destroys the FTL memory pool w/ externally allocated underlying mem buf

 *

 * The external buf is not being freed.

 *

 * @param mpool The memory pool to be destroyed

 */

void ftl_mempool_destroy_ext(struct ftl_mempool *mpool);



/**

 * @brief Initialize the FTL memory pool w/ externally allocated mem buf.

 *

 * The pool is initialized to contain only elements that were not claimed.

 * All claimed elements are considered to be in use and will be returned

 * to the pool via ftl_mempool_put() after initialization.

 * After the pool is initialized, it is only accessible via

 * ftl_mempool_get() and ftl_mempool_put() APIs.

 *

 * This function should only be called on an uninitialized pool (ie. created via ftl_mempool_create_ext).

 * Any attempt to initialize an already initialized pool (whether after calling ftl_mempool_create, or

 * calling ftl_mempool_initialize_ext twice) will result in an assert.

 *

 * Depending on the memory pool being initialized or not, the use of the

 * following APIs is as follows:

 * API					uninitialized pool		initialized pool

 * ftl_mempool_get()			disallowed			allowed

 * ftl_mempool_put()			disallowed			allowed

 * ftl_mempool_claim_df()		allowed				disallowed

 * ftl_mempool_release_df()		allowed				disallowed

 *

 * @param mpool The memory pool

 */

void ftl_mempool_initialize_ext(struct ftl_mempool *mpool);



/**

 * @brief Return a df object id for a given pool element.

 *

 * @param mpool			The memory pool

 * @param df_obj_ptr		Pointer to the pool element

 *

 * @return df object id

 */

ftl_df_obj_id ftl_mempool_get_df_obj_id(struct ftl_mempool *mpool, void *df_obj_ptr);



/**

 * @brief Return an element pointer for a given df object id.

 *

 * @param mpool			The memory pool

 * @param df_obj_id		Df object id of a pool element

 *

 * @return Element ptr

 */

void *ftl_mempool_get_df_ptr(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id);



/**

 * @brief Claim an element for use.

 *

 * Allowed only for the uninitialized memory pool.

 *

 * @param mpool			The memory pool

 * @param df_obj_id		Df object id of a pool element to claim

 *

 * @return Element ptr

 */

void *ftl_mempool_claim_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id);



/**

 * @brief Release an element to the pool.

 *

 * Allowed only for the uninitialized memory pool.

 *

 * @param mpool			The memory pool

 * @param df_obj_id		Df object id of a pool element to claim

 */

void ftl_mempool_release_df(struct ftl_mempool *mpool, ftl_df_obj_id df_obj_id);



/**

 * @brief Return an index for a given element in the memory pool.

 *

 * @param mpool			The memory pool

 * @param df_obj_ptr		Element from df memory pool. The pointer may be offset from the beginning of the element.

 *

 * @return Index (offset / element_size) of the element parameter from the beginning of the pool

 */

size_t ftl_mempool_get_df_obj_index(struct ftl_mempool *mpool, void *df_obj_ptr);

#endif /* FTL_MEMPOOL_H */

test/unit/lib/ftl/ftl_mempool.c/ftl_mempool_ut.c

+6 −0
Original line number Diff line number Diff line
@@ -15,6 +15,12 @@ 
#define SIZE (ALIGNMENT * 2)

#define SOCKET_ID_ANY -1



DEFINE_STUB(ftl_bitmap_create, struct ftl_bitmap *, (void *buf, size_t size), NULL);

DEFINE_STUB_V(ftl_bitmap_destroy, (struct ftl_bitmap *bitmap));

DEFINE_STUB(ftl_bitmap_get, bool, (const struct ftl_bitmap *bitmap, uint64_t bit), true);

DEFINE_STUB_V(ftl_bitmap_set, (struct ftl_bitmap *bitmap, uint64_t bit));

DEFINE_STUB_V(ftl_bitmap_clear, (struct ftl_bitmap *bitmap, uint64_t bit));



static struct ftl_mempool *g_mpool;



static void