Commit 94b7f8d8 authored by Kozlowski Mateusz's avatar Kozlowski Mateusz Committed by Ben Walker
Browse files

FTL: Add L2P cache eviction logic 



Adds eviction of least recently used pages from the L2P cache - dirty
pages will be persisted.

Signed-off-by: default avatarKozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: default avatarArtur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Ic646f7e9da777d077b5cb9b409c3f03ef05b1273
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13357


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 73f9b4f5

lib/ftl/ftl_l2p_cache.c

+175 −0
Original line number Diff line number Diff line
@@ -140,6 +140,7 @@ typedef void (*ftl_l2p_cache_sync_cb)(struct spdk_ftl_dev *dev, int status, void 
static bool page_set_is_done(struct ftl_l2p_page_set *page_set);

static void page_set_end(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,

			 struct ftl_l2p_page_set *page_set);

static void page_out_io_retry(void *arg);

static void page_in_io_retry(void *arg);



static inline void
@@ -216,6 +217,18 @@ ftl_l2p_cache_page_remove(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page 
	ftl_mempool_put(cache->l2_ctx_pool, page);

}



static inline struct ftl_l2p_page *

ftl_l2p_cache_get_coldest_page(struct ftl_l2p_cache *cache)

{

	return TAILQ_LAST(&cache->lru_list, l2p_lru_list);

}



static inline struct ftl_l2p_page *

ftl_l2p_cache_get_hotter_page(struct ftl_l2p_page *page)

{

	return TAILQ_PREV(page, l2p_lru_list, list_entry);

}



static inline uint64_t

ftl_l2p_cache_page_get_bdev_offset(struct ftl_l2p_cache *cache,

				   struct ftl_l2p_page *page)
@@ -258,6 +271,14 @@ ftl_l2p_cache_page_alloc(struct ftl_l2p_cache *cache, size_t page_no) 
	return page;

}



static inline bool

ftl_l2p_cache_page_can_remove(struct ftl_l2p_page *page)

{

	return (!page->updates &&

		page->state != L2P_CACHE_PAGE_INIT &&

		!page->pin_ref_cnt);

}



static inline ftl_addr

ftl_l2p_cache_get_addr(struct spdk_ftl_dev *dev,

		       struct ftl_l2p_cache *cache, struct ftl_l2p_page *page, uint64_t lba)
@@ -302,6 +323,21 @@ ftl_l2p_cache_page_unpin(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page) 
	}

}



static inline bool

ftl_l2p_cache_page_can_evict(struct ftl_l2p_page *page)

{

	return (page->state == L2P_CACHE_PAGE_FLUSHING ||

		page->state == L2P_CACHE_PAGE_PERSISTING ||

		page->state == L2P_CACHE_PAGE_INIT ||

		page->pin_ref_cnt) ? false : true;

}



static bool

ftl_l2p_cache_evict_continue(struct ftl_l2p_cache *cache)

{

	return cache->l2_pgs_avail + cache->l2_pgs_evicting < cache->evict_keep;

}



static void *

_ftl_l2p_cache_init(struct spdk_ftl_dev *dev, size_t addr_size, uint64_t l2p_size)

{
@@ -900,6 +936,143 @@ ftl_l2p_cache_process_page_sets(struct spdk_ftl_dev *dev, struct ftl_l2p_cache * 
	return 0;

}



static struct ftl_l2p_page *

eviction_get_page(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)

{

	uint64_t i = 0;

	struct ftl_l2p_page *page = ftl_l2p_cache_get_coldest_page(cache);



	while (page) {

		ftl_bug(L2P_CACHE_PAGE_READY != page->state);

		ftl_bug(page->pin_ref_cnt);



		if (ftl_l2p_cache_page_can_evict(page)) {

			ftl_l2p_cache_lru_remove_page(cache, page);

			return page;

		}



		/*

		 * Practically only one iteration is needed to find a page. It is because

		 * the rank of pages contains only ready and unpinned pages

		 */

		ftl_bug(++i > 1024);



		page = ftl_l2p_cache_get_hotter_page(page);

	}



	return NULL;

}



static void

page_out_io_complete(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,

		     struct ftl_l2p_page *page, bool success)

{

	cache->l2_pgs_evicting--;



	ftl_bug(page->ctx.updates > page->updates);

	ftl_bug(!TAILQ_EMPTY(&page->ppe_list));

	ftl_bug(page->on_lru_list);



	if (spdk_likely(success)) {

		page->updates -= page->ctx.updates;

	}



	if (success && ftl_l2p_cache_page_can_remove(page)) {

		ftl_l2p_cache_page_remove(cache, page);

	} else {

		if (!page->pin_ref_cnt) {

			ftl_l2p_cache_lru_add_page(cache, page);

		}

		page->state = L2P_CACHE_PAGE_READY;

	}

}



static void

page_out_io_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)

{

	struct ftl_l2p_page *page = cb_arg;

	struct ftl_l2p_cache *cache = page->ctx.cache;

	struct spdk_ftl_dev *dev = cache->dev;



	spdk_bdev_free_io(bdev_io);

	page_out_io_complete(dev, cache, page, success);

}



static void

page_out_io(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,

	    struct ftl_l2p_page *page)

{

	struct spdk_io_channel *ioch;

	struct spdk_bdev *bdev;

	struct spdk_bdev_io_wait_entry *bdev_io_wait;

	int rc;



	page->ctx.cache = cache;



	rc = ftl_nv_cache_bdev_write_blocks_with_md(dev, ftl_l2p_cache_get_bdev_desc(cache),

			ftl_l2p_cache_get_bdev_iochannel(cache),

			page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),

			1, page_out_io_cb, page);



	cache->l2_pgs_evicting++;

	if (spdk_likely(0 == rc)) {

		return;

	}



	if (rc == -ENOMEM) {

		ioch = ftl_l2p_cache_get_bdev_iochannel(cache);

		bdev = spdk_bdev_desc_get_bdev(ftl_l2p_cache_get_bdev_desc(cache));

		bdev_io_wait = &page->ctx.bdev_io_wait;

		bdev_io_wait->bdev = bdev;

		bdev_io_wait->cb_fn = page_out_io_retry;

		bdev_io_wait->cb_arg = page;



		rc = spdk_bdev_queue_io_wait(bdev, ioch, bdev_io_wait);

		ftl_bug(rc);

	} else {

		ftl_abort();

	}

}



static void

page_out_io_retry(void *arg)

{

	struct ftl_l2p_page *page = arg;

	struct ftl_l2p_cache *cache = page->ctx.cache;

	struct spdk_ftl_dev *dev = cache->dev;



	cache->l2_pgs_evicting--;

	page_out_io(dev, cache, page);

}



static void

ftl_l2p_cache_process_eviction(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)

{

	struct ftl_l2p_page *page;



	if (!ftl_l2p_cache_evict_continue(cache)) {

		return;

	}



	if (cache->l2_pgs_evicting > 512) {

		return;

	}



	page = eviction_get_page(dev, cache);

	if (spdk_unlikely(!page)) {

		return;

	}



	if (page->updates) {

		page->state = L2P_CACHE_PAGE_FLUSHING;

		page->ctx.updates = page->updates;

		page_out_io(dev, cache, page);

	} else {

		/* Page clean and we can remove it */

		ftl_l2p_cache_page_remove(cache, page);

	}

}



void

ftl_l2p_cache_process(struct spdk_ftl_dev *dev)

{
@@ -915,4 +1088,6 @@ ftl_l2p_cache_process(struct spdk_ftl_dev *dev) 
			break;

		}

	}



	ftl_l2p_cache_process_eviction(dev, cache);

}