Commit 711759a0 authored by Kozlowski Mateusz's avatar Kozlowski Mateusz Committed by Jim Harris
Browse files

FTL: Add reloc helper functions 



Adds functions for reading end metadata and initializing band reloc state.

Signed-off-by: default avatarKozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: default avatarArtur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: I3d12c4a7edd36f0437bf10316114c83efe449f0f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13343


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent f45c0075

lib/ftl/ftl_band.c

+221 −0
Original line number Diff line number Diff line
@@ -66,6 +66,7 @@ _ftl_band_set_free(struct ftl_band *band) 


	/* Add the band to the free band list */

	TAILQ_INSERT_TAIL(&dev->free_bands, band, queue_entry);

	band->reloc = false;



	dev->num_free++;

	ftl_apply_limits(dev);
@@ -364,3 +365,223 @@ ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev) 
	/* Map pool element holds the whole tail md */

	return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE;

}



static double

_band_invalidity(struct ftl_band *band)

{

	double valid = band->p2l_map.num_valid;

	double count = ftl_band_user_blocks(band);



	return 1.0 - (valid / count);

}



static void

dump_bands_under_relocation(struct spdk_ftl_dev *dev)

{

	uint64_t i = dev->sb_shm->gc_info.current_band_id;

	uint64_t end = dev->sb_shm->gc_info.current_band_id + dev->num_logical_bands_in_physical;



	for (; i < end; i++) {

		struct ftl_band *band = &dev->bands[i];



		FTL_DEBUGLOG(dev, "Band, id %u, phys_is %u, wr cnt = %u, invalidity = %u%%\n",

			     band->id, band->phys_id, (uint32_t)band->md->wr_cnt,

			     (uint32_t)(_band_invalidity(band) * 100));

	}

}



static bool

is_band_relocateable(struct ftl_band *band)

{

	/* Can only move data from closed bands */

	if (FTL_BAND_STATE_CLOSED != band->md->state) {

		return false;

	}



	/* Band is already under relocation, skip it */

	if (band->reloc) {

		return false;

	}



	return true;

}



static void

get_band_phys_info(struct spdk_ftl_dev *dev, uint64_t phys_id,

		   double *invalidity, double *wr_cnt)

{

	struct ftl_band *band;

	uint64_t band_id = phys_id * dev->num_logical_bands_in_physical;



	*wr_cnt = *invalidity = 0.0L;

	for (; band_id < ftl_get_num_bands(dev); band_id++) {

		band = &dev->bands[band_id];



		if (phys_id != band->phys_id) {

			break;

		}



		*wr_cnt += band->md->wr_cnt;



		if (!is_band_relocateable(band)) {

			continue;

		}



		*invalidity += _band_invalidity(band);

	}



	*invalidity /= dev->num_logical_bands_in_physical;

	*wr_cnt /= dev->num_logical_bands_in_physical;

}



static bool

band_cmp(double a_invalidity, double a_wr_cnt,

	 double b_invalidity, double b_wr_cnt,

	 uint64_t a_id, uint64_t b_id)

{

	assert(a_id != FTL_BAND_PHYS_ID_INVALID);

	assert(b_id != FTL_BAND_PHYS_ID_INVALID);

	double diff = a_invalidity - b_invalidity;

	if (diff < 0.0L) {

		diff *= -1.0L;

	}



	/* Use the following metrics for picking bands for GC (in order):

	 * - relative invalidity

	 * - if invalidity is similar (within 10% points), then their write counts (how many times band was written to)

	 * - if write count is equal, then pick based on their placement on base device (lower LBAs win)

	 */

	if (diff > 0.1L) {

		return a_invalidity > b_invalidity;

	}



	if (a_wr_cnt != b_wr_cnt) {

		return a_wr_cnt < b_wr_cnt;

	}



	return a_id < b_id;

}



static void

band_start_gc(struct spdk_ftl_dev *dev, struct ftl_band *band)

{

	ftl_bug(false == is_band_relocateable(band));



	TAILQ_REMOVE(&dev->shut_bands, band, queue_entry);

	band->reloc = true;



	FTL_DEBUGLOG(dev, "Band to GC, id %u\n", band->id);

}



static struct ftl_band *

gc_high_priority_band(struct spdk_ftl_dev *dev)

{

	struct ftl_band *band;

	uint64_t high_prio_id = dev->sb_shm->gc_info.band_id_high_prio;



	if (FTL_BAND_ID_INVALID != high_prio_id) {

		ftl_bug(high_prio_id >= dev->num_bands);



		band = &dev->bands[high_prio_id];

		dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;



		band_start_gc(dev, band);

		FTL_NOTICELOG(dev, "GC takes high priority band, id %u\n", band->id);

		return band;

	}



	return 0;

}



static void

ftl_band_reset_gc_iter(struct spdk_ftl_dev *dev)

{

	dev->sb->gc_info.is_valid = 0;

	dev->sb->gc_info.current_band_id = FTL_BAND_ID_INVALID;

	dev->sb->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;

	dev->sb->gc_info.band_phys_id = FTL_BAND_PHYS_ID_INVALID;



	dev->sb_shm->gc_info = dev->sb->gc_info;

}



struct ftl_band *

ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev)

{

	double invalidity, max_invalidity = 0.0L;

	double wr_cnt, max_wr_cnt = 0.0L;

	uint64_t phys_id = FTL_BAND_PHYS_ID_INVALID;

	struct ftl_band *band;

	uint64_t i, band_count;

	uint64_t phys_count;



	band = gc_high_priority_band(dev);

	if (spdk_unlikely(NULL != band)) {

		return band;

	}



	phys_count = dev->num_logical_bands_in_physical;

	band_count = ftl_get_num_bands(dev);



	for (; dev->sb_shm->gc_info.current_band_id < band_count;) {

		band = &dev->bands[dev->sb_shm->gc_info.current_band_id];

		if (band->phys_id != dev->sb_shm->gc_info.band_phys_id) {

			break;

		}



		if (false == is_band_relocateable(band)) {

			dev->sb_shm->gc_info.current_band_id++;

			continue;

		}



		band_start_gc(dev, band);

		return band;

	}



	for (i = 0; i < band_count; i += phys_count) {

		band = &dev->bands[i];



		/* Calculate entire band physical group invalidity */

		get_band_phys_info(dev, band->phys_id, &invalidity, &wr_cnt);



		if (invalidity != 0.0L) {

			if (phys_id == FTL_BAND_PHYS_ID_INVALID ||

			    band_cmp(invalidity, wr_cnt, max_invalidity, max_wr_cnt,

				     band->phys_id, phys_id)) {

				max_invalidity = invalidity;

				max_wr_cnt = wr_cnt;

				phys_id = band->phys_id;

			}

		}

	}



	if (FTL_BAND_PHYS_ID_INVALID != phys_id) {

		FTL_DEBUGLOG(dev, "Band physical id %"PRIu64" to GC\n", phys_id);

		dev->sb_shm->gc_info.is_valid = 0;

		dev->sb_shm->gc_info.current_band_id = phys_id * phys_count;

		dev->sb_shm->gc_info.band_phys_id = phys_id;

		dev->sb_shm->gc_info.is_valid = 1;

		dump_bands_under_relocation(dev);

		return ftl_band_search_next_to_reloc(dev);

	} else {

		ftl_band_reset_gc_iter(dev);

	}



	return NULL;

}



void

ftl_band_init_gc_iter(struct spdk_ftl_dev *dev)

{

	if (dev->conf.mode & SPDK_FTL_MODE_CREATE) {

		ftl_band_reset_gc_iter(dev);

		return;

	}



	if (dev->sb->clean) {

		dev->sb_shm->gc_info = dev->sb->gc_info;

		return;

	}



	/* We lost GC state due to dirty shutdown, reset GC state to start over */

	ftl_band_reset_gc_iter(dev);

}

lib/ftl/ftl_band.h

+6 −0
Original line number Diff line number Diff line
@@ -97,6 +97,9 @@ struct ftl_band { 
	/* P2L map */

	struct ftl_p2l_map		p2l_map;



	/* Band relocation is in progress */

	bool				reloc;



	/* Band's index */

	uint32_t			id;
@@ -144,6 +147,8 @@ ftl_addr ftl_band_tail_md_addr(struct ftl_band *band); 
int ftl_band_filled(struct ftl_band *band, size_t offset);

int ftl_band_write_prep(struct ftl_band *band);

size_t ftl_p2l_map_pool_elem_size(struct spdk_ftl_dev *dev);

struct ftl_band *ftl_band_search_next_to_reloc(struct spdk_ftl_dev *dev);

void ftl_band_init_gc_iter(struct spdk_ftl_dev *dev);

ftl_addr ftl_band_p2l_map_addr(struct ftl_band *band);

void ftl_band_open(struct ftl_band *band, enum ftl_band_type type);

void ftl_band_close(struct ftl_band *band);
@@ -152,6 +157,7 @@ void ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq); 
void ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq);

void ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq);

void ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq);

void ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx);

void ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx);



static inline void

lib/ftl/ftl_band_ops.c

+84 −1
Original line number Diff line number Diff line
@@ -340,7 +340,7 @@ ftl_band_close(struct ftl_band *band) 
	void *metadata = band->p2l_map.band_map;

	uint64_t num_blocks = ftl_tail_md_num_blocks(dev);



	/* Write LBA map first, after completion, set the state to close on nvcache, then internally */

	/* Write P2L map first, after completion, set the state to close on nvcache, then internally */

	ftl_band_set_state(band, FTL_BAND_STATE_CLOSING);

	ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks);

	ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band);
@@ -382,6 +382,68 @@ ftl_band_free(struct ftl_band *band) 
	/* TODO: The whole band erase code should probably be done here instead */

}



static void

read_md_cb(struct ftl_basic_rq *brq)

{

	struct ftl_band *band = brq->owner.priv;

	struct spdk_ftl_dev *dev = band->dev;

	ftl_band_ops_cb cb;

	uint32_t band_map_crc;

	bool success = true;

	void *priv;



	cb = band->owner.ops_fn;

	priv = band->owner.priv;



	if (!brq->success) {

		ftl_band_basic_rq_read(band, &band->metadata_rq);

		return;

	}



	band_map_crc = spdk_crc32c_update(band->p2l_map.band_map,

					  ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0);

	if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) {

		FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n");

		success = false;

	}

	band->owner.ops_fn = NULL;

	band->owner.priv = NULL;

	cb(band, priv, success);

}



static int

_read_md(struct ftl_band *band)

{

	struct spdk_ftl_dev *dev = band->dev;

	struct ftl_basic_rq *rq = &band->metadata_rq;



	if (ftl_band_alloc_p2l_map(band)) {

		return -ENOMEM;

	}



	/* Read P2L map */

	ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev));

	ftl_basic_rq_set_owner(rq, read_md_cb, band);



	rq->io.band = band;

	rq->io.addr = ftl_band_p2l_map_addr(band);



	ftl_band_basic_rq_read(band, &band->metadata_rq);



	return 0;

}



static void

read_md(void *band)

{

	int rc;



	rc = _read_md(band);

	if (spdk_unlikely(rc)) {

		spdk_thread_send_msg(spdk_get_thread(), read_md, band);

	}

}



static void

read_tail_md_cb(struct ftl_basic_rq *brq)

{
@@ -426,3 +488,24 @@ ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx) 


	ftl_band_basic_rq_read(band, &band->metadata_rq);

}



void

ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx)

{

	struct ftl_band *band = ftl_band_search_next_to_reloc(dev);



	/* if disk is very small, GC start very early that no band is ready for it */

	if (spdk_unlikely(!band)) {

		cb(NULL, cntx, false);

		return;

	}



	/* Only one owner is allowed */

	assert(!band->queue_depth);

	assert(!band->owner.ops_fn);

	assert(!band->owner.priv);

	band->owner.ops_fn = cb;

	band->owner.priv = cntx;



	read_md(band);

}

lib/ftl/ftl_core.h

+2 −0
Original line number Diff line number Diff line
@@ -150,6 +150,8 @@ struct spdk_ftl_dev { 
	/* Writer for GC IOs */

	struct ftl_writer		writer_gc;



	uint32_t			num_logical_bands_in_physical;



	/* Retry init sequence */

	bool				init_retry;

};

lib/ftl/mngt/ftl_mngt_band.c

+66 −1
Original line number Diff line number Diff line
@@ -146,6 +146,8 @@ decorate_bands(struct spdk_ftl_dev *dev) 
		TAILQ_REMOVE(&dev->shut_bands, band, queue_entry);

		i++;

	}



	dev->num_logical_bands_in_physical = num_logical_in_phys;

}



void
@@ -155,6 +157,65 @@ ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 
	ftl_mngt_next_step(mngt);

}



static struct ftl_band *

next_high_prio_band(struct spdk_ftl_dev *dev)

{

	struct ftl_band *result = NULL, *band;

	uint64_t validity = UINT64_MAX;



	TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) {

		if (band->p2l_map.num_valid < validity) {

			result = band;

			validity = result->p2l_map.num_valid;

		}

	}



	return result;

}



static int

finalize_init_gc(struct spdk_ftl_dev *dev)

{

	struct ftl_band *band;

	uint64_t free_blocks, blocks_to_move;



	ftl_band_init_gc_iter(dev);

	dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID;



	if (0 == dev->num_free) {

		/* Get number of available blocks in writer */

		free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc);



		/*

		 * First, check a band candidate to GC

		 */

		band = ftl_band_search_next_to_reloc(dev);

		ftl_bug(NULL == band);

		blocks_to_move = band->p2l_map.num_valid;

		if (blocks_to_move <= free_blocks) {

			/* This GC band can be moved */

			return 0;

		}



		/*

		 * The GC candidate cannot be moved because no enough space. We need to find

		 * another band.

		 */

		band = next_high_prio_band(dev);

		ftl_bug(NULL == band);



		if (band->p2l_map.num_valid > free_blocks) {

			FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n");

			return -1;

		} else {

			/* GC needs to start using this band */

			dev->sb_shm->gc_info.band_id_high_prio = band->id;

		}

	}



	return 0;

}



void

ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)

{
@@ -233,5 +294,9 @@ ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process * 
		return;

	}



	if (finalize_init_gc(dev)) {

		ftl_mngt_fail_step(mngt);

	} else {

		ftl_mngt_next_step(mngt);

	}

}