lib/ftl: consider 3D TLC NAND read unit size (658d118c) · Commits · Public Repositories / spdk

include/spdk/ftl.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -86,6 +86,9 @@ struct spdk_ftl_conf {
		/* User writes limits */
		struct spdk_ftl_limit limits[SPDK_FTL_LIMIT_MAX];
		} defrag;

		/* Number of interleaving units per ws_opt */
		size_t num_interleave_units;
		};

		/* Range of parallel units (inclusive) */

lib/ftl/ftl_core.c

+13 −6

Original line number	Diff line number	Diff line
		@@ -609,17 +609,23 @@ ftl_process_shutdown(struct spdk_ftl_dev *dev)
		{
		size_t size = ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_INTERNAL) +
		ftl_rwb_num_acquired(dev->rwb, FTL_RWB_TYPE_USER);
		size_t num_active = dev->xfer_size * ftl_rwb_get_active_batches(dev->rwb);

		if (size >= dev->xfer_size) {
		num_active = num_active ? num_active : dev->xfer_size;
		if (size >= num_active) {
		return;
		}

		/* If we reach this point we need to remove free bands */
		/* and pad current wptr band to the end */
		if (ftl_rwb_get_active_batches(dev->rwb) <= 1) {
		ftl_remove_free_bands(dev);
		}

		/* Pad write buffer until band is full */
		ftl_rwb_pad(dev, dev->xfer_size - size);
		/* TODO : It would be better to request padding to as many as PUs possible */
		/* instead of requesting to one PU at a time */
		ftl_rwb_pad(dev, num_active - size);
		}

		static int
		@@ -1268,7 +1274,7 @@ static void
		ftl_flush_pad_batch(struct spdk_ftl_dev *dev)
		{
		struct ftl_rwb *rwb = dev->rwb;
		size_t size;
		size_t size, num_entries;

		size = ftl_rwb_num_acquired(rwb, FTL_RWB_TYPE_INTERNAL) +
		ftl_rwb_num_acquired(rwb, FTL_RWB_TYPE_USER);
		@@ -1280,8 +1286,9 @@ ftl_flush_pad_batch(struct spdk_ftl_dev *dev)
		/* Only add padding when there's less than xfer size */
		/* entries in the buffer. Otherwise we just have to wait */
		/* for the entries to become ready. */
		if (size < dev->xfer_size) {
		ftl_rwb_pad(dev, dev->xfer_size - (size % dev->xfer_size));
		num_entries = ftl_rwb_get_active_batches(dev->rwb) * dev->xfer_size;
		if (size < num_entries) {
		ftl_rwb_pad(dev, num_entries - (size % num_entries));
		}
		}

lib/ftl/ftl_init.c

+12 −3

Original line number	Diff line number	Diff line
		@@ -87,6 +87,9 @@ static const struct spdk_ftl_conf g_default_conf = {
		.max_active_relocs = 3,
		/* IO pool size per user thread (this should be adjusted to thread IO qdepth) */
		.user_io_pool_size = 2048,
		/* Number of interleaving units per ws_opt */
		/* 1 for default and 3 for 3D TLC NAND */
		.num_interleave_units = 1,
		};

		static void ftl_dev_free_sync(struct spdk_ftl_dev *dev);
		@@ -116,7 +119,8 @@ ftl_band_init_md(struct ftl_band *band)
		}

		static int
		ftl_check_conf(const struct spdk_ftl_conf *conf)
		ftl_check_conf(const struct spdk_ftl_conf *conf,
		const struct spdk_ocssd_geometry_data *geo)
		{
		size_t i;

		@@ -135,6 +139,9 @@ ftl_check_conf(const struct spdk_ftl_conf *conf)
		if (conf->rwb_size % FTL_BLOCK_SIZE != 0) {
		return -1;
		}
		if (geo->ws_opt % conf->num_interleave_units != 0) {
		return -1;
		}

		for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) {
		if (conf->defrag.limits[i].limit > 100) {
		@@ -157,7 +164,7 @@ ftl_check_init_opts(const struct spdk_ftl_dev_init_opts *opts,
		return -1;
		}

		if (ftl_check_conf(opts->conf)) {
		if (ftl_check_conf(opts->conf, geo)) {
		return -1;
		}

		@@ -952,7 +959,7 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn c
		goto fail_sync;
		}

		dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size);
		dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size, ftl_dev_num_punits(dev));
		if (!dev->rwb) {
		SPDK_ERRLOG("Unable to initialize rwb structures\n");
		goto fail_sync;
		@@ -1095,6 +1102,8 @@ spdk_ftl_dev_free(struct spdk_ftl_dev dev, spdk_ftl_fn cb, void cb_arg)
		dev->halt_arg = cb_arg;
		dev->halt = 1;

		ftl_rwb_disable_interleaving(dev->rwb);

		spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_add_halt_poller, dev);
		return 0;
		}

lib/ftl/ftl_rwb.c

+86 −12

Original line number	Diff line number	Diff line
		@@ -70,6 +70,12 @@ struct ftl_rwb_batch {
		struct ftl_rwb {
		/* Number of batches */
		size_t num_batches;

		/* Information for interleaving */
		size_t interleave_offset;
		/* Maximum number of active batches */
		size_t max_active_batches;

		/* Number of entries per batch */
		size_t xfer_size;
		/* Metadata's size */
		@@ -80,11 +86,16 @@ struct ftl_rwb {
		/* User/internal limits */
		size_t limits[FTL_RWB_TYPE_MAX];

		/* Current batch */
		struct ftl_rwb_batch *current;
		/* Active batch queue */
		STAILQ_HEAD(, ftl_rwb_batch) active_queue;
		/* Number of active batches */
		unsigned int num_active_batches;

		/* Free batch queue */
		STAILQ_HEAD(, ftl_rwb_batch) free_queue;
		/* Number of active batches */
		unsigned int num_free_batches;

		/* Submission batch queue */
		struct spdk_ring *submit_queue;
		/* High-priority batch queue */
		@@ -174,9 +185,9 @@ ftl_rwb_batch_init(struct ftl_rwb rwb, struct ftl_rwb_batch batch, unsigned in
		}

		struct ftl_rwb *
		ftl_rwb_init(const struct spdk_ftl_conf *conf, size_t xfer_size, size_t md_size)
		ftl_rwb_init(const struct spdk_ftl_conf *conf, size_t xfer_size, size_t md_size, size_t num_punits)
		{
		struct ftl_rwb *rwb;
		struct ftl_rwb *rwb = NULL;
		struct ftl_rwb_batch *batch;
		size_t i;

		@@ -194,8 +205,10 @@ ftl_rwb_init(const struct spdk_ftl_conf *conf, size_t xfer_size, size_t md_size)

		assert(conf->rwb_size % xfer_size == 0);
		rwb->xfer_size = xfer_size;
		rwb->interleave_offset = xfer_size / conf->num_interleave_units;
		rwb->max_active_batches = conf->num_interleave_units == 1 ? 1 : num_punits;
		rwb->md_size = md_size;
		rwb->num_batches = conf->rwb_size / (FTL_BLOCK_SIZE * xfer_size);
		rwb->num_batches = conf->rwb_size / (FTL_BLOCK_SIZE * xfer_size) + rwb->max_active_batches;

		rwb->batches = calloc(rwb->num_batches, sizeof(*rwb->batches));
		if (!rwb->batches) {
		@@ -219,6 +232,7 @@ ftl_rwb_init(const struct spdk_ftl_conf *conf, size_t xfer_size, size_t md_size)
		}

		STAILQ_INIT(&rwb->free_queue);
		STAILQ_INIT(&rwb->active_queue);

		for (i = 0; i < rwb->num_batches; ++i) {
		batch = &rwb->batches[i];
		@@ -229,6 +243,7 @@ ftl_rwb_init(const struct spdk_ftl_conf *conf, size_t xfer_size, size_t md_size)
		}

		STAILQ_INSERT_TAIL(&rwb->free_queue, batch, stailq);
		rwb->num_free_batches++;
		}

		for (unsigned int i = 0; i < FTL_RWB_TYPE_MAX; ++i) {
		@@ -293,6 +308,7 @@ ftl_rwb_batch_release(struct ftl_rwb_batch *batch)

		pthread_spin_lock(&rwb->lock);
		STAILQ_INSERT_TAIL(&rwb->free_queue, batch, stailq);
		rwb->num_free_batches++;
		pthread_spin_unlock(&rwb->lock);
		}

		@@ -336,6 +352,12 @@ ftl_rwb_num_acquired(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type)
		return __atomic_load_n(&rwb->num_acquired[type], __ATOMIC_SEQ_CST);
		}

		size_t
		ftl_rwb_get_active_batches(const struct ftl_rwb *rwb)
		{
		return rwb->num_active_batches;
		}

		void
		ftl_rwb_batch_revert(struct ftl_rwb_batch *batch)
		{
		@@ -370,6 +392,28 @@ ftl_rwb_check_limits(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type)
		return ftl_rwb_num_acquired(rwb, type) >= rwb->limits[type];
		}

		static struct ftl_rwb_batch *
		_ftl_rwb_acquire_batch(struct ftl_rwb *rwb)
		{
		struct ftl_rwb_batch *batch;
		size_t i;

		if (rwb->num_free_batches < rwb->max_active_batches) {
		return NULL;
		}

		for (i = 0; i < rwb->max_active_batches; i++) {
		batch = STAILQ_FIRST(&rwb->free_queue);
		STAILQ_REMOVE(&rwb->free_queue, batch, ftl_rwb_batch, stailq);
		rwb->num_free_batches--;

		STAILQ_INSERT_TAIL(&rwb->active_queue, batch, stailq);
		rwb->num_active_batches++;
		}

		return STAILQ_FIRST(&rwb->active_queue);
		}

		struct ftl_rwb_entry *
		ftl_rwb_acquire(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type)
		{
		@@ -382,22 +426,28 @@ ftl_rwb_acquire(struct ftl_rwb *rwb, enum ftl_rwb_entry_type type)

		pthread_spin_lock(&rwb->lock);

		current = rwb->current;
		current = STAILQ_FIRST(&rwb->active_queue);
		if (!current) {
		current = STAILQ_FIRST(&rwb->free_queue);
		current = _ftl_rwb_acquire_batch(rwb);
		if (!current) {
		goto error;
		}

		STAILQ_REMOVE(&rwb->free_queue, current, ftl_rwb_batch, stailq);
		rwb->current = current;
		}

		entry = &current->entries[current->num_acquired++];

		/* If the whole batch is filled, clear the current batch pointer */
		if (current->num_acquired >= rwb->xfer_size) {
		rwb->current = NULL;
		/* If the whole batch is filled, */
		/* remove the current batch from active_queue */
		/* since it will need to move to submit_queue */
		STAILQ_REMOVE(&rwb->active_queue, current, ftl_rwb_batch, stailq);
		rwb->num_active_batches--;
		} else if (current->num_acquired % rwb->interleave_offset == 0) {
		/* If the current batch is filled by the interleaving offset, */
		/* move the current batch at the tail of active_queue */
		/* to place the next logical blocks into another batch. */
		STAILQ_REMOVE(&rwb->active_queue, current, ftl_rwb_batch, stailq);
		STAILQ_INSERT_TAIL(&rwb->active_queue, current, stailq);
		}

		pthread_spin_unlock(&rwb->lock);
		@@ -408,6 +458,30 @@ error:
		return NULL;
		}

		void
		ftl_rwb_disable_interleaving(struct ftl_rwb *rwb)
		{
		struct ftl_rwb_batch batch, temp;

		pthread_spin_lock(&rwb->lock);
		rwb->max_active_batches = 1;
		rwb->interleave_offset = rwb->xfer_size;

		STAILQ_FOREACH_SAFE(batch, &rwb->active_queue, stailq, temp) {
		if (batch->num_acquired == 0) {
		STAILQ_REMOVE(&rwb->active_queue, batch, ftl_rwb_batch, stailq);
		rwb->num_active_batches--;

		assert(batch->num_ready == 0);
		assert(batch->num_acquired == 0);

		STAILQ_INSERT_TAIL(&rwb->free_queue, batch, stailq);
		rwb->num_free_batches++;
		}
		}
		pthread_spin_unlock(&rwb->lock);
		}

		struct ftl_rwb_batch *
		ftl_rwb_pop(struct ftl_rwb *rwb)
		{

lib/ftl/ftl_rwb.h

+4 −1

Original line number	Diff line number	Diff line
		@@ -90,7 +90,9 @@ struct ftl_rwb_entry {
		LIST_ENTRY(ftl_rwb_entry) list_entry;
		};

		struct ftl_rwb ftl_rwb_init(const struct spdk_ftl_conf conf, size_t xfer_size, size_t md_size);
		struct ftl_rwb ftl_rwb_init(const struct spdk_ftl_conf conf, size_t xfer_size,
		size_t md_size, size_t num_punits);
		size_t ftl_rwb_get_active_batches(const struct ftl_rwb *rwb);
		void ftl_rwb_free(struct ftl_rwb *rwb);
		void ftl_rwb_batch_release(struct ftl_rwb_batch *batch);
		void ftl_rwb_push(struct ftl_rwb_entry *entry);
		@@ -110,6 +112,7 @@ void ftl_rwb_batch_revert(struct ftl_rwb_batch *batch);
		struct ftl_rwb_entry ftl_rwb_batch_first_entry(struct ftl_rwb_batch batch);
		void ftl_rwb_batch_get_data(struct ftl_rwb_batch batch);
		void ftl_rwb_batch_get_md(struct ftl_rwb_batch batch);
		void ftl_rwb_disable_interleaving(struct ftl_rwb *rwb);

		static inline void
		_ftl_rwb_entry_set_valid(struct ftl_rwb_entry *entry, bool valid)

Admin message