FTL: Add relocation logic (43a4d47a) · Commits · Public Repositories / spdk

lib/ftl/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ CFLAGS += -I.
		FTL_SUBDIRS := mngt utils

		C_SRCS = ftl_core.c ftl_init.c ftl_layout.c ftl_debug.c ftl_io.c ftl_sb.c ftl_l2p.c ftl_l2p_flat.c
		C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c ftl_rq.c
		C_SRCS += ftl_nv_cache.c ftl_band.c ftl_band_ops.c ftl_writer.c ftl_rq.c ftl_reloc.c
		C_SRCS += mngt/ftl_mngt.c mngt/ftl_mngt_bdev.c mngt/ftl_mngt_shutdown.c mngt/ftl_mngt_startup.c
		C_SRCS += mngt/ftl_mngt_md.c mngt/ftl_mngt_misc.c mngt/ftl_mngt_ioch.c mngt/ftl_mngt_l2p.c
		C_SRCS += mngt/ftl_mngt_band.c

lib/ftl/ftl_core.c

+18 −0

Original line number	Diff line number	Diff line
		@@ -78,6 +78,11 @@ ftl_shutdown_complete(struct spdk_ftl_dev *dev)
		return false;
		}

		if (!ftl_reloc_is_halted(dev->reloc)) {
		ftl_reloc_halt(dev->reloc);
		return false;
		}

		if (!ftl_writer_is_halted(&dev->writer_gc)) {
		ftl_writer_halt(&dev->writer_gc);
		return false;
		@@ -265,6 +270,18 @@ ftl_submit_read(struct ftl_io *io)
		}
		}

		bool
		ftl_needs_reloc(struct spdk_ftl_dev *dev)
		{
		size_t limit = ftl_get_limit(dev, SPDK_FTL_LIMIT_START);

		if (dev->num_free <= limit) {
		return true;
		}

		return false;
		}

		void
		spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev dev, struct spdk_ftl_attrs attrs)
		{
		@@ -493,6 +510,7 @@ ftl_core_poller(void *ctx)
		ftl_process_io_queue(dev);
		ftl_writer_run(&dev->writer_user);
		ftl_writer_run(&dev->writer_gc);
		ftl_reloc(dev->reloc);
		ftl_nv_cache_process(dev);
		ftl_l2p_process(dev);

lib/ftl/ftl_core.h

+5 −0

Original line number	Diff line number	Diff line
		@@ -124,6 +124,9 @@ struct spdk_ftl_dev {
		/* Inflight IO operations */
		uint32_t num_inflight;

		/* Manages data relocation */
		struct ftl_reloc *reloc;

		/* Thread on which the poller is running */
		struct spdk_thread *core_thread;

		@@ -166,6 +169,8 @@ int ftl_io_channel_poll(void *arg);

		struct ftl_io_channel ftl_io_channel_get_ctx(struct spdk_io_channel ioch);

		bool ftl_needs_reloc(struct spdk_ftl_dev *dev);

		struct ftl_band ftl_band_get_next_free(struct spdk_ftl_dev dev);

		static inline uint64_t

lib/ftl/ftl_internal.h

+16 −0

Original line number	Diff line number	Diff line
		@@ -24,6 +24,8 @@
		*/
		typedef uint64_t ftl_addr;

		struct spdk_ftl_dev;

		enum ftl_md_type {
		FTL_MD_TYPE_BAND,
		FTL_MD_TYPE_CHUNK
		@@ -80,4 +82,18 @@ struct ftl_p2l_map {
		};
		};

		struct spdk_ftl_dev;

		struct ftl_reloc ftl_reloc_init(struct spdk_ftl_dev dev);

		void ftl_reloc_free(struct ftl_reloc *reloc);

		void ftl_reloc(struct ftl_reloc *reloc);

		void ftl_reloc_halt(struct ftl_reloc *reloc);

		void ftl_reloc_resume(struct ftl_reloc *reloc);

		bool ftl_reloc_is_halted(const struct ftl_reloc *reloc);

		#endif /* FTL_INTERNAL_H */

lib/ftl/ftl_reloc.c

0 → 100644

+627 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: BSD-3-Clause
		* Copyright (c) Intel Corporation.
		* All rights reserved.
		*/

		#include "ftl_band.h"
		#include "ftl_core.h"
		#include "ftl_debug.h"
		#include "ftl_io.h"
		#include "ftl_internal.h"
		#include "spdk/ftl.h"
		#include "spdk/likely.h"

		struct ftl_reloc;
		struct ftl_band_reloc;

		/* TODO: Should probably change the move naming nomenclature to something more descriptive */
		enum ftl_reloc_move_state {
		FTL_RELOC_STATE_READ = 0,
		FTL_RELOC_STATE_PIN,
		FTL_RELOC_STATE_WRITE,
		FTL_RELOC_STATE_WAIT,
		FTL_RELOC_STATE_HALT,

		FTL_RELOC_STATE_MAX
		};

		struct ftl_reloc_move {
		/* FTL device */
		struct spdk_ftl_dev *dev;

		struct ftl_reloc *reloc;

		/* Request for doing IO */
		struct ftl_rq *rq;

		/* Move state (read, write) */
		enum ftl_reloc_move_state state;

		/* Entry of circular list */
		TAILQ_ENTRY(ftl_reloc_move) qentry;
		};

		struct ftl_reloc {
		/* Device associated with relocate */
		struct spdk_ftl_dev *dev;

		/* Indicates relocate is about to halt */
		bool halt;

		/* Band which are read to relocate */
		struct ftl_band *band;

		/* Bands already read, but waiting for finishing GC */
		TAILQ_HEAD(, ftl_band) band_done;
		size_t band_done_count;

		/* Flags indicating reloc is waiting for a new band */
		bool band_waiting;

		/* Maximum number of IOs per band */
		size_t max_qdepth;

		/* Queue of free move objects */
		struct ftl_reloc_move *move_buffer;

		/* Array of movers queue for each state */
		TAILQ_HEAD(, ftl_reloc_move) move_queue[FTL_RELOC_STATE_MAX];

		};

		static void move_read_cb(struct ftl_rq *rq);
		static void move_write_cb(struct ftl_rq *rq);
		static void move_set_state(struct ftl_reloc_move *mv, enum ftl_reloc_move_state state);
		static void move_write(struct ftl_reloc reloc, struct ftl_reloc_move mv);
		static void move_read_error_cb(struct ftl_rq rq, struct ftl_band band, uint64_t idx,
		uint64_t count);

		static void
		move_deinit(struct ftl_reloc_move *mv)
		{
		assert(mv);
		ftl_rq_del(mv->rq);
		}

		static int
		move_init(struct ftl_reloc reloc, struct ftl_reloc_move mv)
		{
		mv->state = FTL_RELOC_STATE_HALT;
		TAILQ_INSERT_TAIL(&reloc->move_queue[FTL_RELOC_STATE_HALT], mv, qentry);

		mv->reloc = reloc;
		mv->dev = reloc->dev;
		mv->rq = ftl_rq_new(mv->dev, mv->dev->md_size);

		if (!mv->rq) {
		return -ENOMEM;
		}
		mv->rq->owner.priv = mv;

		return 0;
		}

		struct ftl_reloc *
		ftl_reloc_init(struct spdk_ftl_dev *dev)
		{
		struct ftl_reloc *reloc;
		struct ftl_reloc_move *move;
		size_t i, count;

		reloc = calloc(1, sizeof(*reloc));
		if (!reloc) {
		return NULL;
		}

		reloc->dev = dev;
		reloc->halt = true;
		reloc->max_qdepth = dev->sb->max_reloc_qdepth;

		reloc->move_buffer = calloc(reloc->max_qdepth, sizeof(*reloc->move_buffer));
		if (!reloc->move_buffer) {
		FTL_ERRLOG(dev, "Failed to initialize reloc moves pool");
		goto error;
		}

		/* Initialize movers queues */
		count = SPDK_COUNTOF(reloc->move_queue);
		for (i = 0; i < count; ++i) {
		TAILQ_INIT(&reloc->move_queue[i]);
		}

		for (i = 0; i < reloc->max_qdepth; ++i) {
		move = &reloc->move_buffer[i];

		if (move_init(reloc, move)) {
		goto error;
		}
		}

		TAILQ_INIT(&reloc->band_done);

		return reloc;
		error:
		ftl_reloc_free(reloc);
		return NULL;
		}

		struct ftl_reloc_task_fini {
		struct ftl_reloc_task *task;
		spdk_msg_fn cb;
		void *cb_arg;
		};

		void
		ftl_reloc_free(struct ftl_reloc *reloc)
		{
		size_t i;

		if (!reloc) {
		return;
		}

		if (reloc->move_buffer) {
		for (i = 0; i < reloc->max_qdepth; ++i) {
		move_deinit(&reloc->move_buffer[i]);
		}
		}

		free(reloc->move_buffer);
		free(reloc);
		}

		void
		ftl_reloc_halt(struct ftl_reloc *reloc)
		{
		reloc->halt = true;
		}

		void
		ftl_reloc_resume(struct ftl_reloc *reloc)
		{
		struct ftl_reloc_move mv, next;
		reloc->halt = false;

		TAILQ_FOREACH_SAFE(mv, &reloc->move_queue[FTL_RELOC_STATE_HALT], qentry,
		next) {
		move_set_state(mv, FTL_RELOC_STATE_READ);
		}
		}

		static void
		move_set_state(struct ftl_reloc_move *mv, enum ftl_reloc_move_state state)
		{
		struct ftl_reloc *reloc = mv->reloc;

		switch (state) {
		case FTL_RELOC_STATE_READ:
		mv->rq->owner.cb = move_read_cb;
		mv->rq->owner.error = move_read_error_cb;
		mv->rq->iter.idx = 0;
		mv->rq->iter.count = 0;
		mv->rq->success = true;
		break;

		case FTL_RELOC_STATE_WRITE:
		mv->rq->owner.cb = move_write_cb;
		mv->rq->owner.error = NULL;
		break;

		case FTL_RELOC_STATE_PIN:
		case FTL_RELOC_STATE_WAIT:
		case FTL_RELOC_STATE_HALT:
		break;

		default:
		ftl_abort();
		break;
		}

		if (mv->state != state) {
		/* Remove the mover from previous queue */
		TAILQ_REMOVE(&reloc->move_queue[mv->state], mv, qentry);
		/* Insert the mover to the new queue */
		TAILQ_INSERT_TAIL(&reloc->move_queue[state], mv, qentry);
		/* Update state */
		mv->state = state;
		}
		}

		static void
		move_get_band_cb(struct ftl_band band, void cntx, bool status)
		{
		struct ftl_reloc *reloc = cntx;

		if (spdk_likely(status)) {
		reloc->band = band;
		ftl_band_iter_init(band);
		}
		reloc->band_waiting = false;
		}

		static void
		move_grab_new_band(struct ftl_reloc *reloc)
		{
		if (!reloc->band_waiting) {
		if (!ftl_needs_reloc(reloc->dev)) {
		return;
		}

		/* Limit number of simultaneously relocated bands */
		if (reloc->band_done_count > 2) {
		return;
		}

		reloc->band_waiting = true;
		ftl_band_get_next_gc(reloc->dev, move_get_band_cb, reloc);
		}
		}

		static struct ftl_band *
		move_get_band(struct ftl_reloc *reloc)
		{
		struct ftl_band *band = reloc->band;

		if (!band) {
		move_grab_new_band(reloc);
		return NULL;
		}

		if (!ftl_band_filled(band, band->md->iter.offset)) {
		/* Band still not read, we can continue reading */
		return band;
		}

		TAILQ_INSERT_TAIL(&reloc->band_done, band, queue_entry);
		reloc->band_done_count++;
		reloc->band = NULL;

		return NULL;
		}

		static void
		move_advance_rq(struct ftl_rq *rq)
		{
		struct ftl_band *band = rq->io.band;
		uint64_t offset, i;
		struct ftl_rq_entry *entry = &rq->entries[rq->iter.idx];

		assert(rq->iter.idx + rq->iter.count <= rq->num_blocks);

		for (i = 0; i < rq->iter.count; i++) {
		offset = ftl_band_block_offset_from_addr(band, rq->io.addr);

		assert(offset < ftl_get_num_blocks_in_band(band->dev));

		entry->lba = band->p2l_map.band_map[offset];
		entry->addr = rq->io.addr;
		entry->owner.priv = band;

		entry++;
		rq->io.addr = ftl_band_next_addr(band, rq->io.addr, 1);
		band->owner.cnt++;
		}

		/* Increase QD for the request */
		rq->iter.qd++;

		/* Advanced request iterator */
		rq->iter.idx += rq->iter.count;
		}

		static void
		move_init_entries(struct ftl_rq *rq, uint64_t idx, uint64_t count)
		{
		uint64_t i = 0;
		struct ftl_rq_entry *iter = &rq->entries[idx];

		assert(idx + count <= rq->num_blocks);

		i = 0;
		while (i < count) {
		iter->addr = FTL_ADDR_INVALID;
		iter->owner.priv = NULL;
		iter->lba = FTL_LBA_INVALID;
		iter++;
		i++;
		}
		}

		static void
		move_read_error_cb(struct ftl_rq rq, struct ftl_band band, uint64_t idx, uint64_t count)
		{
		move_init_entries(rq, idx, count);
		band->owner.cnt -= count;
		}

		static void
		move_read_cb(struct ftl_rq *rq)
		{
		struct ftl_reloc_move *mv = rq->owner.priv;

		/* Decrease QD of the request */
		assert(rq->iter.qd > 0);
		rq->iter.qd--;

		if (rq->iter.idx != rq->num_blocks \|\| rq->iter.qd) {
		return;
		}

		move_set_state(mv, FTL_RELOC_STATE_PIN);
		}

		static void
		move_rq_pad(struct ftl_rq rq, struct ftl_band band)
		{
		struct ftl_rq_entry *entry = &rq->entries[rq->iter.idx];

		for (; rq->iter.idx < rq->num_blocks; ++rq->iter.idx) {
		entry->addr = rq->io.addr;
		entry->owner.priv = band;
		entry->lba = FTL_LBA_INVALID;
		entry++;
		rq->io.addr = ftl_band_next_addr(band, rq->io.addr, 1);
		band->owner.cnt++;
		}

		assert(rq->iter.idx == rq->num_blocks);
		}

		static void
		move_read(struct ftl_reloc reloc, struct ftl_reloc_move mv, struct ftl_band *band)
		{
		struct ftl_rq *rq = mv->rq;
		uint64_t band_left, rq_left;

		rq_left = rq->num_blocks - rq->iter.idx;
		assert(rq_left > 0);

		band_left = ftl_band_user_blocks_left(band, band->md->iter.offset);
		rq->iter.count = spdk_min(rq_left, band_left);

		ftl_band_rq_read(band, rq);

		move_advance_rq(rq);

		/* Advance band iterator */
		ftl_band_iter_advance(band, rq->iter.count);

		/* If band is fully written pad rest of request */
		if (ftl_band_filled(band, band->md->iter.offset)) {
		move_rq_pad(rq, band);
		}

		if (rq->iter.idx == rq->num_blocks) {
		/*
		* All request entries scheduled for reading,
		* We can change state to waiting
		*/
		move_set_state(mv, FTL_RELOC_STATE_WAIT);
		}
		}

		static void
		move_pin_cb(struct spdk_ftl_dev dev, int status, struct ftl_l2p_pin_ctx pin_ctx)
		{
		struct ftl_reloc_move *mv = pin_ctx->cb_ctx;
		struct ftl_rq *rq = mv->rq;

		if (status) {
		rq->iter.status = status;
		pin_ctx->lba = FTL_LBA_INVALID;
		}

		if (--rq->iter.remaining == 0) {
		if (rq->iter.status) {
		/* unpin and try again */
		ftl_rq_unpin(rq);
		move_set_state(mv, FTL_RELOC_STATE_PIN);
		return;
		}

		move_set_state(mv, FTL_RELOC_STATE_WRITE);
		}
		}

		static void
		move_pin(struct ftl_reloc_move *mv)
		{
		struct ftl_rq *rq = mv->rq;
		struct ftl_rq_entry *entry = rq->entries;
		uint64_t i;

		move_set_state(mv, FTL_RELOC_STATE_WAIT);

		rq->iter.remaining = rq->iter.count = rq->num_blocks;
		rq->iter.status = 0;

		for (i = 0; i < rq->num_blocks; i++) {
		if (entry->lba != FTL_LBA_INVALID) {
		ftl_l2p_pin(rq->dev, entry->lba, 1, move_pin_cb, mv, &entry->l2p_pin_ctx);
		} else {
		ftl_l2p_pin_skip(rq->dev, move_pin_cb, mv, &entry->l2p_pin_ctx);
		}
		entry++;
		}
		}

		static void
		move_finish_write(struct ftl_rq *rq)
		{
		uint64_t i;
		struct spdk_ftl_dev *dev = rq->dev;
		struct ftl_rq_entry *iter = rq->entries;
		ftl_addr addr = rq->io.addr;
		struct ftl_band *rq_band = rq->io.band;
		struct ftl_band *band;

		for (i = 0; i < rq->num_blocks; ++i, ++iter) {
		band = iter->owner.priv;

		if (band) {
		assert(band->owner.cnt > 0);
		band->owner.cnt--;
		}
		if (iter->lba != FTL_LBA_INVALID) {
		/* Update L2P table */
		ftl_l2p_update_base(dev, iter->lba, addr, iter->addr);
		ftl_l2p_unpin(dev, iter->lba, 1);
		}
		addr = ftl_band_next_addr(rq_band, addr, 1);
		}
		}

		static void
		move_write_cb(struct ftl_rq *rq)
		{
		struct ftl_reloc_move *mv = rq->owner.priv;

		assert(rq->iter.qd == 1);
		rq->iter.qd--;

		if (spdk_likely(rq->success)) {
		move_finish_write(rq);
		move_set_state(mv, FTL_RELOC_STATE_READ);
		} else {
		/* Write failed, repeat write */
		move_set_state(mv, FTL_RELOC_STATE_WRITE);
		}
		}

		static void
		move_write(struct ftl_reloc reloc, struct ftl_reloc_move mv)
		{
		struct spdk_ftl_dev *dev = mv->dev;
		struct ftl_rq *rq = mv->rq;

		assert(rq->iter.idx == rq->num_blocks);

		/* Request contains data to be placed on a new location, submit it */
		ftl_writer_queue_rq(&dev->writer_gc, rq);
		rq->iter.qd++;

		move_set_state(mv, FTL_RELOC_STATE_WAIT);
		}

		static void
		move_run(struct ftl_reloc reloc, struct ftl_reloc_move mv)
		{
		struct ftl_band *band;

		switch (mv->state) {
		case FTL_RELOC_STATE_READ: {
		if (spdk_unlikely(reloc->halt)) {
		move_set_state(mv, FTL_RELOC_STATE_HALT);
		break;
		}

		band = move_get_band(reloc);
		if (!band) {
		break;
		}

		move_read(reloc, mv, band);
		}
		break;

		case FTL_RELOC_STATE_PIN:
		move_pin(mv);
		break;

		case FTL_RELOC_STATE_WRITE:
		if (spdk_unlikely(reloc->halt)) {
		ftl_rq_unpin(mv->rq);
		move_set_state(mv, FTL_RELOC_STATE_HALT);
		break;
		}

		move_write(reloc, mv);
		break;

		case FTL_RELOC_STATE_HALT:
		case FTL_RELOC_STATE_WAIT:
		break;

		default:
		assert(0);
		ftl_abort();
		break;
		}
		}

		static void
		move_handle_band_error(struct ftl_band *band)
		{
		struct ftl_reloc *reloc = band->dev->reloc;
		/*
		* Handle band error, it's because an error occurred during reading,
		* Add band to the close band list, will try reloc it in a moment
		*/
		TAILQ_REMOVE(&reloc->band_done, band, queue_entry);
		reloc->band_done_count--;

		band->md->state = FTL_BAND_STATE_CLOSING;
		ftl_band_set_state(band, FTL_BAND_STATE_CLOSED);
		}

		static void
		move_release_bands(struct ftl_reloc *reloc)
		{
		struct ftl_band *band;

		if (TAILQ_EMPTY(&reloc->band_done)) {
		return;
		}

		band = TAILQ_FIRST(&reloc->band_done);

		if (band->owner.cnt \|\| ftl_band_qd(band)) {
		/* Band still in use */
		return;
		}

		if (ftl_band_empty(band)) {
		assert(ftl_band_filled(band, band->md->iter.offset));
		TAILQ_REMOVE(&reloc->band_done, band, queue_entry);
		reloc->band_done_count--;
		ftl_band_free(band);
		} else {
		move_handle_band_error(band);
		}
		}

		bool
		ftl_reloc_is_halted(const struct ftl_reloc *reloc)
		{
		size_t i, count;

		count = SPDK_COUNTOF(reloc->move_queue);
		for (i = 0; i < count; ++i) {
		if (i == FTL_RELOC_STATE_HALT) {
		continue;
		}

		if (!TAILQ_EMPTY(&reloc->move_queue[i])) {
		return false;
		}
		}

		return true;
		}

		void
		ftl_reloc(struct ftl_reloc *reloc)
		{
		size_t i, count;

		count = SPDK_COUNTOF(reloc->move_queue);
		for (i = 0; i < count; ++i) {
		if (TAILQ_EMPTY(&reloc->move_queue[i])) {
		continue;
		}

		move_run(reloc, TAILQ_FIRST(&reloc->move_queue[i]));
		}

		move_release_bands(reloc);
		}