Commit 04814b72 authored by Wojciech Malikowski's avatar Wojciech Malikowski Committed by Jim Harris
Browse files

lib/ftl: Internal IO retry mechanism in case ENOMEM from nvme layer 



Added internal retry IO queue. In case ENOMEM from
nvme layer request is put to retry queue. When some
inflight operations are completed try to make progress
on IOs in retry queue in first place.

Change-Id: Ie9d3d20bd34431ee57f9454f242b0cdca349c804
Signed-off-by: default avatarWojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/448461


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent f8399d45

lib/ftl/ftl_core.c

+52 −8
Original line number Diff line number Diff line
@@ -702,6 +702,15 @@ ftl_read_canceled(int rc) 
}



static void

ftl_add_to_retry_queue(struct ftl_io *io)

{

	if (!(io->flags & FTL_IO_RETRY)) {

		io->flags |= FTL_IO_RETRY;

		TAILQ_INSERT_TAIL(&io->dev->retry_queue, io, retry_entry);

	}

}



static int

ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa,

		void *ctx)

{
@@ -733,11 +742,11 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa, 
					   ftl_io_iovec_addr(io),

					   ftl_ppa_addr_pack(io->dev, ppa), lbk_cnt,

					   ftl_io_cmpl_cb, io, 0);

		if (rc) {

		if (rc == -ENOMEM) {

			ftl_add_to_retry_queue(io);

			break;

		} else if (rc) {

			io->status = rc;

			if (rc != -ENOMEM) {

				SPDK_ERRLOG("spdk_nvme_ns_cmd_read failed with status: %d\n", rc);

			}

			break;

		}
@@ -750,6 +759,8 @@ ftl_submit_read(struct ftl_io *io, ftl_next_ppa_fn next_ppa, 
	if (ftl_io_done(io)) {

		ftl_io_complete(io);

	}



	return rc;

}



static int
@@ -1384,7 +1395,7 @@ spdk_ftl_write(struct spdk_ftl_dev *dev, struct spdk_io_channel *ch, uint64_t lb 
	return _spdk_ftl_write(io);

}



void

int

ftl_io_read(struct ftl_io *io)

{

	struct spdk_ftl_dev *dev = io->dev;
@@ -1397,11 +1408,11 @@ ftl_io_read(struct ftl_io *io) 
			next_ppa = ftl_lba_read_next_ppa;

		}



		ftl_submit_read(io, next_ppa, NULL);

		return;

		return ftl_submit_read(io, next_ppa, NULL);

	}



	spdk_thread_send_msg(ftl_get_read_thread(dev), _ftl_read, io);

	return 0;

}



static void
@@ -1517,12 +1528,39 @@ ftl_process_anm_event(struct ftl_anm_event *event) 
	ftl_anm_event_complete(event);

}



static void

ftl_process_retry_queue(struct spdk_ftl_dev *dev)

{

	struct ftl_io *io;

	int rc;



	while (!TAILQ_EMPTY(&dev->retry_queue)) {

		io = TAILQ_FIRST(&dev->retry_queue);



		/* Retry only if IO is still healthy */

		if (spdk_likely(io->status == 0)) {

			rc = ftl_io_read(io);

			if (rc == -ENOMEM) {

				break;

			}

		}



		io->flags &= ~FTL_IO_RETRY;

		TAILQ_REMOVE(&dev->retry_queue, io, retry_entry);



		if (ftl_io_done(io)) {

			ftl_io_complete(io);

		}

	}

}



int

ftl_task_read(void *ctx)

{

	struct ftl_thread *thread = ctx;

	struct spdk_ftl_dev *dev = thread->dev;

	struct spdk_nvme_qpair *qpair = ftl_get_read_qpair(dev);

	size_t num_completed;



	if (dev->halt) {

		if (ftl_shutdown_complete(dev)) {
@@ -1531,7 +1569,13 @@ ftl_task_read(void *ctx) 
		}

	}



	return spdk_nvme_qpair_process_completions(qpair, 0);

	num_completed = spdk_nvme_qpair_process_completions(qpair, 0);



	if (num_completed && !TAILQ_EMPTY(&dev->retry_queue)) {

		ftl_process_retry_queue(dev);

	}



	return num_completed;

}



int

lib/ftl/ftl_core.h

+3 −1
Original line number Diff line number Diff line
@@ -200,6 +200,8 @@ struct spdk_ftl_dev { 


	/* Inflight IO operations */

	uint32_t				num_inflight;

	/* Queue of IO awaiting retry */

	TAILQ_HEAD(, ftl_io)			retry_queue;



	/* Manages data relocation */

	struct ftl_reloc			*reloc;
@@ -215,7 +217,7 @@ struct spdk_ftl_dev { 
typedef void (*ftl_restore_fn)(struct spdk_ftl_dev *, struct ftl_restore *, int);



void	ftl_apply_limits(struct spdk_ftl_dev *dev);

void	ftl_io_read(struct ftl_io *io);

int	ftl_io_read(struct ftl_io *io);

int	ftl_io_write(struct ftl_io *io);

int	ftl_io_erase(struct ftl_io *io);

int	ftl_io_flush(struct ftl_io *io);

lib/ftl/ftl_init.c

+1 −0
Original line number Diff line number Diff line
@@ -838,6 +838,7 @@ spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb 
				sizeof(struct ftl_io_channel),

				NULL);



	TAILQ_INIT(&dev->retry_queue);

	dev->ioch = spdk_get_io_channel(dev);

	dev->init_cb = cb;

	dev->init_arg = cb_arg;

lib/ftl/ftl_io.h

+5 −1
Original line number Diff line number Diff line
@@ -68,6 +68,8 @@ enum ftl_io_flags { 
	FTL_IO_PPA_MODE		= (1 << 6),

	/* Indicates that IO contains noncontiguous LBAs */

	FTL_IO_VECTOR_LBA	= (1 << 7),

	/* Indicates that IO is being retried */

	FTL_IO_RETRY		= (1 << 8),

};



enum ftl_io_type {
@@ -197,6 +199,8 @@ struct ftl_io { 


	/* Trace group id */

	uint64_t				trace;



	TAILQ_ENTRY(ftl_io)			retry_entry;

};



/* Metadata IO */
@@ -232,7 +236,7 @@ ftl_io_mode_lba(const struct ftl_io *io) 
static inline bool

ftl_io_done(const struct ftl_io *io)

{

	return io->req_cnt == 0;

	return io->req_cnt == 0 && !(io->flags & FTL_IO_RETRY);

}



struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch);

lib/ftl/ftl_reloc.c

+7 −2
Original line number Diff line number Diff line
@@ -473,6 +473,7 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) 
{

	struct ftl_ppa ppa;

	size_t num_lbks;

	int rc;



	num_lbks = ftl_reloc_next_lbks(breloc, &ppa);
@@ -486,8 +487,12 @@ ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) 
		return -1;

	}



	ftl_io_read(io);

	return 0;

	rc = ftl_io_read(io);

	if (rc == -ENOMEM) {

		rc = 0;

	}



	return rc;

}



static void