Commit 1d293860 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Darek Stojaczyk
Browse files

lib/ftl: stop splitting internal IO data buffers 



Some of the internal IOs split the contiguous data buffer into multiple
iovecs to separate the requests. It's unnecessary and can be replaced
by checking the position within the data buffer.

Change-Id: I9255ea0072fee6c4e6a7dca21e4008780bc45610
Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455519


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarWojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
parent b52a5a45

lib/ftl/ftl_band.c

+5 −9
Original line number Diff line number Diff line
@@ -734,8 +734,7 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_md *md, void *data, str 
		.size		= sizeof(*io),

		.flags		= FTL_IO_MD | FTL_IO_PPA_MODE,

		.type		= FTL_IO_READ,

		.iov_cnt	= 1,

		.req_size	= lbk_cnt,

		.lbk_cnt	= lbk_cnt,

		.fn		= ftl_read_md_cb,

		.data		= data,

	};
@@ -756,7 +755,7 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_md *md, void *data, str 


static struct ftl_io *

ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band,

		     void *data, size_t req_cnt, spdk_ftl_fn cb)

		     void *data, size_t lbk_cnt, spdk_ftl_fn cb)

{

	struct ftl_io_init_opts opts = {

		.dev		= dev,
@@ -766,8 +765,7 @@ ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 
		.size		= sizeof(struct ftl_io),

		.flags		= FTL_IO_MD | FTL_IO_PPA_MODE,

		.type		= FTL_IO_WRITE,

		.iov_cnt	= req_cnt,

		.req_size	= dev->xfer_size,

		.lbk_cnt	= lbk_cnt,

		.fn		= cb,

		.data		= data,

		.md		= NULL,
@@ -784,8 +782,7 @@ ftl_band_write_md(struct ftl_band *band, void *data, size_t lbk_cnt, 
	struct ftl_io *io;

	int rc;



	io = ftl_io_init_md_write(dev, band, data,

				  spdk_divide_round_up(lbk_cnt, dev->xfer_size), cb);

	io = ftl_io_init_md_write(dev, band, data, lbk_cnt, cb);

	if (!io) {

		return -ENOMEM;

	}
@@ -842,8 +839,7 @@ ftl_band_read_md(struct ftl_band *band, struct ftl_md *md, void *data, size_t lb 
		return -ENOENT;

	}



	io = ftl_io_init_md_read(dev, md, data, start_ppa, band, lbk_cnt,

				 unpack_fn, cb);

	io = ftl_io_init_md_read(dev, md, data, start_ppa, band, lbk_cnt, unpack_fn, cb);

	if (!io) {

		return -ENOMEM;

	}

lib/ftl/ftl_core.c

+9 −13
Original line number Diff line number Diff line
@@ -923,9 +923,8 @@ ftl_alloc_io_nv_cache(struct ftl_io *parent, size_t num_lbks) 
	struct ftl_io_init_opts opts = {

		.dev		= parent->dev,

		.parent		= parent,

		.iov_cnt	= 1,

		.data		= ftl_io_iovec_addr(parent),

		.req_size	= num_lbks,

		.lbk_cnt	= num_lbks,

		.flags		= FTL_IO_CACHE,

	};
@@ -1009,7 +1008,7 @@ _ftl_write_nv_cache(void *ctx) 


		/* Shrink the IO if there isn't enough room in the cache to fill the whole iovec */

		if (spdk_unlikely(num_lbks != ftl_io_iovec_len_left(io))) {

			ftl_io_shrink_iovec(child, ftl_io_iovec_addr(child), 1, num_lbks);

			ftl_io_shrink_iovec(child, num_lbks);

		}



		ftl_submit_nv_cache(child);
@@ -1171,8 +1170,7 @@ ftl_io_init_child_write(struct ftl_io *parent, struct ftl_ppa ppa, 
		.size		= sizeof(struct ftl_io),

		.flags		= 0,

		.type		= FTL_IO_WRITE,

		.iov_cnt	= 1,

		.req_size	= dev->xfer_size,

		.lbk_cnt	= dev->xfer_size,

		.fn		= cb,

		.data		= data,

		.md		= md,
@@ -1203,11 +1201,10 @@ ftl_submit_child_write(struct ftl_wptr *wptr, struct ftl_io *io, int lbk_cnt) 
{

	struct spdk_ftl_dev	*dev = io->dev;

	struct ftl_io		*child;

	struct iovec		*iov = ftl_io_iovec(io);

	int			rc;



	/* Split IO to child requests and release chunk immediately after child is completed */

	child = ftl_io_init_child_write(io, wptr->ppa, iov[io->iov_pos].iov_base,

	child = ftl_io_init_child_write(io, wptr->ppa, ftl_io_iovec_addr(io),

					ftl_io_get_md(io), ftl_io_child_write_cb);

	if (!child) {

		return -EAGAIN;
@@ -1238,12 +1235,11 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) 
	struct spdk_ftl_dev	*dev = io->dev;

	struct iovec		*iov = ftl_io_iovec(io);

	int			rc = 0;

	size_t			lbk_cnt;



	assert(io->lbk_cnt % dev->xfer_size == 0);



	while (io->iov_pos < io->iov_cnt) {

		lbk_cnt = iov[io->iov_pos].iov_len / PAGE_SIZE;

		assert(iov[io->iov_pos].iov_len > 0);

		assert(lbk_cnt == dev->xfer_size);



		/* There are no guarantees of the order of completion of NVMe IO submission queue */

		/* so wait until chunk is not busy before submitting another write */
@@ -1253,7 +1249,7 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) 
			break;

		}



		rc = ftl_submit_child_write(wptr, io, lbk_cnt);

		rc = ftl_submit_child_write(wptr, io, dev->xfer_size);



		if (rc == -EAGAIN) {

			wptr->current_io = io;
@@ -1263,8 +1259,8 @@ ftl_submit_write(struct ftl_wptr *wptr, struct ftl_io *io) 
			break;

		}



		ftl_trace_submission(dev, io, wptr->ppa, lbk_cnt);

		ftl_wptr_advance(wptr, lbk_cnt);

		ftl_trace_submission(dev, io, wptr->ppa, dev->xfer_size);

		ftl_wptr_advance(wptr, dev->xfer_size);

	}



	if (ftl_io_done(io)) {

lib/ftl/ftl_io.c

+12 −48
Original line number Diff line number Diff line
@@ -160,49 +160,24 @@ ftl_io_iovec_len_left(struct ftl_io *io) 
}



static void

_ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size)

ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t lbk_cnt)

{

	struct iovec *iov;

	size_t i;



	io->iov_pos = 0;

	io->iov_cnt = iov_cnt;

	io->lbk_cnt = iov_cnt * req_size;



	iov = ftl_io_iovec(io);

	for (i = 0; i < iov_cnt; ++i) {

		iov[i].iov_base = (char *)buf + i * req_size * PAGE_SIZE;

		iov[i].iov_len = req_size * PAGE_SIZE;

	}

}



static int

ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size)

{

	if (iov_cnt > 1) {

		io->iov.vector = calloc(iov_cnt, sizeof(struct iovec));

		if (!io->iov.vector) {

			return -ENOMEM;

		}

	}



	_ftl_io_init_iovec(io, buf, iov_cnt, req_size);

	io->lbk_cnt = lbk_cnt;

	io->iov_cnt = 1;



	return 0;

	io->iov.single.iov_base = buf;

	io->iov.single.iov_len = lbk_cnt * PAGE_SIZE;

}



void

ftl_io_shrink_iovec(struct ftl_io *io, char *buf, size_t iov_cnt, size_t req_size)

ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt)

{

	assert(io->iov_cnt >= iov_cnt);

	assert(io->lbk_cnt >= iov_cnt * req_size);

	assert(io->iov_cnt == 1);

	assert(io->lbk_cnt >= lbk_cnt);

	assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0);



	if (iov_cnt == 1 && io->iov_cnt > 1) {

		free(io->iov.vector);

	}



	_ftl_io_init_iovec(io, buf, iov_cnt, req_size);

	ftl_io_init_iovec(io, ftl_io_iovec_addr(io), lbk_cnt);

}



static void
@@ -244,12 +219,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) 
	io->band = opts->band;

	io->md = opts->md;



	if (ftl_io_init_iovec(io, opts->data, opts->iov_cnt, opts->req_size)) {

		if (!opts->io) {

			ftl_io_free(io);

		}

		return NULL;

	}

	ftl_io_init_iovec(io, opts->data, opts->lbk_cnt);



	return io;

}
@@ -266,8 +236,7 @@ ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band, 
		.size		= sizeof(struct ftl_io),

		.flags		= 0,

		.type		= FTL_IO_WRITE,

		.iov_cnt	= 1,

		.req_size	= dev->xfer_size,

		.lbk_cnt	= dev->xfer_size,

		.fn		= cb,

		.data		= ftl_rwb_batch_get_data(batch),

		.md		= ftl_rwb_batch_get_md(batch),
@@ -288,8 +257,7 @@ ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb) 
		.size		= sizeof(struct ftl_io),

		.flags		= FTL_IO_PPA_MODE,

		.type		= FTL_IO_ERASE,

		.iov_cnt	= 0,

		.req_size	= 1,

		.lbk_cnt	= 1,

		.fn		= cb,

		.data		= NULL,

		.md		= NULL,
@@ -336,10 +304,6 @@ _ftl_io_free(struct ftl_io *io) 


	assert(LIST_EMPTY(&io->children));



	if ((io->flags & FTL_IO_INTERNAL) && io->iov_cnt > 1) {

		free(io->iov.vector);

	}



	if (pthread_spin_destroy(&io->lock)) {

		SPDK_ERRLOG("pthread_spin_destroy failed\n");

	}

lib/ftl/ftl_io.h

+3 −6
Original line number Diff line number Diff line
@@ -98,17 +98,14 @@ struct ftl_io_init_opts { 
	/* IO type */

	enum ftl_io_type			type;



	/* Number of split requests */

	size_t                                  iov_cnt;



	/* RWB entry */

	struct ftl_rwb_batch			*rwb_batch;



	/* Band to which the IO is directed */

	struct ftl_band				*band;



	/* Request size */

	size_t                                  req_size;

	/* Number of logical blocks */

	size_t                                  lbk_cnt;



	/* Data */

	void                                    *data;
@@ -285,7 +282,7 @@ void ftl_io_user_init(struct spdk_ftl_dev *dev, struct ftl_io *io, uint64_t lba, 
		      spdk_ftl_fn fn, void *cb_arg, int type);

void *ftl_io_get_md(const struct ftl_io *io);

void ftl_io_complete(struct ftl_io *io);

void ftl_io_shrink_iovec(struct ftl_io *io, char *buf, size_t iov_cnt, size_t req_size);

void ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt);

void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status);

void ftl_io_reset(struct ftl_io *io);

lib/ftl/ftl_reloc.c

+1 −2
Original line number Diff line number Diff line
@@ -443,8 +443,7 @@ ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_io *io, 
		.size		= sizeof(*io),

		.flags		= FTL_IO_KEEP_ALIVE | FTL_IO_INTERNAL | FTL_IO_PPA_MODE,

		.type		= FTL_IO_READ,

		.iov_cnt	= 1,

		.req_size	= num_lbks,

		.lbk_cnt	= num_lbks,

		.fn		= ftl_reloc_read_cb,

	};