Commit 0291b284 authored by Artur Paszkiewicz's avatar Artur Paszkiewicz Committed by Ben Walker
Browse files

FTL: Add read path 



Signed-off-by: default avatarKozlowski Mateusz <mateusz.kozlowski@intel.com>
Signed-off-by: default avatarArtur Paszkiewicz <artur.paszkiewicz@intel.com>
Change-Id: Ib5bac109b59d5a21a7dad1f8e79b5da7633ffa9d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/13334


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent a05d3047

autotest.sh

+4 −0
Original line number Diff line number Diff line
@@ -305,6 +305,10 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then 
		run_test "ocf" ./test/ocf/ocf.sh

	fi



	if [ $SPDK_TEST_FTL -eq 1 ]; then

		run_test "ftl" ./test/ftl/ftl.sh

	fi



	if [ $SPDK_TEST_VMD -eq 1 ]; then

		run_test "vmd" ./test/vmd/vmd.sh

	fi

include/spdk/ftl.h

+19 −0
Original line number Diff line number Diff line
@@ -155,6 +155,25 @@ void spdk_ftl_conf_deinit(struct spdk_ftl_conf *conf); 
 */

void spdk_ftl_get_default_conf(struct spdk_ftl_conf *conf);



/**

 * Submits a read to the specified device.

 *

 * \param dev Device

 * \param io Allocated ftl_io

 * \param ch I/O channel

 * \param lba Starting LBA to read the data

 * \param lba_cnt Number of sectors to read

 * \param iov Single IO vector or pointer to IO vector table

 * \param iov_cnt Number of IO vectors

 * \param cb_fn Callback function to invoke when the I/O is completed

 * \param cb_arg Argument to pass to the callback function

 *

 * \return 0 if successfully submitted, negative errno otherwise.

 */

int spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,

		   uint64_t lba, uint64_t lba_cnt,

		   struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg);



/**

 * Submits a write to the specified device.

 *

lib/ftl/ftl_core.c

+212 −2
Original line number Diff line number Diff line
@@ -26,6 +26,23 @@ spdk_ftl_io_size(void) 
	return sizeof(struct ftl_io);

}



static void

ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)

{

	struct ftl_io *io = cb_arg;



	if (spdk_unlikely(!success)) {

		io->status = -EIO;

	}



	ftl_io_dec_req(io);

	if (ftl_io_done(io)) {

		ftl_io_complete(io);

	}



	spdk_bdev_free_io(bdev_io);

}



static void

ftl_band_erase(struct ftl_band *band)

{
@@ -118,6 +135,126 @@ ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr) 
	}

}



static int

ftl_read_canceled(int rc)

{

	return rc == -EFAULT;

}



static int

ftl_get_next_read_addr(struct ftl_io *io, ftl_addr *addr)

{

	struct spdk_ftl_dev *dev = io->dev;

	ftl_addr next_addr;

	size_t i;

	bool addr_cached = false;



	*addr = ftl_l2p_get(dev, ftl_io_current_lba(io));

	io->map[io->pos] = *addr;



	/* If the address is invalid, skip it */

	if (*addr == FTL_ADDR_INVALID) {

		return -EFAULT;

	}



	addr_cached = ftl_addr_in_nvc(dev, *addr);



	for (i = 1; i < ftl_io_iovec_len_left(io); ++i) {

		next_addr = ftl_l2p_get(dev, ftl_io_get_lba(io, io->pos + i));



		if (next_addr == FTL_ADDR_INVALID) {

			break;

		}



		/* It's not enough to check for contiguity, if user data is on the last block

		 * of base device and first nvc, then they're 'contiguous', but can't be handled

		 * with one read request.

		 */

		if (addr_cached != ftl_addr_in_nvc(dev, next_addr)) {

			break;

		}



		if (*addr + i != next_addr) {

			break;

		}



		io->map[io->pos + i] = next_addr;

	}



	return i;

}



static void ftl_submit_read(struct ftl_io *io);



static void

_ftl_submit_read(void *_io)

{

	struct ftl_io *io = _io;



	ftl_submit_read(io);

}



static void

ftl_submit_read(struct ftl_io *io)

{

	struct spdk_ftl_dev *dev = io->dev;

	ftl_addr addr;

	int rc = 0, num_blocks;



	while (io->pos < io->num_blocks) {

		num_blocks = ftl_get_next_read_addr(io, &addr);

		rc = num_blocks;



		/* User LBA doesn't hold valid data (trimmed or never written to), fill with 0 and skip this block */

		if (ftl_read_canceled(rc)) {

			memset(ftl_io_iovec_addr(io), 0, FTL_BLOCK_SIZE);

			ftl_io_advance(io, 1);

			continue;

		}



		assert(num_blocks > 0);



		if (ftl_addr_in_nvc(dev, addr)) {

			rc = ftl_nv_cache_read(io, addr, num_blocks, ftl_io_cmpl_cb, io);

		} else {

			rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch,

						   ftl_io_iovec_addr(io),

						   addr, num_blocks, ftl_io_cmpl_cb, io);

		}



		if (spdk_unlikely(rc)) {

			if (rc == -ENOMEM) {

				struct spdk_bdev *bdev;

				struct spdk_io_channel *ch;



				if (ftl_addr_in_nvc(dev, addr)) {

					bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc);

					ch = dev->nv_cache.cache_ioch;

				} else {

					bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);

					ch = dev->base_ioch;

				}

				io->bdev_io_wait.bdev = bdev;

				io->bdev_io_wait.cb_fn = _ftl_submit_read;

				io->bdev_io_wait.cb_arg = io;

				spdk_bdev_queue_io_wait(bdev, ch, &io->bdev_io_wait);

				return;

			} else {

				ftl_abort();

			}

		}



		ftl_io_inc_req(io);

		ftl_io_advance(io, num_blocks);

	}



	/* If we didn't have to read anything from the device, */

	/* complete the request right away */

	if (ftl_io_done(io)) {

		ftl_io_complete(io);

	}

}



void

spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs)

{
@@ -126,6 +263,38 @@ spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *at 
	attrs->optimum_io_size = dev->xfer_size;

}



static void

ftl_io_pin_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx)

{

	struct ftl_io *io = pin_ctx->cb_ctx;



	if (spdk_unlikely(status != 0)) {

		/* Retry on the internal L2P fault */

		io->status = -EAGAIN;

		ftl_io_complete(io);

		return;

	}



	io->flags |= FTL_IO_PINNED;

	ftl_submit_read(io);

}



static void

ftl_io_pin(struct ftl_io *io)

{

	if (spdk_unlikely(io->flags & FTL_IO_PINNED)) {

		/*

		 * The IO is in a retry path and it had been pinned already.

		 * Continue with further processing.

		 */

		ftl_l2p_pin_skip(io->dev, ftl_io_pin_cb, io, &io->l2p_pin_ctx);

	} else {

		/* First time when pinning the IO */

		ftl_l2p_pin(io->dev, io->lba, io->num_blocks,

			    ftl_io_pin_cb, io, &io->l2p_pin_ctx);

	}

}



static void

start_io(struct ftl_io *io)

{
@@ -141,8 +310,7 @@ start_io(struct ftl_io *io) 


	switch (io->type) {

	case FTL_IO_READ:

		io->status = -EOPNOTSUPP;

		ftl_io_complete(io);

		TAILQ_INSERT_TAIL(&dev->rd_sq, io, queue_entry);

		break;

	case FTL_IO_WRITE:

		TAILQ_INSERT_TAIL(&dev->wr_sq, io, queue_entry);
@@ -201,6 +369,38 @@ spdk_ftl_writev(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_chan 
	return queue_io(dev, io);

}



int

spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,

	       uint64_t lba, uint64_t lba_cnt, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg)

{

	int rc;



	if (iov_cnt == 0) {

		return -EINVAL;

	}



	if (lba_cnt == 0) {

		return -EINVAL;

	}



	if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) {

		FTL_ERRLOG(dev, "Invalid IO vector to handle, device %s, LBA %"PRIu64"\n",

			   dev->conf.name, lba);

		return -EINVAL;

	}



	if (!dev->initialized) {

		return -EBUSY;

	}



	rc = ftl_io_init(ch, io, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_READ);

	if (rc) {

		return rc;

	}



	return queue_io(dev, io);

}



#define FTL_IO_QUEUE_BATCH 16

int

ftl_io_channel_poll(void *arg)
@@ -245,6 +445,16 @@ ftl_process_io_queue(struct spdk_ftl_dev *dev) 
	struct ftl_io_channel *ioch;

	struct ftl_io *io;



	/* TODO: Try to figure out a mechanism to batch more requests at the same time,

	 * with keeping enough resources (pinned pages), between reads, writes and gc/compaction

	 */

	if (!TAILQ_EMPTY(&dev->rd_sq)) {

		io = TAILQ_FIRST(&dev->rd_sq);

		TAILQ_REMOVE(&dev->rd_sq, io, queue_entry);

		assert(io->type == FTL_IO_READ);

		ftl_io_pin(io);

	}



	if (!ftl_nv_cache_full(&dev->nv_cache) && !TAILQ_EMPTY(&dev->wr_sq)) {

		io = TAILQ_FIRST(&dev->wr_sq);

		TAILQ_REMOVE(&dev->wr_sq, io, queue_entry);

lib/ftl/spdk_ftl.map

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
	spdk_ftl_conf_deinit;

	spdk_ftl_get_io_channel;

	spdk_ftl_io_size;

	spdk_ftl_readv;

	spdk_ftl_writev;



	local: *;

module/bdev/ftl/bdev_ftl.c

+27 −1
Original line number Diff line number Diff line
@@ -105,6 +105,31 @@ bdev_ftl_cb(void *arg, int rc) 
	spdk_bdev_io_complete(bdev_io, status);

}



static void

bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,

		    bool success)

{

	struct ftl_bdev *ftl_bdev;

	int rc;



	ftl_bdev = bdev_io->bdev->ctxt;



	if (!success) {

		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);

		return;

	}



	rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx,

			    ch,

			    bdev_io->u.bdev.offset_blocks,

			    bdev_io->u.bdev.num_blocks,

			    bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io);



	if (spdk_unlikely(rc != 0)) {

		spdk_bdev_io_complete(bdev_io, rc);

	}

}



static int

_bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)

{
@@ -112,7 +137,8 @@ _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_i 


	switch (bdev_io->type) {

	case SPDK_BDEV_IO_TYPE_READ:

		spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);

		spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb,

				     bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);

		return 0;



	case SPDK_BDEV_IO_TYPE_WRITE: