Commit e49ae312 authored by Mateusz Kozlowski's avatar Mateusz Kozlowski Committed by Ben Walker
Browse files

lib/ftl: Add ftl_io argument to internal I/O callbacks 



Almost all I/O callbacks in ftl utilize data in ftl_io, which is
initialized as callback argument in ftl_io_init_internal. This patch
changes the behavior to always returning the ftl_io struct in addition
to an extra opaque buffer.

Signed-off-by: default avatarMateusz Kozlowski <mateusz.kozlowski@intel.com>
Change-Id: I611ab1b33575f599798a2bb65c231a724c852c7f
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/455831


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarWojciech Malikowski <wojciech.malikowski@intel.com>
Reviewed-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 5875bb0e

lib/ftl/ftl_band.c

+26 −26
Original line number Diff line number Diff line
@@ -722,9 +722,9 @@ ftl_band_release_lba_map(struct ftl_band *band) 
}



static void

ftl_read_md_cb(void *arg, int status)

ftl_read_md_cb(struct ftl_io *io, void *arg, int status)

{

	struct ftl_md_io *md_io = arg;

	struct ftl_md_io *md_io = (struct ftl_md_io *)io;



	if (!status) {

		status = md_io->pack_fn(md_io->io.band);
@@ -732,13 +732,13 @@ ftl_read_md_cb(void *arg, int status) 
		status = FTL_MD_IO_FAILURE;

	}



	md_io->cb.fn(md_io->cb.ctx, status);

	md_io->cb_fn(io, md_io->cb_ctx, status);

}



static struct ftl_md_io *

ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_ppa ppa,

		    struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn fn,

		    ftl_md_pack_fn pack_fn, struct ftl_cb cb)

		    struct ftl_band *band, size_t lbk_cnt, ftl_io_fn fn,

		    ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx)

{

	struct ftl_md_io *io;

	struct ftl_io_init_opts opts = {
@@ -750,7 +750,7 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 
		.flags		= FTL_IO_MD | FTL_IO_PPA_MODE,

		.type		= FTL_IO_READ,

		.lbk_cnt	= lbk_cnt,

		.fn		= fn,

		.cb_fn		= fn,

		.data		= band->lba_map.dma_buf,

	};
@@ -761,14 +761,15 @@ ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 


	io->io.ppa = ppa;

	io->pack_fn = pack_fn;

	io->cb = cb;

	io->cb_fn = cb_fn;

	io->cb_ctx = cb_ctx;



	return io;

}



static struct ftl_io *

ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band,

		     void *data, size_t lbk_cnt, spdk_ftl_fn cb)

		     void *data, size_t lbk_cnt, ftl_io_fn cb)

{

	struct ftl_io_init_opts opts = {

		.dev		= dev,
@@ -779,7 +780,7 @@ ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 
		.flags		= FTL_IO_MD | FTL_IO_PPA_MODE,

		.type		= FTL_IO_WRITE,

		.lbk_cnt	= lbk_cnt,

		.fn		= cb,

		.cb_fn		= cb,

		.data		= data,

		.md		= NULL,

	};
@@ -789,7 +790,7 @@ ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 


static int

ftl_band_write_md(struct ftl_band *band, size_t lbk_cnt,

		  ftl_md_pack_fn md_fn, spdk_ftl_fn cb)

		  ftl_md_pack_fn md_fn, ftl_io_fn cb)

{

	struct spdk_ftl_dev *dev = band->dev;

	struct ftl_io *io;
@@ -815,14 +816,14 @@ ftl_band_md_clear(struct ftl_band *band) 
}



int

ftl_band_write_head_md(struct ftl_band *band, spdk_ftl_fn cb)

ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb)

{

	return ftl_band_write_md(band, ftl_head_md_num_lbks(band->dev),

				 ftl_pack_head_md, cb);

}



int

ftl_band_write_tail_md(struct ftl_band *band, spdk_ftl_fn cb)

ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb)

{

	return ftl_band_write_md(band, ftl_tail_md_num_lbks(band->dev),

				 ftl_pack_tail_md, cb);
@@ -839,7 +840,7 @@ ftl_band_lba_map_ppa(struct ftl_band *band, size_t offset) 


static int

ftl_band_read_md(struct ftl_band *band, size_t lbk_cnt, struct ftl_ppa start_ppa,

		 spdk_ftl_fn fn, ftl_md_pack_fn pack_fn, struct ftl_cb cb)

		 ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx)

{

	struct spdk_ftl_dev *dev = band->dev;

	struct ftl_md_io *io;
@@ -848,7 +849,7 @@ ftl_band_read_md(struct ftl_band *band, size_t lbk_cnt, struct ftl_ppa start_ppa 
		return -ENOENT;

	}



	io = ftl_io_init_md_read(dev, start_ppa, band, lbk_cnt, fn, pack_fn, cb);

	io = ftl_io_init_md_read(dev, start_ppa, band, lbk_cnt, fn, pack_fn, cb_fn, cb_ctx);

	if (!io) {

		return -ENOMEM;

	}
@@ -858,10 +859,10 @@ ftl_band_read_md(struct ftl_band *band, size_t lbk_cnt, struct ftl_ppa start_ppa 
}



int

ftl_band_read_tail_md(struct ftl_band *band, struct ftl_ppa ppa, struct ftl_cb cb)

ftl_band_read_tail_md(struct ftl_band *band, struct ftl_ppa ppa, ftl_io_fn cb_fn, void *cb_ctx)

{

	return ftl_band_read_md(band, ftl_tail_md_num_lbks(band->dev), ppa,

				ftl_read_md_cb, ftl_unpack_tail_md, cb);

				ftl_read_md_cb, ftl_unpack_tail_md, cb_fn, cb_ctx);

}



static size_t
@@ -877,10 +878,9 @@ ftl_lba_map_offset_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 
}



static void

ftl_read_lba_map_cb(void *arg, int status)

ftl_read_lba_map_cb(struct ftl_io *io, void *arg, int status)

{

	struct ftl_md_io *md_io = arg;

	struct ftl_io *io = &md_io->io;

	struct ftl_md_io *md_io = (struct ftl_md_io *)io;

	struct ftl_lba_map *lba_map = &io->band->lba_map;

	uint64_t offset;
@@ -892,12 +892,12 @@ ftl_read_lba_map_cb(void *arg, int status) 
		       io->lbk_cnt * FTL_BLOCK_SIZE);

	}



	md_io->cb.fn(md_io->cb.ctx, status);

	md_io->cb_fn(io, md_io->cb_ctx, status);

}



int

ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt,

		      struct ftl_cb cb)

		      ftl_io_fn cb_fn, void *cb_ctx)

{

	size_t lbk_cnt, lbk_off;
@@ -907,18 +907,19 @@ ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt, 
	assert(lbk_off + lbk_cnt <= ftl_lba_map_num_lbks(band->dev));



	return ftl_band_read_md(band, lbk_cnt, ftl_band_lba_map_ppa(band, lbk_off),

				ftl_read_lba_map_cb, NULL, cb);

				ftl_read_lba_map_cb, NULL, cb_fn, cb_ctx);

}



int

ftl_band_read_head_md(struct ftl_band *band, struct ftl_cb cb)

ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx)

{

	return ftl_band_read_md(band,

				ftl_head_md_num_lbks(band->dev),

				ftl_band_head_md_ppa(band),

				ftl_read_md_cb,

				ftl_unpack_head_md,

				cb);

				cb_fn,

				cb_ctx);

}



static void
@@ -943,9 +944,8 @@ ftl_erase_fail(struct ftl_io *io, int status) 
}



static void

ftl_band_erase_cb(void *ctx, int status)

ftl_band_erase_cb(struct ftl_io *io, void *ctx, int status)

{

	struct ftl_io *io = ctx;

	struct ftl_chunk *chunk;



	if (spdk_unlikely(status)) {

lib/ftl/ftl_band.h

+6 −6
Original line number Diff line number Diff line
@@ -38,10 +38,10 @@ 
#include "spdk/bit_array.h"

#include "spdk/queue.h"



#include "ftl_io.h"

#include "ftl_ppa.h"



struct spdk_ftl_dev;

struct ftl_cb;



enum ftl_chunk_state {

	FTL_CHUNK_STATE_FREE,
@@ -170,7 +170,7 @@ void ftl_band_clear_lba_map(struct ftl_band *band); 
void		ftl_band_release_lba_map(struct ftl_band *band);

int		ftl_band_read_lba_map(struct ftl_band *band,

				      size_t offset, size_t lba_cnt,

				      struct ftl_cb cb);

				      ftl_io_fn cb_fn, void *cb_ctx);

struct ftl_ppa ftl_band_next_xfer_ppa(struct ftl_band *band, struct ftl_ppa ppa,

				      size_t num_lbks);

struct ftl_ppa ftl_band_next_ppa(struct ftl_band *band, struct ftl_ppa ppa,
@@ -184,10 +184,10 @@ struct ftl_band *ftl_band_from_ppa(struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 
struct ftl_chunk *ftl_band_chunk_from_ppa(struct ftl_band *band, struct ftl_ppa);

void		ftl_band_md_clear(struct ftl_band *band);

int		ftl_band_read_tail_md(struct ftl_band *band, struct ftl_ppa,

				      struct ftl_cb cb);

int		ftl_band_read_head_md(struct ftl_band *band, struct ftl_cb cb);

int		ftl_band_write_tail_md(struct ftl_band *band, spdk_ftl_fn cb);

int		ftl_band_write_head_md(struct ftl_band *band, spdk_ftl_fn cb);

				      ftl_io_fn cb_fn, void *cb_ctx);

int		ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx);

int		ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb);

int		ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb);

struct ftl_ppa ftl_band_tail_md_ppa(struct ftl_band *band);

struct ftl_ppa ftl_band_head_md_ppa(struct ftl_band *band);

void		ftl_band_write_failed(struct ftl_band *band);

lib/ftl/ftl_core.c

+9 −9
Original line number Diff line number Diff line
@@ -79,7 +79,10 @@ struct ftl_flush { 
	size_t				num_req;



	/* Callback */

	struct ftl_cb			cb;

	struct {

		spdk_ftl_fn		fn;

		void			*ctx;

	} cb;



	/* Batch bitmap */

	struct spdk_bit_array		*bmap;
@@ -189,9 +192,8 @@ ftl_md_write_fail(struct ftl_io *io, int status) 
}



static void

ftl_md_write_cb(void *arg, int status)

ftl_md_write_cb(struct ftl_io *io, void *arg, int status)

{

	struct ftl_io *io = arg;

	struct spdk_ftl_dev *dev = io->dev;

	struct ftl_nv_cache *nv_cache = &dev->nv_cache;

	struct ftl_wptr *wptr;
@@ -1055,9 +1057,8 @@ ftl_write_fail(struct ftl_io *io, int status) 
}



static void

ftl_write_cb(void *arg, int status)

ftl_write_cb(struct ftl_io *io, void *arg, int status)

{

	struct ftl_io *io = arg;

	struct spdk_ftl_dev *dev = io->dev;

	struct ftl_rwb_batch *batch = io->rwb_batch;

	struct ftl_rwb_entry *entry;
@@ -1157,7 +1158,7 @@ ftl_update_l2p(struct spdk_ftl_dev *dev, const struct ftl_rwb_entry *entry, 


static struct ftl_io *

ftl_io_init_child_write(struct ftl_io *parent, struct ftl_ppa ppa,

			void *data, void *md, spdk_ftl_fn cb)

			void *data, void *md, ftl_io_fn cb)

{

	struct ftl_io *io;

	struct spdk_ftl_dev *dev = parent->dev;
@@ -1171,7 +1172,7 @@ ftl_io_init_child_write(struct ftl_io *parent, struct ftl_ppa ppa, 
		.flags		= 0,

		.type		= FTL_IO_WRITE,

		.lbk_cnt	= dev->xfer_size,

		.fn		= cb,

		.cb_fn		= cb,

		.data		= data,

		.md		= md,

	};
@@ -1187,10 +1188,9 @@ ftl_io_init_child_write(struct ftl_io *parent, struct ftl_ppa ppa, 
}



static void

ftl_io_child_write_cb(void *ctx, int status)

ftl_io_child_write_cb(struct ftl_io *io, void *ctx, int status)

{

	struct ftl_chunk *chunk;

	struct ftl_io *io = ctx;



	chunk = ftl_band_chunk_from_ppa(io->band, io->ppa);

	chunk->busy = false;

lib/ftl/ftl_io.c

+21 −14
Original line number Diff line number Diff line
@@ -256,15 +256,15 @@ ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt) 


static void

ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev,

	    spdk_ftl_fn fn, void *ctx, int flags, int type)

	    ftl_io_fn fn, void *ctx, int flags, int type)

{

	io->flags |= flags | FTL_IO_INITIALIZED;

	io->type = type;

	io->dev = dev;

	io->lba.single = FTL_LBA_INVALID;

	io->ppa.ppa = FTL_PPA_INVALID;

	io->cb.fn = fn;

	io->cb.ctx = ctx;

	io->cb_fn = fn;

	io->cb_ctx = ctx;

	io->trace = ftl_trace_alloc_id(dev);

}
@@ -291,7 +291,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) 
	}



	ftl_io_clear(io);

	ftl_io_init(io, dev, opts->fn, io, opts->flags | FTL_IO_INTERNAL, opts->type);

	ftl_io_init(io, dev, opts->cb_fn, opts->cb_ctx, opts->flags | FTL_IO_INTERNAL, opts->type);



	io->rwb_batch = opts->rwb_batch;

	io->band = opts->band;
@@ -309,7 +309,7 @@ ftl_io_init_internal(const struct ftl_io_init_opts *opts) 


struct ftl_io *

ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,

		struct ftl_rwb_batch *batch, spdk_ftl_fn cb)

		struct ftl_rwb_batch *batch, ftl_io_fn cb)

{

	struct ftl_io_init_opts opts = {

		.dev		= dev,
@@ -320,7 +320,7 @@ ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band, 
		.flags		= 0,

		.type		= FTL_IO_WRITE,

		.lbk_cnt	= dev->xfer_size,

		.fn		= cb,

		.cb_fn		= cb,

		.data		= ftl_rwb_batch_get_data(batch),

		.md		= ftl_rwb_batch_get_md(batch),

	};
@@ -329,7 +329,7 @@ ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band, 
}



struct ftl_io *

ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb)

ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb)

{

	struct ftl_io *io;

	struct ftl_io_init_opts opts = {
@@ -341,7 +341,7 @@ ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb) 
		.flags		= FTL_IO_PPA_MODE,

		.type		= FTL_IO_ERASE,

		.lbk_cnt	= 1,

		.fn		= cb,

		.cb_fn		= cb,

		.data		= NULL,

		.md		= NULL,

	};
@@ -356,9 +356,15 @@ ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb) 
	return io;

}



static void

_ftl_user_cb(struct ftl_io *io, void *arg, int status)

{

	io->user_fn(arg, status);

}



struct ftl_io *

ftl_io_user_init(struct spdk_io_channel *_ioch, uint64_t lba, size_t lbk_cnt, struct iovec *iov,

		 size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg, int type)

		 size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)

{

	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(_ioch);

	struct spdk_ftl_dev *dev = ioch->dev;
@@ -369,8 +375,9 @@ ftl_io_user_init(struct spdk_io_channel *_ioch, uint64_t lba, size_t lbk_cnt, st 
		return NULL;

	}



	ftl_io_init(io, dev, cb_fn, cb_arg, 0, type);

	ftl_io_init(io, dev, _ftl_user_cb, cb_ctx, 0, type);

	io->lba.single = lba;

	io->user_fn = cb_fn;



	if (ftl_io_init_iovec(io, iov, iov_cnt, lbk_cnt)) {

		ftl_io_free(io);
@@ -425,8 +432,8 @@ ftl_io_complete(struct ftl_io *io) 
	pthread_spin_unlock(&io->lock);



	if (complete) {

		if (io->cb.fn) {

			io->cb.fn(io->cb.ctx, io->status);

		if (io->cb_fn) {

			io->cb_fn(io, io->cb_ctx, io->status);

		}



		if (parent && ftl_io_remove_child(io)) {
@@ -511,10 +518,10 @@ ftl_io_alloc(struct spdk_io_channel *ch) 
}



void

ftl_io_reinit(struct ftl_io *io, spdk_ftl_fn fn, void *ctx, int flags, int type)

ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, void *ctx, int flags, int type)

{

	ftl_io_clear(io);

	ftl_io_init(io, io->dev, fn, ctx, flags, type);

	ftl_io_init(io, io->dev, cb, ctx, flags, type);

}



void

lib/ftl/ftl_io.h

+20 −15
Original line number Diff line number Diff line
@@ -47,6 +47,7 @@ struct ftl_band; 
struct ftl_io;



typedef int (*ftl_md_pack_fn)(struct ftl_band *);

typedef void (*ftl_io_fn)(struct ftl_io *, void *, int);



/* IO flags */

enum ftl_io_flags {
@@ -112,16 +113,11 @@ struct ftl_io_init_opts { 
	/* Metadata */

	void                                    *md;



	/* Callback */

	spdk_ftl_fn				fn;

};



struct ftl_cb {

	/* Callback function */

	spdk_ftl_fn				fn;

	/* Callback's function */

	ftl_io_fn				cb_fn;



	/* Callback's context */

	void					*ctx;

	void					*cb_ctx;

};



struct ftl_io_channel {
@@ -187,8 +183,14 @@ struct ftl_io { 
	/* Number of split requests */

	size_t					req_cnt;



	/* Completion callback */

	struct ftl_cb				cb;

	/* Callback's function */

	ftl_io_fn				cb_fn;



	/* Callback's context */

	void					*cb_ctx;



	/* User callback function */

	spdk_ftl_fn				user_fn;



	/* Flags */

	int					flags;
@@ -222,8 +224,11 @@ struct ftl_md_io { 
	/* Serialization/deserialization callback */

	ftl_md_pack_fn				pack_fn;



	/* User's callback */

	struct ftl_cb				cb;

	/* Callback's function */

	ftl_io_fn				cb_fn;



	/* Callback's context */

	void					*cb_ctx;

};



static inline bool
@@ -251,7 +256,7 @@ struct ftl_io *ftl_io_alloc_child(struct ftl_io *parent); 
void ftl_io_fail(struct ftl_io *io, int status);

void ftl_io_free(struct ftl_io *io);

struct ftl_io *ftl_io_init_internal(const struct ftl_io_init_opts *opts);

void ftl_io_reinit(struct ftl_io *io, spdk_ftl_fn cb,

void ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb,

		   void *ctx, int flags, int type);

void ftl_io_clear(struct ftl_io *io);

void ftl_io_inc_req(struct ftl_io *io);
@@ -264,8 +269,8 @@ size_t ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt); 
void *ftl_io_iovec_addr(struct ftl_io *io);

size_t ftl_io_iovec_len_left(struct ftl_io *io);

struct ftl_io *ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,

			       struct ftl_rwb_batch *entry, spdk_ftl_fn cb);

struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb);

			       struct ftl_rwb_batch *entry, ftl_io_fn cb);

struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb);

struct ftl_io *ftl_io_user_init(struct spdk_io_channel *ioch, uint64_t lba, size_t lbk_cnt,

				struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn,

				void *cb_arg, int type);