Commit 0345729e authored by Ahriben Gonzalez's avatar Ahriben Gonzalez Committed by Tomasz Zawadzki
Browse files

nvme: Add metadata support to io commands 



Adding metadata support for io commands. Currently metadata is ignored
even if present in the cmd struct. Making metadata adress
readable/writable depending on data transfer bits. Adding extra unit
test to make sure metadata fields are populated.

Signed-off-by: default avatarAhriben Gonzalez <ahribeng@gmail.com>
Change-Id: I1d01974a6b2831c82b43e94073065d235eea429a
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10854


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
parent 9e14341b

lib/nvme/nvme_cuse.c

+143 −38
Original line number Diff line number Diff line
@@ -70,9 +70,14 @@ struct cuse_io_ctx { 


	uint64_t			lba;

	uint32_t			lba_count;

	uint16_t			apptag;

	uint16_t			appmask;



	void				*data;

	void				*metadata;



	int				data_len;

	int				metadata_len;



	fuse_req_t			req;

};
@@ -81,6 +86,7 @@ static void 
cuse_io_ctx_free(struct cuse_io_ctx *ctx)

{

	spdk_free(ctx->data);

	spdk_free(ctx->metadata);

	free(ctx);

}
@@ -97,7 +103,7 @@ static void 
cuse_nvme_passthru_cmd_cb(void *arg, const struct spdk_nvme_cpl *cpl)

{

	struct cuse_io_ctx *ctx = arg;

	struct iovec out_iov[2];

	struct iovec out_iov[3];

	struct spdk_nvme_cpl _cpl;

	int out_iovcnt = 0;

	uint16_t status_field = cpl->status_raw >> 1; /* Drop out phase bit */
@@ -107,11 +113,18 @@ cuse_nvme_passthru_cmd_cb(void *arg, const struct spdk_nvme_cpl *cpl) 
	out_iov[out_iovcnt].iov_len = sizeof(_cpl.cdw0);

	out_iovcnt += 1;



	if (ctx->data_transfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST && ctx->data_len > 0) {

	if (ctx->data_transfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {

		if (ctx->data_len > 0) {

			out_iov[out_iovcnt].iov_base = ctx->data;

			out_iov[out_iovcnt].iov_len = ctx->data_len;

			out_iovcnt += 1;

		}

		if (ctx->metadata_len > 0) {

			out_iov[out_iovcnt].iov_base = ctx->metadata;

			out_iov[out_iovcnt].iov_len = ctx->metadata_len;

			out_iovcnt += 1;

		}

	}



	fuse_reply_ioctl_iov(ctx->req, status_field, out_iov, out_iovcnt);

	cuse_io_ctx_free(ctx);
@@ -124,8 +137,9 @@ cuse_nvme_passthru_cmd_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, voi 
	struct cuse_io_ctx *ctx = arg;



	if (nsid != 0) {

		rc = spdk_nvme_ctrlr_cmd_io_raw(ctrlr, ctrlr->external_io_msgs_qpair, &ctx->nvme_cmd, ctx->data,

						ctx->data_len, cuse_nvme_passthru_cmd_cb, (void *)ctx);

		rc = spdk_nvme_ctrlr_cmd_io_raw_with_md(ctrlr, ctrlr->external_io_msgs_qpair, &ctx->nvme_cmd,

							ctx->data,

							ctx->data_len, ctx->metadata, cuse_nvme_passthru_cmd_cb, (void *)ctx);

	} else {

		rc = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &ctx->nvme_cmd, ctx->data, ctx->data_len,

						   cuse_nvme_passthru_cmd_cb, (void *)ctx);
@@ -138,7 +152,7 @@ cuse_nvme_passthru_cmd_execute(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, voi 


static void

cuse_nvme_passthru_cmd_send(fuse_req_t req, struct nvme_passthru_cmd *passthru_cmd,

			    const void *data, int cmd)

			    const void *data, const void *metadata, int cmd)

{

	struct cuse_io_ctx *ctx;

	struct cuse_device *cuse_device = fuse_req_userdata(req);
@@ -165,6 +179,7 @@ cuse_nvme_passthru_cmd_send(fuse_req_t req, struct nvme_passthru_cmd *passthru_c 
	ctx->nvme_cmd.cdw15 = passthru_cmd->cdw15;



	ctx->data_len = passthru_cmd->data_len;

	ctx->metadata_len = passthru_cmd->metadata_len;



	if (ctx->data_len > 0) {

		ctx->data = spdk_malloc(ctx->data_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
@@ -179,6 +194,19 @@ cuse_nvme_passthru_cmd_send(fuse_req_t req, struct nvme_passthru_cmd *passthru_c 
		}

	}



	if (ctx->metadata_len > 0) {

		ctx->metadata = spdk_malloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);

		if (!ctx->metadata) {

			SPDK_ERRLOG("Cannot allocate memory for metadata\n");

			fuse_reply_err(req, ENOMEM);

			cuse_io_ctx_free(ctx);

			return;

		}

		if (metadata != NULL) {

			memcpy(ctx->metadata, metadata, ctx->metadata_len);

		}

	}



	if ((unsigned int)cmd != NVME_IOCTL_ADMIN_CMD) {

		/* Send NS for IO IOCTLs */

		rv = nvme_io_msg_send(cuse_device->ctrlr, passthru_cmd->nsid, cuse_nvme_passthru_cmd_execute, ctx);
@@ -200,9 +228,9 @@ cuse_nvme_passthru_cmd(fuse_req_t req, int cmd, void *arg, 
		       const void *in_buf, size_t in_bufsz, size_t out_bufsz)

{

	struct nvme_passthru_cmd *passthru_cmd;

	struct iovec in_iov[2], out_iov[2];

	struct iovec in_iov[3], out_iov[3];

	int in_iovcnt = 0, out_iovcnt = 0;

	const void *dptr = NULL;

	const void *dptr = NULL, *mdptr = NULL;

	enum spdk_nvme_data_transfer data_transfer;



	in_iov[in_iovcnt].iov_base = (void *)arg;
@@ -223,19 +251,32 @@ cuse_nvme_passthru_cmd(fuse_req_t req, int cmd, void *arg, 
			in_iov[in_iovcnt].iov_len = passthru_cmd->data_len;

			in_iovcnt += 1;

		}

		/* Make metadata pointer accessible (RO) */

		if (passthru_cmd->metadata != 0) {

			in_iov[in_iovcnt].iov_base = (void *)passthru_cmd->metadata;

			in_iov[in_iovcnt].iov_len = passthru_cmd->metadata_len;

			in_iovcnt += 1;

		}

	}



	/* Always make result field writable regardless of data transfer bits */

	/* Always make result field writeable regardless of data transfer bits */

	out_iov[out_iovcnt].iov_base = &((struct nvme_passthru_cmd *)arg)->result;

	out_iov[out_iovcnt].iov_len = sizeof(uint32_t);

	out_iovcnt += 1;



	if (data_transfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {

		/* Make data pointer accessible (WO) */

		if (passthru_cmd->data_len > 0) {

			out_iov[out_iovcnt].iov_base = (void *)passthru_cmd->addr;

			out_iov[out_iovcnt].iov_len = passthru_cmd->data_len;

			out_iovcnt += 1;

		}

		/* Make metadata pointer accessible (WO) */

		if (passthru_cmd->metadata_len > 0) {

			out_iov[out_iovcnt].iov_base = (void *)passthru_cmd->metadata;

			out_iov[out_iovcnt].iov_len = passthru_cmd->metadata_len;

			out_iovcnt += 1;

		}

	}



	if (out_bufsz == 0) {
@@ -250,9 +291,11 @@ cuse_nvme_passthru_cmd(fuse_req_t req, int cmd, void *arg, 


	if (data_transfer == SPDK_NVME_DATA_HOST_TO_CONTROLLER) {

		dptr = (passthru_cmd->addr == 0) ? NULL : in_buf + sizeof(*passthru_cmd);

		mdptr = (passthru_cmd->metadata == 0) ? NULL : in_buf + sizeof(*passthru_cmd) +

			passthru_cmd->data_len;

	}



	cuse_nvme_passthru_cmd_send(req, passthru_cmd, dptr, cmd);

	cuse_nvme_passthru_cmd_send(req, passthru_cmd, dptr, mdptr, cmd);

}



static void
@@ -365,10 +408,11 @@ cuse_nvme_submit_io_write_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void 
	struct cuse_io_ctx *ctx = arg;

	struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);



	rc = spdk_nvme_ns_cmd_write(ns, ctrlr->external_io_msgs_qpair, ctx->data,

	rc = spdk_nvme_ns_cmd_write_with_md(ns, ctrlr->external_io_msgs_qpair, ctx->data, ctx->metadata,

					    ctx->lba, /* LBA start */

					    ctx->lba_count, /* number of LBAs */

				    cuse_nvme_submit_io_write_done, ctx, 0);

					    cuse_nvme_submit_io_write_done, ctx, 0,

					    ctx->appmask, ctx->apptag);



	if (rc != 0) {

		SPDK_ERRLOG("write failed: rc = %d\n", rc);
@@ -380,7 +424,7 @@ cuse_nvme_submit_io_write_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void 


static void

cuse_nvme_submit_io_write(struct cuse_device *cuse_device, fuse_req_t req, int cmd, void *arg,

			  struct fuse_file_info *fi, unsigned flags, uint32_t block_size,

			  struct fuse_file_info *fi, unsigned flags, uint32_t block_size, uint32_t md_size,

			  const void *in_buf, size_t in_bufsz, size_t out_bufsz)

{

	const struct nvme_user_io *user_io = in_buf;
@@ -410,6 +454,25 @@ cuse_nvme_submit_io_write(struct cuse_device *cuse_device, fuse_req_t req, int c 


	memcpy(ctx->data, in_buf + sizeof(*user_io), ctx->data_len);



	if (user_io->metadata) {

		ctx->apptag = user_io->apptag;

		ctx->appmask = user_io->appmask;

		ctx->metadata_len = md_size * ctx->lba_count;

		ctx->metadata = spdk_zmalloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);



		if (ctx->metadata == NULL) {

			SPDK_ERRLOG("Cannot allocate memory for metadata\n");

			if (ctx->metadata_len == 0) {

				SPDK_ERRLOG("Device format does not support metadata\n");

			}

			fuse_reply_err(req, ENOMEM);

			cuse_io_ctx_free(ctx);

			return;

		}



		memcpy(ctx->metadata, in_buf + sizeof(*user_io) + ctx->data_len, ctx->metadata_len);

	}



	rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_write_cb,

			      ctx);

	if (rc < 0) {
@@ -423,13 +486,21 @@ static void 
cuse_nvme_submit_io_read_done(void *ref, const struct spdk_nvme_cpl *cpl)

{

	struct cuse_io_ctx *ctx = (struct cuse_io_ctx *)ref;

	struct iovec iov;

	struct iovec iov[2];

	int iovcnt = 0;

	uint16_t status_field = cpl->status_raw >> 1; /* Drop out phase bit */



	iov.iov_base = ctx->data;

	iov.iov_len = ctx->data_len;

	iov[iovcnt].iov_base = ctx->data;

	iov[iovcnt].iov_len = ctx->data_len;

	iovcnt += 1;



	if (ctx->metadata) {

		iov[iovcnt].iov_base = ctx->metadata;

		iov[iovcnt].iov_len = ctx->metadata_len;

		iovcnt += 1;

	}



	fuse_reply_ioctl_iov(ctx->req, status_field, &iov, 1);

	fuse_reply_ioctl_iov(ctx->req, status_field, iov, iovcnt);



	cuse_io_ctx_free(ctx);

}
@@ -441,10 +512,11 @@ cuse_nvme_submit_io_read_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void * 
	struct cuse_io_ctx *ctx = arg;

	struct spdk_nvme_ns *ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);



	rc = spdk_nvme_ns_cmd_read(ns, ctrlr->external_io_msgs_qpair, ctx->data,

	rc = spdk_nvme_ns_cmd_read_with_md(ns, ctrlr->external_io_msgs_qpair, ctx->data, ctx->metadata,

					   ctx->lba, /* LBA start */

					   ctx->lba_count, /* number of LBAs */

				   cuse_nvme_submit_io_read_done, ctx, 0);

					   cuse_nvme_submit_io_read_done, ctx, 0,

					   ctx->appmask, ctx->apptag);



	if (rc != 0) {

		SPDK_ERRLOG("read failed: rc = %d\n", rc);
@@ -456,7 +528,7 @@ cuse_nvme_submit_io_read_cb(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, void * 


static void

cuse_nvme_submit_io_read(struct cuse_device *cuse_device, fuse_req_t req, int cmd, void *arg,

			 struct fuse_file_info *fi, unsigned flags, uint32_t block_size,

			 struct fuse_file_info *fi, unsigned flags, uint32_t block_size, uint32_t md_size,

			 const void *in_buf, size_t in_bufsz, size_t out_bufsz)

{

	int rc;
@@ -484,6 +556,23 @@ cuse_nvme_submit_io_read(struct cuse_device *cuse_device, fuse_req_t req, int cm 
		return;

	}



	if (user_io->metadata) {

		ctx->apptag = user_io->apptag;

		ctx->appmask = user_io->appmask;

		ctx->metadata_len = md_size * ctx->lba_count;

		ctx->metadata = spdk_zmalloc(ctx->metadata_len, 4096, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);



		if (ctx->metadata == NULL) {

			SPDK_ERRLOG("Cannot allocate memory for metadata\n");

			if (ctx->metadata_len == 0) {

				SPDK_ERRLOG("Device format does not support metadata\n");

			}

			fuse_reply_err(req, ENOMEM);

			cuse_io_ctx_free(ctx);

			return;

		}

	}



	rc = nvme_io_msg_send(cuse_device->ctrlr, cuse_device->nsid, cuse_nvme_submit_io_read_cb, ctx);

	if (rc < 0) {

		SPDK_ERRLOG("Cannot send read io\n");
@@ -499,15 +588,18 @@ cuse_nvme_submit_io(fuse_req_t req, int cmd, void *arg, 
		    const void *in_buf, size_t in_bufsz, size_t out_bufsz)

{

	const struct nvme_user_io *user_io;

	struct iovec in_iov[2], out_iov;

	struct iovec in_iov[3], out_iov[2];

	int in_iovcnt = 0, out_iovcnt = 0;

	struct cuse_device *cuse_device = fuse_req_userdata(req);

	struct spdk_nvme_ns *ns;

	uint32_t block_size;

	uint32_t md_size;



	in_iov[0].iov_base = (void *)arg;

	in_iov[0].iov_len = sizeof(*user_io);

	in_iov[in_iovcnt].iov_base = (void *)arg;

	in_iov[in_iovcnt].iov_len = sizeof(*user_io);

	in_iovcnt += 1;

	if (in_bufsz == 0) {

		fuse_reply_ioctl_retry(req, in_iov, 1, NULL, 0);

		fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, NULL, 0);

		return;

	}
@@ -515,29 +607,42 @@ cuse_nvme_submit_io(fuse_req_t req, int cmd, void *arg, 


	ns = spdk_nvme_ctrlr_get_ns(cuse_device->ctrlr, cuse_device->nsid);

	block_size = spdk_nvme_ns_get_sector_size(ns);

	md_size = spdk_nvme_ns_get_md_size(ns);



	switch (user_io->opcode) {

	case SPDK_NVME_OPC_READ:

		out_iov.iov_base = (void *)user_io->addr;

		out_iov.iov_len = (user_io->nblocks + 1) * block_size;

		out_iov[out_iovcnt].iov_base = (void *)user_io->addr;

		out_iov[out_iovcnt].iov_len = (user_io->nblocks + 1) * block_size;

		out_iovcnt += 1;

		if (user_io->metadata != 0) {

			out_iov[out_iovcnt].iov_base = (void *)user_io->metadata;

			out_iov[out_iovcnt].iov_len = (user_io->nblocks + 1) * md_size;

			out_iovcnt += 1;

		}

		if (out_bufsz == 0) {

			fuse_reply_ioctl_retry(req, in_iov, 1, &out_iov, 1);

			fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, out_iov, out_iovcnt);

			return;

		}



		cuse_nvme_submit_io_read(cuse_device, req, cmd, arg, fi, flags,

					 block_size, in_buf, in_bufsz, out_bufsz);

					 block_size, md_size, in_buf, in_bufsz, out_bufsz);

		break;

	case SPDK_NVME_OPC_WRITE:

		in_iov[1].iov_base = (void *)user_io->addr;

		in_iov[1].iov_len = (user_io->nblocks + 1) * block_size;

		in_iov[in_iovcnt].iov_base = (void *)user_io->addr;

		in_iov[in_iovcnt].iov_len = (user_io->nblocks + 1) * block_size;

		in_iovcnt += 1;

		if (user_io->metadata != 0) {

			in_iov[in_iovcnt].iov_base = (void *)user_io->metadata;

			in_iov[in_iovcnt].iov_len = (user_io->nblocks + 1) * md_size;

			in_iovcnt += 1;

		}

		if (in_bufsz == sizeof(*user_io)) {

			fuse_reply_ioctl_retry(req, in_iov, 2, NULL, 0);

			fuse_reply_ioctl_retry(req, in_iov, in_iovcnt, NULL, out_iovcnt);

			return;

		}



		cuse_nvme_submit_io_write(cuse_device, req, cmd, arg, fi, flags,

					  block_size, in_buf, in_bufsz, out_bufsz);

					  block_size, md_size, in_buf, in_bufsz, out_bufsz);

		break;

	default:

		SPDK_ERRLOG("SUBMIT_IO: opc:%d not valid\n", user_io->opcode);

test/nvme/cuse/cuse.c

+12 −8
Original line number Diff line number Diff line
@@ -47,8 +47,8 @@ DEFINE_STUB(spdk_nvme_ctrlr_cmd_admin_raw, int, (struct spdk_nvme_ctrlr *ctrlr, 
		struct spdk_nvme_cmd *cmd, void *buf, uint32_t len,

		spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0);



DEFINE_STUB(spdk_nvme_ctrlr_cmd_io_raw, int, (struct spdk_nvme_ctrlr *ctrlr,

		struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd, void *buf, uint32_t len,

DEFINE_STUB(spdk_nvme_ctrlr_cmd_io_raw_with_md, int, (struct spdk_nvme_ctrlr *ctrlr,

		struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd, void *buf, uint32_t len, void *md_buf,

		spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0);



DEFINE_STUB(spdk_nvme_ctrlr_get_num_ns, uint32_t, (struct spdk_nvme_ctrlr *ctrlr), 128);
@@ -84,20 +84,24 @@ DEFINE_STUB(spdk_nvme_ctrlr_reset, int, (struct spdk_nvme_ctrlr *ctrlr), 0); 


DEFINE_STUB(spdk_nvme_ctrlr_reset_subsystem, int, (struct spdk_nvme_ctrlr *ctrlr), 0);



DEFINE_STUB(spdk_nvme_ns_cmd_read, int, (struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,

		void *payload,

DEFINE_STUB(spdk_nvme_ns_cmd_read_with_md, int, (struct spdk_nvme_ns *ns,

		struct spdk_nvme_qpair *qpair,

		void *payload, void *metadata,

		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,

		uint32_t io_flags), 0);

		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag), 0);



DEFINE_STUB(spdk_nvme_ns_cmd_write, int, (struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,

		void *payload,

DEFINE_STUB(spdk_nvme_ns_cmd_write_with_md, int, (struct spdk_nvme_ns *ns,

		struct spdk_nvme_qpair *qpair,

		void *payload, void *metadata,

		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,

		uint32_t io_flags), 0);

		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag), 0);



DEFINE_STUB(spdk_nvme_ns_get_num_sectors, uint64_t, (struct spdk_nvme_ns *ns), 0);



DEFINE_STUB(spdk_nvme_ns_get_sector_size, uint32_t, (struct spdk_nvme_ns *ns), 0);



DEFINE_STUB(spdk_nvme_ns_get_md_size, uint32_t, (struct spdk_nvme_ns *ns), 0);



DEFINE_STUB_V(spdk_unaffinitize_thread, (void));



DEFINE_STUB(spdk_nvme_ctrlr_get_ns, struct spdk_nvme_ns *, (struct spdk_nvme_ctrlr *ctrlr,

test/unit/lib/nvme/nvme_cuse.c/nvme_cuse_ut.c

+162 −22
Original line number Diff line number Diff line
@@ -45,27 +45,31 @@ DEFINE_STUB(spdk_nvme_ctrlr_cmd_admin_raw, int, (struct spdk_nvme_ctrlr *ctrlr, 
		struct spdk_nvme_cmd *cmd, void *buf, uint32_t len,

		spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0);



DEFINE_STUB(spdk_nvme_ctrlr_cmd_io_raw, int, (struct spdk_nvme_ctrlr *ctrlr,

		struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd, void *buf, uint32_t len,

DEFINE_STUB(spdk_nvme_ctrlr_cmd_io_raw_with_md, int, (struct spdk_nvme_ctrlr *ctrlr,

		struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd, void *buf, uint32_t len, void *md_buf,

		spdk_nvme_cmd_cb cb_fn, void *cb_arg), 0);



DEFINE_STUB(spdk_nvme_ctrlr_reset, int, (struct spdk_nvme_ctrlr *ctrlr), 0);



DEFINE_STUB(spdk_nvme_ctrlr_reset_subsystem, int, (struct spdk_nvme_ctrlr *ctrlr), 0);



DEFINE_STUB(spdk_nvme_ns_cmd_read, int,

	    (struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,

	     void *payload, uint64_t lba, uint32_t lba_count,

	     spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags), 0);

DEFINE_STUB(spdk_nvme_ns_cmd_read_with_md, int, (struct spdk_nvme_ns *ns,

		struct spdk_nvme_qpair *qpair,

		void *payload, void *metadata,

		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,

		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag), 0);



DEFINE_STUB(spdk_nvme_ns_cmd_write, int,

	    (struct spdk_nvme_ns *ns, struct spdk_nvme_qpair *qpair,

	     void *payload, uint64_t lba, uint32_t lba_count,

	     spdk_nvme_cmd_cb cb_fn, void *cb_arg, uint32_t io_flags), 0);

DEFINE_STUB(spdk_nvme_ns_cmd_write_with_md, int, (struct spdk_nvme_ns *ns,

		struct spdk_nvme_qpair *qpair,

		void *payload, void *metadata,

		uint64_t lba, uint32_t lba_count, spdk_nvme_cmd_cb cb_fn, void *cb_arg,

		uint32_t io_flags, uint16_t apptag_mask, uint16_t apptag), 0);



DEFINE_STUB(spdk_nvme_ns_get_num_sectors, uint64_t,

	    (struct spdk_nvme_ns *ns), 0);



DEFINE_STUB(spdk_nvme_ns_get_md_size, uint32_t, (struct spdk_nvme_ns *ns), 0);



DEFINE_STUB_V(spdk_unaffinitize_thread, (void));



DEFINE_STUB(nvme_io_msg_ctrlr_register, int,
@@ -178,6 +182,7 @@ test_cuse_nvme_submit_io_read_write(void) 
	fuse_req_t req = (void *)0xDEEACDFF;

	unsigned flags = FUSE_IOCTL_DIR;

	uint32_t block_size = 4096;

	uint32_t md_size = 0;

	size_t in_bufsz = 4096;

	size_t out_bufsz = 4096;
@@ -192,7 +197,69 @@ test_cuse_nvme_submit_io_read_write(void) 


	/* Submit IO read */

	cuse_nvme_submit_io_read(&cuse_device, req, 0, arg, &fi, flags,

				 block_size, user_io, in_bufsz, out_bufsz);

				 block_size, md_size, user_io, in_bufsz, out_bufsz);

	CU_ASSERT(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->req == req);

	CU_ASSERT(g_ut_ctx->lba = user_io->slba);

	CU_ASSERT(g_ut_ctx->lba_count == (uint32_t)(user_io->nblocks + 1));

	CU_ASSERT(g_ut_ctx->data_len ==

		  (int)((user_io->nblocks + 1) * block_size));

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len == 0);

	CU_ASSERT(g_ut_ctx->metadata == NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0);

	CU_ASSERT(g_ut_ctx->apptag == 0);

	cuse_io_ctx_free(g_ut_ctx);



	/* Submit IO write */

	g_ut_ctx = NULL;



	cuse_nvme_submit_io_write(&cuse_device, req, 0, arg, &fi, flags,

				  block_size, md_size, user_io, in_bufsz, out_bufsz);

	CU_ASSERT(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->req == req);

	CU_ASSERT(g_ut_ctx->lba = user_io->slba);

	CU_ASSERT(g_ut_ctx->lba_count == (uint32_t)(user_io->nblocks + 1));

	CU_ASSERT(g_ut_ctx->data_len ==

		  (int)((user_io->nblocks + 1) * block_size));

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len == 0);

	CU_ASSERT(g_ut_ctx->metadata == NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0);

	CU_ASSERT(g_ut_ctx->apptag == 0);

	cuse_io_ctx_free(g_ut_ctx);

	free(user_io);

}



static void

test_cuse_nvme_submit_io_read_write_with_md(void)

{

	struct cuse_device cuse_device = {};

	struct fuse_file_info fi = {};

	struct nvme_user_io *user_io = NULL;

	char arg[1024] = {};

	fuse_req_t req = (void *)0xDEEACDFF;

	unsigned flags = FUSE_IOCTL_DIR;

	uint32_t block_size = 4096;

	uint32_t md_size = 8;

	size_t in_bufsz = 4096;

	size_t out_bufsz = 4096;



	/* Allocate memory to avoid stack buffer overflow */

	user_io = calloc(4, 4096);

	SPDK_CU_ASSERT_FATAL(user_io != NULL);

	cuse_device.ctrlr = (void *)0xDEADBEEF;

	cuse_device.nsid = 1;

	user_io->slba = 1024;

	user_io->nblocks = 1;

	user_io->appmask = 0xF00D;

	user_io->apptag = 0xC0DE;

	user_io->metadata = 0xDEADDEAD;

	g_ut_ctx = NULL;



	/* Submit IO read */

	cuse_nvme_submit_io_read(&cuse_device, req, 0, arg, &fi, flags,

				 block_size, md_size, user_io, in_bufsz, out_bufsz);

	CU_ASSERT(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->req == req);

	CU_ASSERT(g_ut_ctx->lba = user_io->slba);
@@ -200,13 +267,18 @@ test_cuse_nvme_submit_io_read_write(void) 
	CU_ASSERT(g_ut_ctx->data_len ==

		  (int)((user_io->nblocks + 1) * block_size));

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len ==

		  (int)((user_io->nblocks + 1) * md_size));

	CU_ASSERT(g_ut_ctx->metadata != NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0xF00D);

	CU_ASSERT(g_ut_ctx->apptag == 0xC0DE);

	cuse_io_ctx_free(g_ut_ctx);



	/* Submit IO write */

	g_ut_ctx = NULL;



	cuse_nvme_submit_io_write(&cuse_device, req, 0, arg, &fi, flags,

				  block_size, user_io, in_bufsz, out_bufsz);

				  block_size, md_size, user_io, in_bufsz, out_bufsz);

	CU_ASSERT(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->req == req);

	CU_ASSERT(g_ut_ctx->lba = user_io->slba);
@@ -214,6 +286,11 @@ test_cuse_nvme_submit_io_read_write(void) 
	CU_ASSERT(g_ut_ctx->data_len ==

		  (int)((user_io->nblocks + 1) * block_size));

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len ==

		  (int)((user_io->nblocks + 1) * md_size));

	CU_ASSERT(g_ut_ctx->metadata != NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0xF00D);

	CU_ASSERT(g_ut_ctx->apptag == 0xC0DE);

	cuse_io_ctx_free(g_ut_ctx);

	free(user_io);

}
@@ -237,6 +314,56 @@ test_cuse_nvme_submit_passthru_cmd(void) 
	passthru_cmd->opcode       = SPDK_NVME_DATA_CONTROLLER_TO_HOST;

	passthru_cmd->nsid         = 1;

	passthru_cmd->data_len     = 512;

	passthru_cmd->metadata_len = 0;

	passthru_cmd->cdw10        = 0xc0de1010;

	passthru_cmd->cdw11        = 0xc0de1111;

	passthru_cmd->cdw12        = 0xc0de1212;

	passthru_cmd->cdw13        = 0xc0de1313;

	passthru_cmd->cdw14        = 0xc0de1414;

	passthru_cmd->cdw15        = 0xc0de1515;



	/* Send IO Command IOCTL */

	cuse_nvme_passthru_cmd_send(req, passthru_cmd, NULL, NULL, NVME_IOCTL_IO_CMD);

	SPDK_CU_ASSERT_FATAL(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata == NULL);

	CU_ASSERT(g_ut_ctx->req               == req);

	CU_ASSERT(g_ut_ctx->data_len          == 512);

	CU_ASSERT(g_ut_ctx->metadata_len      == 0);

	CU_ASSERT(g_ut_ctx->nvme_cmd.opc      == SPDK_NVME_DATA_CONTROLLER_TO_HOST);

	CU_ASSERT(g_ut_ctx->nvme_cmd.nsid     == 1);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw10    == 0xc0de1010);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw11    == 0xc0de1111);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw12    == 0xc0de1212);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw13    == 0xc0de1313);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw14    == 0xc0de1414);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw15    == 0xc0de1515);



	cuse_io_ctx_free(g_ut_ctx);

	free(passthru_cmd);

	free(g_cuse_device);

}



static void

test_cuse_nvme_submit_passthru_cmd_with_md(void)

{

	struct nvme_passthru_cmd *passthru_cmd = NULL;

	fuse_req_t req = (void *)0xDEEACDFF;



	passthru_cmd = calloc(1, sizeof(struct nvme_passthru_cmd));

	g_cuse_device = calloc(1, sizeof(struct cuse_device));



	/* Use fatal or we'll segfault if we didn't get memory */

	SPDK_CU_ASSERT_FATAL(passthru_cmd != NULL);

	SPDK_CU_ASSERT_FATAL(g_cuse_device != NULL);

	g_cuse_device->ctrlr = (void *)0xDEADBEEF;



	g_ut_ctx = NULL;

	/* Passthrough command */

	passthru_cmd->opcode       = SPDK_NVME_DATA_CONTROLLER_TO_HOST;

	passthru_cmd->nsid         = 1;

	passthru_cmd->data_len     = 512;

	passthru_cmd->metadata_len = 8;

	passthru_cmd->cdw10        = 0xc0de1010;

	passthru_cmd->cdw11        = 0xc0de1111;

	passthru_cmd->cdw12        = 0xc0de1212;
@@ -245,11 +372,13 @@ test_cuse_nvme_submit_passthru_cmd(void) 
	passthru_cmd->cdw15        = 0xc0de1515;



	/* Send IO Command IOCTL */

	cuse_nvme_passthru_cmd_send(req, passthru_cmd, NULL, NVME_IOCTL_IO_CMD);

	cuse_nvme_passthru_cmd_send(req, passthru_cmd, NULL, NULL, NVME_IOCTL_IO_CMD);

	SPDK_CU_ASSERT_FATAL(g_ut_ctx != NULL);

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata != NULL);

	CU_ASSERT(g_ut_ctx->req               == req);

	CU_ASSERT(g_ut_ctx->data_len          == 512);

	CU_ASSERT(g_ut_ctx->metadata_len      == 8);

	CU_ASSERT(g_ut_ctx->nvme_cmd.opc      == SPDK_NVME_DATA_CONTROLLER_TO_HOST);

	CU_ASSERT(g_ut_ctx->nvme_cmd.nsid     == 1);

	CU_ASSERT(g_ut_ctx->nvme_cmd.cdw10    == 0xc0de1010);
@@ -334,6 +463,11 @@ test_cuse_nvme_submit_io(void) 
	CU_ASSERT(g_ut_ctx->lba_count == 2);

	CU_ASSERT(g_ut_ctx->data_len == 2 * 4096);

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len == 0);

	CU_ASSERT(g_ut_ctx->metadata == NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0);

	CU_ASSERT(g_ut_ctx->apptag == 0);



	cuse_io_ctx_free(g_ut_ctx);



	/* Write */
@@ -348,6 +482,10 @@ test_cuse_nvme_submit_io(void) 
	CU_ASSERT(g_ut_ctx->lba_count == 2);

	CU_ASSERT(g_ut_ctx->data_len == 2 * 4096);

	CU_ASSERT(g_ut_ctx->data != NULL);

	CU_ASSERT(g_ut_ctx->metadata_len == 0);

	CU_ASSERT(g_ut_ctx->metadata == NULL);

	CU_ASSERT(g_ut_ctx->appmask == 0);

	CU_ASSERT(g_ut_ctx->apptag == 0);

	cuse_io_ctx_free(g_ut_ctx);



	/* Invalid */
@@ -428,7 +566,9 @@ int main(int argc, char **argv) 


	suite = CU_add_suite("nvme_cuse", NULL, NULL);

	CU_ADD_TEST(suite, test_cuse_nvme_submit_io_read_write);

	CU_ADD_TEST(suite, test_cuse_nvme_submit_io_read_write_with_md);

	CU_ADD_TEST(suite, test_cuse_nvme_submit_passthru_cmd);

	CU_ADD_TEST(suite, test_cuse_nvme_submit_passthru_cmd_with_md);

	CU_ADD_TEST(suite, test_nvme_cuse_get_cuse_ns_device);

	CU_ADD_TEST(suite, test_cuse_nvme_submit_io);

	CU_ADD_TEST(suite, test_cuse_nvme_reset);