Commit bc1d0b91 authored by Changpeng Liu's avatar Changpeng Liu
Browse files

nvmf: add namespace reservation register command support 



Reservations can be used by two or more hosts to coordinate
acccess to a shared namespace, host must register to a namespace
prior to establishing a reservation.  Unregistering by a host
may cause a reservation release, this feature will be supported
after reservation acquire patch.

Change-Id: Id44aa1f82f30d9ecc5999a2a9a7c20b2af77774a
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/436936


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 791d89bf

lib/nvmf/ctrlr.c

+3 −0
Original line number Diff line number Diff line
@@ -2127,6 +2127,9 @@ spdk_nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req) 
		return spdk_nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req);

	case SPDK_NVME_OPC_DATASET_MANAGEMENT:

		return spdk_nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req);

	case SPDK_NVME_OPC_RESERVATION_REGISTER:

		spdk_thread_send_msg(ctrlr->subsys->thread, spdk_nvmf_ns_reservation_request, req);

		return SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS;

	default:

		return spdk_nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req);

	}

lib/nvmf/nvmf_internal.h

+12 −0
Original line number Diff line number Diff line
@@ -171,6 +171,13 @@ struct spdk_nvmf_request { 
	TAILQ_ENTRY(spdk_nvmf_request)	link;

};



struct spdk_nvmf_registrant {

	TAILQ_ENTRY(spdk_nvmf_registrant) link;

	struct spdk_uuid hostid;

	/* Registration key */

	uint64_t rkey;

};



struct spdk_nvmf_ns {

	uint32_t nsid;

	struct spdk_nvmf_subsystem *subsystem;
@@ -179,6 +186,10 @@ struct spdk_nvmf_ns { 
	struct spdk_nvmf_ns_opts opts;

	/* reservation notificaton mask */

	uint32_t mask;

	/* generation code */

	uint32_t gen;

	/* registrants head */

	TAILQ_HEAD(, spdk_nvmf_registrant) registrants;

};



struct spdk_nvmf_qpair {
@@ -327,6 +338,7 @@ void spdk_nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem, 
struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem,

		uint16_t cntlid);

int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr);

void spdk_nvmf_ns_reservation_request(void *ctx);



/*

 * Abort aer is sent on a per controller basis and sends a completion for the aer to the host.

lib/nvmf/subsystem.c

+174 −0
Original line number Diff line number Diff line
@@ -883,6 +883,7 @@ static int 
_spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid)

{

	struct spdk_nvmf_ns *ns;

	struct spdk_nvmf_registrant *reg, *reg_tmp;



	assert(subsystem->state == SPDK_NVMF_SUBSYSTEM_PAUSED ||

	       subsystem->state == SPDK_NVMF_SUBSYSTEM_INACTIVE);
@@ -903,6 +904,10 @@ _spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t n 


	subsystem->ns[nsid - 1] = NULL;



	TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, reg_tmp) {

		TAILQ_REMOVE(&ns->registrants, reg, link);

		free(reg);

	}

	spdk_bdev_module_release_bdev(ns->bdev);

	spdk_bdev_close(ns->desc);

	free(ns);
@@ -1079,6 +1084,7 @@ spdk_nvmf_subsystem_add_ns(struct spdk_nvmf_subsystem *subsystem, struct spdk_bd 
	}

	subsystem->ns[opts.nsid - 1] = ns;

	ns->nsid = opts.nsid;

	TAILQ_INIT(&ns->registrants);



	SPDK_DEBUGLOG(SPDK_LOG_NVMF, "Subsystem %s: bdev %s assigned nsid %" PRIu32 "\n",

		      spdk_nvmf_subsystem_get_nqn(subsystem),
@@ -1268,3 +1274,171 @@ spdk_nvmf_subsystem_get_max_namespaces(const struct spdk_nvmf_subsystem *subsyst 
{

	return subsystem->max_allowed_nsid;

}



static struct spdk_nvmf_registrant *

nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns,

				   struct spdk_uuid *uuid)

{

	struct spdk_nvmf_registrant *reg, *tmp;



	TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {

		if (spdk_uuid_compare(&reg->hostid, uuid) == 0) {

			return reg;

		}

	}



	return NULL;

}



static int

nvmf_ns_reservation_add_registrant(struct spdk_nvmf_ns *ns,

				   struct spdk_nvmf_ctrlr *ctrlr,

				   uint64_t nrkey)

{

	struct spdk_nvmf_registrant *reg;



	reg = calloc(1, sizeof(*reg));

	if (!reg) {

		return -ENOMEM;

	}



	reg->rkey = nrkey;

	/* set hostid for the registrant */

	spdk_uuid_copy(&reg->hostid, &ctrlr->hostid);

	TAILQ_INSERT_TAIL(&ns->registrants, reg, link);

	ns->gen++;



	return 0;

}



static void

nvmf_ns_reservation_remove_registrant(struct spdk_nvmf_ns *ns,

				      struct spdk_nvmf_registrant *reg)

{

	TAILQ_REMOVE(&ns->registrants, reg, link);

	free(reg);

	ns->gen++;

	return;

}



static void

nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns,

			     struct spdk_nvmf_ctrlr *ctrlr,

			     struct spdk_nvmf_request *req)

{

	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	uint8_t rrega, iekey, cptpl;

	struct spdk_nvme_reservation_register_data key;

	struct spdk_nvmf_registrant *reg;

	uint8_t status = SPDK_NVME_SC_SUCCESS;

	int rc;



	rrega = cmd->cdw10 & 0x7u;

	iekey = (cmd->cdw10 >> 3) & 0x1u;

	cptpl = (cmd->cdw10 >> 30) & 0x3u;

	memcpy(&key, req->data, sizeof(key));



	SPDK_DEBUGLOG(SPDK_LOG_NVMF, "REGISTER: RREGA %u, IEKEY %u, CPTPL %u, "

		      "NRKEY 0x%"PRIx64", NRKEY 0x%"PRIx64"\n",

		      rrega, iekey, cptpl, key.crkey, key.nrkey);



	/* TODO: doesn't support for now */

	if (cptpl == SPDK_NVME_RESERVE_PTPL_PERSIST_POWER_LOSS) {

		SPDK_ERRLOG("Can't change persist through power loss for now\n");

		status = SPDK_NVME_SC_INVALID_FIELD;

		goto exit;

	}



	/* current Host Identifier has registrant or not */

	reg = nvmf_ns_reservation_get_registrant(ns, &ctrlr->hostid);



	switch (rrega) {

	case SPDK_NVME_RESERVE_REGISTER_KEY:

		if (!reg) {

			/* register new controller */

			if (key.nrkey == 0) {

				SPDK_ERRLOG("Can't register zeroed new key\n");

				status = SPDK_NVME_SC_INVALID_FIELD;

				goto exit;

			}

			rc = nvmf_ns_reservation_add_registrant(ns, ctrlr, key.nrkey);

			if (rc < 0) {

				status = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

				goto exit;

			}

		} else {

			/* register with same key is not an error */

			if (reg->rkey != key.nrkey) {

				SPDK_ERRLOG("The same host already register a "

					    "key with 0x%"PRIx64"\n",

					    reg->rkey);

				status = SPDK_NVME_SC_RESERVATION_CONFLICT;

				goto exit;

			}

		}

		break;

	case SPDK_NVME_RESERVE_UNREGISTER_KEY:

		if (!reg || (!iekey && reg->rkey != key.crkey)) {

			SPDK_ERRLOG("No registrant or current key doesn't match "

				    "with existing registrant key\n");

			status = SPDK_NVME_SC_RESERVATION_CONFLICT;

			goto exit;

		}

		nvmf_ns_reservation_remove_registrant(ns, reg);

		break;

	case SPDK_NVME_RESERVE_REPLACE_KEY:

		if (!reg || (!iekey && reg->rkey != key.crkey)) {

			SPDK_ERRLOG("No registrant or current key doesn't match "

				    "with existing registrant key\n");

			status = SPDK_NVME_SC_RESERVATION_CONFLICT;

			goto exit;

		}

		if (key.nrkey == 0) {

			SPDK_ERRLOG("Can't register zeroed new key\n");

			status = SPDK_NVME_SC_INVALID_FIELD;

			goto exit;

		}

		reg->rkey = key.nrkey;

		break;

	default:

		status = SPDK_NVME_SC_INVALID_FIELD;

		goto exit;

	}



exit:

	req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;

	req->rsp->nvme_cpl.status.sc = status;

	return;

}



static void

spdk_nvmf_ns_reservation_complete(void *ctx)

{

	struct spdk_nvmf_request *req = ctx;



	spdk_nvmf_request_complete(req);

}



void

spdk_nvmf_ns_reservation_request(void *ctx)

{

	struct spdk_nvmf_request *req = (struct spdk_nvmf_request *)ctx;

	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	struct spdk_nvmf_poll_group *group = req->qpair->group;

	struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;

	uint32_t nsid;

	struct spdk_nvmf_ns *ns;



	nsid = cmd->nsid;

	ns = _spdk_nvmf_subsystem_get_ns(ctrlr->subsys, nsid);

	assert(ns != NULL);



	switch (cmd->opc) {

	case SPDK_NVME_OPC_RESERVATION_REGISTER:

		nvmf_ns_reservation_register(ns, ctrlr, req);

		break;

	default:

		break;

	}

	spdk_thread_send_msg(group->thread, spdk_nvmf_ns_reservation_complete, req);

}

test/unit/lib/nvmf/ctrlr.c/ctrlr_ut.c

+2 −0
Original line number Diff line number Diff line
@@ -156,6 +156,8 @@ DEFINE_STUB(spdk_nvmf_transport_req_complete, 
	    (struct spdk_nvmf_request *req),

	    0);



DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx));



int

spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx)

{

test/unit/lib/nvmf/tcp.c/tcp_ut.c

+2 −0
Original line number Diff line number Diff line
@@ -158,6 +158,8 @@ DEFINE_STUB(spdk_nvmf_transport_req_complete, 
	    (struct spdk_nvmf_request *req),

	    0);



DEFINE_STUB_V(spdk_nvmf_ns_reservation_request, (void *ctx));



struct spdk_trace_histories *g_trace_histories;



struct spdk_bdev {