Commit 78bfb2a1 authored by Changpeng Liu's avatar Changpeng Liu
Browse files

nvmf: generate reservation notification log pages 



A host can use the Asynchronous Event Command to be notified of
the presense of one or more avaiable reservation notification
log pages.  A reservation notificaton log page should be created
whenever an unmasked reservation notification occurs.

Change-Id: I8b83e5319725286dd0a5efc1b22d8ac4673e31e1
Signed-off-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/439931


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 8cc7b0bc

lib/nvmf/ctrlr.c

+58 −0
Original line number Diff line number Diff line
@@ -269,6 +269,7 @@ spdk_nvmf_ctrlr_create(struct spdk_nvmf_subsystem *subsystem, 
		return NULL;

	}



	TAILQ_INIT(&ctrlr->log_head);

	ctrlr->subsys = subsystem;

	ctrlr->thread = req->qpair->group->thread;
@@ -361,8 +362,14 @@ static void 
_spdk_nvmf_ctrlr_destruct(void *ctx)

{

	struct spdk_nvmf_ctrlr *ctrlr = ctx;

	struct spdk_nvmf_reservation_log *log, *log_tmp;



	spdk_nvmf_ctrlr_stop_keep_alive_timer(ctrlr);



	TAILQ_FOREACH_SAFE(log, &ctrlr->log_head, link, log_tmp) {

		TAILQ_REMOVE(&ctrlr->log_head, log, link);

		free(log);

	}

	free(ctrlr);

}
@@ -2080,6 +2087,57 @@ spdk_nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr) 
	ctrlr->aer_req = NULL;

}



void

spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr,

				       struct spdk_nvmf_ns *ns,

				       enum spdk_nvme_reservation_notification_log_page_type type)

{

	struct spdk_nvmf_reservation_log *log;



	switch (type) {

	case SPDK_NVME_RESERVATION_LOG_PAGE_EMPTY:

		return;

	case SPDK_NVME_REGISTRATION_PREEMPTED:

		if (ns->mask & SPDK_NVME_REGISTRATION_PREEMPTED_MASK) {

			return;

		}

		break;

	case SPDK_NVME_RESERVATION_RELEASED:

		if (ns->mask & SPDK_NVME_RESERVATION_RELEASED_MASK) {

			return;

		}

		break;

	case SPDK_NVME_RESERVATION_PREEMPTED:

		if (ns->mask & SPDK_NVME_RESERVATION_PREEMPTED_MASK) {

			return;

		}

		break;

	default:

		return;

	}



	ctrlr->log_page_count++;



	/* Maximum number of queued log pages is 255 */

	if (ctrlr->num_avail_log_pages == 0xff) {

		log = TAILQ_LAST(&ctrlr->log_head, log_page_head);

		log->log.log_page_count = ctrlr->log_page_count;

		return;

	}



	log = calloc(1, sizeof(*log));

	if (!log) {

		SPDK_ERRLOG("Alloc log page failed, ignore the log\n");

		return;

	}



	log->log.log_page_count = ctrlr->log_page_count;

	log->log.type = type;

	log->log.num_avail_log_pages = ctrlr->num_avail_log_pages++;

	log->log.nsid = ns->nsid;

	TAILQ_INSERT_TAIL(&ctrlr->log_head, log, link);

}



/* Check from subsystem poll group's namespace information data structure */

static bool

nvmf_ns_info_ctrlr_is_registrant(struct spdk_nvmf_subsystem_pg_ns_info *ns_info,

lib/nvmf/nvmf_internal.h

+14 −0
Original line number Diff line number Diff line
@@ -241,6 +241,14 @@ struct spdk_nvmf_ctrlr_feat { 
	union spdk_nvme_feat_keep_alive_timer keep_alive_timer;

};



/*

 * NVMf reservation notificaton log page.

 */

struct spdk_nvmf_reservation_log {

	struct spdk_nvme_reservation_notification_log	log;

	TAILQ_ENTRY(spdk_nvmf_reservation_log)		link;

};



/*

 * This structure represents an NVMe-oF controller,

 * which is like a "session" in networking terms.
@@ -268,6 +276,9 @@ struct spdk_nvmf_ctrlr { 


	uint16_t changed_ns_list_count;

	struct spdk_nvme_ns_list changed_ns_list;

	uint64_t log_page_count;

	uint8_t num_avail_log_pages;

	TAILQ_HEAD(log_page_head, spdk_nvmf_reservation_log) log_head;



	/* Time to trigger keep-alive--poller_time = now_tick + period */

	uint64_t last_keep_alive_tick;
@@ -360,6 +371,9 @@ struct spdk_nvmf_ctrlr *spdk_nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem 
		uint16_t cntlid);

int spdk_nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr);

void spdk_nvmf_ns_reservation_request(void *ctx);

void spdk_nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr,

		struct spdk_nvmf_ns *ns,

		enum spdk_nvme_reservation_notification_log_page_type type);



/*

 * Abort aer is sent on a per controller basis and sends a completion for the aer to the host.

lib/nvmf/subsystem.c

+161 −1
Original line number Diff line number Diff line
@@ -1303,6 +1303,87 @@ nvmf_ns_reservation_get_registrant(struct spdk_nvmf_ns *ns, 
	return NULL;

}



/* Generate reservation notice log to registered HostID controllers */

static void

nvmf_subsystem_gen_ctrlr_notification(struct spdk_nvmf_subsystem *subsystem,

				      struct spdk_nvmf_ns *ns,

				      struct spdk_uuid *hostid_list,

				      uint32_t num_hostid,

				      enum spdk_nvme_reservation_notification_log_page_type type)

{

	struct spdk_nvmf_ctrlr *ctrlr;

	uint32_t i;



	for (i = 0; i < num_hostid; i++) {

		TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) {

			if (!spdk_uuid_compare(&ctrlr->hostid, &hostid_list[i])) {

				spdk_nvmf_ctrlr_reservation_notice_log(ctrlr, ns, type);

			}

		}

	}

}



/* Get all registrants' hostid other than the controller who issued the command */

static uint32_t

nvmf_ns_reservation_get_all_other_hostid(struct spdk_nvmf_ns *ns,

		struct spdk_uuid *hostid_list,

		uint32_t max_num_hostid,

		struct spdk_uuid *current_hostid)

{

	struct spdk_nvmf_registrant *reg, *tmp;

	uint32_t num_hostid = 0;



	TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) {

		if (spdk_uuid_compare(&reg->hostid, current_hostid)) {

			if (num_hostid == max_num_hostid) {

				assert(false);

				return max_num_hostid;

			}

			hostid_list[num_hostid++] = reg->hostid;

		}

	}



	return num_hostid;

}



/* Calculate the unregistered HostID list according to list

 * prior to execute preempt command and list after executing

 * preempt command.

 */

static uint32_t

nvmf_ns_reservation_get_unregistered_hostid(struct spdk_uuid *old_hostid_list,

		uint32_t old_num_hostid,

		struct spdk_uuid *remaining_hostid_list,

		uint32_t remaining_num_hostid)

{

	struct spdk_uuid temp_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t i, j, num_hostid = 0;

	bool found;



	if (!remaining_num_hostid) {

		return old_num_hostid;

	}



	for (i = 0; i < old_num_hostid; i++) {

		found = false;

		for (j = 0; j < remaining_num_hostid; j++) {

			if (!spdk_uuid_compare(&old_hostid_list[i], &remaining_hostid_list[j])) {

				found = true;

				break;

			}

		}

		if (!found) {

			spdk_uuid_copy(&temp_hostid_list[num_hostid++], &old_hostid_list[i]);

		}

	}



	if (num_hostid) {

		memcpy(old_hostid_list, temp_hostid_list, sizeof(struct spdk_uuid) * num_hostid);

	}



	return num_hostid;

}



/* current reservation type is all registrants or not */

static bool

nvmf_ns_reservation_all_registrants_type(struct spdk_nvmf_ns *ns)
@@ -1452,11 +1533,13 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, 
			     struct spdk_nvmf_request *req)

{

	struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;

	uint8_t rrega, iekey, cptpl;

	uint8_t rrega, iekey, cptpl, rtype;

	struct spdk_nvme_reservation_register_data key;

	struct spdk_nvmf_registrant *reg;

	uint8_t status = SPDK_NVME_SC_SUCCESS;

	bool update_sgroup = false;

	struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t num_hostid = 0;

	int rc;



	rrega = cmd->cdw10 & 0x7u;
@@ -1510,7 +1593,21 @@ nvmf_ns_reservation_register(struct spdk_nvmf_ns *ns, 
			status = SPDK_NVME_SC_RESERVATION_CONFLICT;

			goto exit;

		}



		rtype = ns->rtype;

		num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,

				SPDK_NVMF_MAX_NUM_REGISTRANTS,

				&ctrlr->hostid);



		nvmf_ns_reservation_remove_registrant(ns, reg);



		if (!ns->rtype && num_hostid && (rtype == SPDK_NVME_RESERVE_WRITE_EXCLUSIVE_REG_ONLY ||

						 rtype == SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS_REG_ONLY)) {

			nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,

							      hostid_list,

							      num_hostid,

							      SPDK_NVME_RESERVATION_RELEASED);

		}

		update_sgroup = true;

		break;

	case SPDK_NVME_RESERVE_REPLACE_KEY:
@@ -1550,6 +1647,11 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
	bool all_regs = false;

	uint32_t count = 0;

	bool update_sgroup = true;

	struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t num_hostid = 0;

	struct spdk_uuid new_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t new_num_hostid = 0;

	bool reservation_released = false;

	uint8_t status = SPDK_NVME_SC_SUCCESS;



	racqa = cmd->cdw10 & 0x7u;
@@ -1603,18 +1705,24 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
			nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey);

			break;

		}

		num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,

				SPDK_NVMF_MAX_NUM_REGISTRANTS,

				&ctrlr->hostid);



		/* only 1 reservation holder and reservation key is valid */

		if (!all_regs) {

			/* preempt itself */

			if (nvmf_ns_reservation_registrant_is_holder(ns, reg) &&

			    ns->crkey == key.prkey) {

				ns->rtype = rtype;

				reservation_released = true;

				break;

			}



			if (ns->crkey == key.prkey) {

				nvmf_ns_reservation_remove_registrant(ns, ns->holder);

				nvmf_ns_reservation_acquire_reservation(ns, key.crkey, rtype, reg);

				reservation_released = true;

			} else if (key.prkey != 0) {

				nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey);

			} else {
@@ -1647,6 +1755,36 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
	}



exit:

	if (update_sgroup && racqa == SPDK_NVME_RESERVE_PREEMPT) {

		new_num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, new_hostid_list,

				 SPDK_NVMF_MAX_NUM_REGISTRANTS,

				 &ctrlr->hostid);

		/* Preempt notification occurs on the unregistered controllers

		 * other than the controller who issued the command.

		 */

		num_hostid = nvmf_ns_reservation_get_unregistered_hostid(hostid_list,

				num_hostid,

				new_hostid_list,

				new_num_hostid);

		if (num_hostid) {

			nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,

							      hostid_list,

							      num_hostid,

							      SPDK_NVME_REGISTRATION_PREEMPTED);



		}

		/* Reservation released notification occurs on the

		 * controllers which are the remaining registrants other than

		 * the controller who issued the command.

		 */

		if (reservation_released && new_num_hostid) {

			nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,

							      new_hostid_list,

							      new_num_hostid,

							      SPDK_NVME_RESERVATION_RELEASED);



		}

	}

	req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;

	req->rsp->nvme_cpl.status.sc = status;

	return update_sgroup;
@@ -1663,6 +1801,8 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, 
	uint64_t crkey;

	uint8_t status = SPDK_NVME_SC_SUCCESS;

	bool update_sgroup = true;

	struct spdk_uuid hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t num_hostid = 0;



	rrela = cmd->cdw10 & 0x7u;

	iekey = (cmd->cdw10 >> 3) & 0x1u;
@@ -1688,6 +1828,10 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, 
		goto exit;

	}



	num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,

			SPDK_NVMF_MAX_NUM_REGISTRANTS,

			&ctrlr->hostid);



	switch (rrela) {

	case SPDK_NVME_RESERVE_RELEASE:

		if (!ns->holder) {
@@ -1706,10 +1850,26 @@ nvmf_ns_reservation_release(struct spdk_nvmf_ns *ns, 
			update_sgroup = false;

			goto exit;

		}



		rtype = ns->rtype;

		nvmf_ns_reservation_release_reservation(ns);



		if (num_hostid && rtype != SPDK_NVME_RESERVE_WRITE_EXCLUSIVE &&

		    rtype != SPDK_NVME_RESERVE_EXCLUSIVE_ACCESS) {

			nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,

							      hostid_list,

							      num_hostid,

							      SPDK_NVME_RESERVATION_RELEASED);

		}

		break;

	case SPDK_NVME_RESERVE_CLEAR:

		nvmf_ns_reservation_clear_all_registrants(ns);

		if (num_hostid) {

			nvmf_subsystem_gen_ctrlr_notification(ns->subsystem, ns,

							      hostid_list,

							      num_hostid,

							      SPDK_NVME_RESERVATION_PREEMPTED);

		}

		break;

	default:

		status = SPDK_NVME_SC_INVALID_FIELD;

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

+47 −1
Original line number Diff line number Diff line
@@ -50,6 +50,10 @@ DEFINE_STUB(spdk_bdev_module_claim_bdev, 
DEFINE_STUB_V(spdk_bdev_module_release_bdev,

	      (struct spdk_bdev *bdev));



DEFINE_STUB_V(spdk_nvmf_ctrlr_reservation_notice_log,

	      (struct spdk_nvmf_ctrlr *ctrlr, struct spdk_nvmf_ns *ns,

	       enum spdk_nvme_reservation_notification_log_page_type type));



DEFINE_STUB(spdk_bdev_get_block_size, uint32_t,

	    (const struct spdk_bdev *bdev), 512);
@@ -458,7 +462,7 @@ test_spdk_nvmf_subsystem_set_sn(void) 
 *     |            NAMESPACE 1               |

 *      --------------------------------------

 */



static struct spdk_nvmf_subsystem g_subsystem;

static struct spdk_nvmf_ctrlr g_ctrlr1_A, g_ctrlr2_A, g_ctrlr_B, g_ctrlr_C;

static struct spdk_nvmf_ns g_ns;

struct spdk_nvmf_subsystem_pg_ns_info g_ns_info;
@@ -466,29 +470,71 @@ struct spdk_nvmf_subsystem_pg_ns_info g_ns_info; 
static void

ut_reservation_init(void)

{



	TAILQ_INIT(&g_subsystem.ctrlrs);



	memset(&g_ns, 0, sizeof(g_ns));

	TAILQ_INIT(&g_ns.registrants);

	g_ns.subsystem = &g_subsystem;



	/* Host A has two controllers */

	spdk_uuid_generate(&g_ctrlr1_A.hostid);

	TAILQ_INIT(&g_ctrlr1_A.log_head);

	g_ctrlr1_A.subsys = &g_subsystem;

	TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr1_A, link);

	spdk_uuid_copy(&g_ctrlr2_A.hostid, &g_ctrlr1_A.hostid);

	TAILQ_INIT(&g_ctrlr2_A.log_head);

	g_ctrlr2_A.subsys = &g_subsystem;

	TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr2_A, link);



	/* Host B has 1 controller */

	spdk_uuid_generate(&g_ctrlr_B.hostid);

	TAILQ_INIT(&g_ctrlr_B.log_head);

	g_ctrlr_B.subsys = &g_subsystem;

	TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_B, link);



	/* Host C has 1 controller */

	spdk_uuid_generate(&g_ctrlr_C.hostid);

	TAILQ_INIT(&g_ctrlr_C.log_head);

	g_ctrlr_C.subsys = &g_subsystem;

	TAILQ_INSERT_TAIL(&g_subsystem.ctrlrs, &g_ctrlr_C, link);

}



static void

ut_reservation_deinit(void)

{

	struct spdk_nvmf_registrant *reg, *tmp;

	struct spdk_nvmf_reservation_log *log, *log_tmp;

	struct spdk_nvmf_ctrlr *ctrlr, *ctrlr_tmp;



	TAILQ_FOREACH_SAFE(reg, &g_ns.registrants, link, tmp) {

		TAILQ_REMOVE(&g_ns.registrants, reg, link);

		free(reg);

	}

	TAILQ_FOREACH_SAFE(log, &g_ctrlr1_A.log_head, link, log_tmp) {

		TAILQ_REMOVE(&g_ctrlr1_A.log_head, log, link);

		free(log);

	}

	g_ctrlr1_A.num_avail_log_pages = 0;

	TAILQ_FOREACH_SAFE(log, &g_ctrlr2_A.log_head, link, log_tmp) {

		TAILQ_REMOVE(&g_ctrlr2_A.log_head, log, link);

		free(log);

	}

	g_ctrlr2_A.num_avail_log_pages = 0;

	TAILQ_FOREACH_SAFE(log, &g_ctrlr_B.log_head, link, log_tmp) {

		TAILQ_REMOVE(&g_ctrlr_B.log_head, log, link);

		free(log);

	}

	g_ctrlr_B.num_avail_log_pages = 0;

	TAILQ_FOREACH_SAFE(log, &g_ctrlr_C.log_head, link, log_tmp) {

		TAILQ_REMOVE(&g_ctrlr_C.log_head, log, link);

		free(log);

	}

	g_ctrlr_C.num_avail_log_pages = 0;



	TAILQ_FOREACH_SAFE(ctrlr, &g_subsystem.ctrlrs, link, ctrlr_tmp) {

		TAILQ_REMOVE(&g_subsystem.ctrlrs, ctrlr, link);

	}

}



static struct spdk_nvmf_request *