Commit ede81afa authored by Joel Cunningham's avatar Joel Cunningham Committed by Tomasz Zawadzki
Browse files

lib/nvmf: add ns preempt hostids list 



Add initial handling for preempt-and-abort command which builds a list of
preempted hostids. This list was already being built for reservation log
notifications but we need to save it in struct spdk_nvmf_ns for later
processing by poll groups.

This introduces a new structure spdk_nvmf_reserv_preempt_abort_info which
will contain the hostids list and in future commits more state related to
preempt-and-abort. The structure is linked in to struct spdk_nvmf_ns as
a pointer to avoid growing the structure for every use case of
namespaces.  The hostid list requires 256 bytes.

Change-Id: I8c3daad1333ef3d18eac087bcd97846332f62bf6
Signed-off-by: default avatarJoel Cunningham <joel.cunningham@oracle.com>
Reviewed-on: https://review.spdk.io/c/spdk/spdk/+/26672


Tested-by: default avatarSPDK Automated Test System <spdkbot@gmail.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Community-CI: Mellanox Build Bot
Reviewed-by: default avatarChangpeng Liu <changpeliu@tencent.com>
parent 09a167d9

lib/nvmf/nvmf_internal.h

+9 −0
Original line number Diff line number Diff line
@@ -167,6 +167,14 @@ struct spdk_nvmf_registrant { 
	uint16_t cntlid;

};



struct spdk_nvmf_reservation_preempt_abort_info {

	/* preempted controllers */

	struct spdk_uuid hostids[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint8_t hostids_cnt;

	uint32_t hostids_gen; /* Generational counter every time the list changes */

};

SPDK_STATIC_ASSERT(SPDK_NVMF_MAX_NUM_REGISTRANTS <= UINT8_MAX, "hostids_cnt storage type");



struct spdk_nvmf_ns {

	uint32_t nsid;

	uint32_t anagrpid;
@@ -188,6 +196,7 @@ struct spdk_nvmf_ns { 
	enum spdk_nvme_reservation_type rtype;

	/* current reservation holder, only valid if reservation type can only have one holder */

	struct spdk_nvmf_registrant *holder;

	struct spdk_nvmf_reservation_preempt_abort_info *preempt_abort;

	/* Persist Through Power Loss file which contains the persistent reservation */

	char *ptpl_file;

	/* Persist Through Power Loss feature is enabled */

lib/nvmf/subsystem.c

+36 −4
Original line number Diff line number Diff line
@@ -1814,6 +1814,7 @@ spdk_nvmf_subsystem_remove_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t ns 
	}



	free(ns->ptpl_file);

	free(ns->preempt_abort);

	nvmf_ns_reservation_clear_all_registrants(ns);

	spdk_bdev_module_release_bdev(ns->bdev);

	spdk_bdev_close(ns->desc);
@@ -3355,6 +3356,8 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
	struct spdk_uuid new_hostid_list[SPDK_NVMF_MAX_NUM_REGISTRANTS];

	uint32_t new_num_hostid = 0;

	bool reservation_released = false;

	bool is_preempt = false;

	bool is_abort = false;

	uint8_t status = SPDK_NVME_SC_SUCCESS;



	racqa = cmd->cdw10_bits.resv_acquire.racqa;
@@ -3412,15 +3415,33 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
		}

		break;

	case SPDK_NVME_RESERVE_PREEMPT:

	case SPDK_NVME_RESERVE_PREEMPT_ABORT:

		is_preempt = true;

		is_abort = (racqa == SPDK_NVME_RESERVE_PREEMPT_ABORT);



		/* Allocate memory for performing preempt-and-abort on first abort received */

		if (is_abort && !ns->preempt_abort) {

			ns->preempt_abort = calloc(1, sizeof(*ns->preempt_abort));

			if (!ns->preempt_abort) {

				status = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;

				update_sgroup = false;

				goto exit;

			}

		}



		/* Build copy of current other hosts so we can generate a delta

		 * of registrants removed due to the prempt.

		 */

		num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,

				SPDK_NVMF_MAX_NUM_REGISTRANTS,

				&ctrlr->hostid);



		/* no reservation holder */

		if (!ns->holder) {

			/* unregister with PRKEY */

			nvmf_ns_reservation_remove_registrants_by_key(ns, key.prkey);

			break;

		}

		num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, hostid_list,

				SPDK_NVMF_MAX_NUM_REGISTRANTS,

				&ctrlr->hostid);



		/* only 1 reservation holder and reservation key is valid */

		if (!all_regs) {
@@ -3468,7 +3489,7 @@ nvmf_ns_reservation_acquire(struct spdk_nvmf_ns *ns, 
	}



exit:

	if (update_sgroup && racqa == SPDK_NVME_RESERVE_PREEMPT) {

	if (update_sgroup && is_preempt) {

		new_num_hostid = nvmf_ns_reservation_get_all_other_hostid(ns, new_hostid_list,

				 SPDK_NVMF_MAX_NUM_REGISTRANTS,

				 &ctrlr->hostid);
@@ -3497,6 +3518,17 @@ exit: 
							      SPDK_NVME_RESERVATION_RELEASED);



		}



		/* For Preempt-and-abort copy the hostids for evaluation

		 * of outstanding IO on those controllers on each poll group */

		if (is_abort) {

			struct spdk_nvmf_reservation_preempt_abort_info *p_info = ns->preempt_abort;

			assert(num_hostid <= SPDK_NVMF_MAX_NUM_REGISTRANTS);

			memcpy(p_info->hostids, hostid_list,

			       sizeof(struct spdk_uuid) * num_hostid);

			p_info->hostids_cnt = (uint8_t)num_hostid;

			p_info->hostids_gen++;

		}

	}

	req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;

	req->rsp->nvme_cpl.status.sc = status;

test/unit/lib/nvmf/subsystem.c/subsystem_ut.c

+3 −0
Original line number Diff line number Diff line
@@ -862,6 +862,9 @@ ut_reservation_deinit(void) 
	TAILQ_FOREACH_SAFE(ctrlr, &g_subsystem.ctrlrs, link, ctrlr_tmp) {

		TAILQ_REMOVE(&g_subsystem.ctrlrs, ctrlr, link);

	}



	free(g_ns.preempt_abort);

	g_ns.preempt_abort = NULL;

}



static struct spdk_nvmf_request *