Commit 46316bb5 authored by Tomasz Kulasek's avatar Tomasz Kulasek Committed by Tomasz Zawadzki
Browse files

lib/nvme: prevent creating existing cuse device 



This patch attempts to solve naming conflict between
CUSE devices created by different SPDK instances.

Each NVMe device is enumerated by SPDK process from 0
up to 127. When process attempts to start cuse device
tries to set exclusive lock on temporary file
"/tmp/spdk_nvme_cuse_lock_<index>" and keep it until
device will be stopped.
If setting lock fails, index is incremented.

It prevents to use the same controller index from
few SPDK instances.

Change-Id: If744ac23f813bd992efb80ae2b61a1acefb5054c
Signed-off-by: default avatarTomasz Kulasek <tomaszx.kulasek@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/474829


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom SPDK FC-NVMe CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent b7b45bc7

lib/nvme/nvme_cuse.c

+78 −4
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ 
struct cuse_device {

	char				dev_name[128];

	uint32_t			index;

	int				claim_fd;

	char				lock_name[64];



	struct spdk_nvme_ctrlr		*ctrlr;		/**< NVMe controller */

	uint32_t			nsid;		/**< NVMe name space id, or 0 */
@@ -711,6 +713,68 @@ cuse_nvme_ns_start(struct cuse_device *ctrlr_device, uint32_t nsid, const char * 
	return 0;

}



static int

nvme_cuse_claim(struct cuse_device *ctrlr_device, uint32_t index)

{

	int dev_fd;

	int pid;

	void *dev_map;

	struct flock cusedev_lock = {

		.l_type = F_WRLCK,

		.l_whence = SEEK_SET,

		.l_start = 0,

		.l_len = 0,

	};



	snprintf(ctrlr_device->lock_name, sizeof(ctrlr_device->lock_name),

		 "/tmp/spdk_nvme_cuse_lock_%" PRIu32, index);



	dev_fd = open(ctrlr_device->lock_name, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);

	if (dev_fd == -1) {

		fprintf(stderr, "could not open %s\n", ctrlr_device->lock_name);

		return -errno;

	}



	if (ftruncate(dev_fd, sizeof(int)) != 0) {

		fprintf(stderr, "could not truncate %s\n", ctrlr_device->lock_name);

		close(dev_fd);

		return -errno;

	}



	dev_map = mmap(NULL, sizeof(int), PROT_READ | PROT_WRITE,

		       MAP_SHARED, dev_fd, 0);

	if (dev_map == MAP_FAILED) {

		fprintf(stderr, "could not mmap dev %s (%d)\n", ctrlr_device->lock_name, errno);

		close(dev_fd);

		return -errno;

	}



	if (fcntl(dev_fd, F_SETLK, &cusedev_lock) != 0) {

		pid = *(int *)dev_map;

		fprintf(stderr, "Cannot create lock on device %s, probably"

			" process %d has claimed it\n", ctrlr_device->lock_name, pid);

		munmap(dev_map, sizeof(int));

		close(dev_fd);

		/* F_SETLK returns unspecified errnos, normalize them */

		return -EACCES;

	}



	*(int *)dev_map = (int)getpid();

	munmap(dev_map, sizeof(int));

	ctrlr_device->claim_fd = dev_fd;

	ctrlr_device->index = index;

	/* Keep dev_fd open to maintain the lock. */

	return 0;

}



static void

nvme_cuse_unclaim(struct cuse_device *ctrlr_device)

{

	close(ctrlr_device->claim_fd);

	ctrlr_device->claim_fd = -1;

	unlink(ctrlr_device->lock_name);

}



static void

cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device)

{
@@ -730,6 +794,7 @@ cuse_nvme_ctrlr_stop(struct cuse_device *ctrlr_device) 
	if (spdk_bit_array_count_set(g_ctrlr_started) == 0) {

		spdk_bit_array_free(&g_ctrlr_started);

	}

	nvme_cuse_unclaim(ctrlr_device);

	free(ctrlr_device);

}
@@ -760,11 +825,20 @@ nvme_cuse_start(struct spdk_nvme_ctrlr *ctrlr) 
	TAILQ_INIT(&ctrlr_device->ns_devices);

	ctrlr_device->ctrlr = ctrlr;



	ctrlr_device->index = spdk_bit_array_find_first_clear(g_ctrlr_started, 0);

	/* Check if device already exists, if not increment index until success */

	ctrlr_device->index = 0;

	while (1) {

		ctrlr_device->index = spdk_bit_array_find_first_clear(g_ctrlr_started, ctrlr_device->index);

		if (ctrlr_device->index == UINT32_MAX) {

			SPDK_ERRLOG("Too many registered controllers\n");

			goto err2;

		}



		if (nvme_cuse_claim(ctrlr_device, ctrlr_device->index) == 0) {

			break;

		}

		ctrlr_device->index++;

	}

	spdk_bit_array_set(g_ctrlr_started, ctrlr_device->index);

	snprintf(ctrlr_device->dev_name, sizeof(ctrlr_device->dev_name), "spdk/nvme%d",

		 ctrlr_device->index);