Commit 527cd3ed authored by Dariusz Stojaczyk's avatar Dariusz Stojaczyk Committed by Jim Harris
Browse files

bdev/virtio: cleanup virtio device management 



The virtio device can now be
removed with a single call.

Change-Id: I221d2c0578eeddb3de9c2a31eefc0f64fda38939
Signed-off-by: default avatarDariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/390110


Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
parent e7818804

lib/bdev/virtio/bdev_virtio.c

+121 −63
Original line number Diff line number Diff line
@@ -71,8 +71,14 @@ struct virtio_scsi_dev { 
	/* Generic virtio device data. */

	struct virtio_dev		vdev;



	/** Detected SCSI LUNs */

	TAILQ_HEAD(, virtio_scsi_disk)	luns;



	/** Context for the SCSI target scan. */

	struct virtio_scsi_scan_base	*scan_ctx;



	/** Device marked for removal. */

	bool removed;

};



struct virtio_scsi_io_ctx {
@@ -134,6 +140,12 @@ struct bdev_virtio_io_channel { 
	struct virtqueue	*vq;

};



/** Module finish in progress */

static bool g_bdev_virtio_finish = false;



static void virtio_scsi_dev_unregister_cb(void *io_device);

static void virtio_scsi_dev_remove(struct virtio_scsi_dev *svdev);



static int

virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx)

{
@@ -150,6 +162,10 @@ virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx) 
		return -1;

	}



	TAILQ_INIT(&svdev->luns);

	svdev->scan_ctx = NULL;

	svdev->removed = false;



	vdev = &svdev->vdev;

	name = spdk_sprintf_alloc("VirtioScsi%"PRIu32, pci_dev_counter++);

	if (name == NULL) {
@@ -169,9 +185,42 @@ virtio_pci_scsi_dev_create_cb(struct virtio_pci_ctx *pci_ctx) 
				   &num_queues, sizeof(num_queues));

	vdev->max_queues = SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED + num_queues;



	TAILQ_INSERT_TAIL(&g_virtio_driver.scsi_devs, vdev, tailq);

	return 0;

}



static struct virtio_scsi_dev *

virtio_user_scsi_dev_create(const char *name, const char *path,

			    uint16_t num_queues, uint32_t queue_size)

{

	struct virtio_scsi_dev *svdev;

	struct virtio_dev *vdev;

	int rc;



	svdev = calloc(1, sizeof(*svdev));

	if (svdev == NULL) {

		SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", name, path);

		return NULL;

	}



	TAILQ_INIT(&svdev->luns);

	svdev->scan_ctx = NULL;

	svdev->removed = false;



	vdev = &svdev->vdev;

	rc = virtio_user_dev_init(vdev, name, path, num_queues, queue_size,

				  SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to create virito device %s: %s\n", name, path);

		free(svdev);

		return NULL;

	}





	TAILQ_INSERT_TAIL(&g_virtio_driver.scsi_devs, vdev, tailq);

	return svdev;

}



static int scan_target(struct virtio_scsi_scan_base *base);



static int
@@ -432,8 +481,18 @@ bdev_virtio_get_io_channel(void *ctx) 
}



static int

bdev_virtio_destruct(void *ctx)

bdev_virtio_disk_destruct(void *ctx)

{

	struct virtio_scsi_disk *disk = ctx;

	struct virtio_scsi_dev *svdev = disk->svdev;



	TAILQ_REMOVE(&svdev->luns, disk, link);

	free(disk);



	if (svdev->removed && TAILQ_EMPTY(&svdev->luns)) {

		spdk_io_device_unregister(svdev, virtio_scsi_dev_unregister_cb);

	}



	return 0;

}
@@ -447,7 +506,7 @@ bdev_virtio_dump_json_config(void *ctx, struct spdk_json_write_ctx *w) 
}



static const struct spdk_bdev_fn_table virtio_fn_table = {

	.destruct		= bdev_virtio_destruct,

	.destruct		= bdev_virtio_disk_destruct,

	.submit_request		= bdev_virtio_submit_request,

	.io_type_supported	= bdev_virtio_io_type_supported,

	.get_io_channel		= bdev_virtio_get_io_channel,
@@ -624,30 +683,35 @@ bdev_virtio_scsi_ch_destroy_cb(void *io_device, void *ctx_buf) 
static void

scan_target_abort(struct virtio_scsi_scan_base *base, int error)

{

	struct virtio_dev *vdev = &base->svdev->vdev;

	struct virtio_scsi_dev *svdev = base->svdev;

	struct virtio_dev *vdev = &svdev->vdev;

	struct virtio_scsi_disk *disk;

	struct virtqueue *vq;



	while ((disk = TAILQ_FIRST(&base->found_disks))) {

		TAILQ_REMOVE(&base->found_disks, disk, link);

		free(disk);

	}



	TAILQ_REMOVE(&g_virtio_driver.scsi_devs, vdev, tailq);

	virtio_dev_stop(vdev);

	virtio_dev_destruct(vdev);

	free(vdev);

	vq = base->vq;

	spdk_poller_unregister(&vq->poller);

	spdk_dma_free(vq->poller_ctx);

	vq->poller_ctx = NULL;

	virtio_dev_release_queue(vdev, vq->vq_queue_index);



	if (base->cb_fn) {

		base->cb_fn(base->cb_arg, error, NULL, 0);

	}



	spdk_dma_free(base);

	svdev->scan_ctx = NULL;

}



static void

scan_target_finish(struct virtio_scsi_scan_base *base)

{

	struct virtio_dev *vdev = &base->svdev->vdev;

	struct virtio_scsi_dev *svdev = base->svdev;

	struct virtio_dev *vdev = &svdev->vdev;

	size_t bdevs_cnt = 0;

	struct spdk_bdev *bdevs[BDEV_VIRTIO_MAX_TARGET];

	struct virtio_scsi_disk *disk;
@@ -672,16 +736,15 @@ scan_target_finish(struct virtio_scsi_scan_base *base) 
				      SPDK_ENV_SOCKET_ID_ANY);

	if (ctrlq_ring == NULL) {

		SPDK_ERRLOG("Failed to allocate send ring for the controlq.\n");

		scan_target_abort(base, -ENOMEM);

		virtio_scsi_dev_remove(svdev);

		return;

	}



	rc = virtio_dev_acquire_queue(vdev, VIRTIO_SCSI_CONTROLQ);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to acquire the controlq.\n");

		assert(false);

		spdk_ring_free(ctrlq_ring);

		scan_target_abort(base, rc);

		virtio_scsi_dev_remove(svdev);

		return;

	}
@@ -703,6 +766,7 @@ scan_target_finish(struct virtio_scsi_scan_base *base) 
			scan_target_abort(base, rc);

			return;

		}

		TAILQ_INSERT_TAIL(&svdev->luns, disk, link);

		bdevs[bdevs_cnt] = &disk->bdev;

		bdevs_cnt++;

	}
@@ -1207,26 +1271,18 @@ bdev_virtio_process_config(void) 
			num_queues = 1;

		}



		svdev = calloc(1, sizeof(*svdev));

		if (svdev == NULL) {

			SPDK_ERRLOG("virtio device calloc failed\n");

			goto out;

		}



		name = spdk_conf_section_get_val(sp, "Name");

		if (name == NULL) {

			default_name = spdk_sprintf_alloc("VirtioScsi%u", vdev_num);

			name = default_name;

		}



		rc = virtio_user_dev_init(&svdev->vdev, name, path, num_queues, 512,

					  SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED);



		svdev = virtio_user_scsi_dev_create(name, path, num_queues, 512);

		free(default_name);

		default_name = NULL;



		if (rc != 0) {

			free(svdev);

		if (svdev == NULL) {

			rc = -1;

			goto out;

		}

	}
@@ -1245,31 +1301,12 @@ out: 
	return rc;

}



static void

bdev_virtio_scsi_dev_free(struct virtio_scsi_dev *svdev)

{

	struct virtio_dev *vdev = &svdev->vdev;

	struct virtqueue *vq;



	if (virtio_dev_queue_is_acquired(vdev, VIRTIO_SCSI_REQUESTQ)) {

		vq = vdev->vqs[VIRTIO_SCSI_REQUESTQ];

		spdk_poller_unregister(&vq->poller);

		spdk_dma_free(vq->poller_ctx);

		vq->poller_ctx = NULL;

		virtio_dev_release_queue(vdev, VIRTIO_SCSI_REQUESTQ);

	}



	virtio_dev_stop(vdev);

	virtio_dev_destruct(vdev);

	free(svdev);

}



static int

bdev_virtio_scsi_scan(struct virtio_scsi_dev *svdev, bdev_virtio_create_cb cb_fn, void *cb_arg)

bdev_virtio_scsi_dev_scan(struct virtio_scsi_dev *svdev, bdev_virtio_create_cb cb_fn,

			  void *cb_arg)

{

	struct virtio_dev *vdev = &svdev->vdev;

	struct virtio_scsi_scan_base *base = spdk_dma_zmalloc(sizeof(struct virtio_scsi_scan_base), 64,

					     NULL);

	struct virtio_scsi_scan_base *base = spdk_dma_zmalloc(sizeof(*base), 64, NULL);

	struct virtqueue *vq;

	int rc;
@@ -1349,7 +1386,7 @@ bdev_virtio_initialize(void) 
	/* Initialize all created devices and scan available targets */

	TAILQ_FOREACH(vdev, &g_virtio_driver.scsi_devs, tailq) {

		svdev = virtio_dev_to_scsi(vdev);

		rc = bdev_virtio_scsi_scan(svdev, bdev_virtio_initial_scan_complete, NULL);

		rc = bdev_virtio_scsi_dev_scan(svdev, bdev_virtio_initial_scan_complete, NULL);

		if (rc != 0) {

			goto out;

		}
@@ -1362,7 +1399,7 @@ out: 
	TAILQ_FOREACH_SAFE(vdev, &g_virtio_driver.scsi_devs, tailq, next_vdev) {

		svdev = virtio_dev_to_scsi(vdev);

		TAILQ_REMOVE(&g_virtio_driver.scsi_devs, vdev, tailq);

		bdev_virtio_scsi_dev_free(svdev);

		virtio_scsi_dev_remove(svdev);

	}



	spdk_bdev_module_init_done(SPDK_GET_BDEV_MODULE(virtio_scsi));
@@ -1403,24 +1440,55 @@ virtio_scsi_dev_unregister_cb(void *io_device) 
	TAILQ_REMOVE(&g_virtio_driver.scsi_devs, vdev, tailq);

	finish_module = TAILQ_EMPTY(&g_virtio_driver.scsi_devs);



	if (finish_module) {

	if (g_bdev_virtio_finish && finish_module) {

		spdk_bdev_module_finish_done();

	}

}



static void

virtio_scsi_dev_remove(struct virtio_scsi_dev *svdev)

{

	struct virtio_scsi_scan_base *scan_ctx;

	struct virtio_scsi_disk *disk, *disk_tmp;

	bool do_remove = true;



	if (svdev->removed) {

		/** device removal in progress */

		return;

	}



	svdev->removed = true;



	scan_ctx = svdev->scan_ctx;

	if (scan_ctx) {

		scan_target_abort(scan_ctx, -EINTR);

	}



	TAILQ_FOREACH_SAFE(disk, &svdev->luns, link, disk_tmp) {

		spdk_bdev_unregister(&disk->bdev, NULL, NULL);

		do_remove = false;

	}



	if (do_remove) {

		spdk_io_device_unregister(svdev, virtio_scsi_dev_unregister_cb);

	}

}



static void

bdev_virtio_finish(void)

{

	struct virtio_dev *vdev, *next;



	g_bdev_virtio_finish = true;



	if (TAILQ_EMPTY(&g_virtio_driver.scsi_devs)) {

		spdk_bdev_module_finish_done();

		return;

	}



	/* Defer module finish until all controllers are removed. */

	TAILQ_FOREACH_SAFE(vdev, &g_virtio_driver.scsi_devs, tailq, next) {

		spdk_io_device_unregister(virtio_dev_to_scsi(vdev),

					  virtio_scsi_dev_unregister_cb);

		virtio_scsi_dev_remove(virtio_dev_to_scsi(vdev));

	}

}
@@ -1431,24 +1499,14 @@ bdev_virtio_scsi_dev_create(const char *base_name, const char *path, unsigned nu 
	struct virtio_scsi_dev *svdev;

	int rc;



	svdev = calloc(1, sizeof(*svdev));

	svdev = virtio_user_scsi_dev_create(base_name, path, num_queues, queue_size);

	if (svdev == NULL) {

		SPDK_ERRLOG("calloc failed for virtio device %s: %s\n", base_name, path);

		return -ENOMEM;

	}



	rc = virtio_user_dev_init(&svdev->vdev, base_name, path, num_queues, queue_size,

				  SPDK_VIRTIO_SCSI_QUEUE_NUM_FIXED);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to create virito device %s: %s\n", base_name, path);

		free(svdev);

		return -EINVAL;

		return -1;

	}



	rc = bdev_virtio_scsi_scan(svdev, cb_fn, cb_arg);

	rc = bdev_virtio_scsi_dev_scan(svdev, cb_fn, cb_arg);

	if (rc) {

		TAILQ_REMOVE(&g_virtio_driver.scsi_devs, &svdev->vdev, tailq);

		bdev_virtio_scsi_dev_free(svdev);

		virtio_scsi_dev_remove(svdev);

	}



	return rc;

lib/bdev/virtio/rte_virtio/virtio_pci.c

+0 −1
Original line number Diff line number Diff line
@@ -510,7 +510,6 @@ virtio_pci_dev_init(struct virtio_dev *vdev, const char *name, 
	vdev->modern = 1;

	vdev->max_queues = SPDK_VIRTIO_MAX_VIRTQUEUES;



	TAILQ_INSERT_TAIL(&g_virtio_driver.scsi_devs, vdev, tailq);

	return 0;

}

lib/bdev/virtio/rte_virtio/virtio_user.c

+0 −1
Original line number Diff line number Diff line
@@ -449,7 +449,6 @@ virtio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path 
		goto err;

	}



	TAILQ_INSERT_TAIL(&g_virtio_driver.scsi_devs, vdev, tailq);

	return 0;



err: