Commit e28f3397 authored by Dariusz Stojaczyk's avatar Dariusz Stojaczyk Committed by Jim Harris
Browse files

vhost: added spdk_vhost_event_fn 



The next step towards fixing synchronization issues between RPC and
vhost controller reactor.

This patch makes changes to already present vhost timed event API
to conform it to the upcoming external spdk_events API. The timed_event
API shall be removed from headers in future. See next patch for details.

Change-Id: I31b0d7c383b39c32326daa750663ebdeb4edd562
Signed-off-by: default avatarDariusz Stojaczyk <dariuszx.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/377584


Tested-by: default avatarSPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 147ea6b2

include/spdk/vhost.h

+2 −0
Original line number Diff line number Diff line
@@ -55,6 +55,8 @@ void spdk_vhost_shutdown_cb(void); 
/* Forward declaration */

struct spdk_vhost_dev;



typedef int (*spdk_vhost_event_fn)(struct spdk_vhost_dev *, void *);



/**

 * Get handle to next controller.

 * \param prev Previous controller or NULL to get first one.

lib/vhost/vhost.c

+28 −14
Original line number Diff line number Diff line
@@ -49,12 +49,21 @@ static char dev_dirname[PATH_MAX] = ""; 


#define MAX_VHOST_DEVICES	64



struct spdk_vhost_timed_event {

struct spdk_vhost_dev_event_ctx {

	/** Pointer to the controller obtained before enqueuing the event */

	struct spdk_vhost_dev *vdev;



	/** Name of the ctrlr to send event to */

	char *ctrlr_name;



	/** User callback function to be executed on given lcore. */

	spdk_vhost_timed_event_fn cb_fn;

	spdk_vhost_event_fn cb_fn;



	/** Semaphore used to signal that event is done. */

	sem_t sem;



	/** Response to be written by enqueued event. */

	int response;

};



static int new_connection(int vid);
@@ -531,34 +540,36 @@ spdk_vhost_allocate_reactor(uint64_t cpumask) 
}



static void

vhost_timed_event_fn(void *arg1, void *arg2)

spdk_vhost_event_cb(void *arg1, void *arg2)

{

	struct spdk_vhost_timed_event *ctx = arg1;



	if (ctx->cb_fn) {

		ctx->cb_fn(arg2);

	}

	struct spdk_vhost_dev_event_ctx *ctx = arg1;



	ctx->response = ctx->cb_fn(ctx->vdev, arg2);

	sem_post(&ctx->sem);

}



void

spdk_vhost_timed_event_send(int32_t lcore, spdk_vhost_timed_event_fn cb_fn, void *arg,

int

spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, void *arg,

		      unsigned timeout_sec, const char *errmsg)

{

	struct spdk_vhost_timed_event ev_ctx = {0};

	struct spdk_vhost_dev_event_ctx ev_ctx = {0};

	struct spdk_event *ev;

	struct timespec timeout;

	int rc;



	if (sem_init(&ev_ctx.sem, 0, 0) < 0)

	rc = sem_init(&ev_ctx.sem, 0, 0);

	if (rc != 0) {

		SPDK_ERRLOG("Failed to initialize semaphore for vhost timed event\n");

		return -errno;

	}



	ev_ctx.vdev = vdev;

	ev_ctx.ctrlr_name = strdup(vdev->name);

	ev_ctx.cb_fn = cb_fn;

	clock_gettime(CLOCK_REALTIME, &timeout);

	timeout.tv_sec += timeout_sec;



	ev = spdk_event_allocate(lcore, vhost_timed_event_fn, &ev_ctx, arg);

	ev = spdk_event_allocate(vdev->lcore, spdk_vhost_event_cb, &ev_ctx, arg);

	assert(ev);

	spdk_event_call(ev);
@@ -569,6 +580,9 @@ spdk_vhost_timed_event_send(int32_t lcore, spdk_vhost_timed_event_fn cb_fn, void 
	}



	sem_destroy(&ev_ctx.sem);

	free(ev_ctx.ctrlr_name);



	return ev_ctx.response;

}



static void

lib/vhost/vhost_blk.c

+30 −20
Original line number Diff line number Diff line
@@ -359,11 +359,10 @@ no_bdev_vdev_worker(void *arg) 
	}

}



static void

add_vdev_cb(void *arg)

static int

add_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_blk_dev *bvdev = arg;

	struct spdk_vhost_dev *vdev = &bvdev->vdev;



	spdk_vhost_dev_mem_register(&bvdev->vdev);
@@ -378,10 +377,11 @@ add_vdev_cb(void *arg) 
	spdk_poller_register(&bvdev->requestq_poller, bvdev->bdev ? vdev_worker : no_bdev_vdev_worker,

			     bvdev, vdev->lcore, 0);

	SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vdev->name, vdev->lcore);

	return 0;

}



static void

remove_vdev_cb(void *arg)

static int

remove_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_blk_dev *bvdev = arg;
@@ -393,14 +393,16 @@ remove_vdev_cb(void *arg) 
	}



	spdk_vhost_dev_mem_unregister(&bvdev->vdev);

	return 0;

}



static void

unregister_vdev_cb(void *arg)

static int

unregister_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_blk_dev *bvdev = arg;



	spdk_poller_unregister(&bvdev->requestq_poller, NULL);

	return 0;

}



static struct spdk_vhost_blk_dev *
@@ -428,16 +430,10 @@ spdk_vhost_blk_get_dev(struct spdk_vhost_dev *vdev) 
	return bvdev->bdev;

}



static void

bdev_remove_cb(void *remove_ctx)

static int

_bdev_remove_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_blk_dev *bvdev = remove_ctx;



	if (bvdev->vdev.lcore != -1 && (uint32_t)bvdev->vdev.lcore != spdk_env_get_current_core()) {

		/* Call self on proper core. */

		spdk_vhost_timed_event_send(bvdev->vdev.lcore, bdev_remove_cb, bvdev, 1, "vhost blk hot remove");

		return;

	}

	struct spdk_vhost_blk_dev *bvdev = arg;



	SPDK_WARNLOG("Controller %s: Hot-removing bdev - all further requests will fail.\n",

		     bvdev->vdev.name);
@@ -447,6 +443,21 @@ bdev_remove_cb(void *remove_ctx) 
	}



	bvdev->bdev = NULL;

	return 0;

}



static void

bdev_remove_cb(void *remove_ctx)

{

	struct spdk_vhost_blk_dev *bvdev = remove_ctx;



	if (bvdev->vdev.lcore != -1 && (uint32_t)bvdev->vdev.lcore != spdk_env_get_current_core()) {

		/* Call on proper core. */

		spdk_vhost_event_send(&bvdev->vdev, _bdev_remove_cb, bvdev, 1, "vhost blk hot remove");

		return;

	}



	_bdev_remove_cb(&bvdev->vdev, bvdev);

}
@@ -545,8 +556,7 @@ new_device(struct spdk_vhost_dev *vdev) 
		return -1;

	}



	spdk_vhost_timed_event_send(bvdev->vdev.lcore, add_vdev_cb, bvdev, 1, "add blk vdev");



	spdk_vhost_event_send(vdev, add_vdev_cb, bvdev, 1, "add blk vdev");

	return 0;

}
@@ -562,7 +572,7 @@ destroy_device(struct spdk_vhost_dev *vdev) 
		return -1;

	}



	spdk_vhost_timed_event_send(vdev->lcore, unregister_vdev_cb, bvdev, 1, "unregister vdev");

	spdk_vhost_event_send(vdev, unregister_vdev_cb, bvdev, 1, "unregister vdev");



	/* Wait for all tasks to finish */

	for (i = 1000; i && vdev->task_cnt > 0; i--) {
@@ -574,7 +584,7 @@ destroy_device(struct spdk_vhost_dev *vdev) 
		abort();

	}



	spdk_vhost_timed_event_send(vdev->lcore, remove_vdev_cb, bvdev, 1, "remove vdev");

	spdk_vhost_event_send(vdev, remove_vdev_cb, bvdev, 1, "remove vdev");



	free_task_pool(bvdev);

	return 0;

lib/vhost/vhost_internal.h

+2 −4
Original line number Diff line number Diff line
@@ -135,9 +135,7 @@ int spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, const char *name, cons 
			     enum spdk_vhost_dev_type type, const struct spdk_vhost_dev_backend *backend);

int spdk_vhost_dev_remove(struct spdk_vhost_dev *vdev);



typedef void (*spdk_vhost_timed_event_fn)(void *);



void spdk_vhost_timed_event_send(int32_t lcore, spdk_vhost_timed_event_fn cn_fn, void *arg,

int spdk_vhost_event_send(struct spdk_vhost_dev *vdev, spdk_vhost_event_fn cb_fn, void *arg,

			  unsigned timeout_sec, const char *errmsg);



int spdk_vhost_blk_controller_construct(void);

lib/vhost/vhost_scsi.c

+12 −11
Original line number Diff line number Diff line
@@ -688,11 +688,10 @@ vdev_worker(void *arg) 
	}

}



static void

add_vdev_cb(void *arg)

static int

add_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_scsi_dev *svdev = arg;

	struct spdk_vhost_dev *vdev = &svdev->vdev;

	uint32_t i;



	for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
@@ -708,10 +707,11 @@ add_vdev_cb(void *arg) 
	spdk_poller_register(&svdev->requestq_poller, vdev_worker, svdev, vdev->lcore, 0);

	spdk_poller_register(&svdev->mgmt_poller, vdev_mgmt_worker, svdev, vdev->lcore,

			     MGMT_POLL_PERIOD_US);

	return 0;

}



static void

remove_vdev_cb(void *arg)

static int

remove_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_scsi_dev *svdev = arg;

	uint32_t i;
@@ -725,15 +725,17 @@ remove_vdev_cb(void *arg) 


	SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", svdev->vdev.name);

	spdk_vhost_dev_mem_unregister(&svdev->vdev);

	return 0;

}



static void

unregister_vdev_cb(void *arg)

static int

unregister_vdev_cb(struct spdk_vhost_dev *vdev, void *arg)

{

	struct spdk_vhost_scsi_dev *svdev = arg;



	spdk_poller_unregister(&svdev->requestq_poller, NULL);

	spdk_poller_unregister(&svdev->mgmt_poller, NULL);

	return 0;

}



static struct spdk_vhost_scsi_dev *
@@ -1124,8 +1126,7 @@ new_device(struct spdk_vhost_dev *vdev) 
		return -1;

	}



	spdk_vhost_timed_event_send(vdev->lcore, add_vdev_cb, svdev, 1, "add scsi vdev");



	spdk_vhost_event_send(vdev, add_vdev_cb, svdev, 1, "add scsi vdev");

	return 0;

}
@@ -1142,7 +1143,7 @@ destroy_device(struct spdk_vhost_dev *vdev) 
		return -1;

	}



	spdk_vhost_timed_event_send(vdev->lcore, unregister_vdev_cb, svdev, 1, "unregister scsi vdev");

	spdk_vhost_event_send(vdev, unregister_vdev_cb, svdev, 1, "unregister scsi vdev");



	/* Wait for all tasks to finish */

	for (i = 1000; i && vdev->task_cnt > 0; i--) {
@@ -1153,7 +1154,7 @@ destroy_device(struct spdk_vhost_dev *vdev) 
		SPDK_ERRLOG("%s: pending tasks did not finish in 1s.\n", vdev->name);

	}



	spdk_vhost_timed_event_send(vdev->lcore, remove_vdev_cb, svdev, 1, "remove scsi vdev");

	spdk_vhost_event_send(vdev, remove_vdev_cb, svdev, 1, "remove scsi vdev");



	/* Flush not sent events */

	while (spdk_ring_dequeue(svdev->vhost_events, &ev, 1) == 1) {