ublk: add ublk target creation and destruction (a1944e01) · Commits · Public Repositories / spdk

doc/jsonrpc.md

+77 −0

Original line number	Diff line number	Diff line
		@@ -10672,6 +10672,83 @@ Example response:
		}
		~~~

		## Linux Userspace Block Device (UBLK) {#jsonrpc_components_ublk}

		SPDK supports exporting bdevs though Linux ublk. The motivation behind it is to back a Linux kernel block device
		with an SPDK user space bdev.

		To export a device over ublk, first make sure the Linux kernel ublk driver is loaded by running 'modprobe ublk_drv'.

		### ublk_create_target {#rpc_ublk_create_target}

		Start to create ublk threads and initialize ublk target. It will return an error if user calls this RPC twice without
		ublk_destroy_target in between. It will use current cpumask in SPDK when user does not specify cpumask option.

		#### Parameters

		Name \| Optional \| Type \| Description
		----------------------- \| -------- \| ----------- \| -----------
		cpumask \| Optional \| string \| Cpumask for ublk target

		#### Response

		True if ublk target initialization is successful; False if failed.

		#### Example

		Example request:

		~~~json
		{
		"params": {
		"cpumask": "0x2"
		},
		"jsonrpc": "2.0",
		"method": "ublk_create_target",
		"id": 1
		}
		~~~

		Example response:

		~~~json
		{
		"jsonrpc": "2.0",
		"id": 1,
		"result": true
		}
		~~~

		### ublk_destroy_target {#rpc_ublk_destroy_target}

		Release all UBLK devices and destroy ublk target.

		#### Response

		True if ublk target destruction is successful; False if failed.

		#### Example

		Example request:

		~~~json
		{
		"jsonrpc": "2.0",
		"method": "ublk_destroy_target",
		"id": 1
		}
		~~~

		Example response:

		~~~json
		{
		"jsonrpc": "2.0",
		"id": 1,
		"result": true
		}
		~~~

		## Linux Network Block Device (NBD) {#jsonrpc_components_nbd}

		SPDK supports exporting bdevs through Linux nbd. These devices then appear as standard Linux kernel block devices

lib/ublk/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
		SO_VER := 1
		SO_MINOR := 0

		C_SRCS = ublk.c
		C_SRCS = ublk.c ublk_rpc.c
		LIBNAME = ublk
		LOCAL_SYS_LIBS = -luring

lib/ublk/ublk.c

+201 −7

Original line number	Diff line number	Diff line
		@@ -6,37 +6,231 @@
		#include <linux/ublk_cmd.h>
		#include <liburing.h>

		#include "spdk/stdinc.h"
		#include "spdk/string.h"
		#include "spdk/bdev.h"
		#include "spdk/endian.h"
		#include "spdk/env.h"
		#include "spdk/likely.h"
		#include "spdk/log.h"
		#include "spdk/util.h"
		#include "spdk/queue.h"
		#include "spdk/json.h"
		#include "spdk/ublk.h"
		#include "spdk/thread.h"

		struct ublk_tgt {
		#include "ublk_internal.h"

		#define UBLK_CTRL_DEV "/dev/ublk-control"

		#define UBLK_CTRL_RING_DEPTH 32
		#define UBLK_THREAD_MAX 128

		spdk_ublk_fini_cb fini_cb_fn;
		void *fini_cb_arg;
		static uint32_t g_num_ublk_threads = 0;
		static struct spdk_cpuset g_core_mask;

		struct ublk_tgt {
		int ctrl_fd;
		bool active;
		bool is_destroying;
		spdk_ublk_fini_cb cb_fn;
		void *cb_arg;
		struct io_uring ctrl_ring;
		struct spdk_thread *ublk_threads[UBLK_THREAD_MAX];
		};

		static TAILQ_HEAD(, spdk_ublk_dev) g_ublk_bdevs = TAILQ_HEAD_INITIALIZER(g_ublk_bdevs);
		static struct ublk_tgt g_ublk_tgt;

		/* helpers for using io_uring */
		static inline int
		ublk_setup_ring(uint32_t depth, struct io_uring *r, unsigned flags)
		{
		struct io_uring_params p = {};

		p.flags = flags \| IORING_SETUP_CQSIZE;
		p.cq_entries = depth;

		return io_uring_queue_init_params(depth, r, &p);
		}

		void
		spdk_ublk_init(void)
		{
		uint32_t i;

		assert(spdk_get_thread() == spdk_thread_get_app_thread());

		spdk_cpuset_zero(&g_core_mask);
		SPDK_ENV_FOREACH_CORE(i) {
		spdk_cpuset_set_cpu(&g_core_mask, i, true);
		}
		}

		static int
		ublk_open(void)
		{
		int rc;

		g_ublk_tgt.ctrl_fd = open(UBLK_CTRL_DEV, O_RDWR);
		if (g_ublk_tgt.ctrl_fd < 0) {
		rc = errno;
		SPDK_ERRLOG("UBLK conrol dev %s can't be opened, error=%s\n", UBLK_CTRL_DEV, spdk_strerror(errno));
		return -rc;
		}

		rc = ublk_setup_ring(UBLK_CTRL_RING_DEPTH, &g_ublk_tgt.ctrl_ring, IORING_SETUP_SQE128);
		if (rc < 0) {
		SPDK_ERRLOG("UBLK ctrl queue_init: %s\n", spdk_strerror(-rc));
		close(g_ublk_tgt.ctrl_fd);
		return rc;
		}

		return 0;
		}

		static int
		ublk_parse_core_mask(const char mask, struct spdk_cpuset cpumask)
		{
		int rc;
		struct spdk_cpuset tmp_mask;

		if (cpumask == NULL) {
		return -EPERM;
		}

		if (mask == NULL) {
		spdk_cpuset_copy(cpumask, &g_core_mask);
		return 0;
		}

		rc = spdk_cpuset_parse(cpumask, mask);
		if (rc < 0) {
		SPDK_ERRLOG("invalid cpumask %s\n", mask);
		return -rc;
		}

		if (spdk_cpuset_count(cpumask) == 0) {
		SPDK_ERRLOG("no cpus specified\n");
		return -EINVAL;
		}

		spdk_cpuset_copy(&tmp_mask, cpumask);
		spdk_cpuset_and(&tmp_mask, &g_core_mask);

		if (!spdk_cpuset_equal(&tmp_mask, cpumask)) {
		SPDK_ERRLOG("one of selected cpu is outside of core mask(=%s)\n",
		spdk_cpuset_fmt(&g_core_mask));
		return -EINVAL;
		}

		return 0;
		}

		int
		ublk_create_target(const char *cpumask_str)
		{
		int rc;
		uint32_t i;
		char thread_name[32];
		struct spdk_cpuset cpuset = {};
		struct spdk_cpuset thd_cpuset = {};

		if (g_ublk_tgt.active == true) {
		SPDK_ERRLOG("UBLK target has been created\n");
		return -EBUSY;
		}

		rc = ublk_parse_core_mask(cpumask_str, &cpuset);
		if (rc != 0) {
		return rc;
		}

		rc = ublk_open();
		if (rc != 0) {
		SPDK_ERRLOG("Fail to open UBLK, error=%s\n", spdk_strerror(-rc));
		return rc;
		}

		SPDK_ENV_FOREACH_CORE(i) {
		if (spdk_cpuset_get_cpu(&cpuset, i)) {
		spdk_cpuset_zero(&thd_cpuset);
		spdk_cpuset_set_cpu(&thd_cpuset, i, true);
		snprintf(thread_name, sizeof(thread_name), "ublk_thread%u", i);
		g_ublk_tgt.ublk_threads[g_num_ublk_threads] = spdk_thread_create(thread_name, &thd_cpuset);
		g_num_ublk_threads++;
		}
		}
		g_ublk_tgt.active = true;
		SPDK_NOTICELOG("UBLK target created successfully\n");

		return 0;
		}

		static void
		_ublk_fini_done(void *args)
		{
		g_num_ublk_threads = 0;
		g_ublk_tgt.is_destroying = false;
		g_ublk_tgt.active = false;
		if (g_ublk_tgt.cb_fn) {
		g_ublk_tgt.cb_fn(g_ublk_tgt.cb_arg);
		g_ublk_tgt.cb_fn = NULL;
		g_ublk_tgt.cb_arg = NULL;
		}
		}

		static void
		ublk_thread_exit(void *args)
		{
		struct spdk_thread *ublk_thread = spdk_get_thread();
		uint32_t i;

		for (i = 0; i < g_num_ublk_threads; i++) {
		if (g_ublk_tgt.ublk_threads[i] == ublk_thread) {
		spdk_thread_exit(ublk_thread);
		}
		}
		}

		/* This function will be used and extended in next patch */
		static void
		_ublk_fini(void *args)
		{
		spdk_for_each_thread(ublk_thread_exit, NULL, _ublk_fini_done);
		}

		int
		spdk_ublk_fini(spdk_ublk_fini_cb cb_fn, void *cb_arg)
		{
		assert(spdk_get_thread() == spdk_thread_get_app_thread());
		g_ublk_tgt.fini_cb_fn = cb_fn;
		g_ublk_tgt.fini_cb_arg = cb_arg;
		g_ublk_tgt.fini_cb_fn(g_ublk_tgt.fini_cb_arg);

		if (g_ublk_tgt.is_destroying == true) {
		/* UBLK target is being destroying */
		return -EBUSY;
		}
		g_ublk_tgt.cb_fn = cb_fn;
		g_ublk_tgt.cb_arg = cb_arg;
		g_ublk_tgt.is_destroying = true;
		_ublk_fini(NULL);

		return 0;
		}

		int
		ublk_destroy_target(spdk_ublk_fini_cb cb_fn, void *cb_arg)
		{
		int rc;

		if (g_ublk_tgt.active == false) {
		/* UBLK target has not been created */
		return -ENOENT;
		}

		rc = spdk_ublk_fini(cb_fn, cb_arg);

		return rc;
		}

		void
		spdk_ublk_write_config_json(struct spdk_json_write_ctx *w)
		{

lib/ublk/ublk_internal.h

0 → 100644

+24 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: BSD-3-Clause
		* Copyright (C) 2022 Intel Corporation.
		* All rights reserved.
		*/
		/** \file
		* Userspace block device layer
		*/
		#ifndef SPDK_UBLK_INTERNAL_H
		#define SPDK_UBLK_INTERNAL_H

		#include "spdk/ublk.h"

		#ifdef __cplusplus
		extern "C" {
		#endif

		int ublk_create_target(const char *cpumask_str);
		int ublk_destroy_target(spdk_ublk_fini_cb cb_fn, void *cb_arg);

		#ifdef __cplusplus
		}
		#endif

		#endif /* SPDK_UBLK_INTERNAL_H */

lib/ublk/ublk_rpc.c

0 → 100644

+77 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: BSD-3-Clause
		* Copyright (C) 2022 Intel Corporation.
		* All rights reserved.
		*/

		#include "spdk/string.h"
		#include "spdk/env.h"
		#include "spdk/rpc.h"
		#include "spdk/util.h"
		#include "spdk/log.h"

		#include "ublk_internal.h"

		struct rpc_ublk_create_target {
		char *cpumask;
		};

		static const struct spdk_json_object_decoder rpc_ublk_create_target_decoders[] = {
		{"cpumask", offsetof(struct rpc_ublk_create_target, cpumask), spdk_json_decode_string, true},
		};

		static void
		free_rpc_ublk_create_target(struct rpc_ublk_create_target *req)
		{
		free(req->cpumask);
		}

		static void
		rpc_ublk_create_target(struct spdk_jsonrpc_request request, const struct spdk_json_val params)
		{
		int rc = 0;
		struct rpc_ublk_create_target req = {};

		if (params != NULL) {
		if (spdk_json_decode_object(params, rpc_ublk_create_target_decoders,
		SPDK_COUNTOF(rpc_ublk_create_target_decoders),
		&req)) {
		SPDK_ERRLOG("spdk_json_decode_object failed\n");
		rc = -EINVAL;
		goto invalid;
		}
		}
		rc = ublk_create_target(req.cpumask);
		if (rc != 0) {
		goto invalid;
		}
		spdk_jsonrpc_send_bool_response(request, true);
		free_rpc_ublk_create_target(&req);
		return;
		invalid:
		SPDK_ERRLOG("Can't create ublk target: %s\n", spdk_strerror(-rc));
		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, spdk_strerror(-rc));
		free_rpc_ublk_create_target(&req);
		}
		SPDK_RPC_REGISTER("ublk_create_target", rpc_ublk_create_target, SPDK_RPC_RUNTIME)

		static void
		ublk_destroy_target_done(void *arg)
		{
		struct spdk_jsonrpc_request *req = arg;

		spdk_jsonrpc_send_bool_response(req, true);
		SPDK_NOTICELOG("ublk target has been destroyed\n");
		}

		static void
		rpc_ublk_destroy_target(struct spdk_jsonrpc_request request, const struct spdk_json_val params)
		{
		int rc = 0;

		rc = ublk_destroy_target(ublk_destroy_target_done, request);
		if (rc != 0) {
		spdk_jsonrpc_send_error_response(request, SPDK_JSONRPC_ERROR_INTERNAL_ERROR, spdk_strerror(-rc));
		SPDK_ERRLOG("Can't destroy ublk target: %s\n", spdk_strerror(-rc));
		}
		}
		SPDK_RPC_REGISTER("ublk_destroy_target", rpc_ublk_destroy_target, SPDK_RPC_RUNTIME)

Admin message