doc: describe ublk target in user guide

                         spdk_top.md \

                         ssd_internals.md \

                         system_configuration.md \

                         ublk.md \

                         usdt.md \

                         userspace.md \

                         vagrant.md \

<?xml version="1.0"?>

<svg width="680" height="420" xmlns="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg">

 <!-- Created with SVG-edit - https://github.com/SVG-Edit/svgedit-->

 <g class="layer">

  <title>Layer 1</title>

  <rect fill="#ffffff" height="369" id="svg_1" stroke="#000000" width="635.87" x="22.74" y="26.61"/>

  <rect fill="#aaffff" height="0" id="svg_2" stroke="#000000" width="0" x="191.24" y="101.36">Application A</rect>

  <rect fill="#aaffff" height="88.96" id="svg_3" stroke="#000000" width="171" x="400.9" y="67.61">ublk Server</rect>

  <line fill="none" id="svg_4" stroke="#000000" stroke-dasharray="5,5" stroke-width="2" x1="23.11" x2="660.11" y1="199.03" y2="198.03">ublk Server</line>

  <text fill="#000000" font-family="Serif" font-size="21" font-weight="bold" id="svg_5" stroke="#000000" stroke-width="0" text-anchor="middle" transform="matrix(1 0 0 1 0 0)" x="488.28" xml:space="preserve" y="122.24">ublk Server</text>

  <rect fill="#aaffff" height="62" id="svg_6" stroke="#000000" transform="matrix(1 0 0 1 0 0)" width="161" x="384.38" y="311.2"/>

  <text fill="#000000" font-family="Serif" font-size="21" font-weight="bold" id="svg_7" stroke="#000000" stroke-width="0" text-anchor="middle" transform="matrix(1 0 0 1 0 0)" x="468.93" xml:space="preserve" y="349.7">ublk Driver</text>

  <rect fill="#ffff00" height="32" id="svg_8" stroke="#000000" width="98" x="144.36" y="212.94"/>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_9" stroke="#000000" stroke-width="0" text-anchor="middle" x="194.36" xml:space="preserve" y="235.94">/dev/ublkb3</text>

  <rect fill="#ffffff" height="0" id="svg_10" stroke="#000000" width="0" x="175.36" y="246.94"/>

  <rect fill="#ffff00" height="33" id="svg_11" stroke="#000000" width="97" x="200.03" y="239.6"/>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_12" stroke="#000000" stroke-width="0" text-anchor="middle" x="249.36" xml:space="preserve" y="263.27">/dev/ublkb2</text>

  <rect fill="#ffffff" height="0" id="svg_13" stroke="#000000" width="0" x="174.36" y="264.94"/>

  <rect fill="#ffff00" height="33" id="svg_14" stroke="#000000" width="97" x="33.99" y="244.06"/>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_15" stroke="#000000" stroke-width="0" text-anchor="middle" x="82.99" xml:space="preserve" y="267.06">/dev/ublkb1</text>

  <rect fill="#00ff00" height="32" id="svg_16" stroke="#000000" width="93" x="35.99" y="206.31">le/dev/ublkb1</rect>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_17" stroke="#000000" stroke-width="0" text-anchor="middle" x="80.99" xml:space="preserve" y="226.31">Filesystem</text>

  <path d="m383.94,359.38l-298.65,-1.66c0,0 -1.68,-79.96 -1.68,-79.96" fill="none" id="svg_22" stroke="#000000" stroke-linejoin="bevel" stroke-width="4"/>

  <path d="m384.83,334.28l-148.14,-0.2c0,0 3.33,-62.12 3.33,-62.12" fill="none" id="svg_26" stroke="#000000" stroke-linejoin="bevel" stroke-width="4" transform="matrix(1 0 0 1 0 0)"/>

  <path d="m384.69,347.33l-201.99,-0.22l0,-102.04" fill="none" id="svg_27" stroke="#000000" stroke-linejoin="bevel" stroke-width="4" transform="matrix(1 0 0 1 0 0)"/>

  <path d="m454.33,155.75c0,0 0.48,154.94 0.32,154.69c-0.16,-0.25 -0.32,-154.69 -0.32,-154.69z" fill="none" id="svg_28" stroke="#000000" stroke-linejoin="bevel" stroke-width="3"/>

  <path d="m468.6,156.42l0.18,155.99l-0.18,-155.99z" fill="none" id="svg_29" stroke="#000000" stroke-linejoin="bevel" stroke-width="3"/>

  <path d="m482.69,157.08l-0.32,154.03l0.32,-154.03z" fill="none" id="svg_30" stroke="#000000" stroke-linecap="square" stroke-linejoin="bevel" stroke-width="3">ublk Server</path>

  <rect fill="#aaffff" height="35.63" id="svg_40" stroke="#000000" width="109.37" x="65.74" y="91.86">Application A</rect>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_41" stroke="#000000" stroke-width="0" style="cursor: move;" text-anchor="middle" x="119.36" xml:space="preserve" y="112.19">Application D</text>

  <rect fill="#aaffff" height="30.63" id="svg_42" stroke="#000000" width="109.37" x="89.49" y="115.61">Application A</rect>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_43" stroke="#000000" stroke-width="0" text-anchor="middle" x="143.11" xml:space="preserve" y="136.56">Application C</text>

  <rect fill="#aaffff" height="31.25" id="svg_44" stroke="#000000" width="109.37" x="114.49" y="139.99">Application A</rect>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_45" stroke="#000000" stroke-width="0" style="cursor: move;" text-anchor="middle" x="169.36" xml:space="preserve" y="160.31">Application B</text>

  <rect fill="#aaffff" height="30.63" id="svg_46" stroke="#000000" width="109.37" x="145.74" y="164.99">Application A</rect>

  <text fill="#000000" font-family="Serif" font-size="18" id="svg_47" stroke="#000000" stroke-width="0" text-anchor="middle" x="201.24" xml:space="preserve" y="186.56">Application A</text>

  <text fill="#000000" font-family="Serif" font-size="21" font-weight="bold" id="svg_50" stroke="#000000" stroke-width="0" text-anchor="middle" transform="matrix(1 0 0 1 0 0)" x="161.4" xml:space="preserve" y="82.24">ublk Workload</text>

  <text fill="#000000" font-family="Serif" font-size="19" font-style="italic" font-weight="normal" id="svg_51" stroke="#000000" stroke-width="0" text-anchor="middle" x="602.65" xml:space="preserve" y="222.24">Kernel Space</text>

  <text fill="#000000" font-family="Serif" font-size="19" font-style="italic" font-weight="normal" id="svg_52" stroke="#000000" stroke-width="0" text-anchor="middle" transform="matrix(1 0 0 1 0 0)" x="602.03" xml:space="preserve" y="188.49">Userspace</text>

 </g>

</svg>

# ublk Target {#ublk}

## Table of Contents {#ublk_toc}

- @ref ublk_intro

- @ref ublk_internal

- @ref ublk_impl

- @ref ublk_op

## Introduction {#ublk_intro}

[ublk](https://docs.kernel.org/block/ublk.html) (or ubd) is a generic framework for

implementing generic userspace block device based on `io_uring`.  It is designed to

create a highly efficient data path for userspace storage software to provide

high-performance block device service in local host.

The whole ublk service involves three parts: ublk driver, ublk server and ublk workload.

![ublk service stack](img/ublk_service.svg)

* __ublk driver__ is a kernel driver added to kernel 6.0.  It delivers I/O requests

  from a ublk block device(`/dev/ublkbN`) into a ublk server.

* __ublk workload__ can be any local host process which submits I/O requests to a ublk

  block device or a kernel filesystem on top of the ublk block device.

* __ublk server__ is the userspace storage software that fetches the I/O requests delivered

  by the ublk driver.  The ublk server will process the I/O requests with its specific block

  service logic and connected backends.  Once the ublk server gets the response from the

  connected backends, it communicates with the ublk driver and completes the I/O requests.

SPDK ublk target acts as a ublk server.  It can handle ublk I/O requests within the whole

SPDK userspace storage software stack.

A typical usage scenario is for container attached storage:

* Real storage resources are assigned to SPDK, like physical NVMe devices and

  distributed block storage.

* SPDK creates refined block devices via ublk kernel module on top of its organized

  storage resources, based on user configuration.

* Container orchestrator and runtime can then mount and stage the ublk block devices

  for container instances to use.

## ublk Internal {#ublk_internal}

Previously, the design of putting I/O processing logic into userspace software always has a

noticeable interaction overhead between the kernel module and userspace part.

ublk utilizes `io_uring` which has been proven to be very efficient in decreasing the

interaction overhead.  The I/O request is delivered to the userspace ublk server via the

newly added `io_uring` command.  A shared buffer via `mmap` is used for sharing I/O descriptor

to userspace from the kernel driver.  The I/O data is copied only once between the specified

userspace buffer address and request/bio's pages by the ublk driver.

### Control Plane

A control device is create by ublk kernel module at `/dev/ublk-control`.  Userspace server

sends control commands to kernel module via the control device using `io_uring`.

Control commands includes add, configure, and start new ublk block device.

Retrieving device information, stop and delete existing ublk block device are also there.

The add device command creates a bulk char device `/dev/ublkcN`.

It will be used by the ublk userspace server to `mmap` I/O descriptor buffer.

The start device command exposes a ublk block device `/dev/ublkbN`.

The block device can be formatted and mounted by a kernel filesystem,

or read/written directly by other processes.

### Data Plane

The datapath between ublk server and kernel driver includes `io_uring` and shared

memory buffer.  The shared memory buffer is an array of I/O descriptors.

Each SQE (Submission Queue Entry) in `io_uring` is assigned one I/O descriptor and

one user buffer address.  When ublk kernel driver receives I/O requests from upper

layer, the information of I/O requests will be filled into I/O descriptors by ublk

kernel driver.  The I/O data is copied between the specified user buffer address and

request/bio's pages at the proper time.

At start, the ublk server needs to fill the `io_uring` SQ (Submission Queue).  Each

SQE is marked with an operation flag `UBLK_IO_FETCH_REQ` which means the SQE is

ready to get I/O request.

When a CQE (Completion Queue Entry) is returned from the `io_uring` indicating I/O

request, the ublk server gets the position of the I/O descriptor from CQE.

The ublk server handles the I/O request based on information in the I/O descriptor.

After the ublk server completes the I/O request, it updates the I/O's completion status

and ublk operation flag.  This time, the operation flag is `UBLK_IO_COMMIT_AND_FETCH_REQ`

which informs kernel module that one I/O request is completed, and also the SQE slot

is free to fetch new I/O request.

`UBLK_IO_COMMIT_AND_FETCH_REQ` is designed for efficiency in ublk. In runtime, the ublk

server needs to commit I/O results back, and then provide new free SQE slots for fetching

new I/O requests.  Without `UBLK_IO_COMMIT_AND_FETCH_REQ` flag, `io_uring_submit()` should

be called twice,  once for committing I/O results back, once for providing free SQE slots.

With `UBLK_IO_COMMIT_AND_FETCH_REQ` flag, calling `io_uring_submit()` once is enough because

the ublk driver realizes that the submitted SQEs are reused both for committing back I/O

results and fetching new requests.

## SPDK Implementation {#ublk_impl}

SPDK ublk target is implemented as a high performance ublk server.

It creates one ublk spdk_thread on each spdk_reactor by default or on user specified

reactors.  When adding a new ublk block device, SPDK ublk target will assign queues

of ublk block device to ublk spdk_threads in round-robin.

That means one ublk device queue will only be processed by one spdk_thread.

One ublk device with multiple queues can get multiple spdk reactors involved

to process its I/O requests;

One spdk_thread created by ublk target may process multiple queues, each from

different ublk devices.

In this way, spdk reactors can be fully utilized to achieve best performance,

when there are only a few ublk devices.

ublk is `io_uring` based. All ublk I/O queues are mapped to `io_uring`.

ublk spdk_thread gets I/O requests from available CQEs by polling all its assigned

`io_uring`s.

When there are completed I/O requests, ublk spdk_thread will submit them as SQE back

to `io_uring` in batch.

Currently, ublk driver has a system thread context limitation that one ublk device queue

can be only processed in the context of system thread which initialized the it.  SPDK

can't schedule ublk spdk_thread between different SPDK reactors.  In other words, SPDK

dynamic scheduler can't rebalance ublk workload by rescheduling ublk spdk_thread.

## Operation {#ublk_op}

### Enabling SPDK ublk target

Build SPDK with SPDK ublk target enabled.

~~~{.sh}

./configure --with-ublk

make -j

~~~

SPDK ublk target related libaries will then be linked into SPDK appliation `spdk_tgt`.

Setup some hugepages for the SPDK, and then run the SPDK application `spdk_tgt`.

~~~{.sh}

scripts/setup.sh

build/bin/spdk_tgt &

~~~

Once the `spdk_tgt` is initialized, user can enable SPDK ublk feature

by creating ublk target. However, before creating ublk target, ublk kernel module

`ublk_drv` should be loaded using `modprobe`.

~~~{.sh}

modprobe ublk_drv

scripts/rpc.py ublk_create_target

~~~

### Creating ublk block device

SPDK bdevs are block devices which will be exposed to the local host kernel

as ublk block devices.  SPDK supports several different types of storage backends,

including NVMe, Linux AIO, malloc ramdisk and Ceph RBD.  Refer to @ref bdev for

additional information on configuring SPDK storage backends.

This guide will use a malloc bdev (ramdisk) named Malloc0. The following RPC

will create a 256MB malloc bdev with 512-byte block size.

~~~{.sh}

scripts/rpc.py bdev_malloc_create 256 512 -b Malloc0

~~~

The following RPC will create a ublk block device exposing Malloc0 bdev.

The created ublk block device has ID 1.  It internally has 2 queues with

queue depth 128.

~~~{.sh}

scripts/rpc.py ublk_start_disk Malloc0 1 -q 2 -d 128

~~~

This RPC will reply back the ID of ublk block device.

~~~

1

~~~

The position of ublk block device is determined by its ID. It is created at `/dev/ublkb${ID}`.

So the device we just created will be accessible to other processes via `/dev/ublkb1`.

Now applications like FIO or DD can work on `/dev/ublkb1` directly.

~~~{.sh}

dd of=/dev/ublkb1 if=/dev/zero bs=512 count=64

~~~

A ublk block device is a generic kernel block device that can be formatted and

mounted by kernel file system.

~~~{.sh}

mkfs /dev/ublkb1

mount /dev/ublkb1 /mnt/

mkdir /mnt/testdir

echo "Hello，SPDK ublk Target" > /mnt/testdir/testfile

umount /mnt

~~~

### Deleting ublk block device and exit

After usage, ublk block device can be stopped and deleted by RPC `ublk_stop_disk` with its ID.

Specify ID 1, then device `/dev/ublkb1` will be removed.

~~~{.sh}

scripts/rpc.py ublk_stop_disk 1

~~~

If ublk is not used anymore, SPDK ublk target can be destroyed to free related SPDK

resources.

~~~{.sh}

scripts/rpc.py ublk_destroy_target

~~~

Of course, SPDK ublk target and all ublk block devices would be destroyed automatically

when SPDK application is terminated.

- @subpage usdt

- @subpage nvme_multipath

- @subpage sma

- @subpage ublk