module/raid: write initial superblock (77b8f7b6) · Commits · Public Repositories / spdk

doc/bdev.md

+5 −4

Original line number	Diff line number	Diff line
		@@ -536,10 +536,11 @@ Example commands
		## RAID {#bdev_ug_raid}

		RAID virtual bdev module provides functionality to combine any SPDK bdevs into
		one RAID bdev. Currently SPDK supports only RAID 0. RAID functionality does not
		store on-disk metadata on the member disks, so user must recreate the RAID
		volume when restarting application. User may specify member disks to create RAID
		volume event if they do not exists yet - as the member disks are registered at
		one RAID bdev. Currently SPDK supports only RAID 0. RAID metadata may be stored
		on member disks if enabled when creating the RAID bdev, so user does not have to
		recreate the RAID volume when restarting application. It is not enabled by
		default for backward compatibility. User may specify member disks to create
		RAID volume even if they do not exist yet - as the member disks are registered at
		a later time, the RAID module will claim them and will surface the RAID volume
		after all of the member disks are available. It is allowed to use disks of
		different sizes - the smallest disk size will be the amount of space used on

module/bdev/raid/Makefile

+1 −1

Original line number	Diff line number	Diff line
		@@ -10,7 +10,7 @@ SO_VER := 5
		SO_MINOR := 0

		CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/
		C_SRCS = bdev_raid.c bdev_raid_rpc.c raid0.c raid1.c concat.c
		C_SRCS = bdev_raid.c bdev_raid_rpc.c bdev_raid_sb.c raid0.c raid1.c concat.c

		ifeq ($(CONFIG_RAID5F),y)
		C_SRCS += raid5f.c

module/bdev/raid/bdev_raid.c

+124 −49

Original line number	Diff line number	Diff line
		@@ -189,6 +189,7 @@ raid_bdev_cleanup(struct raid_bdev *raid_bdev)
		static void
		raid_bdev_free(struct raid_bdev *raid_bdev)
		{
		spdk_dma_free(raid_bdev->sb);
		spdk_spin_destroy(&raid_bdev->base_bdev_lock);
		free(raid_bdev->base_bdev_info);
		free(raid_bdev->bdev.name);
		@@ -208,8 +209,7 @@ raid_bdev_cleanup_and_free(struct raid_bdev *raid_bdev)
		* params:
		* base_info - raid base bdev info
		* returns:
		* 0 - success
		* non zero - failure
		* none
		*/
		static void
		raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
		@@ -228,6 +228,8 @@ raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
		spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(base_info->desc));
		spdk_bdev_close(base_info->desc);
		base_info->desc = NULL;
		spdk_put_io_channel(base_info->app_thread_ch);
		base_info->app_thread_ch = NULL;

		assert(raid_bdev->num_base_bdevs_discovered);
		raid_bdev->num_base_bdevs_discovered--;
		@@ -606,7 +608,7 @@ raid_bdev_write_info_json(struct raid_bdev *raid_bdev, struct spdk_json_write_ct
		spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
		spdk_json_write_named_string(w, "state", raid_bdev_state_to_str(raid_bdev->state));
		spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
		spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
		spdk_json_write_named_bool(w, "superblock", raid_bdev->sb != NULL);
		spdk_json_write_named_uint32(w, "num_base_bdevs", raid_bdev->num_base_bdevs);
		spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", raid_bdev->num_base_bdevs_discovered);
		spdk_json_write_name(w, "base_bdevs_list");
		@@ -664,7 +666,7 @@ raid_bdev_write_config_json(struct spdk_bdev bdev, struct spdk_json_write_ctx

		assert(spdk_get_thread() == spdk_thread_get_app_thread());

		if (raid_bdev->superblock_enabled) {
		if (raid_bdev->sb != NULL) {
		/* raid bdev configuration is stored in the superblock */
		return;
		}
		@@ -679,7 +681,7 @@ raid_bdev_write_config_json(struct spdk_bdev bdev, struct spdk_json_write_ctx
		spdk_json_write_named_string(w, "uuid", uuid_str);
		spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
		spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
		spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
		spdk_json_write_named_bool(w, "superblock", raid_bdev->sb != NULL);

		spdk_json_write_named_array_begin(w, "base_bdevs");
		RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
		@@ -936,23 +938,8 @@ raid_bdev_init(void)
		return 0;
		}

		/*
		* brief:
		* raid_bdev_create allocates raid bdev based on passed configuration
		* params:
		* name - name for raid bdev
		* strip_size - strip size in KB
		* num_base_bdevs - number of base bdevs
		* level - raid level
		* superblock_enabled - true if raid should have superblock
		* uuid - uuid to set for the bdev
		* raid_bdev_out - the created raid bdev
		* returns:
		* 0 - success
		* non zero - failure
		*/
		int
		raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		static int
		_raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
		struct raid_bdev **raid_bdev_out)
		{
		@@ -962,6 +949,11 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		struct raid_base_bdev_info *base_info;
		uint8_t min_operational;

		if (strnlen(name, RAID_BDEV_SB_NAME_SIZE) == RAID_BDEV_SB_NAME_SIZE) {
		SPDK_ERRLOG("Raid bdev name '%s' exceeds %d characters\n", name, RAID_BDEV_SB_NAME_SIZE - 1);
		return -EINVAL;
		}

		if (raid_bdev_find_by_name(name) != NULL) {
		SPDK_ERRLOG("Duplicate raid bdev name found: %s\n", name);
		return -EEXIST;
		@@ -1048,7 +1040,15 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
		raid_bdev->level = level;
		raid_bdev->min_base_bdevs_operational = min_operational;
		raid_bdev->superblock_enabled = superblock_enabled;

		if (superblock_enabled) {
		raid_bdev->sb = spdk_dma_zmalloc(RAID_BDEV_SB_MAX_LENGTH, 0x1000, NULL);
		if (!raid_bdev->sb) {
		SPDK_ERRLOG("Failed to allocate raid bdev sb buffer\n");
		raid_bdev_free(raid_bdev);
		return -ENOMEM;
		}
		}

		raid_bdev_gen = &raid_bdev->bdev;

		@@ -1073,6 +1073,47 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		return 0;
		}

		/*
		* brief:
		* raid_bdev_create allocates raid bdev based on passed configuration
		* params:
		* name - name for raid bdev
		* strip_size - strip size in KB
		* num_base_bdevs - number of base bdevs
		* level - raid level
		* superblock_enabled - true if raid should have superblock
		* uuid - uuid to set for the bdev
		* raid_bdev_out - the created raid bdev
		* returns:
		* 0 - success
		* non zero - failure
		*/
		int
		raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
		enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
		struct raid_bdev **raid_bdev_out)
		{
		struct raid_bdev *raid_bdev;
		int rc;

		assert(uuid != NULL);

		rc = _raid_bdev_create(name, strip_size, num_base_bdevs, level, superblock_enabled, uuid,
		&raid_bdev);
		if (rc != 0) {
		return rc;
		}

		if (superblock_enabled && spdk_uuid_is_null(uuid)) {
		/* we need to have the uuid to store in the superblock before the bdev is registered */
		spdk_uuid_generate(&raid_bdev->bdev.uuid);
		}

		*raid_bdev_out = raid_bdev;

		return 0;
		}

		/*
		* brief:
		* Check underlying block devices against support for metadata. Do not configure
		@@ -1116,6 +1157,48 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev)
		return 0;
		}

		static void
		raid_bdev_configure_cont(struct raid_bdev *raid_bdev)
		{
		struct spdk_bdev *raid_bdev_gen = &raid_bdev->bdev;
		int rc;

		raid_bdev->state = RAID_BDEV_STATE_ONLINE;
		SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
		SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
		raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
		spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
		sizeof(struct raid_bdev_io_channel),
		raid_bdev_gen->name);
		rc = spdk_bdev_register(raid_bdev_gen);
		if (rc != 0) {
		SPDK_ERRLOG("Unable to register raid bdev and stay at configuring state\n");
		if (raid_bdev->module->stop != NULL) {
		raid_bdev->module->stop(raid_bdev);
		}
		spdk_io_device_unregister(raid_bdev, NULL);
		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
		return;
		}
		SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
		SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
		raid_bdev_gen->name, raid_bdev);
		}

		static void
		raid_bdev_configure_write_sb_cb(int status, struct raid_bdev raid_bdev, void ctx)
		{
		if (status == 0) {
		raid_bdev_configure_cont(raid_bdev);
		} else {
		SPDK_ERRLOG("Failed to write raid bdev '%s' superblock: %s\n",
		raid_bdev->bdev.name, spdk_strerror(-status));
		if (raid_bdev->module->stop != NULL) {
		raid_bdev->module->stop(raid_bdev);
		}
		}
		}

		/*
		* brief:
		* If raid bdev config is complete, then only register the raid bdev to
		@@ -1131,7 +1214,6 @@ static int
		raid_bdev_configure(struct raid_bdev *raid_bdev)
		{
		uint32_t blocklen = 0;
		struct spdk_bdev *raid_bdev_gen;
		struct raid_base_bdev_info *base_info;
		struct spdk_bdev *base_bdev;
		int rc = 0;
		@@ -1166,9 +1248,7 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
		}
		raid_bdev->strip_size_shift = spdk_u32log2(raid_bdev->strip_size);
		raid_bdev->blocklen_shift = spdk_u32log2(blocklen);

		raid_bdev_gen = &raid_bdev->bdev;
		raid_bdev_gen->blocklen = blocklen;
		raid_bdev->bdev.blocklen = blocklen;

		rc = raid_bdev_configure_md(raid_bdev);
		if (rc != 0) {
		@@ -1181,26 +1261,13 @@ raid_bdev_configure(struct raid_bdev *raid_bdev)
		SPDK_ERRLOG("raid module startup callback failed\n");
		return rc;
		}
		raid_bdev->state = RAID_BDEV_STATE_ONLINE;
		SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
		SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
		raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
		spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
		sizeof(struct raid_bdev_io_channel),
		raid_bdev->bdev.name);
		rc = spdk_bdev_register(raid_bdev_gen);
		if (rc != 0) {
		SPDK_ERRLOG("Unable to register raid bdev and stay at configuring state\n");
		if (raid_bdev->module->stop != NULL) {
		raid_bdev->module->stop(raid_bdev);
		}
		spdk_io_device_unregister(raid_bdev, NULL);
		raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
		return rc;

		if (raid_bdev->sb != NULL) {
		raid_bdev_init_superblock(raid_bdev);
		raid_bdev_write_superblock(raid_bdev, raid_bdev_configure_write_sb_cb, NULL);
		} else {
		raid_bdev_configure_cont(raid_bdev);
		}
		SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
		SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
		raid_bdev_gen->name, raid_bdev);

		return 0;
		}
		@@ -1555,6 +1622,14 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)

		assert(raid_bdev->state != RAID_BDEV_STATE_ONLINE);

		base_info->app_thread_ch = spdk_bdev_get_io_channel(desc);
		if (base_info->app_thread_ch == NULL) {
		SPDK_ERRLOG("Failed to get io channel\n");
		spdk_bdev_module_release_bdev(bdev);
		spdk_bdev_close(desc);
		return -ENOMEM;
		}

		base_info->desc = desc;
		base_info->blockcnt = bdev->blockcnt;
		base_info->data_offset = 0;
		@@ -1562,7 +1637,7 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)
		raid_bdev->num_base_bdevs_discovered++;
		assert(raid_bdev->num_base_bdevs_discovered <= raid_bdev->num_base_bdevs);

		if (raid_bdev->superblock_enabled) {
		if (raid_bdev->sb != NULL) {
		assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % bdev->blocklen) == 0);
		base_info->data_offset = RAID_BDEV_MIN_DATA_OFFSET_SIZE / bdev->blocklen;

module/bdev/raid/bdev_raid.h

+95 −3

Original line number	Diff line number	Diff line
		@@ -81,6 +81,9 @@ struct raid_base_bdev_info {

		/* Hold the number of blocks to know how large the base bdev is resized. */
		uint64_t blockcnt;

		/* io channel for the app thread */
		struct spdk_io_channel *app_thread_ch;
		};

		/*
		@@ -154,14 +157,14 @@ struct raid_bdev {
		/* Set to true if destroy of this raid bdev is started. */
		bool destroy_started;

		/* Set to true if superblock metadata is enabled on this raid bdev */
		bool superblock_enabled;

		/* Module for RAID-level specific operations */
		struct raid_bdev_module *module;

		/* Private data for the raid module */
		void *module_private;

		/* Superblock */
		struct raid_bdev_superblock *sb;
		};

		#define RAID_FOR_EACH_BASE_BDEV(r, i) \
		@@ -338,4 +341,93 @@ raid_bdev_flush_blocks(struct raid_base_bdev_info *base_info, struct spdk_io_cha
		num_blocks, cb, cb_arg);
		}

		/*
		* Definitions related to raid bdev superblock
		*/

		#define RAID_BDEV_SB_VERSION_MAJOR 1
		#define RAID_BDEV_SB_VERSION_MINOR 0

		#define RAID_BDEV_SB_NAME_SIZE 64

		enum raid_bdev_sb_base_bdev_state {
		RAID_SB_BASE_BDEV_MISSING = 0,
		RAID_SB_BASE_BDEV_CONFIGURED = 1,
		RAID_SB_BASE_BDEV_FAILED = 2,
		RAID_SB_BASE_BDEV_SPARE = 3,
		};

		struct raid_bdev_sb_base_bdev {
		/* uuid of the base bdev */
		struct spdk_uuid uuid;
		/* offset in blocks from base device start to the start of raid data area */
		uint64_t data_offset;
		/* size in blocks of the base device raid data area */
		uint64_t data_size;
		/* state of the base bdev */
		uint32_t state;
		/* feature/status flags */
		uint32_t flags;
		/* slot number of this base bdev in the raid */
		uint8_t slot;

		uint8_t reserved[23];
		};
		SPDK_STATIC_ASSERT(sizeof(struct raid_bdev_sb_base_bdev) == 64, "incorrect size");

		struct raid_bdev_superblock {
		#define RAID_BDEV_SB_SIG "SPDKRAID"
		uint8_t signature[8];
		struct {
		/* incremented when a breaking change in the superblock structure is made */
		uint16_t major;
		/* incremented for changes in the superblock that are backward compatible */
		uint16_t minor;
		} version;
		/* length in bytes of the entire superblock */
		uint32_t length;
		/* crc32c checksum of the entire superblock */
		uint32_t crc;
		/* feature/status flags */
		uint32_t flags;
		/* unique id of the raid bdev */
		struct spdk_uuid uuid;
		/* name of the raid bdev */
		uint8_t name[RAID_BDEV_SB_NAME_SIZE];
		/* size of the raid bdev in blocks */
		uint64_t raid_size;
		/* the raid bdev block size - must be the same for all base bdevs */
		uint32_t block_size;
		/* the raid level */
		uint32_t level;
		/* strip (chunk) size in blocks */
		uint32_t strip_size;
		/* state of the raid */
		uint32_t state;
		/* sequence number, incremented on every superblock update */
		uint64_t seq_number;
		/* number of raid base devices */
		uint8_t num_base_bdevs;

		uint8_t reserved[118];

		/* size of the base bdevs array */
		uint8_t base_bdevs_size;
		/* array of base bdev descriptors */
		struct raid_bdev_sb_base_bdev base_bdevs[];
		};
		SPDK_STATIC_ASSERT(sizeof(struct raid_bdev_superblock) == 256, "incorrect size");

		#define RAID_BDEV_SB_MAX_LENGTH \
		SPDK_ALIGN_CEIL((sizeof(struct raid_bdev_superblock) + UINT8_MAX * sizeof(struct raid_bdev_sb_base_bdev)), 0x1000)

		SPDK_STATIC_ASSERT(RAID_BDEV_SB_MAX_LENGTH < RAID_BDEV_MIN_DATA_OFFSET_SIZE,
		"Incorrect min data offset");

		typedef void (raid_bdev_write_sb_cb)(int status, struct raid_bdev raid_bdev, void *ctx);

		void raid_bdev_init_superblock(struct raid_bdev *raid_bdev);
		void raid_bdev_write_superblock(struct raid_bdev *raid_bdev, raid_bdev_write_sb_cb cb,
		void *cb_ctx);

		#endif /* SPDK_BDEV_RAID_INTERNAL_H */

module/bdev/raid/bdev_raid_sb.c

0 → 100644

+158 −0

Original line number	Diff line number	Diff line
		/* SPDX-License-Identifier: BSD-3-Clause
		* Copyright (C) 2022 Intel Corporation.
		* All rights reserved.
		*/

		#include "spdk/bdev_module.h"
		#include "spdk/crc32.h"
		#include "spdk/env.h"
		#include "spdk/log.h"
		#include "spdk/string.h"
		#include "spdk/util.h"

		#include "bdev_raid.h"

		struct raid_bdev_write_sb_ctx {
		struct raid_bdev *raid_bdev;
		int status;
		uint64_t nbytes;
		uint8_t submitted;
		uint8_t remaining;
		raid_bdev_write_sb_cb cb;
		void *cb_ctx;
		struct spdk_bdev_io_wait_entry wait_entry;
		};

		void
		raid_bdev_init_superblock(struct raid_bdev *raid_bdev)
		{
		struct raid_bdev_superblock *sb = raid_bdev->sb;
		struct raid_base_bdev_info *base_info;
		struct raid_bdev_sb_base_bdev *sb_base_bdev;

		memset(sb, 0, RAID_BDEV_SB_MAX_LENGTH);

		memcpy(&sb->signature, RAID_BDEV_SB_SIG, sizeof(sb->signature));
		sb->version.major = RAID_BDEV_SB_VERSION_MAJOR;
		sb->version.minor = RAID_BDEV_SB_VERSION_MINOR;
		spdk_uuid_copy(&sb->uuid, &raid_bdev->bdev.uuid);
		snprintf(sb->name, RAID_BDEV_SB_NAME_SIZE, "%s", raid_bdev->bdev.name);
		sb->raid_size = raid_bdev->bdev.blockcnt;
		sb->block_size = raid_bdev->bdev.blocklen;
		sb->level = raid_bdev->level;
		sb->strip_size = raid_bdev->strip_size;
		/* TODO: sb->state */
		sb->num_base_bdevs = sb->base_bdevs_size = raid_bdev->num_base_bdevs;
		sb->length = sizeof(sb) + sizeof(sb_base_bdev) * sb->base_bdevs_size;

		sb_base_bdev = &sb->base_bdevs[0];
		RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
		spdk_uuid_copy(&sb_base_bdev->uuid, spdk_bdev_get_uuid(spdk_bdev_desc_get_bdev(base_info->desc)));
		sb_base_bdev->data_offset = base_info->data_offset;
		sb_base_bdev->data_size = base_info->data_size;
		sb_base_bdev->state = RAID_SB_BASE_BDEV_CONFIGURED;
		sb_base_bdev->slot = base_info - raid_bdev->base_bdev_info;
		sb_base_bdev++;
		}
		}

		static void
		raid_bdev_sb_update_crc(struct raid_bdev_superblock *sb)
		{
		sb->crc = 0;
		sb->crc = spdk_crc32c_update(sb, sb->length, 0);
		}

		static void
		raid_bdev_write_sb_base_bdev_done(int status, struct raid_bdev_write_sb_ctx *ctx)
		{
		if (status != 0) {
		ctx->status = status;
		}

		if (--ctx->remaining == 0) {
		ctx->cb(ctx->status, ctx->raid_bdev, ctx->cb_ctx);
		free(ctx);
		}
		}

		static void
		raid_bdev_write_superblock_cb(struct spdk_bdev_io bdev_io, bool success, void cb_arg)
		{
		struct raid_bdev_write_sb_ctx *ctx = cb_arg;
		int status = 0;

		if (!success) {
		SPDK_ERRLOG("Failed to save superblock on bdev %s\n", bdev_io->bdev->name);
		status = -EIO;
		}

		spdk_bdev_free_io(bdev_io);

		raid_bdev_write_sb_base_bdev_done(status, ctx);
		}

		static void
		_raid_bdev_write_superblock(void *_ctx)
		{
		struct raid_bdev_write_sb_ctx *ctx = _ctx;
		struct raid_bdev *raid_bdev = ctx->raid_bdev;
		struct raid_base_bdev_info *base_info;
		uint8_t i;
		int rc;

		for (i = ctx->submitted; i < raid_bdev->num_base_bdevs; i++) {
		base_info = &raid_bdev->base_bdev_info[i];
		rc = spdk_bdev_write(base_info->desc, base_info->app_thread_ch,
		(void *)raid_bdev->sb, 0, ctx->nbytes,
		raid_bdev_write_superblock_cb, ctx);
		if (rc != 0) {
		struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(base_info->desc);

		if (rc == -ENOMEM) {
		ctx->wait_entry.bdev = bdev;
		ctx->wait_entry.cb_fn = _raid_bdev_write_superblock;
		ctx->wait_entry.cb_arg = ctx;
		spdk_bdev_queue_io_wait(bdev, base_info->app_thread_ch, &ctx->wait_entry);
		return;
		}

		assert(ctx->remaining > 1);
		raid_bdev_write_sb_base_bdev_done(rc, ctx);
		}

		ctx->submitted++;
		}

		raid_bdev_write_sb_base_bdev_done(0, ctx);
		}

		void
		raid_bdev_write_superblock(struct raid_bdev raid_bdev, raid_bdev_write_sb_cb cb, void cb_ctx)
		{
		struct raid_bdev_write_sb_ctx *ctx;
		struct raid_bdev_superblock *sb = raid_bdev->sb;

		assert(spdk_get_thread() == spdk_thread_get_app_thread());
		assert(sb != NULL);
		assert(cb != NULL);

		ctx = calloc(1, sizeof(*ctx));
		if (!ctx) {
		cb(-ENOMEM, raid_bdev, cb_ctx);
		return;
		}

		ctx->raid_bdev = raid_bdev;
		ctx->nbytes = SPDK_ALIGN_CEIL(sb->length, spdk_bdev_get_block_size(&raid_bdev->bdev));
		ctx->remaining = raid_bdev->num_base_bdevs + 1;
		ctx->cb = cb;
		ctx->cb_ctx = cb_ctx;

		sb->seq_number++;
		raid_bdev_sb_update_crc(sb);

		_raid_bdev_write_superblock(ctx);
		}

		SPDK_LOG_REGISTER_COMPONENT(bdev_raid_sb)