Commit 3d2a3ee4 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Tomasz Zawadzki
Browse files

bdev/error: Support injecting data corruption into I/O buffer 



Support injecting data corruption for processing read/write I/O.

Add two parameters, corrupt_offset and corrupt_value, and an error
type corrupt_data.

For write I/O, inject data corruption before submitting write I/O
to the underlying bdev. For read I/O, inject data corruption after
read I/O successfully returned.

Cause the data corruption by xoring the value at the corrupt_offset
with the corrupt_value. corrupt_value is required to be non-zero to
cause data corruption necessarily.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I67d8d252b06d7a221413e74996f7f894e6519556
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15028


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Community-CI: Mellanox Build Bot
parent e0daee98

doc/jsonrpc.md

+3 −1
Original line number Diff line number Diff line
@@ -4596,8 +4596,10 @@ Name | Optional | Type | Description 
----------------------- | -------- | ----------- | -----------

name                    | Required | string      | Name of the error injection bdev

io_type                 | Required | string      | io type 'clear' 'read' 'write' 'unmap' 'flush' 'all'

error_type              | Required | string      | error type 'failure' 'pending'

error_type              | Required | string      | error type 'failure' 'pending' 'corrupt_data'

num                     | Optional | int         | the number of commands you want to fail.(default:1)

corrupt_offset          | Optional | int         | the offset in bytes to xor with corrupt_value

corrupt_value           | Optional | int         | the value for xor (1-255, 0 is invalid)



#### Example

module/bdev/error/vbdev_error.c

+59 −0
Original line number Diff line number Diff line
@@ -30,6 +30,8 @@ static TAILQ_HEAD(, spdk_vbdev_error_config) g_error_config 
struct vbdev_error_info {

	uint32_t			error_type;

	uint32_t			error_num;

	uint64_t			corrupt_offset;

	uint8_t				corrupt_value;

};



/* Context for each error bdev */
@@ -81,6 +83,14 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts) 
	uint32_t i;

	int rc = 0;



	if (opts->error_type == VBDEV_IO_CORRUPT_DATA) {

		if (opts->corrupt_value == 0) {

			/* If corrupt_value is 0, XOR cannot cause data corruption. */

			SPDK_ERRLOG("corrupt_value should be non-zero.\n");

			return -EINVAL;

		}

	}



	pthread_mutex_lock(&g_vbdev_error_mutex);



	rc = spdk_bdev_open_ext(name, false, dummy_bdev_event_cb, NULL, &desc);
@@ -109,6 +119,8 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts) 
		for (i = 0; i < SPDK_COUNTOF(error_disk->error_vector); i++) {

			error_disk->error_vector[i].error_type = opts->error_type;

			error_disk->error_vector[i].error_num = opts->error_num;

			error_disk->error_vector[i].corrupt_offset = opts->corrupt_offset;

			error_disk->error_vector[i].corrupt_value = opts->corrupt_value;

		}

	} else if (0 == opts->io_type) {

		for (i = 0; i < SPDK_COUNTOF(error_disk->error_vector); i++) {
@@ -117,6 +129,8 @@ vbdev_error_inject_error(char *name, const struct vbdev_error_inject_opts *opts) 
	} else {

		error_disk->error_vector[opts->io_type].error_type = opts->error_type;

		error_disk->error_vector[opts->io_type].error_num = opts->error_num;

		error_disk->error_vector[opts->io_type].corrupt_offset = opts->corrupt_offset;

		error_disk->error_vector[opts->io_type].corrupt_value = opts->corrupt_value;

	}



exit:
@@ -156,10 +170,46 @@ vbdev_error_get_error_type(struct error_disk *error_disk, uint32_t io_type) 
	return 0;

}



static void

vbdev_error_corrupt_io_data(struct spdk_bdev_io *bdev_io, uint64_t corrupt_offset,

			    uint8_t corrupt_value)

{

	uint8_t *buf;

	int i;



	if (bdev_io->u.bdev.iovs == NULL || bdev_io->u.bdev.iovs[0].iov_base == NULL) {

		return;

	}



	for (i = 0; i < bdev_io->u.bdev.iovcnt; i++) {

		if (bdev_io->u.bdev.iovs[i].iov_len > corrupt_offset) {

			buf = (uint8_t *)bdev_io->u.bdev.iovs[i].iov_base;



			buf[corrupt_offset] ^= corrupt_value;

			break;

		}



		corrupt_offset -= bdev_io->u.bdev.iovs[i].iov_len;

	}

}



static void

vbdev_error_complete_request(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)

{

	int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;

	struct error_disk *error_disk = bdev_io->bdev->ctxt;

	uint32_t error_type;



	if (success && bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {

		error_type = vbdev_error_get_error_type(error_disk, bdev_io->type);

		if (error_type == VBDEV_IO_CORRUPT_DATA) {

			error_disk->error_vector[bdev_io->type].error_num--;



			vbdev_error_corrupt_io_data(bdev_io,

						    error_disk->error_vector[bdev_io->type].corrupt_offset,

						    error_disk->error_vector[bdev_io->type].corrupt_value);

		}

	}



	spdk_bdev_io_complete(bdev_io, status);

}
@@ -187,6 +237,15 @@ vbdev_error_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bde 
		TAILQ_INSERT_TAIL(&error_disk->pending_ios, bdev_io, module_link);

		error_disk->error_vector[bdev_io->type].error_num--;

		break;

	case VBDEV_IO_CORRUPT_DATA:

		if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {

			error_disk->error_vector[bdev_io->type].error_num--;



			vbdev_error_corrupt_io_data(bdev_io,

						    error_disk->error_vector[bdev_io->type].corrupt_offset,

						    error_disk->error_vector[bdev_io->type].corrupt_value);

		}

	/* fallthrough */

	case 0:

		rc = spdk_bdev_part_submit_request_ext(&ch->part_ch, bdev_io,

						       vbdev_error_complete_request);

module/bdev/error/vbdev_error.h

+3 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 
enum vbdev_error_type {

	VBDEV_IO_FAILURE = 1,

	VBDEV_IO_PENDING,

	VBDEV_IO_CORRUPT_DATA,

};



typedef void (*spdk_delete_error_complete)(void *cb_arg, int bdeverrno);
@@ -38,6 +39,8 @@ struct vbdev_error_inject_opts { 
	uint32_t io_type;

	uint32_t error_type;

	uint32_t error_num;

	uint64_t corrupt_offset;

	uint8_t corrupt_value;

};



/**

module/bdev/error/vbdev_error_rpc.c

+4 −0
Original line number Diff line number Diff line
@@ -45,6 +45,8 @@ rpc_error_bdev_decode_error_type(const struct spdk_json_val *val, void *out) 
		*error_type = VBDEV_IO_FAILURE;

	} else if (spdk_json_strequal(val, "pending") == true) {

		*error_type = VBDEV_IO_PENDING;

	} else if (spdk_json_strequal(val, "corrupt_data") == true) {

		*error_type = VBDEV_IO_CORRUPT_DATA;

	} else {

		SPDK_NOTICELOG("Invalid parameter value: error_type\n");

		return -EINVAL;
@@ -153,6 +155,8 @@ static const struct spdk_json_object_decoder rpc_error_information_decoders[] = 
	{"io_type", offsetof(struct rpc_error_information, opts.io_type), rpc_error_bdev_decode_io_type},

	{"error_type", offsetof(struct rpc_error_information, opts.error_type), rpc_error_bdev_decode_error_type},

	{"num", offsetof(struct rpc_error_information, opts.error_num), spdk_json_decode_uint32, true},

	{"corrupt_offset", offsetof(struct rpc_error_information, opts.corrupt_offset), spdk_json_decode_uint64, true},

	{"corrupt_value", offsetof(struct rpc_error_information, opts.corrupt_value), spdk_json_decode_uint8, true},

};



static void

python/spdk/rpc/bdev.py

+9 −2
Original line number Diff line number Diff line
@@ -1505,14 +1505,17 @@ def bdev_get_histogram(client, name): 
    return client.call('bdev_get_histogram', params)





def bdev_error_inject_error(client, name, io_type, error_type, num):

def bdev_error_inject_error(client, name, io_type, error_type, num,

                            corrupt_offset, corrupt_value):

    """Inject an error via an error bdev.



    Args:

        name: name of error bdev

        io_type: one of "clear", "read", "write", "unmap", "flush", or "all"

        error_type: one of "failure" or "pending"

        error_type: one of "failure", "pending", or "corrupt_data"

        num: number of commands to fail

        corrupt_offset: offset in bytes to xor with corrupt_value

        corrupt_value: value for xor (1-255, 0 is invalid)

    """

    params = {

        'name': name,
@@ -1522,6 +1525,10 @@ def bdev_error_inject_error(client, name, io_type, error_type, num): 


    if num:

        params['num'] = num

    if corrupt_offset:

        params['corrupt_offset'] = corrupt_offset

    if corrupt_value:

        params['corrupt_value'] = corrupt_value



    return client.call('bdev_error_inject_error', params)