Commit b0ad3322 authored by Ziye Yang's avatar Ziye Yang Committed by Ben Walker
Browse files

bdev/nvme: RPC call can now specify remote NVMe-oF devices 



Change-Id: Ief3cf71489f0725249d51851b8d3270cb4e034db
Signed-off-by: default avatarZiye Yang <ziye.yang@intel.com>
parent 0829424e

autotest.sh

+1 −0
Original line number Diff line number Diff line
@@ -135,6 +135,7 @@ if [ $(uname -s) = Linux ] && [ $SPDK_TEST_ISCSI -eq 1 ]; then 
	fi

	run_test ./test/iscsi_tgt/ext4test/ext4test.sh

	run_test ./test/iscsi_tgt/rbd/rbd.sh

	run_test ./test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

	timing_exit iscsi_tgt



	run_test test/lib/iscsi/iscsi.sh

lib/bdev/nvme/blockdev_nvme.c

+3 −0
Original line number Diff line number Diff line
@@ -543,6 +543,7 @@ probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 
	}



	if (!claim_device) {

		SPDK_ERRLOG("Not claiming device at %s\n", trid->traddr);

		return false;

	}
@@ -675,11 +676,13 @@ spdk_bdev_nvme_create(struct spdk_nvme_transport_id *trid, 
	probe_ctx.count = 1;

	probe_ctx.trids[0] = *trid;

	if (spdk_nvme_probe(trid, &probe_ctx, probe_cb, attach_cb, NULL)) {

		SPDK_ERRLOG("Failed to probe for new devices\n");

		return -1;

	}



	nvme_ctrlr = nvme_ctrlr_get(trid);

	if (!nvme_ctrlr) {

		SPDK_ERRLOG("Failed to find new NVMe controller\n");

		return -1;

	}

lib/bdev/nvme/blockdev_nvme_rpc.c

+48 −6
Original line number Diff line number Diff line
@@ -34,23 +34,38 @@ 
#include <string.h>



#include "blockdev_nvme.h"



#include "spdk/string.h"

#include "spdk/rpc.h"

#include "spdk/util.h"



#include "spdk_internal/log.h"



struct rpc_construct_nvme {

	char *pci_address;

	char *trtype;

	char *adrfam;

	char *traddr;

	char *trsvcid;

	char *subnqn;

};



static void

free_rpc_construct_nvme(struct rpc_construct_nvme *req)

{

	free(req->pci_address);

	free(req->trtype);

	free(req->adrfam);

	free(req->traddr);

	free(req->trsvcid);

	free(req->subnqn);

}



static const struct spdk_json_object_decoder rpc_construct_nvme_decoders[] = {

	{"pci_address", offsetof(struct rpc_construct_nvme, pci_address), spdk_json_decode_string},

	{"trtype", offsetof(struct rpc_construct_nvme, trtype), spdk_json_decode_string},

	{"traddr", offsetof(struct rpc_construct_nvme, traddr), spdk_json_decode_string},



	{"adrfam", offsetof(struct rpc_construct_nvme, adrfam), spdk_json_decode_string, true},

	{"trsvcid", offsetof(struct rpc_construct_nvme, trsvcid), spdk_json_decode_string, true},

	{"subnqn", offsetof(struct rpc_construct_nvme, subnqn), spdk_json_decode_string, true},

};



#define NVME_MAX_BLOCKDEVS_PER_RPC 32
@@ -66,16 +81,43 @@ spdk_rpc_construct_nvme_bdev(struct spdk_jsonrpc_server_conn *conn, 
	const char *names[NVME_MAX_BLOCKDEVS_PER_RPC];

	size_t count = 0;

	size_t i;

	int rc;



	if (spdk_json_decode_object(params, rpc_construct_nvme_decoders,

				    SPDK_COUNTOF(rpc_construct_nvme_decoders),

				    &req)) {

		SPDK_TRACELOG(SPDK_TRACE_DEBUG, "spdk_json_decode_object failed\n");

		SPDK_ERRLOG("spdk_json_decode_object failed\n");

		goto invalid;

	}



	trid.trtype = SPDK_NVME_TRANSPORT_PCIE;

	snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.pci_address);

	/* Parse trtype */

	rc = spdk_nvme_transport_id_parse_trtype(&trid.trtype, req.trtype);

	if (rc < 0) {

		SPDK_ERRLOG("Failed to parse trtype: %s\n", req.trtype);

		goto invalid;

	}



	/* Parse traddr */

	snprintf(trid.traddr, sizeof(trid.traddr), "%s", req.traddr);



	/* Parse adrfam */

	if (req.adrfam) {

		rc = spdk_nvme_transport_id_parse_adrfam(&trid.adrfam, req.adrfam);

		if (rc < 0) {

			SPDK_ERRLOG("Failed to parse adrfam: %s\n", req.adrfam);

			goto invalid;

		}

	}



	/* Parse trsvcid */

	if (req.trsvcid) {

		snprintf(trid.trsvcid, sizeof(trid.trsvcid), "%s", req.trsvcid);

	}



	/* Parse subnqn */

	if (req.subnqn) {

		snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", req.subnqn);

	}



	count = NVME_MAX_BLOCKDEVS_PER_RPC;

	if (spdk_bdev_nvme_create(&trid, names, &count)) {

scripts/rpc.py

+19 −3
Original line number Diff line number Diff line
@@ -173,10 +173,26 @@ p.add_argument('fname', help='Path to device or file (ex: /dev/sda)') 
p.set_defaults(func=construct_aio_bdev)



def construct_nvme_bdev(args):

    params = {'pci_address': args.pci_address}

    print_array(jsonrpc_call('construct_nvme_bdev', params))

    params = {'trtype': args.trtype,

              'traddr': args.traddr}



    if args.adrfam:

        params['adrfam'] = args.adrfam



    if args.trsvcid:

        params['trsvcid'] = args.trsvcid



    if args.subnqn:

        params['subnqn'] = args.subnqn



    jsonrpc_call('construct_nvme_bdev', params)



p = subparsers.add_parser('construct_nvme_bdev', help='Add bdev with nvme backend')

p.add_argument('pci_address', help='PCI address domain:bus:device.function')

p.add_argument('-t', '--trtype', help='NVMe-oF target trtype: e.g., rdma, pcie', required=True)

p.add_argument('-a', '--traddr', help='NVMe-oF target address: e.g., an ip address or BDF', required=True)

p.add_argument('-f', '--adrfam', help='NVMe-oF target adrfam: e.g., ipv4, ipv6, ib, fc, intra_host')

p.add_argument('-s', '--trsvcid', help='NVMe-oF target trsvcid: e.g., a port number')

p.add_argument('-n', '--subnqn', help='NVMe-oF target subnqn')

p.set_defaults(func=construct_nvme_bdev)



def construct_rbd_bdev(args):

test/iscsi_tgt/nvme_remote/fio_remote_nvme.sh

0 → 100755
+77 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



set -e



testdir=$(readlink -f $(dirname $0))

rootdir=$testdir/../../..

source $rootdir/scripts/autotest_common.sh

source $rootdir/test/nvmf/common.sh



if [ -z "$TARGET_IP" ]; then

	echo "TARGET_IP not defined in environment"

	exit 1

fi



if [ -z "$INITIATOR_IP" ]; then

	echo "INITIATOR_IP not defined in environment"

	exit 1

fi



if ! rdma_nic_available; then

	echo "no NIC for nvmf test"

	exit 0

fi



rpc_py="python $rootdir/scripts/rpc.py"

fio_py="python $rootdir/scripts/fio.py"



ISCSI_PORT=3260

NVMF_PORT=4420



timing_enter nvme_remote



# Start the NVMf target

$rootdir/app/nvmf_tgt/nvmf_tgt -c $rootdir/test/nvmf/nvmf.conf -m 0x2 -p 1 -s 512 &

nvmfpid=$!

echo "NVMf target launched. pid: $nvmfpid"

trap "killprocess $nvmfpid; exit 1" SIGINT SIGTERM EXIT

waitforlisten $nvmfpid 5260

echo "NVMf target has started."

bdevs=$($rpc_py construct_malloc_bdev 64 512)

$rpc_py construct_nvmf_subsystem Virtual nqn.2016-06.io.spdk:cnode1 'transport:RDMA traddr:192.168.100.8 trsvcid:4420' '' -s SPDK00000000000001 -n "$bdevs"

echo "NVMf subsystem created."



# Start the iSCSI target

$rootdir/app/iscsi_tgt/iscsi_tgt -c $testdir/iscsi.conf -m 0x1 -p 0 -s 512 &

iscsipid=$!

echo "iSCSI target launched. pid: $iscsipid"

trap "process_core; killprocess $iscsipid; exit 1" SIGINT SIGTERM EXIT

# The configuration file for the iSCSI target told it to use port 5261 for RPC

waitforlisten $iscsipid 5261

echo "iSCSI target has started."



echo "Creating an iSCSI target node."

$rpc_py -p 5261 add_portal_group 1 $TARGET_IP:$ISCSI_PORT

$rpc_py -p 5261 add_initiator_group 1 ALL $INITIATOR_IP/32

$rpc_py -p 5261 construct_nvme_bdev -t "rdma" -f "ipv4" -a 192.168.100.8 -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1

$rpc_py -p 5261 construct_target_node Target1 Target1_alias 'Nvme0n1:0' '1:1' 64 1 0 0 0

sleep 1



echo "Logging in to iSCSI target."

iscsiadm -m discovery -t sendtargets -p $TARGET_IP:$ISCSI_PORT

iscsiadm -m node --login -p $TARGET_IP:$ISCSI_PORT

trap "iscsicleanup; process_core; killprocess $pid; exit 1" SIGINT SIGTERM EXIT

sleep 1



echo "Running FIO"

$fio_py 4096 1 randrw 1 verify



rm -f ./local-job0-0-verify.state

trap - SIGINT SIGTERM EXIT



iscsicleanup

killprocess $iscsipid

$rpc_py delete_nvmf_subsystem nqn.2016-06.io.spdk:cnode1

killprocess $nvmfpid



timing_exit nvme_remote