Commit 2796687d authored Nov 30, 2022 by Parameswaran Krishnamurthy Committed by Jim Harris Jan 12, 2023

nvme: Added support for TP-8009, Auto-discovery of Discovery controllers for...


nvme: Added support for TP-8009, Auto-discovery of Discovery controllers for NVME initiator using mDNS using Avahi

Approach:
Avahi Daemon needs to be running to provide the mDNS server service. In the SPDK, Avahi-client library based client API is implemented.
The client API will connect to the Avahi-daemon and receive events for new discovery and removal of an existing discovery entry.

Following sets on new RPCs have been introduced.

scripts/rpc.py bdev_nvme_start_mdns_discovery -b cdc_auto -s _nvme-disc._tcp

User shall initiate an mDNS based discovery using this RPC. This will start a Avahi-client based poller
looking for new discovery events from the Avahi server. On a new discovery of the discovery controller,
the existing bdev_nvme_start_discovery API will be invoked with the trid of the discovery controller learnt.
This will enable automatic connection of the initiator to the subsystems discovered from the discovery controller.
Multiple mdns discovery instances can be run by specifying a unique bdev-prefix and a unique servicename to discover as parameters.

scripts/rpc.py bdev_nvme_stop_mdns_discovery -b cdc_auto

This will stop the Avahi poller that was started for the specified service.Internally bdev_nvme_stop_discovery
API will be invoked for each of the discovery controllers learnt automatically by this instance of mdns discovery service.
This will result in termination of connections to all the subsystems learnt by this mdns discovery instance.

scripts/rpc.py bdev_nvme_get_mdns_discovery_info

This RPC will display the list of mdns discovery instances running and the trid of the controllers discovered by these instances.

Test Result:

root@ubuntu-pm-18-226:~/param-spdk/spdk/build/bin# ./nvmf_tgt -i 1 -s 2048 -m 0xF
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_start_mdns_discovery -b cdc_auto -s _nvme-disc._tcp
root@ubuntu-pm-18-226:~/param-spdk/spdk#
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_get_mdns_discovery_info
[
  {
    "name": "cdc_auto",
    "svcname": "_nvme-disc._tcp",
    "referrals": [
      {
        "name": "cdc_auto0",
        "trid": {
          "trtype": "TCP",
          "adrfam": "IPv4",
          "traddr": "66.1.2.21",
          "trsvcid": "8009",
          "subnqn": "nqn.2014-08.org.nvmexpress.discovery"
        }
      },
      {
        "name": "cdc_auto1",
        "trid": {
          "trtype": "TCP",
          "adrfam": "IPv4",
          "traddr": "66.1.1.21",
          "trsvcid": "8009",
          "subnqn": "nqn.2014-08.org.nvmexpress.discovery"
        }
      }
    ]
  }
]
root@ubuntu-pm-18-226:~/param-spdk/spdk#
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_get_discovery_info
[
  {
    "name": "cdc_auto0",
    "trid": {
      "trtype": "TCP",
      "adrfam": "IPv4",
      "traddr": "66.1.2.21",
      "trsvcid": "8009",
      "subnqn": "nqn.2014-08.org.nvmexpress.discovery"
    },
    "referrals": []
  },
  {
    "name": "cdc_auto1",
    "trid": {
      "trtype": "TCP",
      "adrfam": "IPv4",
      "traddr": "66.1.1.21",
      "trsvcid": "8009",
      "subnqn": "nqn.2014-08.org.nvmexpress.discovery"
    },
    "referrals": []
  }
]
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_get_bdevs
[
  {
    "name": "cdc_auto02n1",
    "aliases": [
      "600110d6-1681-1681-0403-000045805c45"
    ],
    "product_name": "NVMe disk",
    "block_size": 512,
    "num_blocks": 32768,
    "uuid": "600110d6-1681-1681-0403-000045805c45",
    "assigned_rate_limits": {
      "rw_ios_per_sec": 0,
      "rw_mbytes_per_sec": 0,
      "r_mbytes_per_sec": 0,
      "w_mbytes_per_sec": 0
    },
    "claimed": false,
    "zoned": false,
    "supported_io_types": {
      "read": true,
      "write": true,
      "unmap": true,
      "write_zeroes": true,
      "flush": true,
      "reset": true,
      "compare": true,
      "compare_and_write": true,
      "abort": true,
      "nvme_admin": true,
      "nvme_io": true
    },
    "driver_specific": {
      "nvme": [
        {
          "trid": {
            "trtype": "TCP",
            "adrfam": "IPv4",
            "traddr": "66.1.1.40",
            "trsvcid": "4420",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.0"
          },
          "ctrlr_data": {
            "cntlid": 3,
            "vendor_id": "0x0000",
            "model_number": "SANBlaze VLUN P3T0",
            "serial_number": "00-681681dc681681dc",
            "firmware_revision": "V10.5",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.0",
            "oacs": {
              "security": 0,
              "format": 1,
              "firmware": 0,
              "ns_manage": 1
            },
            "multi_ctrlr": true,
            "ana_reporting": true
          },
          "vs": {
            "nvme_version": "2.0"
          },
          "ns_data": {
            "id": 1,
            "ana_state": "optimized",
            "can_share": true
          }
        }
      ],
      "mp_policy": "active_passive"
    }
  },
  {
    "name": "cdc_auto00n1",
    "aliases": [
      "600110da-09a6-09a6-0302-00005eeb19b4"
    ],
    "product_name": "NVMe disk",
    "block_size": 512,
    "num_blocks": 2048,
    "uuid": "600110da-09a6-09a6-0302-00005eeb19b4",
    "assigned_rate_limits": {
      "rw_ios_per_sec": 0,
      "rw_mbytes_per_sec": 0,
      "r_mbytes_per_sec": 0,
      "w_mbytes_per_sec": 0
    },
    "claimed": false,
    "zoned": false,
    "supported_io_types": {
      "read": true,
      "write": true,
      "unmap": true,
      "write_zeroes": true,
      "flush": true,
      "reset": true,
      "compare": true,
      "compare_and_write": true,
      "abort": true,
      "nvme_admin": true,
      "nvme_io": true
    },
    "driver_specific": {
      "nvme": [
        {
          "trid": {
            "trtype": "TCP",
            "adrfam": "IPv4",
            "traddr": "66.1.2.40",
            "trsvcid": "4420",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.2.0"
          },
          "ctrlr_data": {
            "cntlid": 1,
            "vendor_id": "0x0000",
            "model_number": "SANBlaze VLUN P2T0",
            "serial_number": "00-ab09a6f5ab09a6f5",
            "firmware_revision": "V10.5",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.2.0",
            "oacs": {
              "security": 0,
              "format": 1,
              "firmware": 0,
              "ns_manage": 1
            },
            "multi_ctrlr": true,
            "ana_reporting": true
          },
          "vs": {
            "nvme_version": "2.0"
          },
          "ns_data": {
            "id": 1,
            "ana_state": "optimized",
            "can_share": true
          }
        }
      ],
      "mp_policy": "active_passive"
    }
  },
  {
    "name": "cdc_auto01n1",
    "aliases": [
      "600110d6-dce8-dce8-0403-00010b2d3d8c"
    ],
    "product_name": "NVMe disk",
    "block_size": 512,
    "num_blocks": 32768,
    "uuid": "600110d6-dce8-dce8-0403-00010b2d3d8c",
    "assigned_rate_limits": {
      "rw_ios_per_sec": 0,
      "rw_mbytes_per_sec": 0,
      "r_mbytes_per_sec": 0,
      "w_mbytes_per_sec": 0
    },
    "claimed": false,
    "zoned": false,
    "supported_io_types": {
      "read": true,
      "write": true,
      "unmap": true,
      "write_zeroes": true,
      "flush": true,
      "reset": true,
      "compare": true,
      "compare_and_write": true,
      "abort": true,
      "nvme_admin": true,
      "nvme_io": true
    },
    "driver_specific": {
      "nvme": [
        {
          "trid": {
            "trtype": "TCP",
            "adrfam": "IPv4",
            "traddr": "66.1.1.40",
            "trsvcid": "4420",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.1"
          },
          "ctrlr_data": {
            "cntlid": 3,
            "vendor_id": "0x0000",
            "model_number": "SANBlaze VLUN P3T1",
            "serial_number": "01-6ddce86d6ddce86d",
            "firmware_revision": "V10.5",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.1",
            "oacs": {
              "security": 0,
              "format": 1,
              "firmware": 0,
              "ns_manage": 1
            },
            "multi_ctrlr": true,
            "ana_reporting": true
          },
          "vs": {
            "nvme_version": "2.0"
          },
          "ns_data": {
            "id": 1,
            "ana_state": "optimized",
            "can_share": true
          }
        }
      ],
      "mp_policy": "active_passive"
    }
  },
  {
    "name": "cdc_auto01n2",
    "aliases": [
      "600110d6-dce8-dce8-0403-00010b2d3d8d"
    ],
    "product_name": "NVMe disk",
    "block_size": 512,
    "num_blocks": 32768,
    "uuid": "600110d6-dce8-dce8-0403-00010b2d3d8d",
    "assigned_rate_limits": {
      "rw_ios_per_sec": 0,
      "rw_mbytes_per_sec": 0,
      "r_mbytes_per_sec": 0,
      "w_mbytes_per_sec": 0
    },
    "claimed": false,
    "zoned": false,
    "supported_io_types": {
      "read": true,
      "write": true,
      "unmap": true,
      "write_zeroes": true,
      "flush": true,
      "reset": true,
      "compare": true,
      "compare_and_write": true,
      "abort": true,
      "nvme_admin": true,
      "nvme_io": true
    },
    "driver_specific": {
      "nvme": [
        {
          "trid": {
            "trtype": "TCP",
            "adrfam": "IPv4",
            "traddr": "66.1.1.40",
            "trsvcid": "4420",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.1"
          },
          "ctrlr_data": {
            "cntlid": 3,
            "vendor_id": "0x0000",
            "model_number": "SANBlaze VLUN P3T1",
            "serial_number": "01-6ddce86d6ddce86d",
            "firmware_revision": "V10.5",
            "subnqn": "nqn.2014-08.com.sanblaze:virtualun.virtualun.3.1",
            "oacs": {
              "security": 0,
              "format": 1,
              "firmware": 0,
              "ns_manage": 1
            },
            "multi_ctrlr": true,
            "ana_reporting": true
          },
          "vs": {
            "nvme_version": "2.0"
          },
          "ns_data": {
            "id": 2,
            "ana_state": "optimized",
            "can_share": true
          }
        }
      ],
      "mp_policy": "active_passive"
    }
  }
]
root@ubuntu-pm-18-226:~/param-spdk/spdk#

root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_stop_mdns_discovery -b cdc_auto
root@ubuntu-pm-18-226:~/param-spdk/spdk#
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_get_mdns_discovery_info
[]
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_nvme_get_discovery_info
[]
root@ubuntu-pm-18-226:~/param-spdk/spdk# scripts/rpc.py bdev_get_bdevs
[]
root@ubuntu-pm-18-226:~/param-spdk/spdk#

Signed-off-by: Parameswaran Krishnamurthy <parameswaran.krishna@dell.com>
Change-Id: Ic2c2e614e2549a655c7f81ae844b80d8505a4f02
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15703


Reviewed-by: Aleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: Shuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
Reviewed-by: Boris Glimcher <Boris.Glimcher@emc.com>
Reviewed-by:  <qun.wan@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>

parent b3785e87

CHANGELOG.md

+4 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,10 @@ decrypt operations. New RPC `dpdk_cryptodev_scan_accel_module` has been added to

		### bdev

		Added RPCs bdev_nvme_start_mdns_discovery, bdev_nvme_get_mdns_discovery_info and
		bdev_nvme_stop_mdns_discovery to perform Avahi based mDNS discovery service,
		as per NVMe TP 8009 - Automated Discovery of Ethernet Discovery Controllers.

		Both of interleaved and separated metadata are now supported by the malloc bdev module.

		Protection information is now supported by the malloc bdev module.

CONFIG

+4 −0

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@
		# Copyright (C) 2015 Intel Corporation.
		# All rights reserved.
		# Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
		# Copyright (c) 2022 Dell Inc, or its subsidiaries.
		#

		# configure options: __CONFIGURE_OPTIONS__
		@@ -208,3 +209,6 @@ CONFIG_IPSEC_MB_DIR=

		# Generate Storage Management Agent's protobuf interface
		CONFIG_SMA=n

		# Build with Avahi support
		CONFIG_AVAHI=n

configure

+19 −0

Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@
		# SPDX-License-Identifier: BSD-3-Clause
		# Copyright (C) 2017 Intel Corporation
		# All rights reserved.
		# Copyright (c) 2022 Dell Inc, or its subsidiaries.
		#

		set -e
		@@ -120,6 +121,8 @@ function usage() {
		echo " (Typically /usr/lib/llvm-VER/lib/clang/VER/lib/linux/libclang_rt.fuzzer_no_main-ARCH.a)"
		echo " --with-sma Generate Storage Management Agent's protobuf interface"
		echo " --without-sma No path required."
		echo " --with-avahi Build with Avahi mDNS discovery client service enabled in bdev-nvme module."
		echo " --without-avahi No path required."
		echo ""
		echo "Environment variables:"
		echo ""
		@@ -641,6 +644,12 @@ for i in "$@"; do
		--without-sma)
		CONFIG[SMA]=n
		;;
		--with-avahi)
		CONFIG[AVAHI]=y
		;;
		--without-avahi)
		CONFIG[AVAHI]=n
		;;
		--)
		break
		;;
		@@ -1217,6 +1226,16 @@ if [[ "${CONFIG[SMA]}" = "y" ]]; then
		fi
		fi

		if [[ "${CONFIG[AVAHI]}" = "y" ]]; then
		if ! echo -e '#include <avahi-client/client.h>\n#include <avahi-common/malloc.h>\n' \
		'int main(void) { return 0; }\n' \
		\| "${BUILD_CMD[@]}" -lavahi-client -lavahi-common - 2> /dev/null; then
		echo "--with-avahi requires libavahi-client and libavahi-common."
		echo "Please install then re-run this script."
		exit 1
		fi
		fi

		# For ARM Neoverse-N1 platform, debug build needs gcc version newer than 8.4
		if [[ "${CONFIG[DEBUG]}" = "y" && $arch = aarch64* && "$CC_TYPE" = "gcc" ]]; then
		GCC_VERSION=$($CC -dumpfullversion)

doc/jsonrpc.md

+140 −0

Original line number	Diff line number	Diff line
		@@ -11379,3 +11379,143 @@ Example response:
		"result": true
		}
		~~~

		### bdev_nvme_start_mdns_discovery {#rpc_bdev_nvme_start_mdns_discovery}

		Starts an mDNS based discovery service for the specified service type for the
		auto-discovery of discovery controllers (NVMe TP-8009).

		The discovery service will listen for the mDNS discovery events from the
		Avahi-daemon and will connect to the newly learnt discovery controllers.
		Then the discovery service will automatically attach to any subsystem found in the
		discovery log page from the discovery controllers learnt using mDNS.
		When determining a controller name to use when attaching, it will use
		the 'name' parameter as a prefix, followed by a unique integer assigned for each of the
		discovery controllers learnt through mDNS, followed by a unique integer for that discovery
		service. If the discovery service identifies a subsystem that has been previously
		attached but is listed with a different path, it will use the same controller name
		as the previous entry, and connect as a multipath.

		When the discovery service sees that a subsystem entry has been removed
		from the log page, it will automatically detach from that controller as well.

		The 'name' is also used to later stop the discovery service.

		#### Parameters

		Name \| Optional \| Type \| Description
		-------------------------- \| -------- \| ----------- \| -----------
		name \| Required \| string \| Prefix for NVMe discovery services found
		svcname \| Required \| string \| Service to discover: e.g. _nvme-disc._tcp
		hostnqn \| Optional \| string \| NVMe-oF hostnqn

		#### Example

		Example request:

		~~~json
		{
		"jsonrpc": "2.0",
		"method": "bdev_nvme_start_mdns_discovery",
		"id": 1,
		"params": {
		"name": "cdc_auto",
		"svcname": "_nvme-disc._tcp",
		"hostnqn": "nqn.2021-12.io.spdk:host1",
		}
		}
		~~~

		Example response:

		~~~json
		{
		"jsonrpc": "2.0",
		"id": 1,
		"result": true
		}
		~~~

		### bdev_nvme_stop_mdns_discovery {#rpc_bdev_nvme_stop_mdns_discovery}

		Stops a mDNS discovery service. This includes detaching any controllers that were
		discovered via the service that is being stopped.

		#### Parameters

		Name \| Optional \| Type \| Description
		-------------------------- \| -------- \| ----------- \| -----------
		name \| Required \| string \| Name of mDNS discovery instance to stop

		#### Example

		Example request:

		~~~json
		{
		"jsonrpc": "2.0",
		"method": "bdev_nvme_stop_mdns_discovery",
		"id": 1,
		"params": {
		"name": "cdc_auto"
		}
		}
		~~~

		Example response:

		~~~json
		{
		"jsonrpc": "2.0",
		"id": 1,
		"result": true
		}
		~~~

		### bdev_nvme_get_mdns_discovery_info {#rpc_bdev_nvme_get_mdns_discovery_info}

		Get the information about the mDNS discovery service instances.

		#### Example

		Example request:
		~~~json
		{
		"jsonrpc": "2.0",
		"method": "bdev_nvme_get_mdns_discovery_info",
		"id": 1
		}
		~~~

		Example response:

		~~~json
		[
		{
		"name": "cdc_auto",
		"svcname": "_nvme-disc._tcp",
		"referrals": [
		{
		"name": "cdc_auto0",
		"trid": {
		"trtype": "TCP",
		"adrfam": "IPv4",
		"traddr": "66.1.2.21",
		"trsvcid": "8009",
		"subnqn": "nqn.2014-08.org.nvmexpress.discovery"
		}
		},
		{
		"name": "cdc_auto1",
		"trid": {
		"trtype": "TCP",
		"adrfam": "IPv4",
		"traddr": "66.1.1.21",
		"trsvcid": "8009",
		"subnqn": "nqn.2014-08.org.nvmexpress.discovery"
		}
		}
		]
		}
		]
		~~~

mk/spdk.common.mk

+5 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@
		# Copyright (c) 2019, 2021 Mellanox Corporation.
		# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES
		# All rights reserved.
		# Copyright (c) 2022 Dell Inc, or its subsidiaries.
		#

		ifeq ($(wildcard $(SPDK_ROOT_DIR)/mk/config.mk),)
		@@ -175,6 +176,10 @@ LDFLAGS += -L$(CONFIG_URING_PATH)
		endif
		endif

		ifeq ($(CONFIG_AVAHI),y)
		SYS_LIBS += -lavahi-common -lavahi-client
		endif

		IPSEC_MB_DIR=$(CONFIG_IPSEC_MB_DIR)

		ISAL_DIR=$(SPDK_ROOT_DIR)/isa-l