Commit 38ab383a authored by Evgeniy Kochetov's avatar Evgeniy Kochetov Committed by Jim Harris
Browse files

nvmf/rdma: Add RDMA polling statistics 



RDMA polling statistics: number of polls and number of completion
entries returned.

Signed-off-by: default avatarEvgeniy Kochetov <evgeniik@mellanox.com>
Change-Id: Iabcf2cb6f6a35f595b89b58cdfcd177a637dda13
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/445289


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 43bb4e6b

doc/jsonrpc.md

+13 −1
Original line number Diff line number Diff line
@@ -4176,7 +4176,19 @@ Example response: 
        "pending_bdev_io": 1721,

        "transports": [

          {

            "trtype": "RDMA"

            "trtype": "RDMA",

            "devices": [

              {

                "name": "mlx5_1",

                "polls": 1536729,

                "completions": 0

              },

              {

                "name": "mlx5_0",

                "polls": 1536729,

                "completions": 18667357

              }

            ]

          }

        ]

      }

include/spdk/nvmf.h

+8 −1
Original line number Diff line number Diff line
@@ -85,11 +85,18 @@ struct spdk_nvmf_poll_group_stat { 
	uint64_t pending_bdev_io;

};



struct spdk_nvmf_rdma_device_stat {

	const char *name;

	uint64_t polls;

	uint64_t completions;

};



struct spdk_nvmf_transport_poll_group_stat {

	spdk_nvme_transport_type_t trtype;

	union {

		struct {

			int dummy;

			uint64_t num_devices;

			struct spdk_nvmf_rdma_device_stat *devices;

		} rdma;

	};

};

lib/event/subsystems/nvmf/nvmf_rpc.c

+12 −0
Original line number Diff line number Diff line
@@ -1612,11 +1612,23 @@ static void 
write_nvmf_transport_stats(struct spdk_json_write_ctx *w,

			   struct spdk_nvmf_transport_poll_group_stat *stat)

{

	uint64_t i;



	spdk_json_write_object_begin(w);

	spdk_json_write_named_string(w, "trtype",

				     spdk_nvme_transport_id_trtype_str(stat->trtype));

	switch (stat->trtype) {

	case SPDK_NVME_TRANSPORT_RDMA:

		spdk_json_write_named_array_begin(w, "devices");

		for (i = 0; i < stat->rdma.num_devices; ++i) {

			spdk_json_write_object_begin(w);

			spdk_json_write_named_string(w, "name", stat->rdma.devices[i].name);

			spdk_json_write_named_uint64(w, "polls", stat->rdma.devices[i].polls);

			spdk_json_write_named_uint64(w, "completions", stat->rdma.devices[i].completions);

			spdk_json_write_object_end(w);

		}

		spdk_json_write_array_end(w);

		break;

	default:

		break;

	}

lib/nvmf/rdma.c

+37 −0
Original line number Diff line number Diff line
@@ -409,6 +409,11 @@ struct spdk_nvmf_rdma_qpair { 
	bool					last_wqe_reached;

};



struct spdk_nvmf_rdma_poller_stat {

	uint64_t				completions;

	uint64_t				polls;

};



struct spdk_nvmf_rdma_poller {

	struct spdk_nvmf_rdma_device		*device;

	struct spdk_nvmf_rdma_poll_group	*group;
@@ -424,6 +429,7 @@ struct spdk_nvmf_rdma_poller { 
	struct ibv_srq				*srq;



	struct spdk_nvmf_rdma_resources		*resources;

	struct spdk_nvmf_rdma_poller_stat	stat;



	TAILQ_HEAD(, spdk_nvmf_rdma_qpair)	qpairs;
@@ -3420,6 +3426,9 @@ spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport, 
		return -1;

	}



	rpoller->stat.polls++;

	rpoller->stat.completions += reaped;



	for (i = 0; i < reaped; i++) {



		rdma_wr = (struct spdk_nvmf_rdma_wr *)wc[i].wr_id;
@@ -3654,6 +3663,10 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 
	struct spdk_io_channel *ch;

	struct spdk_nvmf_poll_group *group;

	struct spdk_nvmf_transport_poll_group *tgroup;

	struct spdk_nvmf_rdma_poll_group *rgroup;

	struct spdk_nvmf_rdma_poller *rpoller;

	struct spdk_nvmf_rdma_device_stat *device_stat;

	uint64_t num_devices = 0;



	if (tgt == NULL || stat == NULL) {

		return -EINVAL;
@@ -3669,6 +3682,27 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 
				return -ENOMEM;

			}

			(*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA;



			rgroup = SPDK_CONTAINEROF(tgroup, struct spdk_nvmf_rdma_poll_group, group);

			/* Count devices to allocate enough memory */

			TAILQ_FOREACH(rpoller, &rgroup->pollers, link) {

				++num_devices;

			}

			(*stat)->rdma.devices = calloc(num_devices, sizeof(struct spdk_nvmf_rdma_device_stat));

			if (!(*stat)->rdma.devices) {

				SPDK_ERRLOG("Failed to allocate NVMf RDMA devices statistics\n");

				free(*stat);

				return -ENOMEM;

			}



			(*stat)->rdma.num_devices = num_devices;

			num_devices = 0;

			TAILQ_FOREACH(rpoller, &rgroup->pollers, link) {

				device_stat = &(*stat)->rdma.devices[num_devices++];

				device_stat->name = ibv_get_device_name(rpoller->device->context->device);

				device_stat->polls = rpoller->stat.polls;

				device_stat->completions = rpoller->stat.completions;

			}

			return 0;

		}

	}
@@ -3678,6 +3712,9 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 
static void

spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat)

{

	if (stat) {

		free(stat->rdma.devices);

	}

	free(stat);

}