Commit 38ab383a authored by Evgeniy Kochetov's avatar Evgeniy Kochetov Committed by Jim Harris
Browse files

nvmf/rdma: Add RDMA polling statistics



RDMA polling statistics: number of polls and number of completion
entries returned.

Signed-off-by: default avatarEvgeniy Kochetov <evgeniik@mellanox.com>
Change-Id: Iabcf2cb6f6a35f595b89b58cdfcd177a637dda13
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/445289


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 43bb4e6b
Loading
Loading
Loading
Loading
+13 −1
Original line number Diff line number Diff line
@@ -4176,7 +4176,19 @@ Example response:
        "pending_bdev_io": 1721,
        "transports": [
          {
            "trtype": "RDMA"
            "trtype": "RDMA",
            "devices": [
              {
                "name": "mlx5_1",
                "polls": 1536729,
                "completions": 0
              },
              {
                "name": "mlx5_0",
                "polls": 1536729,
                "completions": 18667357
              }
            ]
          }
        ]
      }
+8 −1
Original line number Diff line number Diff line
@@ -85,11 +85,18 @@ struct spdk_nvmf_poll_group_stat {
	uint64_t pending_bdev_io;
};

struct spdk_nvmf_rdma_device_stat {
	const char *name;
	uint64_t polls;
	uint64_t completions;
};

struct spdk_nvmf_transport_poll_group_stat {
	spdk_nvme_transport_type_t trtype;
	union {
		struct {
			int dummy;
			uint64_t num_devices;
			struct spdk_nvmf_rdma_device_stat *devices;
		} rdma;
	};
};
+12 −0
Original line number Diff line number Diff line
@@ -1612,11 +1612,23 @@ static void
write_nvmf_transport_stats(struct spdk_json_write_ctx *w,
			   struct spdk_nvmf_transport_poll_group_stat *stat)
{
	uint64_t i;

	spdk_json_write_object_begin(w);
	spdk_json_write_named_string(w, "trtype",
				     spdk_nvme_transport_id_trtype_str(stat->trtype));
	switch (stat->trtype) {
	case SPDK_NVME_TRANSPORT_RDMA:
		spdk_json_write_named_array_begin(w, "devices");
		for (i = 0; i < stat->rdma.num_devices; ++i) {
			spdk_json_write_object_begin(w);
			spdk_json_write_named_string(w, "name", stat->rdma.devices[i].name);
			spdk_json_write_named_uint64(w, "polls", stat->rdma.devices[i].polls);
			spdk_json_write_named_uint64(w, "completions", stat->rdma.devices[i].completions);
			spdk_json_write_object_end(w);
		}
		spdk_json_write_array_end(w);
		break;
	default:
		break;
	}
+37 −0
Original line number Diff line number Diff line
@@ -409,6 +409,11 @@ struct spdk_nvmf_rdma_qpair {
	bool					last_wqe_reached;
};

struct spdk_nvmf_rdma_poller_stat {
	uint64_t				completions;
	uint64_t				polls;
};

struct spdk_nvmf_rdma_poller {
	struct spdk_nvmf_rdma_device		*device;
	struct spdk_nvmf_rdma_poll_group	*group;
@@ -424,6 +429,7 @@ struct spdk_nvmf_rdma_poller {
	struct ibv_srq				*srq;

	struct spdk_nvmf_rdma_resources		*resources;
	struct spdk_nvmf_rdma_poller_stat	stat;

	TAILQ_HEAD(, spdk_nvmf_rdma_qpair)	qpairs;

@@ -3420,6 +3426,9 @@ spdk_nvmf_rdma_poller_poll(struct spdk_nvmf_rdma_transport *rtransport,
		return -1;
	}

	rpoller->stat.polls++;
	rpoller->stat.completions += reaped;

	for (i = 0; i < reaped; i++) {

		rdma_wr = (struct spdk_nvmf_rdma_wr *)wc[i].wr_id;
@@ -3654,6 +3663,10 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
	struct spdk_io_channel *ch;
	struct spdk_nvmf_poll_group *group;
	struct spdk_nvmf_transport_poll_group *tgroup;
	struct spdk_nvmf_rdma_poll_group *rgroup;
	struct spdk_nvmf_rdma_poller *rpoller;
	struct spdk_nvmf_rdma_device_stat *device_stat;
	uint64_t num_devices = 0;

	if (tgt == NULL || stat == NULL) {
		return -EINVAL;
@@ -3669,6 +3682,27 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
				return -ENOMEM;
			}
			(*stat)->trtype = SPDK_NVME_TRANSPORT_RDMA;

			rgroup = SPDK_CONTAINEROF(tgroup, struct spdk_nvmf_rdma_poll_group, group);
			/* Count devices to allocate enough memory */
			TAILQ_FOREACH(rpoller, &rgroup->pollers, link) {
				++num_devices;
			}
			(*stat)->rdma.devices = calloc(num_devices, sizeof(struct spdk_nvmf_rdma_device_stat));
			if (!(*stat)->rdma.devices) {
				SPDK_ERRLOG("Failed to allocate NVMf RDMA devices statistics\n");
				free(*stat);
				return -ENOMEM;
			}

			(*stat)->rdma.num_devices = num_devices;
			num_devices = 0;
			TAILQ_FOREACH(rpoller, &rgroup->pollers, link) {
				device_stat = &(*stat)->rdma.devices[num_devices++];
				device_stat->name = ibv_get_device_name(rpoller->device->context->device);
				device_stat->polls = rpoller->stat.polls;
				device_stat->completions = rpoller->stat.completions;
			}
			return 0;
		}
	}
@@ -3678,6 +3712,9 @@ spdk_nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
static void
spdk_nvmf_rdma_poll_group_free_stat(struct spdk_nvmf_transport_poll_group_stat *stat)
{
	if (stat) {
		free(stat->rdma.devices);
	}
	free(stat);
}