Commit 36ac75b9 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

nvmf/rdma: Use new RDMA statistics 



Change-Id: I11c7793301be9dbe9d4c92e482a97fc6146cc355
Signed-off-by: default avatarAlexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/6294


Community-CI: Broadcom CI
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 201aa634

doc/jsonrpc.md

+10 −2
Original line number Diff line number Diff line
@@ -5936,7 +5936,11 @@ Example response: 
                "request_latency": 0,

                "pending_free_request": 0,

                "pending_rdma_read": 0,

                "pending_rdma_write": 0

                "pending_rdma_write": 0,

                "total_send_wrs": 0,

                "send_doorbell_updates": 0,

                "total_recv_wrs": 0,

                "recv_doorbell_updates": 1

              },

              {

                "name": "mlx5_0",
@@ -5946,7 +5950,11 @@ Example response: 
                "request_latency": 1249323766184,

                "pending_free_request": 0,

                "pending_rdma_read": 337602,

                "pending_rdma_write": 0

                "pending_rdma_write": 0,

                "total_send_wrs": 15165875,

                "send_doorbell_updates": 1516587,

                "total_recv_wrs": 15165875,

                "recv_doorbell_updates": 1516587

              }

            ]

          }

include/spdk/nvmf.h

+5 −1
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation. All rights reserved.

 *   Copyright (c) 2018-2019 Mellanox Technologies LTD. All rights reserved.

 *   Copyright (c) 2018-2021 Mellanox Technologies LTD. All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions
@@ -133,6 +133,10 @@ struct spdk_nvmf_rdma_device_stat { 
	uint64_t pending_free_request;

	uint64_t pending_rdma_read;

	uint64_t pending_rdma_write;

	uint64_t total_send_wrs;

	uint64_t send_doorbell_updates;

	uint64_t total_recv_wrs;

	uint64_t recv_doorbell_updates;

};



struct spdk_nvmf_transport_poll_group_stat {

lib/nvmf/nvmf_rpc.c

+9 −1
Original line number Diff line number Diff line
@@ -2,7 +2,7 @@ 
 *   BSD LICENSE

 *

 *   Copyright (c) Intel Corporation. All rights reserved.

 *   Copyright (c) 2018-2020 Mellanox Technologies LTD. All rights reserved.

 *   Copyright (c) 2018-2021 Mellanox Technologies LTD. All rights reserved.

 *

 *   Redistribution and use in source and binary forms, with or without

 *   modification, are permitted provided that the following conditions
@@ -2063,6 +2063,14 @@ write_nvmf_transport_stats(struct spdk_json_write_ctx *w, 
						     stat->rdma.devices[i].pending_rdma_read);

			spdk_json_write_named_uint64(w, "pending_rdma_write",

						     stat->rdma.devices[i].pending_rdma_write);

			spdk_json_write_named_uint64(w, "total_send_wrs",

						     stat->rdma.devices[i].total_send_wrs);

			spdk_json_write_named_uint64(w, "send_doorbell_updates",

						     stat->rdma.devices[i].send_doorbell_updates);

			spdk_json_write_named_uint64(w, "total_recv_wrs",

						     stat->rdma.devices[i].total_recv_wrs);

			spdk_json_write_named_uint64(w, "recv_doorbell_updates",

						     stat->rdma.devices[i].recv_doorbell_updates);

			spdk_json_write_object_end(w);

		}

		spdk_json_write_array_end(w);

lib/nvmf/rdma.c

+7 −0
Original line number Diff line number Diff line
@@ -400,6 +400,7 @@ struct spdk_nvmf_rdma_poller_stat { 
	uint64_t				pending_free_request;

	uint64_t				pending_rdma_read;

	uint64_t				pending_rdma_write;

	struct spdk_rdma_qp_stats		qp_stats;

};



struct spdk_nvmf_rdma_poller {
@@ -1005,6 +1006,7 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	qp_init_attr.cap.max_send_wr	= (uint32_t)rqpair->max_queue_depth * 2;

	qp_init_attr.cap.max_send_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);

	qp_init_attr.cap.max_recv_sge	= spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

	qp_init_attr.stats		= &rqpair->poller->stat.qp_stats;



	if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {

		SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
@@ -3255,6 +3257,7 @@ nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) 
			device->num_srq++;

			memset(&srq_init_attr, 0, sizeof(srq_init_attr));

			srq_init_attr.pd = device->pd;

			srq_init_attr.stats = &poller->stat.qp_stats.recv;

			srq_init_attr.srq_init_attr.attr.max_wr = poller->max_srq_depth;

			srq_init_attr.srq_init_attr.attr.max_sge = spdk_min(device->attr.max_sge, NVMF_DEFAULT_RX_SGE);

			poller->srq = spdk_rdma_srq_create(&srq_init_attr);
@@ -4197,6 +4200,10 @@ nvmf_rdma_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 
				device_stat->pending_free_request = rpoller->stat.pending_free_request;

				device_stat->pending_rdma_read = rpoller->stat.pending_rdma_read;

				device_stat->pending_rdma_write = rpoller->stat.pending_rdma_write;

				device_stat->total_send_wrs = rpoller->stat.qp_stats.send.num_submitted_wrs;

				device_stat->send_doorbell_updates = rpoller->stat.qp_stats.send.doorbell_updates;

				device_stat->total_recv_wrs = rpoller->stat.qp_stats.recv.num_submitted_wrs;

				device_stat->recv_doorbell_updates = rpoller->stat.qp_stats.recv.doorbell_updates;

			}

			return 0;

		}