scripts: add a parser for the dpdk memory stats. (0e9727db) · Commits · Public Repositories / spdk

scripts/dpdk_mem_info.py

0 → 100755

+406 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import os
		from enum import Enum


		class memory:
		def __init__(self, size):
		self.size = size
		self.heaps = []
		self.mempools = []
		self.memzones = []

		def get_size(self):
		return self.size

		def add_mempool(self, pool):
		self.mempools.append(pool)

		def add_memzone(self, zone):
		self.memzones.append(zone)

		def add_heap(self, heap):
		self.heaps.append(heap)

		def get_total_heap_size(self):
		size = 0
		for heap in self.heaps:
		size = size + heap.size
		return size

		def get_total_mempool_size(self):
		size = 0
		for pool in self.mempools:
		size = size + pool.get_memzone_size_sum()
		return size

		def get_total_memzone_size(self):
		size = 0
		for zone in self.memzones:
		size = size + zone.size
		return size

		def print_summary(self):
		print("DPDK memory size {} in {} heap(s)"
		.format(B_to_MiB(self.size), len(self.heaps)))
		print("{} heaps totaling size {}".format(len(self.heaps), B_to_MiB(self.get_total_heap_size())))
		for x in sorted(self.heaps, key=lambda x: x.size, reverse=True):
		x.print_summary(' ')
		print("end heaps----------")
		print("{} mempools totaling size {}".format(len(self.mempools), B_to_MiB(self.get_total_mempool_size())))
		for x in sorted(self.mempools, key=lambda x: x.get_memzone_size_sum(), reverse=True):
		x.print_summary(' ')
		print("end mempools-------")
		print("{} memzones totaling size {}".format(len(self.memzones), B_to_MiB(self.get_total_memzone_size())))
		for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
		x.print_summary(' ')
		print("end memzones-------")

		def print_heap_summary(self, heap_id):
		for heap in self.heaps:
		if heap_id == heap.id:
		heap.print_detailed_stats()
		break
		else:
		print("heap id {} is invalid. please see the summary for valid heaps.\n".format(heap_id))

		def print_mempool_summary(self, name):
		for pool in self.mempools:
		if name == pool.name:
		pool.print_detailed_stats()
		break
		else:
		print("mempool name {} is invalid. please see the summary for valid mempools.\n".format(name))

		def print_memzone_summary(self, name):
		for zone in self.memzones:
		if name == zone.name:
		zone.print_detailed_stats("")
		break
		else:
		print("memzone name {} is invalid. please see the summary for valid memzone.\n".format(name))

		def associate_heap_elements_and_memzones(self):
		for zone in self.memzones:
		for heap_obj in self.heaps:
		for element in heap_obj.busy_malloc_elements:
		if element.check_memzone_compatibility(zone):
		heap_obj.busy_memzone_elements.append(element)
		heap_obj.busy_malloc_elements.remove(element)

		def associate_memzones_and_mempools(self):
		for pool in self.mempools:
		for zone in self.memzones:
		if pool.name in zone.name:
		pool.add_memzone(zone)

		for pool in self.mempools:
		for zone in pool.memzones:
		if zone in self.memzones:
		self.memzones.remove(zone)


		class heap_elem_status(Enum):
		FREE = 0
		BUSY = 1


		class heap_element:
		def __init__(self, size, status, addr):
		self.status = status
		self.size = size
		self.addr = addr
		self.memzone = None

		def print_summary(self, header):
		print("{}element at address: {} with size: {:>15}".format(header, hex(self.addr), B_to_MiB(self.size)))

		def check_memzone_compatibility(self, memzone):
		ele_fini_addr = self.addr + self.size
		memzone_fini_addr = memzone.address + memzone.size
		if (self.addr <= memzone.address and ele_fini_addr >= memzone_fini_addr):
		self.memzone = memzone
		return True
		return False


		class heap:
		def __init__(self, id, size, num_allocations):
		self.id = id
		self.size = size
		self.num_allocations = num_allocations
		self.free_elements = []
		self.busy_malloc_elements = []
		self.busy_memzone_elements = []

		def add_element(self, element):
		if element.status == heap_elem_status.FREE:
		self.free_elements.append(element)
		else:
		self.busy_malloc_elements.append(element)

		def print_element_stats(self, list_to_print, list_type, header):
		print("{}list of {} elements. size: {}".format(header, list_type, B_to_MiB(self.get_element_size(list_to_print))))
		for x in sorted(list_to_print, key=lambda x: x.size, reverse=True):
		x.print_summary("{} ".format(header))
		if x.memzone is not None:
		x.memzone.print_summary(" {}associated memzone info: ".format(header))

		def get_element_size(self, list_to_check):
		size = 0
		for element in list_to_check:
		size = size + element.size
		return size

		def print_summary(self, header):
		print("{}size: {:>15} heap id: {}".format(header, B_to_MiB(self.size), self.id))

		def print_detailed_stats(self):
		print("heap id: {} total size: {} number of busy elements: {} number of free elements: {}"
		.format(self.id, B_to_MiB(self.size), len(self.busy_malloc_elements), len(self.free_elements)))
		self.print_element_stats(self.free_elements, "free", " ")
		self.print_element_stats(self.busy_malloc_elements, "standard malloc", " ")
		self.print_element_stats(self.busy_memzone_elements, "memzone associated", " ")


		class mempool:
		def __init__(self, name, num_objs, num_populated_objs, obj_size):
		self.name = name
		self.num_objs = num_objs
		self.num_populated_objs = num_populated_objs
		self.obj_size = obj_size
		self.memzones = []

		def add_memzone(self, memzone):
		self.memzones.append(memzone)

		def get_memzone_size_sum(self):
		size = 0
		for zone in self.memzones:
		size = size + zone.size
		return size

		def print_summary(self, header):
		print("{}size: {:>15} name: {}"
		.format(header, B_to_MiB(self.get_memzone_size_sum()), self.name))

		def print_detailed_stats(self):
		print("size: {:>15} name: {} comprised of {} memzone(s):"
		.format(B_to_MiB(self.get_memzone_size_sum()), self.name, len(self.memzones)))
		for x in sorted(self.memzones, key=lambda x: x.size, reverse=True):
		x.print_detailed_stats(" ")


		class memzone:
		def __init__(self, name, size, address):
		self.name = name
		self.size = size
		self.address = address
		self.segments = []

		def add_segment(self, segment):
		self.segments.append(segment)

		def print_summary(self, header):
		print("{}size: {:>15} name: {}".format(header, B_to_MiB(self.size), self.name))

		def print_detailed_stats(self, header):
		self.print_summary(header)
		print("{}located at address {}".format(header, hex(self.address)))
		print("{}spanning {} segment(s):".format(header, len(self.segments)))
		for x in sorted(self.segments, key=lambda x: x.size, reverse=True):
		x.print_summary(' ')


		class segment:
		def __init__(self, size, address):
		self.size = size
		self.address = address

		def print_summary(self, header):
		print("{}address: {} length: {:>15}".format(header, hex(self.address), B_to_MiB(self.size)))


		class parse_state(Enum):
		PARSE_MEMORY_SIZE = 0
		PARSE_MEMZONES = 1
		PARSE_MEMZONE_SEGMENTS = 2
		PARSE_MEMPOOLS = 3
		PARSE_MEMPOOL_INFO = 4
		PARSE_HEAPS = 5
		PARSE_HEAP_ELEMENTS = 6


		def B_to_MiB(raw_value):
		raw_value = raw_value / (1024.0 * 1024.0)

		return "%6f %s" % (raw_value, "MiB")


		def parse_zone(line):
		zone, info = line.split(':', 1)
		name, length, addr, trash = info.split(',', 3)

		trash, name = name.split(':', 1)
		name = name.replace("<", "")
		name = name.replace(">", "")
		trash, length = length.split(':', 1)
		trash, addr = addr.split(':', 1)

		return memzone(name, int(length, 0), int(addr, 0))


		def parse_segment(line):
		trash, addr, iova, length, pagesz = line.split(':')
		addr, trash = addr.strip().split(' ')
		length, trash = length.strip().split(' ')

		return segment(int(length, 0), int(addr, 0))


		def parse_mempool_name(line):
		trash, info = line.split()
		name, addr = line.split('@')
		name = name.replace("<", "")
		name = name.replace(">", "")
		trash, name = name.split()

		return name


		def parse_mem_stats(stat_path):
		state = parse_state.PARSE_MEMORY_SIZE
		with open(stat_path, "r") as stats:

		line = stats.readline()
		while line != '':
		if state == parse_state.PARSE_MEMORY_SIZE:
		if "DPDK memory size" in line:
		mem_size = int(line.replace("DPDK memory size ", ""))
		memory_struct = memory(mem_size)
		state = parse_state.PARSE_MEMZONES
		line = stats.readline()

		if state == parse_state.PARSE_MEMZONES:
		if line.find("Zone") == 0:
		zone = parse_zone(line)
		state = parse_state.PARSE_MEMZONE_SEGMENTS
		line = stats.readline()

		if state == parse_state.PARSE_MEMZONE_SEGMENTS:
		if line.find("Zone") == 0:
		memory_struct.add_memzone(zone)
		state = parse_state.PARSE_MEMZONES
		continue
		elif line.lstrip().find("addr:") == 0:
		segment = parse_segment(line)
		zone.add_segment(segment)
		elif "DPDK mempools." in line:
		state = parse_state.PARSE_MEMPOOLS
		continue
		line = stats.readline()

		if state == parse_state.PARSE_MEMPOOLS:
		mempool_info = {}
		if line.find("mempool") == 0:
		mempool_info['name'] = parse_mempool_name(line)
		state = parse_state.PARSE_MEMPOOL_INFO
		line = stats.readline()

		if state == parse_state.PARSE_MEMPOOL_INFO:
		if line.find("mempool") == 0:
		try:
		new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
		int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
		memory_struct.add_mempool(new_mempool)
		except KeyError:
		print("proper key values not provided for mempool.")
		state = parse_state.PARSE_MEMPOOLS
		continue
		elif "cache" in line:
		pass
		elif "DPDK malloc stats." in line:
		try:
		new_mempool = mempool(mempool_info['name'], int(mempool_info['size'], 0),
		int(mempool_info['populated_size'], 0), int(mempool_info['total_obj_size'], 0))
		memory_struct.add_mempool(new_mempool)
		except KeyError:
		print("proper key values not provided for mempool.")
		while "DPDK malloc heaps." not in line:
		line = stats.readline()
		state = parse_state.PARSE_HEAPS
		else:
		try:
		field, value = line.strip().split('=')
		mempool_info[field] = value
		except Exception as e:
		pass
		line = stats.readline()

		if state == parse_state.PARSE_HEAPS:
		trash, heap_id = line.strip().split(':')
		line = stats.readline()
		trash, heap_size = line.split(':')
		line = stats.readline()
		trash, num_allocations = line.split(':')
		if int(heap_size, 0) == 0:
		pass
		else:
		new_heap = heap(heap_id.lstrip(), int(heap_size, 0), int(num_allocations, 0))
		memory_struct.add_heap(new_heap)
		state = parse_state.PARSE_HEAP_ELEMENTS

		line = stats.readline()

		if state == parse_state.PARSE_HEAP_ELEMENTS:
		if line.find("Heap id") == 0:
		state = parse_state.PARSE_HEAPS
		continue
		elif line.find("Malloc element at") == 0:
		trash, address, status = line.rsplit(maxsplit=2)
		line = stats.readline()
		trash, length, trash = line.split(maxsplit=2)
		line = stats.readline()
		if "FREE" in status:
		element = heap_element(int(length, 0), heap_elem_status.FREE, int(address, 0))
		else:
		element = heap_element(int(length, 0), heap_elem_status.BUSY, int(address, 0))
		new_heap.add_element(element)
		line = stats.readline()

		memory_struct.associate_heap_elements_and_memzones()
		memory_struct.associate_memzones_and_mempools()
		return memory_struct


		if __name__ == "__main__":
		parser = argparse.ArgumentParser(description='Dumps memory stats for DPDK. If no arguments are provided, it dumps a general summary.')
		parser.add_argument('-f', dest="stats_file", help='path to a dpdk memory stats file.', default='/tmp/spdk_mem_dump.txt')
		parser.add_argument('-m', '--heap', dest="heap", help='Print detailed information about the given heap.', default=None)
		parser.add_argument('-p', '--mempool', dest="mempool", help='Print detailed information about the given mempool.', default=None)
		parser.add_argument('-z', '--memzone', dest="memzone", help='Print detailed information about the given memzone.', default=None)

		args = parser.parse_args()

		if not os.path.exists(args.stats_file):
		print("Error, specified stats file does not exist. Please make sure you have run the"
		"env_dpdk_get_mem_stats rpc on the spdk app you want to analyze.")
		exit(1)

		mem_info = parse_mem_stats(args.stats_file)

		summary = True
		if args.heap is not None:
		mem_info.print_heap_summary(args.heap)
		summary = False
		if args.mempool is not None:
		mem_info.print_mempool_summary(args.mempool)
		summary = False
		if args.memzone is not None:
		mem_info.print_memzone_summary(args.memzone)
		summary = False

		if summary:
		mem_info.print_summary()