diff --git a/contrib/auto_gdb/README.md b/contrib/auto_gdb/README.md new file mode 100644 index 0000000000..ab75e79594 --- /dev/null +++ b/contrib/auto_gdb/README.md @@ -0,0 +1,45 @@ +# Contents +This directory contains tools to automatically get data about the memory consumption by some objects in dashd process with the help of GDB debugger. + +## dash_dbg.sh +This shell script attaches GDB to the running dashd process (should be built with debug info), executes debug.gdb script and detaches. +By default it uses testnet settings, see script comments to attach it to mainnet dashd. + +## debug.gdb +Contains debugger instructions to execute during attach: loads python code and executes it for the objects we want to investigate. + +## log_size.py +Contains definition of the gdb command log_size. After this script loads it can be called from gdb command line or other gdb scripts. +Command params: +`log_size arg0 arg1` +`arg0` - name of object whose memory will be written in log file +`arg1` - name of the log file +Example: +``` +log_size mnodeman "memlog.txt" +``` + +## used_size.py +Contains definition of the gdb command used_size. After loading of this script it can be called from gdb command line or other gdb scripts. +Command params: +`used_size arg0 arg1` +`arg0` - variable to store memory used by the object +`arg1` - name of object whose memory will be calculated and stored in the first argument +Example: +``` +>(gdb) set $size = 0 +>(gdb) used_size $size mnodeman +>(gdb) p $size +``` + +## stl_containers.py +Contains helper classes to calculate memory used by the STL containers (list, vector, map, set, pair). + +## simple_class_obj.py +Contains a helper class to calculate the memory used by an object as a sum of the memory used by its fields. +All processed objects of such type are listed in the this file, you can add new types you are interested in to this list. +If a type is not listed here, its size is the return of sizeof (except STL containers which are processed in stl_containers.py). + +## common_helpers.py +Several helper functions that are used in other python code. + diff --git a/contrib/auto_gdb/common_helpers.py b/contrib/auto_gdb/common_helpers.py new file mode 100644 index 0000000000..7dcc6c021d --- /dev/null +++ b/contrib/auto_gdb/common_helpers.py @@ -0,0 +1,57 @@ +#!/usr/bin/python +# + +try: + import gdb +except ImportError as e: + raise ImportError("This script must be run in GDB: ", str(e)) +import sys +import os +sys.path.append(os.getcwd()) +import stl_containers +import simple_class_obj + +SIZE_OF_INT = 4 +SIZE_OF_BOOL = 1 +SIZE_OF_INT64 = 8 +SIZE_OF_UINT256 = 32 + + +def get_special_type_obj(gobj): + obj_type = gobj.type.strip_typedefs() + if stl_containers.VectorObj.is_this_type(obj_type): + return stl_containers.VectorObj(gobj) + if stl_containers.ListObj.is_this_type(obj_type): + return stl_containers.ListObj(gobj) + if stl_containers.PairObj.is_this_type(obj_type): + return stl_containers.PairObj(gobj) + if stl_containers.MapObj.is_this_type(obj_type): + return stl_containers.MapObj(gobj) + if stl_containers.SetObj.is_this_type(obj_type): + return stl_containers.SetObj(gobj) + if simple_class_obj.SimpleClassObj.is_this_type(obj_type): + return simple_class_obj.SimpleClassObj(gobj) + return False + + +def is_special_type(obj_type): + if stl_containers.VectorObj.is_this_type(obj_type): + return True + if stl_containers.ListObj.is_this_type(obj_type): + return True + if stl_containers.PairObj.is_this_type(obj_type): + return True + if stl_containers.MapObj.is_this_type(obj_type): + return True + if stl_containers.SetObj.is_this_type(obj_type): + return True + if simple_class_obj.SimpleClassObj.is_this_type(obj_type): + return True + return False + + +def get_instance_size(gobj): + obj = get_special_type_obj(gobj) + if not obj: + return gobj.type.sizeof + return obj.get_used_size() diff --git a/contrib/auto_gdb/dash_dbg.sh b/contrib/auto_gdb/dash_dbg.sh new file mode 100755 index 0000000000..ebe3ed54ec --- /dev/null +++ b/contrib/auto_gdb/dash_dbg.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# use testnet settings, if you need mainnet, use ~/.dashcore/dashd.pid file instead +dash_pid=$(<~/.dashcore/testnet3/dashd.pid) +sudo gdb -batch -ex "source debug.gdb" dashd ${dash_pid} diff --git a/contrib/auto_gdb/debug.gdb b/contrib/auto_gdb/debug.gdb new file mode 100644 index 0000000000..2c79aa89c4 --- /dev/null +++ b/contrib/auto_gdb/debug.gdb @@ -0,0 +1,12 @@ +set pagination off +source used_size.py +source log_size.py +source test_used_size.gdb +#logsize privateSendClient "memlog.txt" +#logsize privateSendServer "memlog.txt" +#logsize mnodeman "memlog.txt" +logsize mnpayments "memlog.txt" +#logsize instantsend "memlog.txt" +#logsize sporkManager "memlog.txt" +#logsize masternodeSync "memlog.txt" +#logsize governance "memlog.txt" diff --git a/contrib/auto_gdb/log_size.py b/contrib/auto_gdb/log_size.py new file mode 100644 index 0000000000..08eaa9048c --- /dev/null +++ b/contrib/auto_gdb/log_size.py @@ -0,0 +1,34 @@ +#!/usr/bin/python +# + +try: + import gdb +except ImportError as e: + raise ImportError("This script must be run in GDB: ", str(e)) +import traceback +import datetime +import sys +import os +sys.path.append(os.getcwd()) +import common_helpers + + +class LogSizeCommand (gdb.Command): + """calc size of the memory used by the object and write it to file""" + + def __init__ (self): + super (LogSizeCommand, self).__init__ ("logsize", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + try: + args = gdb.string_to_argv(arg) + obj = gdb.parse_and_eval(args[0]) + logfile = open(args[1], 'a') + size = common_helpers.get_instance_size(obj) + logfile.write("%s %s: %d\n" % (str(datetime.datetime.now()), args[0], size)) + logfile.close() + except Exception as e: + print(traceback.format_exc()) + raise e + +LogSizeCommand() diff --git a/contrib/auto_gdb/simple_class_obj.py b/contrib/auto_gdb/simple_class_obj.py new file mode 100644 index 0000000000..7aa0166e01 --- /dev/null +++ b/contrib/auto_gdb/simple_class_obj.py @@ -0,0 +1,58 @@ +#!/usr/bin/python +# + +try: + import gdb +except ImportError as e: + raise ImportError("This script must be run in GDB: ", str(e)) +import sys +import os +sys.path.append(os.getcwd()) +import common_helpers + + +simple_types = ["CMasternode", "CMasternodeVerification", + "CMasternodeBroadcast", "CMasternodePing", + "CMasternodeMan", "CDarksendQueue", "CDarkSendEntry", + "CTransaction", "CMutableTransaction", "CPrivateSendBaseSession", + "CPrivateSendBaseManager", "CPrivateSendClientSession", + "CPrivateSendClientManager", "CPrivateSendServer", "CMasternodePayments", + "CMasternodePaymentVote", "CMasternodeBlockPayees", + "CMasternodePayee", "CInstantSend", "CTxLockRequest", + "CTxLockVote", "CTxLockCandidate", "COutPoint", + "COutPointLock", "CSporkManager", "CMasternodeSync", + "CGovernanceManager", "CRateCheckBuffer", "CGovernanceObject", + "CGovernanceVote", "CGovernanceObjectVoteFile"] + +simple_templates = ["CacheMultiMap", "CacheMap"] + + +class SimpleClassObj: + + def __init__ (self, gobj): + self.obj = gobj + + @classmethod + def is_this_type(cls, obj_type): + str_type = str(obj_type) + if str_type in simple_types: + return True + for templ in simple_templates: + if str_type.find(templ + "<") == 0: + return True + return False + + def get_used_size(self): + size = 0 + fields = self.obj.type.fields() + for f in fields: + # check if it is static field + if not hasattr(f, "bitpos"): + continue + # process base class size + if f.is_base_class: + size += common_helpers.get_instance_size(self.obj.cast(f.type.strip_typedefs())) + continue + # process simple field + size += common_helpers.get_instance_size(self.obj[f.name]) + return size diff --git a/contrib/auto_gdb/stl_containers.py b/contrib/auto_gdb/stl_containers.py new file mode 100644 index 0000000000..cb6763c5e0 --- /dev/null +++ b/contrib/auto_gdb/stl_containers.py @@ -0,0 +1,264 @@ +#!/usr/bin/python +# + +try: + import gdb +except ImportError as e: + raise ImportError("This script must be run in GDB: ", str(e)) +import sys +import os +sys.path.append(os.getcwd()) +import common_helpers + + +def find_type(orig, name): + typ = orig.strip_typedefs() + while True: + # Strip cv qualifiers + search = '%s::%s' % (typ.unqualified(), name) + try: + return gdb.lookup_type(search) + except RuntimeError: + pass + # type is not found, try superclass search + field = typ.fields()[0] + if not field.is_base_class: + raise ValueError("Cannot find type %s::%s" % (str(orig), name)) + typ = field.type + + +def get_value_from_aligned_membuf(buf, valtype): + """Returns the value held in a __gnu_cxx::__aligned_membuf.""" + return buf['_M_storage'].address.cast(valtype.pointer()).dereference() + + +def get_value_from_node(node): + valtype = node.type.template_argument(0) + return get_value_from_aligned_membuf(node['_M_storage'], valtype) + + +class VectorObj: + + def __init__ (self, gobj): + self.obj = gobj + + @classmethod + def is_this_type(cls, obj_type): + type_name = str(obj_type) + if type_name.find("std::vector<") == 0: + return True + if type_name.find("std::__cxx11::vector<") == 0: + return True + return False + + def element_type(self): + return self.obj.type.template_argument(0) + + def size(self): + return int(self.obj['_M_impl']['_M_finish'] - + self.obj['_M_impl']['_M_start']) + + def get_used_size(self): + if common_helpers.is_special_type(self.element_type()): + size = self.obj.type.sizeof + item = self.obj['_M_impl']['_M_start'] + finish = self.obj['_M_impl']['_M_finish'] + while item != finish: + elem = item.dereference() + obj = common_helpers.get_special_type_obj(elem) + size += obj.get_used_size() + item = item + 1 + return size + return self.obj.type.sizeof + self.size() * self.element_type().sizeof + + +class ListObj: + + def __init__ (self, gobj): + self.obj = gobj + + @classmethod + def is_this_type(cls, obj_type): + type_name = str(obj_type) + if type_name.find("std::list<") == 0: + return True + if type_name.find("std::__cxx11::list<") == 0: + return True + return False + + def element_type(self): + return self.obj.type.template_argument(0) + + def get_used_size(self): + is_special = common_helpers.is_special_type(self.element_type()) + head = self.obj['_M_impl']['_M_node'] +# nodetype = find_type(self.obj.type, '_Node') + nodetype = head.type + nodetype = nodetype.strip_typedefs().pointer() + current = head['_M_next'] + size = self.obj.type.sizeof + while current != head.address: + if is_special: + elem = current.cast(nodetype).dereference() + size += common_helpers.get_instance_size(elem) + else: + size += self.element_type().sizeof + current = current['_M_next'] + + return size + + +class PairObj: + + def __init__ (self, gobj): + self.obj = gobj + + @classmethod + def is_this_type(cls, obj_type): + type_name = str(obj_type) + if type_name.find("std::pair<") == 0: + return True + if type_name.find("std::__cxx11::pair<") == 0: + return True + return False + + def key_type(self): + return self.obj.type.template_argument(0) + + def value_type(self): + return self.obj.type.template_argument(1) + + def get_used_size(self): + if not common_helpers.is_special_type(self.key_type()) and not common_helpers.is_special_type(self.value_type()): + return self.key_type().sizeof + self.value_type().sizeof + + size = 0 + + if common_helpers.is_special_type(self.key_type()): + obj = common_helpers.get_special_type_obj(self.obj['first']) + size += obj.get_used_size() + else: + size += self.key_type().sizeof + + if common_helpers.is_special_type(self.value_type()): + obj = common_helpers.get_special_type_obj(self.obj['second']) + size += obj.get_used_size() + else: + size += self.value_type().sizeof + + return size + + +class MapObj: + + def __init__ (self, gobj): + self.obj = gobj + self.obj_type = gobj.type + rep_type = find_type(self.obj_type, "_Rep_type") + self.node_type = find_type(rep_type, "_Link_type") + self.node_type = self.node_type.strip_typedefs() + + @classmethod + def is_this_type(cls, obj_type): + type_name = str(obj_type) + if type_name.find("std::map<") == 0: + return True + if type_name.find("std::__cxx11::map<") == 0: + return True + return False + + def key_type(self): + return self.obj_type.template_argument(0).strip_typedefs() + + def value_type(self): + return self.obj_type.template_argument(1).strip_typedefs() + + def size(self): + res = int(self.obj['_M_t']['_M_impl']['_M_node_count']) + return res + + def get_used_size(self): + if not common_helpers.is_special_type(self.key_type()) and not common_helpers.is_special_type(self.value_type()): + return self.obj_type.sizeof + self.size() * (self.key_type().sizeof + self.value_type().sizeof) + if self.size() == 0: + return self.obj_type.sizeof + size = self.obj_type.sizeof + row_node = self.obj['_M_t']['_M_impl']['_M_header']['_M_left'] + for i in range(self.size()): + node_val = row_node.cast(self.node_type).dereference() + pair = get_value_from_node(node_val) + + obj = common_helpers.get_special_type_obj(pair) + size += obj.get_used_size() + + node = row_node + if node.dereference()['_M_right']: + node = node.dereference()['_M_right'] + while node.dereference()['_M_left']: + node = node.dereference()['_M_left'] + else: + parent = node.dereference()['_M_parent'] + while node == parent.dereference()['_M_right']: + node = parent + parent = parent.dereference()['_M_parent'] + if node.dereference()['_M_right'] != parent: + node = parent + row_node = node + return size + + +class SetObj: + + def __init__ (self, gobj): + self.obj = gobj + self.obj_type = gobj.type + rep_type = find_type(self.obj_type, "_Rep_type") + self.node_type = find_type(rep_type, "_Link_type") + self.node_type = self.node_type.strip_typedefs() + + @classmethod + def is_this_type(cls, obj_type): + type_name = str(obj_type) + if type_name.find("std::set<") == 0: + return True + if type_name.find("std::__cxx11::set<") == 0: + return True + return False + + def element_type(self): + return self.obj_type.template_argument(0) + + def size(self): + res = int(self.obj['_M_t']['_M_impl']['_M_node_count']) + return res + + def get_used_size(self): + if not common_helpers.is_special_type(self.element_type()): + return self.obj_type.sizeof + self.size() * self.element_type().sizeof + if self.size() == 0: + return self.obj_type.sizeof + size = self.obj_type.sizeof + row_node = self.obj['_M_t']['_M_impl']['_M_header']['_M_left'] + for i in range(self.size()): + node_val = row_node.cast(self.node_type).dereference() + val = get_value_from_node(node_val) + + obj = common_helpers.get_special_type_obj(val) + size += obj.get_used_size() + + node = row_node + if node.dereference()['_M_right']: + node = node.dereference()['_M_right'] + while node.dereference()['_M_left']: + node = node.dereference()['_M_left'] + else: + parent = node.dereference()['_M_parent'] + while node == parent.dereference()['_M_right']: + node = parent + parent = parent.dereference()['_M_parent'] + if node.dereference()['_M_right'] != parent: + node = parent + row_node = node + return size + + diff --git a/contrib/auto_gdb/test_used_size.gdb b/contrib/auto_gdb/test_used_size.gdb new file mode 100644 index 0000000000..b46d11d2fc --- /dev/null +++ b/contrib/auto_gdb/test_used_size.gdb @@ -0,0 +1,5 @@ +define test_used_size + set $size_ext = 0 + usedsize $size_ext $arg0 + p $size_ext +end diff --git a/contrib/auto_gdb/used_size.py b/contrib/auto_gdb/used_size.py new file mode 100644 index 0000000000..5285965210 --- /dev/null +++ b/contrib/auto_gdb/used_size.py @@ -0,0 +1,44 @@ +#!/usr/bin/python +# + +try: + import gdb +except ImportError as e: + raise ImportError("This script must be run in GDB: ", str(e)) +import traceback +import sys +import os +sys.path.append(os.getcwd()) +import common_helpers + + +class UsedSizeCommand (gdb.Command): + """calc size of the memory used by the object""" + + def __init__ (self): + super (UsedSizeCommand, self).__init__ ("usedsize", gdb.COMMAND_USER) + + @classmethod + def assign_value(cls, obj_name, value): + gdb.execute("set " + obj_name + " = " + str(value)) + + @classmethod + def get_type(cls, obj_name): + return gdb.parse_and_eval(obj_name).type + + def invoke(self, arg, from_tty): + try: + args = gdb.string_to_argv(arg) + obj = gdb.parse_and_eval(args[1]) + obj_type = obj.type + print (args[1] + " is " + str(obj_type)) + size = common_helpers.get_instance_size(obj) + UsedSizeCommand.assign_value(args[0], size) + print (size) + + except Exception as e: + print(traceback.format_exc()) + raise e + +UsedSizeCommand() +