GDB automation with Python script to measure memory usage in dashd (#1609)

* standart STL containers and Masternode-related classes processing

* masternode-related classes

* fix st containers processing,  use reflection to process simple classes

* Content descrioption in README.md

* Increase python scripts performance

Use gdb objects instead strings and type info for creating Pyton wrappers

* fixed simple classes list

* addition to README

* fix nits

* missed `the`

Co-Authored-By: gladcow <sergey@dash.org>

* fix nits

Co-Authored-By: gladcow <sergey@dash.org>

* fixed grammatical issues

Co-Authored-By: gladcow <sergey@dash.org>

* fixed phrase construction

Co-Authored-By: gladcow <sergey@dash.org>

* missed point

Co-Authored-By: gladcow <sergey@dash.org>

* fixed grammatical issues

Co-Authored-By: gladcow <sergey@dash.org>

* remove double space

Co-Authored-By: gladcow <sergey@dash.org>

* fixed grammatical issues

Co-Authored-By: gladcow <sergey@dash.org>

contrib/auto_gdb/README.md

@@ -0,0 +1,45 @@
+# Contents
+This directory contains tools to automatically get data about the memory consumption by some objects in dashd process with the help of GDB debugger.
+
+## dash_dbg.sh
+This shell script attaches GDB to the running dashd process (should be built with debug info), executes debug.gdb script and detaches.
+By default it uses testnet settings, see script comments to attach it to mainnet dashd.
+
+## debug.gdb
+Contains debugger instructions to execute during attach: loads python code and executes it for the objects we want to investigate.
+
+## log_size.py
+Contains definition of the gdb command log_size. After this script loads it can be called from gdb command line or other gdb scripts.
+Command params:
+`log_size arg0 arg1`
+`arg0` - name of object whose memory will be written in log file
+`arg1` - name of the log file
+Example:
+```
+log_size mnodeman "memlog.txt"
+```
+
+## used_size.py
+Contains definition of the gdb command used_size. After loading of this script it can be called from gdb command line or other gdb scripts.
+Command params:
+`used_size arg0 arg1`
+`arg0` - variable to store memory used by the object
+`arg1` - name of object whose memory will be calculated and stored in the first argument
+Example:
+```
+>(gdb) set $size = 0
+>(gdb) used_size $size mnodeman
+>(gdb) p $size
+```
+
+## stl_containers.py
+Contains helper classes to calculate memory used by the STL containers (list, vector, map, set, pair).
+
+## simple_class_obj.py
+Contains a helper class to calculate the memory used by an object as a sum of the memory used by its fields.
+All processed objects of such type are listed in the this file, you can add new types you are interested in to this list.
+If a type is not listed here, its size is the return of sizeof (except STL containers which are processed in stl_containers.py).
+
+## common_helpers.py
+Several helper functions that are used in other python code.
+

contrib/auto_gdb/common_helpers.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python
+#
+
+try:
+    import gdb
+except ImportError as e:
+    raise ImportError("This script must be run in GDB: ", str(e))
+import sys
+import os
+sys.path.append(os.getcwd())
+import stl_containers
+import simple_class_obj
+
+SIZE_OF_INT = 4
+SIZE_OF_BOOL = 1
+SIZE_OF_INT64 = 8
+SIZE_OF_UINT256 = 32
+
+
+def get_special_type_obj(gobj):
+    obj_type = gobj.type.strip_typedefs()
+    if stl_containers.VectorObj.is_this_type(obj_type):
+        return stl_containers.VectorObj(gobj)
+    if stl_containers.ListObj.is_this_type(obj_type):
+        return stl_containers.ListObj(gobj)
+    if stl_containers.PairObj.is_this_type(obj_type):
+        return stl_containers.PairObj(gobj)
+    if stl_containers.MapObj.is_this_type(obj_type):
+        return stl_containers.MapObj(gobj)
+    if stl_containers.SetObj.is_this_type(obj_type):
+        return stl_containers.SetObj(gobj)
+    if simple_class_obj.SimpleClassObj.is_this_type(obj_type):
+        return simple_class_obj.SimpleClassObj(gobj)
+    return False
+
+
+def is_special_type(obj_type):
+    if stl_containers.VectorObj.is_this_type(obj_type):
+        return True
+    if stl_containers.ListObj.is_this_type(obj_type):
+        return True
+    if stl_containers.PairObj.is_this_type(obj_type):
+        return True
+    if stl_containers.MapObj.is_this_type(obj_type):
+        return True
+    if stl_containers.SetObj.is_this_type(obj_type):
+        return True
+    if simple_class_obj.SimpleClassObj.is_this_type(obj_type):
+        return True
+    return False
+
+
+def get_instance_size(gobj):
+    obj = get_special_type_obj(gobj)
+    if not obj:
+        return gobj.type.sizeof
+    return obj.get_used_size()

contrib/auto_gdb/dash_dbg.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+# use testnet settings,  if you need mainnet,  use ~/.dashcore/dashd.pid file instead
+dash_pid=$(<~/.dashcore/testnet3/dashd.pid)
+sudo gdb -batch -ex "source debug.gdb" dashd ${dash_pid}

contrib/auto_gdb/debug.gdb

@@ -0,0 +1,12 @@
+set pagination off
+source used_size.py
+source log_size.py
+source test_used_size.gdb
+#logsize privateSendClient "memlog.txt"
+#logsize privateSendServer "memlog.txt"
+#logsize mnodeman "memlog.txt"
+logsize mnpayments "memlog.txt"
+#logsize instantsend "memlog.txt"
+#logsize sporkManager "memlog.txt"
+#logsize masternodeSync "memlog.txt"
+#logsize governance "memlog.txt"

contrib/auto_gdb/log_size.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python
+#
+
+try:
+    import gdb
+except ImportError as e:
+    raise ImportError("This script must be run in GDB: ", str(e))
+import traceback
+import datetime
+import sys
+import os
+sys.path.append(os.getcwd())
+import common_helpers
+
+
+class LogSizeCommand (gdb.Command):
+    """calc size of the memory used by the object and write it to file"""
+
+    def __init__ (self):
+        super (LogSizeCommand, self).__init__ ("logsize", gdb.COMMAND_USER)
+
+    def invoke(self, arg, from_tty):
+        try:
+            args = gdb.string_to_argv(arg)
+            obj = gdb.parse_and_eval(args[0])
+            logfile = open(args[1], 'a')
+            size = common_helpers.get_instance_size(obj)
+            logfile.write("%s %s: %d\n" % (str(datetime.datetime.now()), args[0], size))
+            logfile.close()
+        except Exception as e:
+            print(traceback.format_exc())
+            raise e
+
+LogSizeCommand()

contrib/auto_gdb/simple_class_obj.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python
+#
+
+try:
+    import gdb
+except ImportError as e:
+    raise ImportError("This script must be run in GDB: ", str(e))
+import sys
+import os
+sys.path.append(os.getcwd())
+import common_helpers
+
+
+simple_types = ["CMasternode", "CMasternodeVerification",
+                "CMasternodeBroadcast", "CMasternodePing",
+                "CMasternodeMan", "CDarksendQueue", "CDarkSendEntry",
+                "CTransaction", "CMutableTransaction", "CPrivateSendBaseSession",
+                "CPrivateSendBaseManager", "CPrivateSendClientSession",
+                "CPrivateSendClientManager", "CPrivateSendServer", "CMasternodePayments",
+                "CMasternodePaymentVote", "CMasternodeBlockPayees",
+                "CMasternodePayee", "CInstantSend", "CTxLockRequest",
+                "CTxLockVote", "CTxLockCandidate", "COutPoint",
+                "COutPointLock", "CSporkManager", "CMasternodeSync",
+                "CGovernanceManager", "CRateCheckBuffer", "CGovernanceObject",
+                "CGovernanceVote", "CGovernanceObjectVoteFile"]
+
+simple_templates = ["CacheMultiMap", "CacheMap"]
+
+
+class SimpleClassObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        str_type = str(obj_type)
+        if str_type in simple_types:
+            return True
+        for templ in simple_templates:
+            if str_type.find(templ + "<") == 0:
+                return True
+        return False
+
+    def get_used_size(self):
+        size = 0
+        fields = self.obj.type.fields()
+        for f in fields:
+            # check if it is static field
+            if not hasattr(f, "bitpos"):
+                continue
+            # process base class size
+            if f.is_base_class:
+                size += common_helpers.get_instance_size(self.obj.cast(f.type.strip_typedefs()))
+                continue
+            # process simple field
+            size += common_helpers.get_instance_size(self.obj[f.name])
+        return size

contrib/auto_gdb/stl_containers.py

@@ -0,0 +1,264 @@
+#!/usr/bin/python
+#
+
+try:
+    import gdb
+except ImportError as e:
+    raise ImportError("This script must be run in GDB: ", str(e))
+import sys
+import os
+sys.path.append(os.getcwd())
+import common_helpers
+
+
+def find_type(orig, name):
+    typ = orig.strip_typedefs()
+    while True:
+        # Strip cv qualifiers
+        search = '%s::%s' % (typ.unqualified(), name)
+        try:
+            return gdb.lookup_type(search)
+        except RuntimeError:
+            pass
+        # type is not found,  try superclass search
+        field = typ.fields()[0]
+        if not field.is_base_class:
+            raise ValueError("Cannot find type %s::%s" % (str(orig), name))
+        typ = field.type
+
+
+def get_value_from_aligned_membuf(buf, valtype):
+    """Returns the value held in a __gnu_cxx::__aligned_membuf."""
+    return buf['_M_storage'].address.cast(valtype.pointer()).dereference()
+
+
+def get_value_from_node(node):
+    valtype = node.type.template_argument(0)
+    return get_value_from_aligned_membuf(node['_M_storage'], valtype)
+
+
+class VectorObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        type_name = str(obj_type)
+        if type_name.find("std::vector<") == 0:
+            return True
+        if type_name.find("std::__cxx11::vector<") == 0:
+            return True
+        return False
+
+    def element_type(self):
+        return self.obj.type.template_argument(0)
+
+    def size(self):
+        return int(self.obj['_M_impl']['_M_finish'] -
+                   self.obj['_M_impl']['_M_start'])
+
+    def get_used_size(self):
+        if common_helpers.is_special_type(self.element_type()):
+            size = self.obj.type.sizeof
+            item = self.obj['_M_impl']['_M_start']
+            finish = self.obj['_M_impl']['_M_finish']
+            while item != finish:
+                elem = item.dereference()
+                obj = common_helpers.get_special_type_obj(elem)
+                size += obj.get_used_size()
+                item = item + 1
+            return size
+        return self.obj.type.sizeof + self.size() * self.element_type().sizeof
+
+
+class ListObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        type_name = str(obj_type)
+        if type_name.find("std::list<") == 0:
+            return True
+        if type_name.find("std::__cxx11::list<") == 0:
+            return True
+        return False
+
+    def element_type(self):
+        return self.obj.type.template_argument(0)
+
+    def get_used_size(self):
+        is_special = common_helpers.is_special_type(self.element_type())
+        head = self.obj['_M_impl']['_M_node']
+#        nodetype = find_type(self.obj.type, '_Node')
+        nodetype = head.type
+        nodetype = nodetype.strip_typedefs().pointer()
+        current = head['_M_next']
+        size = self.obj.type.sizeof
+        while current != head.address:
+            if is_special:
+                elem = current.cast(nodetype).dereference()
+                size += common_helpers.get_instance_size(elem)
+            else:
+                size += self.element_type().sizeof
+            current = current['_M_next']
+
+        return size
+
+
+class PairObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        type_name = str(obj_type)
+        if type_name.find("std::pair<") == 0:
+            return True
+        if type_name.find("std::__cxx11::pair<") == 0:
+            return True
+        return False
+
+    def key_type(self):
+        return self.obj.type.template_argument(0)
+
+    def value_type(self):
+        return self.obj.type.template_argument(1)
+
+    def get_used_size(self):
+        if not common_helpers.is_special_type(self.key_type()) and not common_helpers.is_special_type(self.value_type()):
+            return self.key_type().sizeof + self.value_type().sizeof
+
+        size = 0
+
+        if common_helpers.is_special_type(self.key_type()):
+            obj = common_helpers.get_special_type_obj(self.obj['first'])
+            size += obj.get_used_size()
+        else:
+            size += self.key_type().sizeof
+
+        if common_helpers.is_special_type(self.value_type()):
+            obj = common_helpers.get_special_type_obj(self.obj['second'])
+            size += obj.get_used_size()
+        else:
+            size += self.value_type().sizeof
+
+        return size
+
+
+class MapObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+        self.obj_type = gobj.type
+        rep_type = find_type(self.obj_type, "_Rep_type")
+        self.node_type = find_type(rep_type, "_Link_type")
+        self.node_type = self.node_type.strip_typedefs()
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        type_name = str(obj_type)
+        if type_name.find("std::map<") == 0:
+            return True
+        if type_name.find("std::__cxx11::map<") == 0:
+            return True
+        return False
+
+    def key_type(self):
+        return self.obj_type.template_argument(0).strip_typedefs()
+
+    def value_type(self):
+        return self.obj_type.template_argument(1).strip_typedefs()
+
+    def size(self):
+        res = int(self.obj['_M_t']['_M_impl']['_M_node_count'])
+        return res
+
+    def get_used_size(self):
+        if not common_helpers.is_special_type(self.key_type()) and not common_helpers.is_special_type(self.value_type()):
+            return self.obj_type.sizeof + self.size() * (self.key_type().sizeof + self.value_type().sizeof)
+        if self.size() == 0:
+            return self.obj_type.sizeof
+        size = self.obj_type.sizeof
+        row_node = self.obj['_M_t']['_M_impl']['_M_header']['_M_left']
+        for i in range(self.size()):
+            node_val = row_node.cast(self.node_type).dereference()
+            pair = get_value_from_node(node_val)
+
+            obj = common_helpers.get_special_type_obj(pair)
+            size += obj.get_used_size()
+
+            node = row_node
+            if node.dereference()['_M_right']:
+                node = node.dereference()['_M_right']
+                while node.dereference()['_M_left']:
+                    node = node.dereference()['_M_left']
+            else:
+                parent = node.dereference()['_M_parent']
+                while node == parent.dereference()['_M_right']:
+                    node = parent
+                    parent = parent.dereference()['_M_parent']
+                if node.dereference()['_M_right'] != parent:
+                    node = parent
+            row_node = node
+        return size
+
+
+class SetObj:
+
+    def __init__ (self, gobj):
+        self.obj = gobj
+        self.obj_type = gobj.type
+        rep_type = find_type(self.obj_type, "_Rep_type")
+        self.node_type = find_type(rep_type, "_Link_type")
+        self.node_type = self.node_type.strip_typedefs()
+
+    @classmethod
+    def is_this_type(cls, obj_type):
+        type_name = str(obj_type)
+        if type_name.find("std::set<") == 0:
+            return True
+        if type_name.find("std::__cxx11::set<") == 0:
+            return True
+        return False
+
+    def element_type(self):
+        return self.obj_type.template_argument(0)
+
+    def size(self):
+        res = int(self.obj['_M_t']['_M_impl']['_M_node_count'])
+        return res
+
+    def get_used_size(self):
+        if not common_helpers.is_special_type(self.element_type()):
+            return self.obj_type.sizeof + self.size() * self.element_type().sizeof
+        if self.size() == 0:
+            return self.obj_type.sizeof
+        size = self.obj_type.sizeof
+        row_node = self.obj['_M_t']['_M_impl']['_M_header']['_M_left']
+        for i in range(self.size()):
+            node_val = row_node.cast(self.node_type).dereference()
+            val = get_value_from_node(node_val)
+
+            obj = common_helpers.get_special_type_obj(val)
+            size += obj.get_used_size()
+
+            node = row_node
+            if node.dereference()['_M_right']:
+                node = node.dereference()['_M_right']
+                while node.dereference()['_M_left']:
+                    node = node.dereference()['_M_left']
+            else:
+                parent = node.dereference()['_M_parent']
+                while node == parent.dereference()['_M_right']:
+                    node = parent
+                    parent = parent.dereference()['_M_parent']
+                if node.dereference()['_M_right'] != parent:
+                    node = parent
+            row_node = node
+        return size
+
+

contrib/auto_gdb/test_used_size.gdb

@@ -0,0 +1,5 @@
+define test_used_size
+	set $size_ext = 0
+	usedsize $size_ext $arg0
+	p $size_ext
+end

contrib/auto_gdb/used_size.py

@@ -0,0 +1,44 @@
+#!/usr/bin/python
+#
+
+try:
+    import gdb
+except ImportError as e:
+    raise ImportError("This script must be run in GDB: ", str(e))
+import traceback
+import sys
+import os
+sys.path.append(os.getcwd())
+import common_helpers
+
+
+class UsedSizeCommand (gdb.Command):
+    """calc size of the memory used by the object"""
+
+    def __init__ (self):
+        super (UsedSizeCommand, self).__init__ ("usedsize", gdb.COMMAND_USER)
+
+    @classmethod
+    def assign_value(cls, obj_name, value):
+        gdb.execute("set " + obj_name + " = " + str(value))
+
+    @classmethod
+    def get_type(cls, obj_name):
+        return gdb.parse_and_eval(obj_name).type
+
+    def invoke(self, arg, from_tty):
+        try:
+            args = gdb.string_to_argv(arg)
+            obj = gdb.parse_and_eval(args[1])
+            obj_type = obj.type
+            print (args[1] + " is " + str(obj_type))
+            size = common_helpers.get_instance_size(obj)
+            UsedSizeCommand.assign_value(args[0], size)
+            print (size)
+
+        except Exception as e:
+            print(traceback.format_exc())
+            raise e
+
+UsedSizeCommand()
+