Enable stacktrace support in gitian builds (#3006)

* Remove use of -rdynamic

This causes check-symbols to fail horribly and also turned out to be not
required when using libbacktrace. It was only required when using
"backtrace()" from "<execinfo.h>"

* Remove spurious ], from configure.ac

* Add -DENABLE_STACKTRACES=1 to CMakeLists.txt

* Remove unused method my_backtrace_simple_callback

* Use fs::path().filename() instead of basename()

* Add static g_exeFileName and g_exeFileBaseName

* Use .exe.dbg file when available

* Use uint64_t instead of uintptr_t

* Implement GetBaseAddress() for unix and win32

* Implement unified crash_info and use it everywhere before printing crash info

* Print a serialized version of crash_info when there is no debug info

* Implement "-printcrashinfo" command line option

* Compile stacktrace support unconditionally and only make crash hooks conditional

This also renames the --enable-stacktraces option to --enable-crash-hooks

* Enable crash hooks in win/linux Gitian builds

* Try to load .debug file on MacOS and enable crash hooks for osx Gitian builds

* Check for dsymutil and if it needs --flat

* Create .debug files in osx Gitian build

* Handle review comments

* Also print crash description when no stacktrace is available

* Unconditionally add -g1 debug information

Instead of making it dependent on "--enable-crash-hooks". We will need the
debug info every time now, even in release builds.

* Put MacOS debug info into dSYM symbols instead of plain .debug files

* Implement MacOS specific GetBaseAddress

CMakeLists.txt

@@ -33,6 +33,7 @@ if(DEFINED DEPENDS_PREFIX)
 endif()
 
 add_definitions(
+        -DENABLE_CRASH_HOOKS=1
         -DENABLE_WALLET=1
 )
 

configure.ac

@@ -87,6 +87,7 @@ AC_PATH_PROG(HEXDUMP,hexdump)
 AC_PATH_TOOL(READELF, readelf)
 AC_PATH_TOOL(CPPFILT, c++filt)
 AC_PATH_TOOL(OBJCOPY, objcopy)
+AC_PATH_TOOL(DSYMUTIL, dsymutil)
 
 AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files)
 
@@ -192,12 +193,12 @@ AC_ARG_ENABLE([debug],
     [enable_debug=$enableval],
     [enable_debug=no])
 
-# Enable exception stacktraces
+# Enable crash hooks
-AC_ARG_ENABLE([stacktraces],
+AC_ARG_ENABLE([crash-hooks],
-    [AS_HELP_STRING([--enable-stacktraces],
+    [AS_HELP_STRING([--enable-crash-hooks],
-                    [gather and print exception stack traces (default is no)])],
+                    [hook into exception/signal/assert handling to gather stack traces (default is no)])],
-    [enable_stacktraces=$enableval],
+    [enable_crashhooks=$enableval],
-    [enable_stacktraces=no])
+    [enable_crashhooks=no])
 
 # Enable in-wallet miner
 AC_ARG_ENABLE([miner],
@@ -229,9 +230,10 @@ if test "x$enable_debug" = xyes; then
     if test "x$GXX" = xyes; then
         CXXFLAGS="$CXXFLAGS -g3 -O0"
     fi
-elif test "x$enable_stacktraces" = xyes; then
+else
-    # Enable debug information but don't turn off optimization
+    # We always enable at at least -g1 debug info to support proper stacktraces in crash infos
-    # (stacktraces will be suboptimal, but better than nothing)
+    # Stacktraces will be suboptimal due to optimization, but better than nothing. Also, -fno-omit-frame-pointer
+    # mitigates this a little bit
     if test "x$GCC" = xyes; then
         CFLAGS="$CFLAGS -g1 -fno-omit-frame-pointer"
     fi
@@ -241,17 +243,15 @@ elif test "x$enable_stacktraces" = xyes; then
     fi
 fi
 
-AM_CONDITIONAL([ENABLE_STACKTRACES], [test x$enable_stacktraces = xyes])
+AM_CONDITIONAL([ENABLE_CRASH_HOOKS], [test x$enable_crashhooks = xyes])
-if test "x$enable_stacktraces" = xyes; then
+if test "x$enable_crashhooks" = xyes; then
-    AC_DEFINE(ENABLE_STACKTRACES, 1, [Define this symbol if stacktraces should be enabled])
+    AC_DEFINE(ENABLE_CRASH_HOOKS, 1, [Define this symbol if crash hooks should be enabled])
 fi
 AX_CHECK_LINK_FLAG([-Wl,-wrap=__cxa_allocate_exception], [LINK_WRAP_SUPPORTED=yes],,,)
-AX_CHECK_COMPILE_FLAG([-rdynamic], [RDYNAMIC_SUPPORTED=yes],,,)
+AM_CONDITIONAL([CRASH_HOOKS_WRAPPED_CXX_ABI],[test x$LINK_WRAP_SUPPORTED = xyes])
-AM_CONDITIONAL([STACKTRACE_WRAPPED_CXX_ABI],[test x$LINK_WRAP_SUPPORTED = xyes])
-AM_CONDITIONAL([RDYNAMIC_SUPPORTED],[test x$RDYNAMIC_SUPPORTED = xyes])
 
 if test x$LINK_WRAP_SUPPORTED = "xyes"; then
-  AC_DEFINE(STACKTRACE_WRAPPED_CXX_ABI, 1, [Define this symbol to use wrapped CXX ABIs for exception stacktraces])],
+  AC_DEFINE(CRASH_HOOKS_WRAPPED_CXX_ABI, 1, [Define this symbol to use wrapped CXX ABIs for exception stacktraces])
 fi
 
 # Needed for MinGW targets when debug symbols are enabled as compiled objects get very large
@@ -1142,6 +1142,18 @@ else
   AC_MSG_RESULT([no])
 fi
 
+# When compiled natively on MacOS, we need to specify -flat to avoid producing a dSYM bundle
+# When cross-compiled on linux, we're using a different version of the tool that only supports flat symbol files
+AC_MSG_CHECKING([whether dsymutil needs -flat])
+if test x$DSYMUTIL != x && ($DSYMUTIL --help | grep -q \\-flat); then
+AC_MSG_RESULT([yes])
+	DSYMUTIL_FLAT="$DSYMUTIL -flat"
+else
+	AC_MSG_RESULT([no])
+	DSYMUTIL_FLAT="$DSYMUTIL"
+fi
+AC_MSG_RESULT($dsymutil_needs_flat)
+
 if test x$build_bitcoin_utils$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononono; then
   AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests])
 fi
@@ -1205,6 +1217,7 @@ AC_SUBST(EVENT_PTHREADS_LIBS)
 AC_SUBST(ZMQ_LIBS)
 AC_SUBST(PROTOBUF_LIBS)
 AC_SUBST(QR_LIBS)
+AC_SUBST(DSYMUTIL_FLAT)
 AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini])
 AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py])
 AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
@@ -1277,7 +1290,7 @@ echo "  with test           = $use_tests"
 echo "  with bench          = $use_bench"
 echo "  with upnp           = $use_upnp"
 echo "  debug enabled       = $enable_debug"
-echo "  stacktraces enabled = $enable_stacktraces"
+echo "  crash hooks enabled = $enable_crashhooks"
 echo "  miner enabled       = $enable_miner"
 echo "  werror              = $enable_werror"
 echo 

contrib/gitian-descriptors/gitian-linux.yml

@@ -38,7 +38,7 @@ script: |
 
   WRAP_DIR=$HOME/wrapped
   HOSTS="i686-pc-linux-gnu x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu"
-  CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests"
+  CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="date ar ranlib nm"
   HOST_CFLAGS="-O2 -g"

contrib/gitian-descriptors/gitian-osx.yml

@@ -37,7 +37,7 @@ files:
 script: |
   WRAP_DIR=$HOME/wrapped
   HOSTS="x86_64-apple-darwin11"
-  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage"
+  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg genisoimage"
 
@@ -146,6 +146,7 @@ script: |
 
     CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS}
     make ${MAKEOPTS}
+    make -C src osx_debug
     make install-strip DESTDIR=${INSTALLPATH}
 
     make osx_volname
@@ -170,12 +171,15 @@ script: |
     find . -name "lib*.la" -delete
     find . -name "lib*.a" -delete
     rm -rf ${DISTNAME}/lib/pkgconfig
-    find ${DISTNAME} | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
+    find .. -name *.dSYM -exec cp -ra {} ${DISTNAME}/bin \;
+    find ${DISTNAME} -not -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
+    find ${DISTNAME} -path '*.dSYM*' | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
     cd ../../
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-  mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin11.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin11-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
 
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then

contrib/gitian-descriptors/gitian-win.yml

@@ -31,7 +31,7 @@ files: []
 script: |
   WRAP_DIR=$HOME/wrapped
   HOSTS="i686-w64-mingw32 x86_64-w64-mingw32"
-  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests"
+  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests --enable-crash-hooks"
   FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy"
   FAKETIME_PROGS="date makensis zip"
   HOST_CFLAGS="-O2 -g"

src/Makefile.am

@@ -10,8 +10,8 @@ AM_CXXFLAGS = $(HARDENED_CXXFLAGS) $(ERROR_CXXFLAGS)
 AM_CPPFLAGS = $(HARDENED_CPPFLAGS)
 EXTRA_LIBRARIES =
 
-if ENABLE_STACKTRACES
+if ENABLE_CRASH_HOOKS
-if STACKTRACE_WRAPPED_CXX_ABI
+if CRASH_HOOKS_WRAPPED_CXX_ABI
 # Wrap internal C++ ABI's so that we can attach stacktraces to exceptions
 LDFLAGS_WRAP_EXCEPTIONS = -Wl,-wrap,__cxa_allocate_exception -Wl,-wrap,__cxa_free_exception
 if TARGET_WINDOWS
@@ -20,10 +20,6 @@ else
 LDFLAGS_WRAP_EXCEPTIONS += -Wl,-wrap,__assert_fail
 endif
 endif
-
-if RDYNAMIC_SUPPORTED
-# This gives better stacktraces
-AM_CXXFLAGS += -rdynamic
 endif
 
 if TARGET_WINDOWS
@@ -31,7 +27,6 @@ BACKTRACE_LIB = -ldbghelp -lbacktrace
 else
 BACKTRACE_LIB = -lbacktrace
 endif
-endif
 
 if EMBEDDED_UNIVALUE
 LIBUNIVALUE = univalue/libunivalue.la
@@ -646,6 +641,7 @@ clean-local:
 	-rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno
 	-rm -f config.h
 	-rm -rf test/__pycache__
+	-rm -rf *.dSYM test/*.dSYM bench/*.dSYM qt/*.dSYM qt/test/*.dSYM
 
 .rc.o:
 	@test -f $(WINDRES)
@@ -668,6 +664,10 @@ if HARDEN
 	$(AM_V_at) READELF=$(READELF) OBJDUMP=$(OBJDUMP) $(top_srcdir)/contrib/devtools/security-check.py < $(bin_PROGRAMS)
 endif
 
+
+osx_debug: $(bin_PROGRAMS)
+	for i in $(bin_PROGRAMS); do mkdir -p $$i.dSYM/Contents/Resources/DWARF && $(DSYMUTIL_FLAT) -o $$i.dSYM/Contents/Resources/DWARF/$$(basename $$i) $$i &> /dev/null ; done
+
 %.pb.cc %.pb.h: %.proto
 	@test -f $(PROTOC)
 	$(AM_V_GEN) $(PROTOC) --cpp_out=$(@D) --proto_path=$(<D) $<

src/dash-cli.cpp

@@ -79,6 +79,12 @@ static int AppInitRPC(int argc, char* argv[])
     // Parameters
     //
     gArgs.ParseParameters(argc, argv);
+
+    if (gArgs.IsArgSet("-printcrashinfo")) {
+        std::cout << GetCrashInfoStrFromSerializedStr(gArgs.GetArg("-printcrashinfo", "")) << std::endl;
+        return true;
+    }
+
     if (argc<2 || gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help") || gArgs.IsArgSet("-version")) {
         std::string strUsage = strprintf(_("%s RPC client version"), _(PACKAGE_NAME)) + " " + FormatFullVersion() + "\n";
         if (!gArgs.IsArgSet("-version")) {

src/dash-tx.cpp

@@ -43,6 +43,11 @@ static int AppInitRawTx(int argc, char* argv[])
     //
     gArgs.ParseParameters(argc, argv);
 
+    if (gArgs.IsArgSet("-printcrashinfo")) {
+        std::cout << GetCrashInfoStrFromSerializedStr(gArgs.GetArg("-printcrashinfo", "")) << std::endl;
+        return true;
+    }
+
     // Check for -testnet or -regtest parameter (Params() calls are only valid after this clause)
     try {
         SelectParams(ChainNameFromCommandLine());

src/dashd.cpp

@@ -76,6 +76,11 @@ bool AppInit(int argc, char* argv[])
     // If Qt is used, parameters/dash.conf are parsed in qt/dash.cpp's main()
     gArgs.ParseParameters(argc, argv);
 
+    if (gArgs.IsArgSet("-printcrashinfo")) {
+        std::cout << GetCrashInfoStrFromSerializedStr(gArgs.GetArg("-printcrashinfo", "")) << std::endl;
+        return true;
+    }
+
     // Process help and version before taking care about datadir
     if (gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") ||  gArgs.IsArgSet("-help") || gArgs.IsArgSet("-version"))
     {

src/qt/dash.cpp

@@ -622,6 +622,13 @@ int main(int argc, char *argv[])
     initTranslations(qtTranslatorBase, qtTranslator, translatorBase, translator);
     translationInterface.Translate.connect(Translate);
 
+    if (gArgs.IsArgSet("-printcrashinfo")) {
+        auto crashInfo = GetCrashInfoStrFromSerializedStr(gArgs.GetArg("-printcrashinfo", ""));
+        std::cout << crashInfo << std::endl;
+        QMessageBox::critical(0, QObject::tr(PACKAGE_NAME), QString::fromStdString(crashInfo));
+        return EXIT_SUCCESS;
+    }
+
     // Show help message immediately after parsing command-line options (for "-lang") and setting locale,
     // but before showing splash screen.
     if (gArgs.IsArgSet("-?") || gArgs.IsArgSet("-h") || gArgs.IsArgSet("-help") || gArgs.IsArgSet("-version"))

src/stacktraces.cpp

@@ -3,9 +3,12 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "stacktraces.h"
+#include "fs.h"
 #include "tinyformat.h"
 #include "random.h"
+#include "streams.h"
 #include "util.h"
+#include "utilstrencodings.h"
 
 #include "dash-config.h"
 
@@ -30,25 +33,21 @@
 
 #if !WIN32
 #include <dlfcn.h>
+#if !__APPLE__
+#include <link.h>
+#endif
 #endif
 
 #if __APPLE__
 #include <mach-o/dyld.h>
+#include <mach/mach_init.h>
+#include <sys/sysctl.h>
+#include <mach/mach_vm.h>
 #endif
 
-#ifdef ENABLE_STACKTRACES
 #include <backtrace.h>
-#endif
-#include <libgen.h> // for basename()
 #include <string.h>
 
-static void PrintCrashInfo(const std::string& s)
-{
-    LogPrintf("%s", s);
-    fprintf(stderr, "%s", s.c_str());
-    fflush(stderr);
-}
-
 std::string DemangleSymbol(const std::string& name)
 {
 #if __GNUC__ || __clang__
@@ -71,8 +70,7 @@ std::string DemangleSymbol(const std::string& name)
 // this is the case when the terminate handler or an assert already handled the exception
 static std::atomic<bool> skipAbortSignal(false);
 
-#ifdef ENABLE_STACKTRACES
+static ssize_t GetExeFileNameImpl(char* buf, size_t bufSize)
-ssize_t GetExeFileNameImpl(char* buf, size_t bufSize)
 {
 #if WIN32
     std::vector<TCHAR> tmp(bufSize);
@@ -101,7 +99,7 @@ ssize_t GetExeFileNameImpl(char* buf, size_t bufSize)
 #endif
 }
 
-std::string GetExeFileName()
+static std::string GetExeFileName()
 {
     std::vector<char> buf(1024);
     while (true) {
@@ -116,38 +114,51 @@ std::string GetExeFileName()
     }
 }
 
+static std::string g_exeFileName = GetExeFileName();
+static std::string g_exeFileBaseName = fs::path(g_exeFileName).filename().string();
+
 static void my_backtrace_error_callback (void *data, const char *msg,
                                   int errnum)
 {
-    PrintCrashInfo(strprintf("libbacktrace error: %d - %s\n", errnum, msg));
 }
 
 static backtrace_state* GetLibBacktraceState()
 {
-    static std::string exeFileName = GetExeFileName();
+#if WIN32
-    static const char* exeFileNamePtr = exeFileName.empty() ? nullptr : exeFileName.c_str();
+    // libbacktrace is not able to handle the DWARF debuglink in the .exe
+    // but luckily we can just specify the .dbg file here as it's a valid PE/XCOFF file
+    static std::string debugFileName = g_exeFileName + ".dbg";
+    static const char* exeFileNamePtr = fs::exists(debugFileName) ? debugFileName.c_str() : g_exeFileName.c_str();
+#else
+    static const char* exeFileNamePtr = g_exeFileName.empty() ? nullptr : g_exeFileName.c_str();
+#endif
     static backtrace_state* st = backtrace_create_state(exeFileNamePtr, 1, my_backtrace_error_callback, NULL);
     return st;
 }
 
 #if WIN32
+static uint64_t GetBaseAddress()
+{
+    return 0;
+}
+
 // PC addresses returned by StackWalk64 are in the real mapped space, while libbacktrace expects them to be in the
 // default mapped space starting at 0x400000. This method converts the address.
 // TODO this is probably the same reason libbacktrace is not able to gather the stacktrace on Windows (returns pointers like 0x1 or 0xfffffff)
 // If they ever fix this problem, we might end up converting to invalid addresses here
-static uintptr_t ConvertAddress(uintptr_t addr)
+static uint64_t ConvertAddress(uint64_t addr)
 {
     MEMORY_BASIC_INFORMATION mbi;
 
     if (!VirtualQuery((PVOID)addr, &mbi, sizeof(mbi)))
         return 0;
 
-    uintptr_t hMod = (uintptr_t)mbi.AllocationBase;
+    uint64_t hMod = (uint64_t)mbi.AllocationBase;
-    uintptr_t offset = addr - hMod;
+    uint64_t offset = addr - hMod;
     return 0x400000 + offset;
 }
 
-static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t skip, size_t max_frames, const CONTEXT* pContext = nullptr)
+static __attribute__((noinline)) std::vector<uint64_t> GetStackFrames(size_t skip, size_t max_frames, const CONTEXT* pContext = nullptr)
 {
     // We can't use libbacktrace for stack unwinding on Windows as it returns invalid addresses (like 0x1 or 0xffffffff)
     static BOOL symInitialized = SymInitialize(GetCurrentProcess(), NULL, TRUE);
@@ -195,7 +206,7 @@ static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t sk
 #error unsupported architecture
 #endif
 
-    std::vector<uintptr_t> ret;
+    std::vector<uint64_t> ret;
 
     size_t i = 0;
     while (ret.size() < max_frames) {
@@ -208,7 +219,7 @@ static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t sk
             break;
         }
         if (i >= skip) {
-            uintptr_t pc = ConvertAddress(stackframe.AddrPC.Offset);
+            uint64_t pc = ConvertAddress(stackframe.AddrPC.Offset);
             if (pc == 0) {
                 pc = stackframe.AddrPC.Offset;
             }
@@ -220,18 +231,48 @@ static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t sk
     return ret;
 }
 #else
-static int my_backtrace_simple_callback(void *data, uintptr_t pc)
+
+#if __APPLE__
+static uint64_t GetBaseAddress()
 {
-    auto v = (std::vector<uintptr_t>*)data;
+    mach_port_name_t target_task;
-    v->emplace_back(pc);
+    vm_map_offset_t vmoffset;
-    if (v->size() >= 128) {
+    vm_map_size_t vmsize;
-        // abort
+    uint32_t nesting_depth = 0;
-        return 1;
+    struct vm_region_submap_info_64 vbr;
-    }
+    mach_msg_type_number_t vbrcount = 16;
+    kern_return_t kr;
+
+    kr = task_for_pid(mach_task_self(), getpid(), &target_task);
+    if (kr != KERN_SUCCESS) {
         return 0;
     }
 
-static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t skip, size_t max_frames)
+    kr = mach_vm_region_recurse(target_task, &vmoffset, &vmsize, &nesting_depth, (vm_region_recurse_info_t)&vbr, &vbrcount);
+    if (kr != KERN_SUCCESS) {
+        return 0;
+    }
+
+    return vmoffset;
+}
+#else
+static int dl_iterate_callback(struct dl_phdr_info* info, size_t s, void* data)
+{
+    uint64_t* p = (uint64_t*)data;
+    if (info->dlpi_name == NULL || info->dlpi_name[0] == '\0') {
+        *p = info->dlpi_addr;
+    }
+}
+
+static uint64_t GetBaseAddress()
+{
+    uint64_t basePtr = 0;
+    dl_iterate_phdr(dl_iterate_callback, &basePtr);
+    return basePtr;
+}
+#endif
+
+static __attribute__((noinline)) std::vector<uint64_t> GetStackFrames(size_t skip, size_t max_frames)
 {
     // FYI, this is not using libbacktrace, but "backtrace()" from <execinfo.h>
     std::vector<void*> buf(max_frames);
@@ -241,20 +282,31 @@ static __attribute__((noinline)) std::vector<uintptr_t> GetStackFrames(size_t sk
     }
     buf.resize((size_t)count);
 
-    std::vector<uintptr_t> ret;
+    std::vector<uint64_t> ret;
     ret.reserve(count);
     for (size_t i = skip + 1; i < buf.size(); i++) {
-        ret.emplace_back((uintptr_t) buf[i]);
+        ret.emplace_back((uint64_t) buf[i]);
     }
     return ret;
 }
 #endif
 
 struct stackframe_info {
-    uintptr_t pc;
+    uint64_t pc{0};
     std::string filename;
-    int lineno;
+    int lineno{-1};
     std::string function;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(pc);
+        READWRITE(filename);
+        READWRITE(lineno);
+        READWRITE(function);
+    }
 };
 
 static int my_backtrace_full_callback (void *data, uintptr_t pc, const char *filename, int lineno, const char *function)
@@ -284,7 +336,7 @@ static int my_backtrace_full_callback (void *data, uintptr_t pc, const char *fil
     return 0;
 }
 
-static std::vector<stackframe_info> GetStackFrameInfos(const std::vector<uintptr_t>& stackframes)
+static std::vector<stackframe_info> GetStackFrameInfos(const std::vector<uint64_t>& stackframes)
 {
     std::vector<stackframe_info> infos;
     infos.reserve(stackframes.size());
@@ -298,10 +350,124 @@ static std::vector<stackframe_info> GetStackFrameInfos(const std::vector<uintptr
     return infos;
 }
 
-static std::string GetStackFrameInfosStr(const std::vector<stackframe_info>& sis, size_t spaces = 2)
+struct crash_info_header
 {
-    if (sis.empty()) {
+    std::string magic;
-        return "\n";
+    uint16_t version;
+    std::string exeFileName;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(magic);
+        READWRITE(version);
+        READWRITE(exeFileName);
+    }
+};
+
+struct crash_info
+{
+    std::string crashDescription;
+    std::vector<uint64_t> stackframes;
+    std::vector<stackframe_info> stackframeInfos;
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action)
+    {
+        READWRITE(crashDescription);
+        READWRITE(stackframes);
+        READWRITE(stackframeInfos);
+    }
+
+    void ConvertAddresses(int64_t offset)
+    {
+        for (auto& sf : stackframes) {
+            sf += offset;
+        }
+        for (auto& sfi : stackframeInfos) {
+            sfi.pc += offset;
+        }
+    }
+};
+
+static std::string GetCrashInfoStrNoDebugInfo(crash_info ci)
+{
+    static uint64_t basePtr = GetBaseAddress();
+
+    CDataStream ds(SER_DISK, 0);
+
+    crash_info_header hdr;
+    hdr.magic = "DashCrashInfo";
+    hdr.version = 1;
+    hdr.exeFileName = g_exeFileBaseName;
+    ds << hdr;
+
+    ci.ConvertAddresses(-(int64_t)basePtr);
+    ds << ci;
+
+    auto ciStr = EncodeBase32((const unsigned char*)ds.data(), ds.size());
+    std::string s = ci.crashDescription + "\n";
+    s += strprintf("No debug information available for stacktrace. You should add debug information and then run:\n"
+                   "%s -printcrashinfo=%s\n", g_exeFileBaseName, ciStr);
+    return s;
+}
+
+static std::string GetCrashInfoStr(const crash_info& ci, size_t spaces = 2);
+
+std::string GetCrashInfoStrFromSerializedStr(const std::string& ciStr)
+{
+    static uint64_t basePtr = GetBaseAddress();
+
+    bool dataInvalid = false;
+    auto buf = DecodeBase32(ciStr.c_str(), &dataInvalid);
+    if (buf.empty() || dataInvalid) {
+        return "Error while deserializing crash info";
+    }
+
+    CDataStream ds(buf, SER_DISK, 0);
+
+    crash_info_header hdr;
+    try {
+        ds >> hdr;
+    } catch (...) {
+        return "Error while deserializing crash info header";
+    }
+
+    if (hdr.magic != "DashCrashInfo") {
+        return "Invalid magic string";
+    }
+    if (hdr.version != 1) {
+        return "Unsupported version";
+    }
+    if (hdr.exeFileName != g_exeFileBaseName) {
+        return "Crash info is not for this executable";
+    }
+
+    crash_info ci;
+    try {
+        ds >> ci;
+    } catch (...) {
+        return "Error while deserializing crash info";
+    }
+
+    ci.ConvertAddresses(basePtr);
+
+    if (ci.stackframeInfos.empty()) {
+        std::vector<uint64_t> stackframes(ci.stackframes.begin(), ci.stackframes.end());
+        ci.stackframeInfos = GetStackFrameInfos(stackframes);
+    }
+
+    return GetCrashInfoStr(ci);
+}
+
+static std::string GetCrashInfoStr(const crash_info& ci, size_t spaces)
+{
+    if (ci.stackframeInfos.empty()) {
+        return GetCrashInfoStrNoDebugInfo(ci);
     }
 
     std::string sp;
@@ -310,16 +476,14 @@ static std::string GetStackFrameInfosStr(const std::vector<stackframe_info>& sis
     }
 
     std::vector<std::string> lstrs;
-    lstrs.reserve(sis.size());
+    lstrs.reserve(ci.stackframeInfos.size());
 
-    for (size_t i = 0; i < sis.size(); i++) {
+    for (size_t i = 0; i < ci.stackframeInfos.size(); i++) {
-        auto& si = sis[i];
+        auto& si = ci.stackframeInfos[i];
 
         std::string lstr;
         if (!si.filename.empty()) {
-            std::vector<char> vecFilename(si.filename.size() + 1, '\0');
+            lstr += fs::path(si.filename).filename().string();
-            strcpy(vecFilename.data(), si.filename.c_str());
-            lstr += basename(vecFilename.data());
         } else {
             lstr += "<unknown-file>";
         }
@@ -335,9 +499,9 @@ static std::string GetStackFrameInfosStr(const std::vector<stackframe_info>& sis
 
     std::string fmtStr = strprintf("%%2d#: (0x%%08X) %%-%ds - %%s\n", lstrlen);
 
-    std::string s;
+    std::string s = ci.crashDescription + "\n";
-    for (size_t i = 0; i < sis.size(); i++) {
+    for (size_t i = 0; i < ci.stackframeInfos.size(); i++) {
-        auto& si = sis[i];
+        auto& si = ci.stackframeInfos[i];
 
         auto& lstr = lstrs[i];
 
@@ -356,10 +520,19 @@ static std::string GetStackFrameInfosStr(const std::vector<stackframe_info>& sis
     return s;
 }
 
-static std::mutex g_stacktraces_mutex;
+static void PrintCrashInfo(const crash_info& ci)
-static std::map<void*, std::shared_ptr<std::vector<uintptr_t>>> g_stacktraces;
+{
+    auto str = GetCrashInfoStr(ci);
+    LogPrintf("%s", str);
+    fprintf(stderr, "%s", str.c_str());
+    fflush(stderr);
+}
 
-#if STACKTRACE_WRAPPED_CXX_ABI
+#ifdef ENABLE_CRASH_HOOKS
+static std::mutex g_stacktraces_mutex;
+static std::map<void*, std::shared_ptr<std::vector<uint64_t>>> g_stacktraces;
+
+#if CRASH_HOOKS_WRAPPED_CXX_ABI
 // These come in through -Wl,-wrap
 // It only works on GCC
 extern "C" void* __real___cxa_allocate_exception(size_t thrown_size);
@@ -402,7 +575,7 @@ extern "C" void __real___assert_fail(const char *assertion, const char *file, un
 #endif
 #endif
 
-#if STACKTRACE_WRAPPED_CXX_ABI
+#if CRASH_HOOKS_WRAPPED_CXX_ABI
 #define WRAPPED_NAME(x) __wrap_##x
 #else
 #define WRAPPED_NAME(x) x
@@ -415,7 +588,7 @@ extern "C" void* __attribute__((noinline)) WRAPPED_NAME(__cxa_allocate_exception
 
     // WARNING keep this as it is and don't try to optimize it (no std::move, no std::make_shared, ...)
     // trying to optimize this will cause the optimizer to move the GetStackFrames() call deep into the stl libs
-    std::shared_ptr<std::vector<uintptr_t>> st(new std::vector<uintptr_t>(localSt));
+    std::shared_ptr<std::vector<uint64_t>> st(new std::vector<uint64_t>(localSt));
 
     void* p = __real___cxa_allocate_exception(thrown_size);
 
@@ -432,41 +605,64 @@ extern "C" void __attribute__((noinline)) WRAPPED_NAME(__cxa_free_exception)(voi
     g_stacktraces.erase(thrown_exception);
 }
 
+static __attribute__((noinline)) crash_info GetCrashInfoFromAssertion(const char* assertion, const char* file, int line, const char* function)
+{
+    crash_info ci;
+    ci.stackframes = GetStackFrames(1, 16);
+    ci.crashDescription = "Assertion failure:";
+    if (assertion) {
+        ci.crashDescription += strprintf("\n  assertion: %s", assertion);
+    }
+    if (file) {
+        ci.crashDescription += strprintf("\n  file: %s, line: %d", file, line);
+    }
+    if (function) {
+        ci.crashDescription += strprintf("\n  function: %s", function);
+    }
+    ci.stackframeInfos = GetStackFrameInfos(ci.stackframes);
+    return ci;
+}
+
 #if __clang__
 extern "C" void __attribute__((noinline)) WRAPPED_NAME(__assert_rtn)(const char *function, const char *file, int line, const char *assertion)
 {
-    auto st = GetCurrentStacktraceStr(1);
+    auto ci = GetCrashInfoFromAssertion(assertion, file, line, function);
-    PrintCrashInfo(strprintf("#### assertion failed: %s ####\n%s", assertion, st));
+    PrintCrashInfo(ci);
     skipAbortSignal = true;
     __real___assert_rtn(function, file, line, assertion);
 }
 #elif WIN32
 extern "C" void __attribute__((noinline)) WRAPPED_NAME(_assert)(const char *assertion, const char *file, unsigned int line)
 {
-    auto st = GetCurrentStacktraceStr(1);
+    auto ci = GetCrashInfoFromAssertion(assertion, file, line, nullptr);
-    PrintCrashInfo(strprintf("#### assertion failed: %s ####\n%s", assertion, st));
+    PrintCrashInfo(ci);
     skipAbortSignal = true;
     __real__assert(assertion, file, line);
 }
 extern "C" void __attribute__((noinline)) WRAPPED_NAME(_wassert)(const wchar_t *assertion, const wchar_t *file, unsigned int line)
 {
-    auto st = GetCurrentStacktraceStr(1);
+    auto ci = GetCrashInfoFromAssertion(
-    PrintCrashInfo(strprintf("#### assertion failed: %s ####\n%s", std::string(assertion, assertion + wcslen(assertion)), st));
+            assertion ?  std::string(assertion, assertion + wcslen(assertion)).c_str() : nullptr,
+            file ? std::string(file, file + wcslen(file)).c_str() : nullptr,
+            line, nullptr);
+    PrintCrashInfo(ci);
     skipAbortSignal = true;
     __real__wassert(assertion, file, line);
 }
 #else
 extern "C" void __attribute__((noinline)) WRAPPED_NAME(__assert_fail)(const char *assertion, const char *file, unsigned int line, const char *function)
 {
-    auto st = GetCurrentStacktraceStr(1);
+    auto ci = GetCrashInfoFromAssertion(assertion, file, line, function);
-    PrintCrashInfo(strprintf("#### assertion failed: %s ####\n%s", assertion, st));
+    PrintCrashInfo(ci);
     skipAbortSignal = true;
     __real___assert_fail(assertion, file, line, function);
 }
 #endif
+#endif //ENABLE_CRASH_HOOKS
 
-static std::shared_ptr<std::vector<uintptr_t>> GetExceptionStacktrace(const std::exception_ptr& e)
+static std::shared_ptr<std::vector<uint64_t>> GetExceptionStacktrace(const std::exception_ptr& e)
 {
+#ifdef ENABLE_CRASH_HOOKS
     void* p = *(void**)&e;
 
     std::lock_guard<std::mutex> l(g_stacktraces_mutex);
@@ -475,36 +671,19 @@ static std::shared_ptr<std::vector<uintptr_t>> GetExceptionStacktrace(const std:
         return nullptr;
     }
     return it->second;
-}
-#endif //ENABLE_STACKTRACES
-
-std::string GetExceptionStacktraceStr(const std::exception_ptr& e)
-{
-#ifdef ENABLE_STACKTRACES
-    auto stackframes = GetExceptionStacktrace(e);
-    if (stackframes && !stackframes->empty()) {
-        auto infos = GetStackFrameInfos(*stackframes);
-        return GetStackFrameInfosStr(infos);
-    }
-#endif
-    return "<no stacktrace>\n";
-}
-
-std::string __attribute__((noinline)) GetCurrentStacktraceStr(size_t skip, size_t max_depth)
-{
-#ifdef ENABLE_STACKTRACES
-    auto stackframes = GetStackFrames(skip + 1, max_depth); // +1 to skip current method
-    auto infos = GetStackFrameInfos(stackframes);
-    return GetStackFrameInfosStr(infos);
 #else
-    return "<no stacktrace>\n";
+    return {};
 #endif
 }
 
-std::string GetPrettyExceptionStr(const std::exception_ptr& e)
+crash_info GetCrashInfoFromException(const std::exception_ptr& e)
 {
+    crash_info ci;
+    ci.crashDescription = "Exception: ";
+
     if (!e) {
-        return "<no exception>\n";
+        ci.crashDescription += "<null>";
+        return ci;
     }
 
     std::string type;
@@ -536,35 +715,42 @@ std::string GetPrettyExceptionStr(const std::exception_ptr& e)
         type = DemangleSymbol(type);
     }
 
-    std::string s = strprintf("Exception: type=%s, what=\"%s\"\n", type, what);
+    ci.crashDescription += strprintf("type=%s, what=\"%s\"", type, what);
 
-#if ENABLE_STACKTRACES
+    auto stackframes = GetExceptionStacktrace(e);
-    s += GetExceptionStacktraceStr(e);
+    if (stackframes) {
-#endif
+        ci.stackframes = *stackframes;
+        ci.stackframeInfos = GetStackFrameInfos(ci.stackframes);
+    }
 
-    return s;
+    return ci;
+}
+
+std::string GetPrettyExceptionStr(const std::exception_ptr& e)
+{
+    return GetCrashInfoStr(GetCrashInfoFromException(e));
 }
 
 static void terminate_handler()
 {
     auto exc = std::current_exception();
 
-    std::string s, s2;
+    crash_info ci;
-    s += "#### std::terminate() called, aborting ####\n";
+    ci.crashDescription = "std::terminate() called, aborting";
 
     if (exc) {
-        s += "#### UNCAUGHT EXCEPTION ####\n";
+        auto ci2 = GetCrashInfoFromException(exc);
-        s2 = GetPrettyExceptionStr(exc);
+        ci.crashDescription = strprintf("std::terminate() called due to unhandled exception\n%s", ci2.crashDescription);
+        ci.stackframes = std::move(ci2.stackframes);
+        ci.stackframeInfos = std::move(ci2.stackframeInfos);
     } else {
-        s += "#### UNKNOWN REASON ####\n";
+        ci.crashDescription = "std::terminate() called due unknown reason";
-#ifdef ENABLE_STACKTRACES
+        ci.stackframes = GetStackFrames(0, 16);
-        s2 = GetCurrentStacktraceStr(0);
-#else
-        s2 = "\n";
-#endif
     }
 
-    PrintCrashInfo(strprintf("%s%s", s, s2));
+    ci.stackframeInfos = GetStackFrameInfos(ci.stackframes);
+
+    PrintCrashInfo(ci);
 
     skipAbortSignal = true;
     std::abort();
@@ -575,25 +761,28 @@ void RegisterPrettyTerminateHander()
     std::set_terminate(terminate_handler);
 }
 
-#ifdef ENABLE_STACKTRACES
 #if !WIN32
 static void HandlePosixSignal(int s)
 {
     if (s == SIGABRT && skipAbortSignal) {
         return;
     }
-    std::string st = GetCurrentStacktraceStr(0);
+
     const char* name = strsignal(s);
     if (!name) {
         name = "UNKNOWN";
     }
-    PrintCrashInfo(strprintf("#### signal %s ####\n%s", name, st));
+
+    crash_info ci;
+    ci.crashDescription = strprintf("Posix Signal: %s", name);
+    ci.stackframes = GetStackFrames(0, 16);
+    ci.stackframeInfos = GetStackFrameInfos(ci.stackframes);
+    PrintCrashInfo(ci);
 
     // avoid a signal loop
     skipAbortSignal = true;
     std::abort();
 }
-
 #else
 static void DoHandleWindowsException(EXCEPTION_POINTERS * ExceptionInfo)
 {
@@ -623,11 +812,12 @@ static void DoHandleWindowsException(EXCEPTION_POINTERS * ExceptionInfo)
     default: excType = "UNKNOWN"; break;
     }
 
-    auto stackframes = GetStackFrames(0, 16, ExceptionInfo->ContextRecord);
+    crash_info ci;
-    auto infos = GetStackFrameInfos(stackframes);
+    ci.crashDescription = strprintf("Windows Exception: %s", excType);
-    auto infosStr = GetStackFrameInfosStr(infos);
+    ci.stackframes = GetStackFrames(0, 16, ExceptionInfo->ContextRecord);
+    ci.stackframeInfos = GetStackFrameInfos(ci.stackframes);
 
-    PrintCrashInfo(strprintf("#### Windows Exception %s ####\n%s", excType, infosStr));
+    PrintCrashInfo(ci);
 }
 
 LONG WINAPI HandleWindowsException(EXCEPTION_POINTERS * ExceptionInfo)
@@ -645,11 +835,9 @@ LONG WINAPI HandleWindowsException(EXCEPTION_POINTERS * ExceptionInfo)
     return EXCEPTION_CONTINUE_SEARCH;
 }
 #endif
-#endif // ENABLE_STACKTRACES
 
 void RegisterPrettySignalHandlers()
 {
-#if ENABLE_STACKTRACES
 #if WIN32
     SetUnhandledExceptionFilter(HandleWindowsException);
 #else
@@ -679,5 +867,4 @@ void RegisterPrettySignalHandlers()
         sigaction(s, &sa_segv, NULL);
     }
 #endif
-#endif
 }

src/stacktraces.h

@@ -15,10 +15,8 @@
 
 std::string DemangleSymbol(const std::string& name);
 
-std::string GetCurrentStacktraceStr(size_t skip = 0, size_t max_depth = 16);
-
-std::string GetExceptionStacktraceStr(const std::exception_ptr& e);
 std::string GetPrettyExceptionStr(const std::exception_ptr& e);
+std::string GetCrashInfoStrFromSerializedStr(const std::string& ciStr);
 
 template<typename T>
 std::string GetExceptionWhat(const T& e);

src/util.cpp

@@ -602,7 +602,7 @@ std::string HelpMessageOpt(const std::string &option, const std::string &message
 
 static std::string FormatException(const std::exception_ptr pex, const char* pszThread)
 {
-    return strprintf("EXCEPTION: %s", GetPrettyExceptionStr(pex));
+    return GetPrettyExceptionStr(pex);
 }
 
 void PrintExceptionContinue(const std::exception_ptr pex, const char* pszThread)