merge bitcoin#21664: use LIEF for macOS and Windows symbol & security checks

ci/test/00_setup_env_mac_host.sh

@@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_macos
 export HOST=x86_64-apple-darwin19
-export PIP_PACKAGES="zmq"
+export PIP_PACKAGES="zmq lief"
 export RUN_UNIT_TESTS=true
 export RUN_INTEGRATION_TESTS=false
 export RUN_SECURITY_TESTS="true"

ci/test/04_install.sh

@@ -68,6 +68,9 @@ if [[ $DOCKER_NAME_TAG == centos* ]]; then
 elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
+  if [ -n "$PIP_PACKAGES" ]; then
+    ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+  fi
 fi
 
 if [ "$TRAVIS_OS_NAME" == "osx" ]; then

contrib/containers/ci/Dockerfile

@@ -37,6 +37,7 @@ RUN pip3 install \
     codespell==1.17.1 \
     flake8==3.8.3 \
     jinja2 \
+    lief==0.11.4 \
     pyzmq \
     vulture==2.3 \
     yq \

contrib/devtools/security-check.py

@@ -6,22 +6,13 @@
 Perform basic security checks on a series of executables.
 Exit status will be 0 if successful, and the program will be silent.
 Otherwise the exit status will be 1 and it will log which executables failed which checks.
-Needs `objdump` (for PE) and `otool` (for MACHO).
 '''
-import subprocess
 import sys
-import os
 from typing import List, Optional
 
+import lief
 import pixie
 
-OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
-OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
-
-def run_command(command) -> str:
-    p = subprocess.run(command, stdout=subprocess.PIPE, check=True, universal_newlines=True)
-    return p.stdout
-
 def check_ELF_PIE(executable) -> bool:
     '''
     Check for position independent executable (PIE), allowing for address space randomization.
@@ -143,112 +134,59 @@ def check_ELF_separate_code(executable):
                 return False
     return True
 
-def get_PE_dll_characteristics(executable) -> int:
-    '''Get PE DllCharacteristics bits'''
-    stdout = run_command([OBJDUMP_CMD, '-x',  executable])
-
-    bits = 0
-    for line in stdout.splitlines():
-        tokens = line.split()
-        if len(tokens)>=2 and tokens[0] == 'DllCharacteristics':
-            bits = int(tokens[1],16)
-    return bits
-
-IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA = 0x0020
-IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE    = 0x0040
-IMAGE_DLL_CHARACTERISTICS_NX_COMPAT       = 0x0100
-
 def check_PE_DYNAMIC_BASE(executable) -> bool:
     '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)'''
-    bits = get_PE_dll_characteristics(executable)
+    binary = lief.parse(executable)
-    return (bits & IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE) == IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE
+    return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists
 
 # Must support high-entropy 64-bit address space layout randomization
 # in addition to DYNAMIC_BASE to have secure ASLR.
 def check_PE_HIGH_ENTROPY_VA(executable) -> bool:
     '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR'''
-    bits = get_PE_dll_characteristics(executable)
+    binary = lief.parse(executable)
-    return (bits & IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA) == IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA
+    return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists
 
 def check_PE_RELOC_SECTION(executable) -> bool:
     '''Check for a reloc section. This is required for functional ASLR.'''
-    stdout = run_command([OBJDUMP_CMD, '-h',  executable])
+    binary = lief.parse(executable)
-
+    return binary.has_relocations
-    for line in stdout.splitlines():
-        if '.reloc' in line:
-            return True
-    return False
-
-def check_PE_NX(executable) -> bool:
-    '''NX: DllCharacteristics bit 0x100 signifies nxcompat (DEP)'''
-    bits = get_PE_dll_characteristics(executable)
-    return (bits & IMAGE_DLL_CHARACTERISTICS_NX_COMPAT) == IMAGE_DLL_CHARACTERISTICS_NX_COMPAT
-
-def get_MACHO_executable_flags(executable) -> List[str]:
-    stdout = run_command([OTOOL_CMD, '-vh', executable])
-
-    flags: List[str] = []
-    for line in stdout.splitlines():
-        tokens = line.split()
-        # filter first two header lines
-        if 'magic' in tokens or 'Mach' in tokens:
-            continue
-        # filter ncmds and sizeofcmds values
-        flags += [t for t in tokens if not t.isdigit()]
-    return flags
-
-def check_MACHO_PIE(executable) -> bool:
-    '''
-    Check for position independent executable (PIE), allowing for address space randomization.
-    '''
-    flags = get_MACHO_executable_flags(executable)
-    if 'PIE' in flags:
-        return True
-    return False
 
 def check_MACHO_NOUNDEFS(executable) -> bool:
     '''
     Check for no undefined references.
     '''
-    flags = get_MACHO_executable_flags(executable)
+    binary = lief.parse(executable)
-    if 'NOUNDEFS' in flags:
+    return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS)
-        return True
-    return False
-
-def check_MACHO_NX(executable) -> bool:
-    '''
-    Check for no stack execution
-    '''
-    flags = get_MACHO_executable_flags(executable)
-    if 'ALLOW_STACK_EXECUTION' in flags:
-        return False
-    return True
 
 def check_MACHO_LAZY_BINDINGS(executable) -> bool:
     '''
     Check for no lazy bindings.
     We don't use or check for MH_BINDATLOAD. See #18295.
     '''
-    stdout = run_command([OTOOL_CMD, '-l', executable])
+    binary = lief.parse(executable)
-
+    return binary.dyld_info.lazy_bind == (0,0)
-    for line in stdout.splitlines():
-        tokens = line.split()
-        if 'lazy_bind_off' in tokens or 'lazy_bind_size' in tokens:
-            if tokens[1] != '0':
-                return False
-    return True
 
 def check_MACHO_Canary(executable) -> bool:
     '''
     Check for use of stack canary
     '''
-    stdout = run_command([OTOOL_CMD, '-Iv', executable])
+    binary = lief.parse(executable)
+    return binary.has_symbol('___stack_chk_fail')
 
-    ok = False
+def check_PIE(executable) -> bool:
-    for line in stdout.splitlines():
+    '''
-        if '___stack_chk_fail' in line:
+    Check for position independent executable (PIE),
-            ok = True
+    allowing for address space randomization.
-    return ok
+    '''
+    binary = lief.parse(executable)
+    return binary.is_pie
+
+def check_NX(executable) -> bool:
+    '''
+    Check for no stack execution
+    '''
+    binary = lief.parse(executable)
+    return binary.has_nx
 
 CHECKS = {
 'ELF': [
@@ -259,15 +197,16 @@ CHECKS = {
     ('separate_code', check_ELF_separate_code),
 ],
 'PE': [
+    ('PIE', check_PIE),
     ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE),
     ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA),
-    ('NX', check_PE_NX),
+    ('NX', check_NX),
     ('RELOC_SECTION', check_PE_RELOC_SECTION)
 ],
 'MACHO': [
-    ('PIE', check_MACHO_PIE),
+    ('PIE', check_PIE),
     ('NOUNDEFS', check_MACHO_NOUNDEFS),
-    ('NX', check_MACHO_NX),
+    ('NX', check_NX),
     ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
     ('Canary', check_MACHO_Canary)
 ]
@@ -285,24 +224,24 @@ def identify_executable(executable) -> Optional[str]:
     return None
 
 if __name__ == '__main__':
-    retval = 0
+    retval: int = 0
     for filename in sys.argv[1:]:
         try:
             etype = identify_executable(filename)
             if etype is None:
-                print('%s: unknown format' % filename)
+                print(f'{filename}: unknown format')
                 retval = 1
                 continue
 
-            failed = []
+            failed: List[str] = []
             for (name, func) in CHECKS[etype]:
                 if not func(filename):
                     failed.append(name)
             if failed:
-                print('%s: failed %s' % (filename, ' '.join(failed)))
+                print(f'{filename}: failed {" ".join(failed)}')
                 retval = 1
         except IOError:
-            print('%s: cannot open' % filename)
+            print(f'{filename}: cannot open')
             retval = 1
     sys.exit(retval)
 

contrib/devtools/symbol-check.py

@@ -13,8 +13,9 @@ Example usage:
 import subprocess
 import sys
 import os
-from typing import List, Optional
+from typing import Optional
 
+import lief
 import pixie
 
 # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases
@@ -64,8 +65,6 @@ IGNORE_EXPORTS = {
 '__cxa_demangle'
 }
 CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt')
-OBJDUMP_CMD = os.getenv('OBJDUMP', '/usr/bin/objdump')
-OTOOL_CMD = os.getenv('OTOOL', '/usr/bin/otool')
 
 # Allowed NEEDED libraries
 ELF_ALLOWED_LIBRARIES = {
@@ -213,44 +212,22 @@ def check_ELF_libraries(filename) -> bool:
             ok = False
     return ok
 
-def macho_read_libraries(filename) -> List[str]:
-    p = subprocess.Popen([OTOOL_CMD, '-L', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
-    libraries = []
-    for line in stdout.splitlines():
-        tokens = line.split()
-        if len(tokens) == 1: # skip executable name
-            continue
-        libraries.append(tokens[0].split('/')[-1])
-    return libraries
-
 def check_MACHO_libraries(filename) -> bool:
     ok = True
-    for dylib in macho_read_libraries(filename):
+    binary = lief.parse(filename)
-        if dylib not in MACHO_ALLOWED_LIBRARIES:
+    for dylib in binary.libraries:
-            print('{} is not in ALLOWED_LIBRARIES!'.format(dylib))
+        split = dylib.name.split('/')
+        if split[-1] not in MACHO_ALLOWED_LIBRARIES:
+            print(f'{split[-1]} is not in ALLOWED_LIBRARIES!')
             ok = False
     return ok
 
-def pe_read_libraries(filename) -> List[str]:
-    p = subprocess.Popen([OBJDUMP_CMD, '-x', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True)
-    (stdout, stderr) = p.communicate()
-    if p.returncode:
-        raise IOError('Error opening file')
-    libraries = []
-    for line in stdout.splitlines():
-        if 'DLL Name:' in line:
-            tokens = line.split(': ')
-            libraries.append(tokens[1])
-    return libraries
-
 def check_PE_libraries(filename) -> bool:
     ok = True
-    for dylib in pe_read_libraries(filename):
+    binary = lief.parse(filename)
+    for dylib in binary.libraries:
         if dylib not in PE_ALLOWED_LIBRARIES:
-            print('{} is not in ALLOWED_LIBRARIES!'.format(dylib))
+            print(f'{dylib} is not in ALLOWED_LIBRARIES!')
             ok = False
     return ok
 
@@ -285,7 +262,7 @@ if __name__ == '__main__':
         try:
             etype = identify_executable(filename)
             if etype is None:
-                print('{}: unknown format'.format(filename))
+                print(f'{filename}: unknown format')
                 retval = 1
                 continue
 
@@ -294,9 +271,9 @@ if __name__ == '__main__':
                 if not func(filename):
                     failed.append(name)
             if failed:
-                print('{}: failed {}'.format(filename, ' '.join(failed)))
+                print(f'{filename}: failed {" ".join(failed)}')
                 retval = 1
         except IOError:
-            print('{}: cannot open'.format(filename))
+            print(f'{filename}: cannot open')
             retval = 1
     sys.exit(retval)

contrib/gitian-descriptors/gitian-linux.yml

@@ -23,6 +23,7 @@ packages:
 - "patch"
 - "pkg-config"
 - "python3"
+- "python3-pip"
 - "libxkbcommon0"
 - "ccache"
 # Cross compilation HOSTS:
@@ -109,6 +110,8 @@ script: |
   done
   }
 
+  pip3 install lief==0.11.4
+
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}
   create_global_faketime_wrappers "2000-01-01 12:00:00"

contrib/gitian-descriptors/gitian-osx.yml

@@ -26,6 +26,7 @@ packages:
 - "python3"
 - "python3-dev"
 - "python3-setuptools"
+- "python3-pip"
 - "fonts-tuffy"
 - "ccache"
 - "cmake"
@@ -98,6 +99,8 @@ script: |
   done
   }
 
+  pip3 install lief==0.11.4
+
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}
   create_global_faketime_wrappers "2000-01-01 12:00:00"

contrib/gitian-descriptors/gitian-win.yml

@@ -22,6 +22,7 @@ packages:
 - "zip"
 - "ca-certificates"
 - "python3"
+- "python3-pip"
 - "ccache"
 remotes:
 - "url": "https://github.com/dashpay/dash.git"
@@ -114,6 +115,8 @@ script: |
   done
   }
 
+  pip3 install lief==0.11.4
+
   # Faketime for depends so intermediate results are comparable
   export PATH_orig=${PATH}
   create_global_faketime_wrappers "2000-01-01 12:00:00"

contrib/guix/manifest.scm

@@ -580,6 +580,29 @@ inspecting signatures in Mach-O binaries.")
   (package-with-extra-patches glibc-2.27
     (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch")))
 
+(define-public lief
+  (package
+   (name "python-lief")
+   (version "0.11.4")
+   (source
+    (origin
+     (method git-fetch)
+     (uri (git-reference
+           (url "https://github.com/lief-project/LIEF.git")
+           (commit version)))
+     (file-name (git-file-name name version))
+     (sha256
+      (base32
+       "0h4kcwr9z478almjqhmils8imfpflzk0r7d05g4xbkdyknn162qf"))))
+   (build-system python-build-system)
+   (native-inputs
+    `(("cmake" ,cmake)))
+   (home-page "https://github.com/lief-project/LIEF")
+   (synopsis "Library to Instrument Executable Formats")
+   (description "Python library to to provide a cross platform library which can
+parse, modify and abstract ELF, PE and MachO formats.")
+   (license license:asl2.0)))
+
 (packages->manifest
  (append
   (list ;; The Basics
@@ -616,6 +639,8 @@ inspecting signatures in Mach-O binaries.")
         python-3
         ;; Git
         git
+        ;; Tests
+        lief
         ;; Native gcc 7 toolchain
         gcc-toolchain-7
         (list gcc-toolchain-7 "static"))