sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-25 12:02:48 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity
cdedbea0b4
dash/contrib/gitian-build.py
fanquake 7d78f06a92 partial Merge bitcoin/bitcoin#19817: build: macOS toolchain bump 
a5550f877a2c46d01bb620ae051c0c8ed0fecd0b build: use -stdlib++-isystem with Clang 10 (fanquake)
51d9d1607f2e9d593693ca799393f068192e41aa guix: use Clang 10 for the macOS cross compile (fanquake)
b80a6af9e55325d444e117e85bbfc76d88d898a8 build: no longer patch threading out of ld64 (fanquake)
c29cba44b3706e0a2035e440e560f2d15d50433b build: Xcode 12.1, macOS SDK 10.15.6 (fanquake)
9ed2f19d385aa95f65807999bba2e18417b143dc build: native cctools 973.0.1, ld64 609 (fanquake)
f48f187cce7fa43646fb0d796c244e1515e763ec build: Clang 10.0.1 (Hennadii Stepanov)
9b193cd2a3ca20917611fbed56dfbcd8a39aeab8 build: libtapi 1100.0.11 (fanquake)

Pull request description:

  Bumps our macOS toolchain to be using the following:
  * Clang 10.0.1 (gitian) & Clang 10.0.0 (Guix)
  * ld64 609
  * libtapi 1100.0.11
  * cctools  973.0.1
  * Xcode 12.1
  * macOS SDK 10.15.6

  which are currently the most recent releases available as open source. See upstream [`cctools`](https://github.com/tpoechtrager/cctools-port/commits/973.0.1-ld64-609) and [`libtapi`](https://github.com/tpoechtrager/apple-libtapi/tree/1100.0.11).

  This should improve the possibility of Apple ARM cross-compilation in depends.

  This also removes our [patching out of pthreads usage](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cctools/ld64_disable_threading.patch) in `ld64`. There have been multiple changes since `ld64 450.3`, which have likely fixed the non-determinism we were working around. i.e from [InputFiles.cpp](https://opensource.apple.com/source/ld64/ld64-609/src/ld/InputFiles.cpp.auto.html):
  ```cpp
  // <rdar://problem/15002251> make implicit dylib order be deterministic by sorting by install_name
  std::sort(implicitDylibs.begin(), implicitDylibs.end(), DylibByInstallNameSorter());
  ```

  ```cpp
  // <rdar://problem/42675402> ld64 output is not deterministic due to dylib processing order
  std::sort(unprocessedDylibs.begin(), unprocessedDylibs.end(), [](const ld::dylib::File* lhs, const ld::dylib::File* rhs) {
  return strcmp(lhs->path(), rhs->path()) < 0;
  });
  ```

  Guix Build:
  ```bash
  find guix-build-$(git rev-parse --short=12 HEAD)/output/ -type f -print0 | env LC_ALL=C sort -z | xargs -r0 sha256sum
  f6c3817b8fe5f7370299d1ae2533e4a3acd313ba9f9aa8d423a8956117e52dd5  guix-build-a5550f877a2c/output/dist-archive/bitcoin-a5550f877a2c.tar.gz
  4954dcf563c2d496b8d9fecd48f8e3f7fba2f319ffa254a5bc8ee12cfee6acf0  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx-unsigned.dmg
  8f6095b445c7f1a8e6accd86bb7f0696d5849402084927d2b726b7d557831c3a  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx-unsigned.tar.gz
  cc40f25477b4defc1617ae694313d80f307ddf6742fe6cc85c6bc0e215ef8be0  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx64.tar.gz
  ```

  Gitian Build:
  ```bash
  Generating report
  506a8abdefe559999b43dd9f14905b9b2b5a3363b1cd013d45ae47acc1f7ef6c  bitcoin-a5550f877a2c-osx-unsigned.dmg
  f606997f74026dd12d110d683c6f116b40df324836904ef507dd7ac787e6ebe2  bitcoin-a5550f877a2c-osx-unsigned.tar.gz
  5b495ef15f2c3260c2950921b61326912a9bf533cccd51e13818809fd225489e  bitcoin-a5550f877a2c-osx64.tar.gz
  f6c3817b8fe5f7370299d1ae2533e4a3acd313ba9f9aa8d423a8956117e52dd5  src/bitcoin-a5550f877a2c.tar.gz
  9eb0221e962d2839770963bd03c6c9e98e8bf3078566bee2ae42f06233a710fa  bitcoin-core-osx-22-res.yml
  Done.
  ```

ACKs for top commit:
  hebasto:
    ACK a5550f877a2c46d01bb620ae051c0c8ed0fecd0b

Tree-SHA512: 504c4b0f9cd3b939714a322298320c5bde07e9356a48a9a000060b36f8dce4d6134ed60c3a5188810476a28ec5b108733eabbc6fb8053231b9ea8a494cc91b12
2023-03-26 16:50:26 -05:00

271 lines
15 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# Copyright (c) 2018-2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
import argparse
import os
import subprocess
import sys
def setup():
global args, workdir
programs = ['ruby', 'git', 'make', 'wget', 'curl']
if args.kvm:
programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils']
elif args.docker:
if not os.path.isfile('/lib/systemd/system/docker.service'):
dockers = ['docker.io', 'docker-ce']
for i in dockers:
return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
if return_code == 0:
break
if return_code != 0:
print('Cannot find any way to install Docker.', file=sys.stderr)
sys.exit(1)
else:
programs += ['apt-cacher-ng', 'lxc', 'debootstrap']
subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
if not os.path.isdir('gitian.sigs'):
subprocess.check_call(['git', 'clone', 'https://github.com/dashpay/gitian.sigs.git'])
if not os.path.isdir('dash-detached-sigs'):
subprocess.check_call(['git', 'clone', 'https://github.com/dashpay/dash-detached-sigs.git'])
if not os.path.isdir('gitian-builder'):
subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git'])
if not os.path.isdir('dash'):
subprocess.check_call(['git', 'clone', 'https://github.com/dashpay/dash.git'])
os.chdir('gitian-builder')
make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64']
if args.docker:
make_image_prog += ['--docker']
elif args.lxc:
make_image_prog += ['--lxc', '--disksize', '13000']
subprocess.check_call(make_image_prog)
os.chdir(workdir)
if args.is_focal and not args.kvm and not args.docker:
subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net'])
print('Reboot is required')
sys.exit(0)
def build():
global args, workdir
os.makedirs('dashcore-binaries/' + args.version, exist_ok=True)
print('\nBuilding Dependencies\n')
os.chdir('gitian-builder')
os.makedirs('inputs', exist_ok=True)
subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz'])
subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True)
subprocess.check_call(['make', '-C', '../dash/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common'])
if args.linux:
print('\nCompiling ' + args.version + ' Linux')
subprocess.check_call(['bin/gbuild', '--fetch-tags', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-linux.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-linux.yml'])
subprocess.check_call('mv build/out/dashcore-*.tar.gz build/out/src/dashcore-*.tar.gz ../dashcore-binaries/'+args.version, shell=True)
if args.windows:
print('\nCompiling ' + args.version + ' Windows')
subprocess.check_call(['bin/gbuild', '--fetch-tags', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-win.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-win.yml'])
subprocess.check_call('mv build/out/dashcore-*-win-unsigned.tar.gz inputs/', shell=True)
subprocess.check_call('mv build/out/dashcore-*.zip build/out/dashcore-*.exe build/out/src/dashcore-*.tar.gz ../dashcore-binaries/'+args.version, shell=True)
if args.macos:
print('\nCompiling ' + args.version + ' MacOS')
subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
subprocess.check_call(['bin/gbuild', '--fetch-tags', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)
subprocess.check_call('mv build/out/dashcore-*.tar.gz build/out/dashcore-*.dmg build/out/src/dashcore-*.tar.gz ../dashcore-binaries/'+args.version, shell=True)
os.chdir(workdir)
if args.commit_files:
print('\nCommitting '+args.version+' Unsigned Sigs\n')
os.chdir('gitian.sigs')
subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer])
subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer])
subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer])
subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer])
os.chdir(workdir)
def sign():
global args, workdir
os.chdir('gitian-builder')
if args.windows:
print('\nSigning ' + args.version + ' Windows')
subprocess.check_call('cp inputs/dashcore-' + args.version + '-win-unsigned.tar.gz inputs/dashcore-win-unsigned.tar.gz', shell=True)
subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../dash/contrib/gitian-descriptors/gitian-win-signer.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-win-signer.yml'])
subprocess.check_call('mv build/out/dashcore-*win64-setup.exe ../dashcore-binaries/'+args.version, shell=True)
if args.macos:
print('\nSigning ' + args.version + ' MacOS')
subprocess.check_call('cp inputs/dashcore-' + args.version + '-osx-unsigned.tar.gz inputs/dashcore-osx-unsigned.tar.gz', shell=True)
subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../dash/contrib/gitian-descriptors/gitian-osx-signer.yml'])
subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx-signer.yml'])
subprocess.check_call('mv build/out/dashcore-osx-signed.dmg ../dashcore-binaries/'+args.version+'/dashcore-'+args.version+'-osx.dmg', shell=True)
os.chdir(workdir)
if args.commit_files:
print('\nCommitting '+args.version+' Signed Sigs\n')
os.chdir('gitian.sigs')
subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer])
subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer])
subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer])
os.chdir(workdir)
def verify():
global args, workdir
rc = 0
os.chdir('gitian-builder')
print('\nVerifying v'+args.version+' Linux\n')
if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../dash/contrib/gitian-descriptors/gitian-linux.yml']):
print('Verifying v'+args.version+' Linux FAILED\n')
rc = 1
print('\nVerifying v'+args.version+' Windows\n')
if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../dash/contrib/gitian-descriptors/gitian-win.yml']):
print('Verifying v'+args.version+' Windows FAILED\n')
rc = 1
print('\nVerifying v'+args.version+' MacOS\n')
if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../dash/contrib/gitian-descriptors/gitian-osx.yml']):
print('Verifying v'+args.version+' MacOS FAILED\n')
rc = 1
print('\nVerifying v'+args.version+' Signed Windows\n')
if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../dash/contrib/gitian-descriptors/gitian-win-signer.yml']):
print('Verifying v'+args.version+' Signed Windows FAILED\n')
rc = 1
print('\nVerifying v'+args.version+' Signed MacOS\n')
if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../dash/contrib/gitian-descriptors/gitian-osx-signer.yml']):
print('Verifying v'+args.version+' Signed MacOS FAILED\n')
rc = 1
os.chdir(workdir)
return rc
def main():
global args, workdir
parser = argparse.ArgumentParser(description='Script for running full Gitian builds.')
parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch')
parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request')
parser.add_argument('-u', '--url', dest='url', default='https://github.com/dashpay/dash', help='Specify the URL of the repository. Default is %(default)s')
parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build')
parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build')
parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS')
parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries')
parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS')
parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s')
parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s')
parser.add_argument('-V', '--virtualization', dest='virtualization', default='docker', help='Specify virtualization technology to use: lxc for LXC, kvm for KVM, docker for Docker. Default is %(default)s')
parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)')
parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.')
parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git')
parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file')
parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
args = parser.parse_args()
workdir = os.getcwd()
args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs'])
args.lxc = (args.virtualization == 'lxc')
args.kvm = (args.virtualization == 'kvm')
args.docker = (args.virtualization == 'docker')
script_name = os.path.basename(sys.argv[0])
if not args.lxc and not args.kvm and not args.docker:
print(script_name+': Wrong virtualization option.')
print('Try '+script_name+' --help for more information')
sys.exit(1)
# Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they
# can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm).
os.environ['USE_LXC'] = ''
os.environ['USE_VBOX'] = ''
os.environ['USE_DOCKER'] = ''
if args.docker:
os.environ['USE_DOCKER'] = '1'
elif not args.kvm:
os.environ['USE_LXC'] = '1'
if 'GITIAN_HOST_IP' not in os.environ.keys():
os.environ['GITIAN_HOST_IP'] = '10.0.3.1'
if 'LXC_GUEST_IP' not in os.environ.keys():
os.environ['LXC_GUEST_IP'] = '10.0.3.5'
if args.setup:
setup()
if args.buildsign:
args.build = True
args.sign = True
if not args.build and not args.sign and not args.verify:
sys.exit(0)
args.linux = 'l' in args.os
args.windows = 'w' in args.os
args.macos = 'm' in args.os
# Disable for MacOS if no SDK found
if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
args.macos = False
args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
if not args.signer:
print(script_name+': Missing signer')
print('Try '+script_name+' --help for more information')
sys.exit(1)
if not args.version:
print(script_name+': Missing version')
print('Try '+script_name+' --help for more information')
sys.exit(1)
# Add leading 'v' for tags
if args.commit and args.pull:
raise Exception('Cannot have both commit and pull')
args.commit = ('' if args.commit else 'v') + args.version
os.chdir('dash')
if args.pull:
subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
os.chdir('../gitian-builder/inputs/dash')
subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip()
args.version = 'pull-' + args.version
print(args.commit)
subprocess.check_call(['git', 'fetch'])
subprocess.check_call(['git', 'checkout', args.commit])
os.chdir(workdir)
os.chdir('gitian-builder')
subprocess.check_call(['git', 'pull'])
os.chdir(workdir)
if args.build:
build()
if args.sign:
sign()
if args.verify:
os.chdir('gitian.sigs')
subprocess.check_call(['git', 'pull'])
os.chdir(workdir)
sys.exit(verify())
if __name__ == '__main__':
main()
Reference in New Issue View Git Blame Copy Permalink