merge bitcoin#24534: make gen-sdk deterministic

This commit is contained in:
Kittywhiskers Van Gogh 2022-03-11 16:20:29 +01:00
parent ab10bf9692
commit 2739107de3
No known key found for this signature in database
GPG Key ID: 30CD0C065E5C4AAD
2 changed files with 32 additions and 4 deletions

View File

@ -15,13 +15,16 @@ When complete, it will have produced `Dash-Qt.dmg`.
A free Apple Developer Account is required to proceed.
Our current macOS SDK
(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
extracted from
(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`)
can be extracted from
[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
Alternatively, after logging in to your account go to 'Downloads', then 'More'
and search for [`Xcode_12.2`](https://developer.apple.com/download/all/?q=Xcode%2012.2).
An Apple ID and cookies enabled for the hostname are needed to download this.
The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
The `sha256sum` of the downloaded XIP archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
archive. This makes the SDK less-trivial to extract on non-macOS machines. One
@ -55,7 +58,10 @@ previous stage) as the first argument.
./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
```
The `sha256sum` of the generated TAR.GZ archive should be `e7ca56bc8804d16624fad68be2e71647747d6629cacaaa3de5fbfa7f444e9eae`.
## Deterministic macOS DMG Notes
Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
`binutils` (`ld`, `ar`, etc) and DMG authoring tools.

View File

@ -8,6 +8,21 @@ import gzip
import os
import contextlib
# monkey-patch Python 3.8 and older to fix wrong TAR header handling
# see https://github.com/bitcoin/bitcoin/pull/24534
# and https://github.com/python/cpython/pull/18080 for more info
if sys.version_info < (3, 9):
_old_create_header = tarfile.TarInfo._create_header
def _create_header(info, format, encoding, errors):
buf = _old_create_header(info, format, encoding, errors)
# replace devmajor/devminor with binary zeroes
buf = buf[:329] + bytes(16) + buf[345:]
# recompute checksum
chksum = tarfile.calc_chksums(buf)[0]
buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:]
return buf
tarfile.TarInfo._create_header = staticmethod(_create_header)
@contextlib.contextmanager
def cd(path):
"""Context manager that restores PWD even if an exception was raised."""
@ -75,14 +90,21 @@ def run():
tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
if tarinfo.linkname and tarinfo.linkname.startswith("./"):
tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
# make metadata deterministic
tarinfo.mtime = 0
tarinfo.uid, tarinfo.uname = 0, ''
tarinfo.gid, tarinfo.gname = 0, ''
# don't use isdir() as there are also executable files present
tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0x0644
return tarinfo
with cd(dir_to_add):
# recursion already adds entries in sorted order
tarfp.add(".", recursive=True, filter=change_tarinfo_base)
print("Creating output .tar.gz file...")
with out_sdktgz_path.open("wb") as fp:
with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf:
with tarfile.open(mode="w", fileobj=gzf) as tarfp:
with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp:
print("Adding MacOSX SDK {} files...".format(sdk_version))
tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
print("Adding libc++ headers...")