merge bitcoin#20470: Replace genisoimage with xorriso

.gitignore

@@ -74,6 +74,7 @@ libconftest.dylib*
 *.log
 *.trs
 *.dmg
+*.iso
 
 *.json.h
 *.raw.h

Makefile.am

@@ -34,6 +34,7 @@ space := $(empty) $(empty)
 OSX_APP=Dash-Qt.app
 OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME))
 OSX_DMG = $(OSX_VOLNAME).dmg
+OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso
 OSX_BACKGROUND_SVG=background.svg
 OSX_BACKGROUND_IMAGE=background.tiff
 OSX_BACKGROUND_IMAGE_DPIS=36 72
@@ -144,8 +145,12 @@ $(APP_DIST_DIR)/Applications:
 
 $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Dash-Qt
 
-$(OSX_DMG): $(APP_DIST_EXTRAS)
-	$(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist
+.INTERMEDIATE: $(OSX_TEMP_ISO)
+$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS)
+	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist
+
+$(OSX_DMG): $(OSX_TEMP_ISO)
+	$(DMG) dmg "$<" "$@"
 
 dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG)
 	sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@

ci/Dockerfile.builder

@@ -38,7 +38,7 @@ RUN dpkg --add-architecture i386
 RUN apt-get update && apt-get install $APT_ARGS g++-7-multilib && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS g++-arm-linux-gnueabihf && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS g++-mingw-w64-x86-64 && rm -rf /var/lib/apt/lists/*
-RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis xorriso && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS python3-zmq && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS shellcheck cppcheck && rm -rf /var/lib/apt/lists/*
 RUN apt-get update && apt-get install $APT_ARGS imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools && rm -rf /var/lib/apt/lists/*

configure.ac

@@ -645,7 +645,8 @@ case $host in
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil)
            AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool)
            AC_PATH_TOOL([OTOOL], [otool], otool)
-           AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage)
+           AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs)
+           AC_PATH_PROGS([DMG], [dmg], dmg)
            AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert)
            AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert)
            AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp)

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -7,6 +7,7 @@ architectures:
 - "amd64"
 packages:
 - "faketime"
+- "xorriso"
 remotes:
 - "url": "https://github.com/dashpay/dash-detached-sigs.git"
   "dir": "signature"
@@ -18,7 +19,7 @@ script: |
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
   export PATH="$PWD":$PATH
-  FAKETIME_PROGS="dmg genisoimage"
+  FAKETIME_PROGS="dmg xorrisofs"
 
   # Create global faketime wrappers
   for prog in ${FAKETIME_PROGS}; do
@@ -36,5 +37,5 @@ script: |
   tar -xf ${UNSIGNED}
   OSX_VOLNAME="$(cat osx_volname)"
   ./detached-sig-apply.sh ${UNSIGNED} signature/osx
-  ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app
+  ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app
   ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED}

contrib/gitian-descriptors/gitian-osx.yml

@@ -29,6 +29,7 @@ packages:
 - "fonts-tuffy"
 - "ccache"
 - "cmake"
+- "xorriso"
 remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
@@ -39,9 +40,9 @@ script: |
 
   WRAP_DIR=$HOME/wrapped
   HOSTS="x86_64-apple-darwin18"
-  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
+  CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
-  FAKETIME_PROGS="ar ranlib date dmg genisoimage"
+  FAKETIME_PROGS="ar ranlib date dmg xorrisofs"
 
   export QT_RCC_TEST=1
   export QT_RCC_SOURCE_DATE_OVERRIDE=1
@@ -161,12 +162,11 @@ script: |
 
     make osx_volname
     make deploydir
-    OSX_VOLNAME="$(cat osx_volname)"
     mkdir -p unsigned-app-${i}
     cp osx_volname unsigned-app-${i}/
     cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i}
     cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i}
-    cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i}
+    cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i}
     cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate
     cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff
     mv dist unsigned-app-${i}
@@ -174,8 +174,7 @@ script: |
     find . | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz
     popd
 
-    make deploy
-    ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg
+    make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg"
 
     cd installed
     find . -name "lib*.la" -delete

contrib/macdeploy/README.md

@@ -96,19 +96,15 @@ created using these tools. The build process has been designed to avoid includin
 SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely
 redistributed.
 
-`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been
-patched. A system `genisoimage` will work fine, but it will not be deterministic because
-the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch).
-No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for
-a single invocation, so that's no real concern.
+[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG.
 
-`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the
-`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its
-maintainer has seemingly abandoned the project.
+`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the
+`libdmg-hfsplus` project is used to compress it. There are several bugs in this
+tool and its maintainer has seemingly abandoned the project.
 
 The DMG tool has the ability to create DMGs from scratch as well, but this functionality is
 broken. Only the compression feature is currently used. Ideally, the creation could be fixed
-and `genisoimage` would no longer be necessary.
+and `xorrisofs` would no longer be necessary.
 
 Background images and other features can be added to DMG files by inserting a
 `.DS_Store` before creation. This is generated by the script `contrib/macdeploy/custom_dsstore.py`.

depends/packages/native_cdrkit.mk

@@ -1,27 +0,0 @@
-package=native_cdrkit
-$(package)_version=1.1.11
-$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c
-$(package)_file_name=cdrkit-$($(package)_version).tar.bz2
-$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
-$(package)_patches=cdrkit-deterministic.patch
-$(package)_dependencies=cmake
-
-define $(package)_preprocess_cmds
-  patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch
-endef
-
-define $(package)_config_cmds
-  $(host_prefix)/bin/cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix)
-endef
-
-define $(package)_build_cmds
-  $(MAKE) genisoimage
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install
-endef
-
-define $(package)_postprocess_cmds
-  rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump
-endef

depends/packages/packages.mk

@@ -17,7 +17,7 @@ darwin_native_packages = native_biplist native_ds_store native_mac_alias
 $(host_arch)_$(host_os)_native_packages += native_b2
 
 ifneq ($(build_os),darwin)
-darwin_native_packages += native_cctools native_cdrkit native_libtapi native_libdmg-hfsplus
+darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus
 
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 darwin_native_packages+= native_clang

depends/patches/native_cdrkit/cdrkit-deterministic.patch

@@ -1,86 +0,0 @@
---- cdrkit-1.1.11.old/genisoimage/tree.c	2008-10-21 19:57:47.000000000 -0400
-+++ cdrkit-1.1.11/genisoimage/tree.c	2013-12-06 00:23:18.489622668 -0500
-@@ -1139,8 +1139,9 @@
- scan_directory_tree(struct directory *this_dir, char *path, 
- 						  struct directory_entry *de)
- {
--	DIR		*current_dir;
-+        int             current_file;
- 	char		whole_path[PATH_MAX];
-+        struct dirent  **d_list;
- 	struct dirent	*d_entry;
- 	struct directory *parent;
- 	int		dflag;
-@@ -1164,7 +1165,8 @@
- 	this_dir->dir_flags |= DIR_WAS_SCANNED;
- 
- 	errno = 0;	/* Paranoia */
--	current_dir = opendir(path);
-+	//current_dir = opendir(path);
-+        current_file = scandir(path, &d_list, NULL, alphasort);
- 	d_entry = NULL;
- 
- 	/*
-@@ -1173,12 +1175,12 @@
- 	 */
- 	old_path = path;
- 
--	if (current_dir) {
-+	if (current_file >= 0) {
- 		errno = 0;
--		d_entry = readdir(current_dir);
-+		d_entry = d_list[0];
- 	}
- 
--	if (!current_dir || !d_entry) {
-+	if (current_file < 0 || !d_entry) {
- 		int	ret = 1;
- 
- #ifdef	USE_LIBSCHILY
-@@ -1191,8 +1193,8 @@
- 			de->isorec.flags[0] &= ~ISO_DIRECTORY;
- 			ret = 0;
- 		}
--		if (current_dir)
--			closedir(current_dir);
-+		if(d_list)
-+			free(d_list);
- 		return (ret);
- 	}
- #ifdef	ABORT_DEEP_ISO_ONLY
-@@ -1208,7 +1210,7 @@
- 			errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n");
- 			errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n");
- 		}
--		closedir(current_dir);
-+		free(d_list);
- 		return (1);
- 	}
- #endif
-@@ -1250,13 +1252,13 @@
- 		 * The first time through, skip this, since we already asked
- 		 * for the first entry when we opened the directory.
- 		 */
--		if (dflag)
--			d_entry = readdir(current_dir);
-+		if (dflag && current_file >= 0)
-+			d_entry = d_list[current_file];
- 		dflag++;
- 
--		if (!d_entry)
-+		if (current_file < 0)
- 			break;
--
-+                current_file--;
- 		/* OK, got a valid entry */
- 
- 		/* If we do not want all files, then pitch the backups. */
-@@ -1348,7 +1350,7 @@
- 		insert_file_entry(this_dir, whole_path, d_entry->d_name);
- #endif	/* APPLE_HYB */
- 	}
--	closedir(current_dir);
-+	free(d_list);
- 
- #ifdef APPLE_HYB
- 	/*