From 235b3a789d6c54b042a241ffcfaba4404f6245fa Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 13 May 2015 21:12:04 -0400 Subject: [PATCH] depends: sanity-check sources and cached builds In some cases (Travis), sources and build caches may be moved around in-between builds, and we can't necessarily trust that everything is still intact. This introduces pre-build checks that verify against stashed checksums. Note that this will cause all sources to be re-downloaded, since cached sources weren't trustworthy before this. --- depends/Makefile | 40 +++++++++++++++++++++++++----- depends/README.packages | 4 ++- depends/funcs.mk | 13 +++++++--- depends/packages/native_cctools.mk | 2 ++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index 05ef33f2ee..ef5a20e6c3 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -91,12 +91,12 @@ include funcs.mk toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n $(final_build_id_long) | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) -$(host_prefix)/.stamp_$(final_build_id): | $(native_packages) $(packages) +$(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)rm -rf $(@D) $(AT)mkdir -p $(@D) - $(AT)echo copying packages: $| + $(AT)echo copying packages: $^ $(AT)echo to: $(@D) - $(AT)cd $(@D); $(foreach package,$|, tar xf $($(package)_cached); ) + $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) @@ -121,8 +121,35 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ $< > $@ $(AT)touch $@ -install: $(host_prefix)/share/config.site -download-one: $(all_sources) + +define check_or_remove_cached + mkdir -p $(BASE_CACHE)/$(host)/$(package) && cd $(BASE_CACHE)/$(host)/$(package); \ + $(build_SHA256SUM) -c $($(package)_cached_checksum) >/dev/null 2>/dev/null || \ + ( rm -f $($(package)_cached_checksum); \ + if test -f "$($(package)_cached)"; then echo "Checksum mismatch for $(package). Forcing rebuild.."; rm -f $($(package)_cached_checksum) $($(package)_cached); fi ) +endef + +define check_or_remove_sources + mkdir -p $($(package)_source_dir); cd $($(package)_source_dir); \ + $(build_SHA256SUM) -c $($(package)_fetched) >/dev/null 2>/dev/null || \ + ( if test -f $($(package)_all_sources); then echo "Checksum missing or mismatched for $(package) source. Forcing re-download."; fi; \ + rm -f $($(package)_all_sources) $($(1)_fetched)) +endef + +check-packages: + @$(foreach package,$(all_packages),$(call check_or_remove_cached,$(package));) +check-sources: + @$(foreach package,$(all_packages),$(call check_or_remove_sources,$(package));) + +$(host_prefix)/share/config.site: check-packages + +check-packages: check-sources + +install: check-packages $(host_prefix)/share/config.site + + +download-one: check-sources $(all_sources) + download-osx: @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one download-linux: @@ -130,4 +157,5 @@ download-linux: download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win -.PHONY: install cached download-one download-osx download-linux download-win download + +.PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/depends/README.packages b/depends/README.packages index 5ab7ed7dee..6d87402ebc 100644 --- a/depends/README.packages +++ b/depends/README.packages @@ -30,7 +30,9 @@ These variables are optional: Names of any other packages that this one depends on. $(package)_patches: Filenames of any patches needed to build the package - + $(package)_extra_sources: + Any extra files that will be fetched via $(package)_fetch_cmds. These are + specified so that they can be fetched and verified via 'make download'. Build Variables: After defining the main identifiers, build variables may be added or customized diff --git a/depends/funcs.mk b/depends/funcs.mk index b407737f7f..050a9b1321 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -53,12 +53,14 @@ $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) +$(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps -$(1)_fetched=$$($(1)_source_dir)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name) +$(1)_fetched=$(SOURCES_PATH)/download-stamps/.stamp_fetched-$(1)-$($(1)_file_name).hash $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned @@ -154,7 +156,10 @@ endef define int_add_cmds $($(1)_fetched): $(AT)mkdir -p $$(@D) $(SOURCES_PATH) + $(AT)rm -f $$@ + $(AT)touch $$@ $(AT)cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + $(AT)cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) $(AT)touch $$@ $($(1)_extracted): | $($(1)_fetched) $(AT)echo Extracting $(1)... @@ -195,10 +200,12 @@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) $(AT)rm -rf $$(@D) && mkdir -p $$(@D) $(AT)mv $$($(1)_staging_dir)/$$(@F) $$(@) $(AT)rm -rf $($(1)_staging_dir) +$($(1)_cached_checksum): $($(1)_cached) + $(AT)cd $$(@D); $(build_SHA256SUM) $$( $$(@) .PHONY: $(1) -$(1): | $($(1)_cached) -.SECONDARY: $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) +$(1): | $($(1)_cached_checksum) +.SECONDARY: $($(1)_cached) $($(1)_postprocessed) $($(1)_staged) $($(1)_built) $($(1)_configured) $($(1)_preprocessed) $($(1)_extracted) $($(1)_fetched) endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 951ad4fb29..1c1bcf199a 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -9,6 +9,8 @@ $(package)_clang_download_path=http://llvm.org/releases/$($(package)_clang_versi $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-amd64-Ubuntu-12.04.2.tar.gz $(package)_clang_sha256_hash=60d8f69f032d62ef61bf527857ebb933741ec3352d4d328c5516aa520662dab7 +$(package)_extra_sources=$($(package)_clang_file_name) + define $(package)_fetch_cmds $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \ $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash))