Merge #21304: guix: Add guix-clean script + establish gc-root for container profiles

867a5e172a23899a4a70eca4a396c64f1951745e guix: Register garbage collector root for containers (Carl Dong)
8f8b96fb542701b7717683caa3848390b24f77ab guix: Update hint messages to mention guix-clean (Carl Dong)
44f6d4f56b16e1dc5e8a23318b8e7aad0665f178 guix: Record precious directories and add guix-clean (Carl Dong)
84912d4b24382ae022da3a863bd6caa2b8948d94 build: Remove spaces from variable-printing rules (Carl Dong)

Pull request description:

  ```
  guix: Record precious directories and add guix-clean

  Many users have reported problems that stem from having an unclean
  working tree. To that end, I've written a guix-clean script which should
  help reset the working tree while respecting user-specified precious
  directories.

  Precious directories, such as:

  - SOURCES_PATH
  - BASE_CACHE
  - SDK_PATH
  - OUTDIR

  Should be preserved when cleaning the working tree, and are thus
  recorded in ./contrib/guix/var/precious_dirs.

  The ./contrib/guix/guix-clean script is able to parse that file and make
  sure to avoid them when cleaning out the working tree.
  ```

ACKs for top commit:
  laanwj:
    ACK 867a5e172a23899a4a70eca4a396c64f1951745e

Tree-SHA512: c498fad781ff5e6406639df2b91b687fc528273fdf266bcdba8f6eec3b3b37ecce544b6da0252f0b9c6717f9d88e844e4c7b72d1877bdbabfc6871ddd0172af5
This commit is contained in:
W. J. van der Laan 2021-04-08 23:19:05 +02:00 committed by PastaPastaPasta
parent 27032ca6be
commit 0d5bc0b0f5
6 changed files with 175 additions and 20 deletions

View File

@ -5,7 +5,7 @@
# Pattern rule to print variables, e.g. make print-top_srcdir
print-%:
@echo '$*' = '$($*)'
@echo '$*'='$($*)'
ACLOCAL_AMFLAGS = -I build-aux/m4
SUBDIRS = src

View File

@ -103,6 +103,14 @@ ERR: Build directories for this commit already exist for the following platform
Aborting...
Hint: To blow everything away, you may want to use:
$ ./contrib/guix/guix-clean
Specifically, this will remove all files without an entry in the index,
excluding the SDK directory, the depends download cache, the depends built
packages cache, the garbage collector roots for Guix environments, and the
output directory.
EOF
for host in $hosts_distsrc_exists; do
echo " ${host} '$(distsrc_for_host "$host")'"
@ -119,7 +127,7 @@ fi
for host in $HOSTS; do
case "$host" in
*darwin*)
OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')"
OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
if [ -e "$OSX_SDK" ]; then
echo "Found macOS SDK at '${OSX_SDK}', using..."
else
@ -178,12 +186,6 @@ host_to_commonname() {
esac
}
# Download the depends sources now as we won't have internet access in the build
# container
for host in $HOSTS; do
make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
done
# Determine the reference time used for determinism (overridable by environment)
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}"
@ -201,10 +203,70 @@ time-machine() {
-- "$@"
}
# Precious directories are those which should not be cleaned between successive
# guix builds
depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH'
precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE"
# Usage: contains IFS-SEPARATED-LIST ITEM
contains() {
for i in ${1}; do
if [ "$i" = "${2}" ]; then
return 0 # Found!
fi
done
return 1
}
# If the user explicitly specified a precious directory, create it so we
# can map it into the container
for precious_dir_name in $precious_dir_names; do
precious_dir_path="${!precious_dir_name}"
if [ -n "$precious_dir_path" ]; then
if [ ! -e "$precious_dir_path" ]; then
mkdir -p "$precious_dir_path"
elif [ -L "$precious_dir_path" ]; then
echo "ERR: ${precious_dir_name} cannot be a symbolic link"
exit 1
elif [ ! -d "$precious_dir_path" ]; then
echo "ERR: ${precious_dir_name} must be a directory"
exit 1
fi
fi
done
mkdir -p "$VAR_BASE"
# Record the _effective_ values of precious directories such that guix-clean can
# avoid clobbering them if appropriate.
#
# shellcheck disable=SC2046,SC2086
{
# Get depends precious dir definitions from depends
make -C "${PWD}/depends" \
--no-print-directory \
-- $(printf "print-%s\n" $depends_precious_dir_names)
# Get remaining precious dir definitions from the environment
for precious_dir_name in $precious_dir_names; do
precious_dir_path="${!precious_dir_name}"
if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then
echo "${precious_dir_name}=${precious_dir_path}"
fi
done
} > "${VAR_BASE}/precious_dirs"
# Make sure an output directory exists for our builds
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
mkdir -p "$OUTDIR_BASE"
# Download the depends sources now as we won't have internet access in the build
# container
for host in $HOSTS; do
make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
done
# Usage: outdir_for_host HOST
#
# HOST: The current platform triple we're building for
@ -213,6 +275,14 @@ outdir_for_host() {
echo "${OUTDIR_BASE}/${1}"
}
# Usage: profiledir_for_host HOST COMMAND
#
# HOST: The current platform triple we're building for
#
profiledir_for_host() {
echo "${PROFILES_BASE}/${2}-${1}"
}
#########
# BUILD #
@ -223,24 +293,19 @@ outdir_for_host() {
int_trap() {
cat << EOF
** INT received while building ${1}, you may want to clean up the relevant
output, deploy, and distsrc-* directories before rebuilding
work directories (e.g. distsrc-*) before rebuilding
Hint: To blow everything away, you may want to use:
$ git clean -xdff --exclude='/depends/SDKs/*'
$ ./contrib/guix/guix-clean
Specifically, this will remove all files without an entry in the index,
excluding the SDK directory. Practically speaking, this means that all ignored
and untracked files and directories will be wiped, allowing you to start anew.
excluding the SDK directory, the depends download cache, the depends built
packages cache, the garbage collector roots for Guix environments, and the
output directory.
EOF
}
# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we
# can map them into the container
[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH"
[ -z "$BASE_CACHE" ] || mkdir -p "$BASE_CACHE"
[ -z "$SDK_PATH" ] || mkdir -p "$SDK_PATH"
# Deterministically build Bitcoin Core
# shellcheck disable=SC2153
for host in $HOSTS; do
@ -347,6 +412,7 @@ EOF
--keep-failed \
--fallback \
--link-profile \
--root="$(profiledir_for_host "${HOST}" build)" \
${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
-- env HOST="$host" \

83
contrib/guix/guix-clean Executable file
View File

@ -0,0 +1,83 @@
#!/usr/bin/env bash
export LC_ALL=C
set -e -o pipefail
# Source the common prelude, which:
# 1. Checks if we're at the top directory of the Bitcoin Core repository
# 2. Defines a few common functions and variables
#
# shellcheck source=libexec/prelude.bash
source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
###################
## Sanity Checks ##
###################
################
# Required non-builtin commands should be invokable
################
check_tools cat mkdir make git guix
#############
## Clean ##
#############
# Usage: under_dir MAYBE_PARENT MAYBE_CHILD
#
# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path
# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code.
#
# NOTE: This does not perform any symlink-resolving or path canonicalization.
#
under_dir() {
local path_residue
path_residue="${2##${1}}"
if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then
return 1
else
echo "$path_residue"
fi
}
# Usage: dir_under_git_root MAYBE_CHILD
#
# If MAYBE_CHILD is under the current git repository and exists, print the
# relative path from the git repository's top-level directory to MAYBE_CHILD,
# otherwise, exit with an error code.
#
dir_under_git_root() {
local rv
rv="$(under_dir "$(git_root)" "$1")"
[ -n "$rv" ] && echo "$rv"
}
shopt -s nullglob
found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories...
shopt -u nullglob
exclude_flags=()
for precious_dirs_file in "${found_precious_dirs_files[@]}"; do
# Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH)
# are excluded from git-clean
echo "Found precious_dirs file: '${precious_dirs_file}'"
# Exclude the precious_dirs file itself
if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then
exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" )
fi
# Read each 'name=dir' pair from the precious_dirs file
while IFS='=' read -r name dir; do
# Add an exclusion flag if the precious directory is under the git root.
if under=$(dir_under_git_root "$dir"); then
echo "Avoiding ${name}: ${under}"
exclude_flags+=( --exclude="$under" )
fi
done < "$precious_dirs_file"
done
git clean -xdff "${exclude_flags[@]}"

View File

@ -58,3 +58,9 @@ VERSION_BASE="${version_base_prefix}${VERSION}" # TOP
DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}"
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
var_base_basename="var"
VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}"
profiles_base_basename="profiles"
PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}"

View File

@ -2,7 +2,7 @@
# Pattern rule to print variables, e.g. make print-top_srcdir
print-%:
@echo '$*' = '$($*)'
@echo '$*'='$($*)'
# When invoking a sub-make, keep only the command line variable definitions
# matching the pattern in the filter function.

View File

@ -5,7 +5,7 @@
# Pattern rule to print variables, e.g. make print-top_srcdir
print-%:
@echo '$*' = '$($*)'
@echo '$*'='$($*)'
DIST_SUBDIRS = secp256k1