Merge #21304: guix: Add guix-clean script + establish gc-root for container profiles

867a5e172a23899a4a70eca4a396c64f1951745e guix: Register garbage collector root for containers (Carl Dong)
8f8b96fb542701b7717683caa3848390b24f77ab guix: Update hint messages to mention guix-clean (Carl Dong)
44f6d4f56b16e1dc5e8a23318b8e7aad0665f178 guix: Record precious directories and add guix-clean (Carl Dong)
84912d4b24382ae022da3a863bd6caa2b8948d94 build: Remove spaces from variable-printing rules (Carl Dong)

Pull request description:

  ```
  guix: Record precious directories and add guix-clean

  Many users have reported problems that stem from having an unclean
  working tree. To that end, I've written a guix-clean script which should
  help reset the working tree while respecting user-specified precious
  directories.

  Precious directories, such as:

  - SOURCES_PATH
  - BASE_CACHE
  - SDK_PATH
  - OUTDIR

  Should be preserved when cleaning the working tree, and are thus
  recorded in ./contrib/guix/var/precious_dirs.

  The ./contrib/guix/guix-clean script is able to parse that file and make
  sure to avoid them when cleaning out the working tree.
  ```

ACKs for top commit:
  laanwj:
    ACK 867a5e172a23899a4a70eca4a396c64f1951745e

Tree-SHA512: c498fad781ff5e6406639df2b91b687fc528273fdf266bcdba8f6eec3b3b37ecce544b6da0252f0b9c6717f9d88e844e4c7b72d1877bdbabfc6871ddd0172af5

Makefile.am

@@ -5,7 +5,7 @@
 
 # Pattern rule to print variables, e.g. make print-top_srcdir
 print-%:
-	@echo '$*' = '$($*)'
+	@echo '$*'='$($*)'
 
 ACLOCAL_AMFLAGS = -I build-aux/m4
 SUBDIRS = src

contrib/guix/guix-build

@@ -103,6 +103,14 @@ ERR: Build directories for this commit already exist for the following platform
 
      Aborting...
 
+Hint: To blow everything away, you may want to use:
+
+  $ ./contrib/guix/guix-clean
+
+Specifically, this will remove all files without an entry in the index,
+excluding the SDK directory, the depends download cache, the depends built
+packages cache, the garbage collector roots for Guix environments, and the
+output directory.
 EOF
 for host in $hosts_distsrc_exists; do
     echo "     ${host} '$(distsrc_for_host "$host")'"
@@ -119,7 +127,7 @@ fi
 for host in $HOSTS; do
     case "$host" in
         *darwin*)
-            OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=[[:space:]]\+@@g')"
+            OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')"
             if [ -e "$OSX_SDK" ]; then
                 echo "Found macOS SDK at '${OSX_SDK}', using..."
             else
@@ -178,12 +186,6 @@ host_to_commonname() {
     esac
 }
 
-# Download the depends sources now as we won't have internet access in the build
-# container
-for host in $HOSTS; do
-    make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
-done
-
 # Determine the reference time used for determinism (overridable by environment)
 SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}"
 
@@ -201,10 +203,70 @@ time-machine() {
                       -- "$@"
 }
 
+
+# Precious directories are those which should not be cleaned between successive
+# guix builds
+depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH'
+precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE"
+
+# Usage: contains IFS-SEPARATED-LIST ITEM
+contains() {
+    for i in ${1}; do
+        if [ "$i" = "${2}" ]; then
+            return 0  # Found!
+        fi
+    done
+    return 1
+}
+
+# If the user explicitly specified a precious directory, create it so we
+# can map it into the container
+for precious_dir_name in $precious_dir_names; do
+    precious_dir_path="${!precious_dir_name}"
+    if [ -n "$precious_dir_path" ]; then
+        if [ ! -e "$precious_dir_path" ]; then
+            mkdir -p "$precious_dir_path"
+        elif [ -L "$precious_dir_path" ]; then
+            echo "ERR: ${precious_dir_name} cannot be a symbolic link"
+            exit 1
+        elif [ ! -d "$precious_dir_path" ]; then
+            echo "ERR: ${precious_dir_name} must be a directory"
+            exit 1
+        fi
+    fi
+done
+
+mkdir -p "$VAR_BASE"
+
+# Record the _effective_ values of precious directories such that guix-clean can
+# avoid clobbering them if appropriate.
+#
+# shellcheck disable=SC2046,SC2086
+{
+    # Get depends precious dir definitions from depends
+    make -C "${PWD}/depends" \
+         --no-print-directory \
+         -- $(printf "print-%s\n" $depends_precious_dir_names)
+
+    # Get remaining precious dir definitions from the environment
+    for precious_dir_name in $precious_dir_names; do
+        precious_dir_path="${!precious_dir_name}"
+        if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then
+            echo "${precious_dir_name}=${precious_dir_path}"
+        fi
+    done
+} > "${VAR_BASE}/precious_dirs"
+
 # Make sure an output directory exists for our builds
 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
 mkdir -p "$OUTDIR_BASE"
 
+# Download the depends sources now as we won't have internet access in the build
+# container
+for host in $HOSTS; do
+    make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"}
+done
+
 # Usage: outdir_for_host HOST
 #
 #   HOST: The current platform triple we're building for
@@ -213,6 +275,14 @@ outdir_for_host() {
     echo "${OUTDIR_BASE}/${1}"
 }
 
+# Usage: profiledir_for_host HOST COMMAND
+#
+#   HOST: The current platform triple we're building for
+#
+profiledir_for_host() {
+    echo "${PROFILES_BASE}/${2}-${1}"
+}
+
 
 #########
 # BUILD #
@@ -223,24 +293,19 @@ outdir_for_host() {
 int_trap() {
 cat << EOF
 ** INT received while building ${1}, you may want to clean up the relevant
-   output, deploy, and distsrc-* directories before rebuilding
+   work directories (e.g. distsrc-*) before rebuilding
 
 Hint: To blow everything away, you may want to use:
 
-  $ git clean -xdff --exclude='/depends/SDKs/*'
+  $ ./contrib/guix/guix-clean
 
 Specifically, this will remove all files without an entry in the index,
-excluding the SDK directory. Practically speaking, this means that all ignored
-and untracked files and directories will be wiped, allowing you to start anew.
+excluding the SDK directory, the depends download cache, the depends built
+packages cache, the garbage collector roots for Guix environments, and the
+output directory.
 EOF
 }
 
-# Create SOURCES_PATH, BASE_CACHE, and SDK_PATH if they are non-empty so that we
-# can map them into the container
-[ -z "$SOURCES_PATH" ] || mkdir -p "$SOURCES_PATH"
-[ -z "$BASE_CACHE" ]   || mkdir -p "$BASE_CACHE"
-[ -z "$SDK_PATH" ]     || mkdir -p "$SDK_PATH"
-
 # Deterministically build Bitcoin Core
 # shellcheck disable=SC2153
 for host in $HOSTS; do
@@ -347,6 +412,7 @@ EOF
                                  --keep-failed \
                                  --fallback \
                                  --link-profile \
+                                 --root="$(profiledir_for_host "${HOST}" build)" \
                                  ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \
                                  ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
                                  -- env HOST="$host" \

contrib/guix/guix-clean

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+export LC_ALL=C
+set -e -o pipefail
+
+# Source the common prelude, which:
+#   1. Checks if we're at the top directory of the Bitcoin Core repository
+#   2. Defines a few common functions and variables
+#
+# shellcheck source=libexec/prelude.bash
+source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash"
+
+
+###################
+## Sanity Checks ##
+###################
+
+################
+# Required non-builtin commands should be invokable
+################
+
+check_tools cat mkdir make git guix
+
+
+#############
+##  Clean  ##
+#############
+
+# Usage: under_dir MAYBE_PARENT MAYBE_CHILD
+#
+# If MAYBE_CHILD is a subdirectory of MAYBE_PARENT, print the relative path
+# from MAYBE_PARENT to MAYBE_CHILD. Otherwise, return 1 as the error code.
+#
+# NOTE: This does not perform any symlink-resolving or path canonicalization.
+#
+under_dir() {
+    local path_residue
+    path_residue="${2##${1}}"
+    if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then
+        return 1
+    else
+        echo "$path_residue"
+    fi
+}
+
+# Usage: dir_under_git_root MAYBE_CHILD
+#
+# If MAYBE_CHILD is under the current git repository and exists, print the
+# relative path from the git repository's top-level directory to MAYBE_CHILD,
+# otherwise, exit with an error code.
+#
+dir_under_git_root() {
+    local rv
+    rv="$(under_dir "$(git_root)" "$1")"
+    [ -n "$rv" ] && echo "$rv"
+}
+
+shopt -s nullglob
+found_precious_dirs_files=( "${version_base_prefix}"*/"${var_base_basename}/precious_dirs" ) # This expands to an array of directories...
+shopt -u nullglob
+
+exclude_flags=()
+
+for precious_dirs_file in "${found_precious_dirs_files[@]}"; do
+    # Make sure the precious directories (e.g. SOURCES_PATH, BASE_CACHE, SDK_PATH)
+    # are excluded from git-clean
+    echo "Found precious_dirs file: '${precious_dirs_file}'"
+
+    # Exclude the precious_dirs file itself
+    if dirs_file_exclude_fragment=$(dir_under_git_root "$(dirname "$precious_dirs_file")"); then
+        exclude_flags+=( --exclude="${dirs_file_exclude_fragment}/precious_dirs" )
+    fi
+
+    # Read each 'name=dir' pair from the precious_dirs file
+    while IFS='=' read -r name dir; do
+        # Add an exclusion flag if the precious directory is under the git root.
+        if under=$(dir_under_git_root "$dir"); then
+            echo "Avoiding ${name}: ${under}"
+            exclude_flags+=( --exclude="$under" )
+        fi
+    done < "$precious_dirs_file"
+done
+
+git clean -xdff "${exclude_flags[@]}"

contrib/guix/libexec/prelude.bash

@@ -58,3 +58,9 @@ VERSION_BASE="${version_base_prefix}${VERSION}"  # TOP
 DISTSRC_BASE="${DISTSRC_BASE:-${VERSION_BASE}}"
 
 OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}"
+
+var_base_basename="var"
+VAR_BASE="${VAR_BASE:-${VERSION_BASE}/${var_base_basename}}"
+
+profiles_base_basename="profiles"
+PROFILES_BASE="${PROFILES_BASE:-${VAR_BASE}/${profiles_base_basename}}"

depends/Makefile

@@ -2,7 +2,7 @@
 
 # Pattern rule to print variables, e.g. make print-top_srcdir
 print-%:
-	@echo '$*' = '$($*)'
+	@echo '$*'='$($*)'
 
 # When invoking a sub-make, keep only the command line variable definitions
 # matching the pattern in the filter function.

src/Makefile.am

@@ -5,7 +5,7 @@
 
 # Pattern rule to print variables, e.g. make print-top_srcdir
 print-%:
-	@echo '$*' = '$($*)'
+	@echo '$*'='$($*)'
 
 DIST_SUBDIRS = secp256k1