From 614ff70b4ce05da67bc0f893a8379f1097a9b134 Mon Sep 17 00:00:00 2001
From: Alexander Block <ablock84@gmail.com>
Date: Mon, 19 Nov 2018 07:31:13 +0100
Subject: [PATCH] Let ccache compress the cache by itself instead of
 compressing ccache.tar (#2456)

The cache can become quite large and uncompressing/recompressing it is slow.
Better to let ccache handle compression as it will then only perform it
on files which are actually used.
---
 Jenkinsfile.gitian                          |  2 ++
 contrib/gitian-descriptors/gitian-linux.yml | 11 +++++++----
 contrib/gitian-descriptors/gitian-osx.yml   | 11 +++++++----
 contrib/gitian-descriptors/gitian-win.yml   | 11 +++++++----
 4 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/Jenkinsfile.gitian b/Jenkinsfile.gitian
index f695706fb..793d48807 100644
--- a/Jenkinsfile.gitian
+++ b/Jenkinsfile.gitian
@@ -117,6 +117,8 @@ for(int i = 0; i < targets.size(); i++) {
           archiveArtifacts artifacts: 'dashcore-binaries/*', fingerprint: true
         }
 
+        // TODO remove this in a few days (only needed to prune the old compressed file from Jenkins caches)
+        sh "cd gitian-builder/cache && find -name ccache.tar.gz | xargs rm -f"
         sh "cd gitian-builder/cache && tar czfv ../../cache-${gitianDescriptor.name}.tar.gz common ${gitianDescriptor.name}"
         archiveArtifacts artifacts: "cache-${gitianDescriptor.name}.tar.gz", fingerprint: true
       }
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 6fcda5646..39523dfc3 100755
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -61,12 +61,15 @@ script: |
     export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache
     # As we later wrap the gcc binaries, this is fast
     export CCACHE_COMPILERCHECK="content"
-    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar.gz ]; then
+    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then
       pushd ${GBUILD_PACKAGE_CACHE}
-      tar xzf ccache.tar.gz
-      rm ccache.tar.gz
+      tar xf ccache.tar
+      rm ccache.tar
       popd
     fi
+    # instead of compressing ccache.tar, we let ccache handle it by itself
+    # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use
+    export CCACHE_COMPRESS=1
   else
     CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache"
   fi
@@ -212,7 +215,7 @@ script: |
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then
     pushd ${GBUILD_PACKAGE_CACHE}
-    tar czf ccache.tar.gz ccache
+    tar cf ccache.tar ccache
     rm -rf ccache
     popd
   fi
diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml
index 80fba15d8..5c17c5b46 100644
--- a/contrib/gitian-descriptors/gitian-osx.yml
+++ b/contrib/gitian-descriptors/gitian-osx.yml
@@ -55,12 +55,15 @@ script: |
     # Setup ccache to use correct cache directories
     CONFIGFLAGS="${CONFIGFLAGS} --enable-ccache"
     export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache
-    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar.gz ]; then
+    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then
       pushd ${GBUILD_PACKAGE_CACHE}
-      tar xzf ccache.tar.gz
-      rm ccache.tar.gz
+      tar xf ccache.tar
+      rm ccache.tar
       popd
     fi
+    # instead of compressing ccache.tar, we let ccache handle it by itself
+    # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use
+    export CCACHE_COMPRESS=1
   else
     CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache"
   fi
@@ -176,7 +179,7 @@ script: |
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then
     pushd ${GBUILD_PACKAGE_CACHE}
-    tar czf ccache.tar.gz ccache
+    tar cf ccache.tar ccache
     rm -rf ccache
     popd
   fi
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
index ccb36c7bb..6d1b673a3 100755
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -53,12 +53,15 @@ script: |
     export CCACHE_DIR=${GBUILD_PACKAGE_CACHE}/ccache
     # As we later wrap the gcc binaries, this is fast
     export CCACHE_COMPILERCHECK="content"
-    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar.gz ]; then
+    if [ -f ${GBUILD_PACKAGE_CACHE}/ccache.tar ]; then
       pushd ${GBUILD_PACKAGE_CACHE}
-      tar xzf ccache.tar.gz
-      rm ccache.tar.gz
+      tar xf ccache.tar
+      rm ccache.tar
       popd
     fi
+    # instead of compressing ccache.tar, we let ccache handle it by itself
+    # Otherwise we end up uncompressing/compressing a lot of cache files which we actually never use
+    export CCACHE_COMPRESS=1
   else
     CONFIGFLAGS="${CONFIGFLAGS} --disable-ccache"
   fi
@@ -200,7 +203,7 @@ script: |
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then
     pushd ${GBUILD_PACKAGE_CACHE}
-    tar czf ccache.tar.gz ccache
+    tar cf ccache.tar ccache
     rm -rf ccache
     popd
   fi