diff --git a/.travis.yml b/.travis.yml
index 01fc4721ad..ab605349d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -59,7 +59,8 @@ install:
     - ls -lah $HOST_CACHE_DIR && ls -lah $HOST_CACHE_DIR/depends && ls -lah $HOST_CACHE_DIR/ccache && ls -lah $HOST_CACHE_DIR/docker
     # Load cached builder image
     - if [ -f $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz ]; then zcat $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz | docker load || true; fi
-    - docker build --pull -t $BUILDER_IMAGE_NAME --build-arg=USER_ID=$UID --build-arg=GROUP_ID=$UID --build-arg=BUILD_TARGET=$BUILD_TARGET -f ci/Dockerfile.builder ci
+    - travis_retry docker pull ubuntu:bionic
+    - travis_retry docker build -t $BUILDER_IMAGE_NAME --build-arg=USER_ID=$UID --build-arg=GROUP_ID=$UID --build-arg=BUILD_TARGET=$BUILD_TARGET -f ci/Dockerfile.builder ci
 before_script:
     # Make sure stdout is in blocking mode. Otherwise builds will fail due to large writes to stdout
     # See https://github.com/travis-ci/travis-ci/issues/4704. If this gets fixed, this line can also be removed.
diff --git a/ci/Dockerfile.builder b/ci/Dockerfile.builder
index 250fac8ab7..dd128b475d 100644
--- a/ci/Dockerfile.builder
+++ b/ci/Dockerfile.builder
@@ -2,18 +2,15 @@ FROM ubuntu:bionic
 
 # Build and base stuff
 # (zlib1g-dev and libssl-dev are needed for the Qt host binary builds, but should not be used by target binaries)
-RUN apt-get update && apt-get install -y \
-  git \
-  g++ \
-  autotools-dev libtool m4 automake autoconf pkg-config \
-  zlib1g-dev \
-  libssl1.0-dev \
-  curl \
-  ccache \
-  bsdmainutils \
-  python3-pip python3-dev \
-  cmake \
-  && rm -rf /var/lib/apt/lists
+# We split this up into multiple RUN lines as we might need to retry multiple times on Travis. This way we allow better
+# cache usage.
+RUN apt-get update
+RUN apt-get update && apt-get install -y git
+RUN apt-get update && apt-get install -y g++
+RUN apt-get update && apt-get install -y autotools-dev libtool m4 automake autoconf pkg-config
+RUN apt-get update && apt-get install -y zlib1g-dev libssl1.0-dev curl ccache bsdmainutils cmake
+RUN apt-get update && apt-get install -y python3 python3-dev
+RUN apt-get update && apt-get install -y python3-pip
 
 # Python stuff
 RUN pip3 install pyzmq # really needed?
@@ -36,7 +33,7 @@ ARG BUILD_TARGET=linux64
 ADD matrix.sh /tmp/matrix.sh
 RUN . /tmp/matrix.sh && \
   if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi && \
-  if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES && rm -rf /var/lib/apt/lists; fi
+  if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES; fi
 
 # Make sure std::thread and friends is available
 # Will fail on non-win builds, but we ignore this