Speed up CI by using caches

.circleci/config.yml

@@ -3,10 +3,34 @@ version: 2.0
 references:
   container_config: &container_config
     machine: true
+  cache_init: &cache_init
+    run:
+      name: Initialize Cache
+      command: |
+        echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}" > /tmp/_build_env_vars
+        echo Build env vars used for cache keys:
+        cat /tmp/_build_env_vars
+  container_setup_pre: &container_setup_pre
+    restore_cache:
+      keys:
+        # Find the most recent cache from any branch
+        - v4_container_setup_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
+  container_setup_post: &container_setup_post
+    save_cache:
+      key: v4_container_setup_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
+      paths:
+      - /tmp/aptcache
   container_setup: &container_setup
     run:
       name: Setup Environment
       command: |
+        if [ -d "/tmp/aptcache" ]; then
+          echo Using packages from apt cache
+          sudo cp -R /tmp/aptcache/* /var/cache/apt/archives/
+        else
+          echo No apt cache found
+        fi
+
         sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
         wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
         sudo touch /etc/apt/sources.list.d/clang.list
@@ -24,7 +48,7 @@ references:
         sudo chmod o-w /etc/apt/sources.list.d/clang.list
 
         DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq
-        DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git $APT_COMPILER_PACKAGE cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
+        DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
         # Use /dev/urandom when /dev/random is accessed to use less entropy
         sudo cp -a /dev/urandom /dev/random
 
@@ -38,49 +62,99 @@ references:
           sudo apt-get remove g++-4.8 gcc-4.8
         fi
 
+        # Setup ccache
+        sudo ln -s /usr/bin/ccache /usr/local/bin/$CC
+        sudo ln -s /usr/bin/ccache /usr/local/bin/$CXX
+        sudo mkdir /ccache_data
+        sudo chown circleci:circleci /ccache_data
+        echo 'export CCACHE_COMPILERCHECK=content' >> $BASH_ENV
+        echo 'export CCACHE_COMPRESS=1' >> $BASH_ENV
+        echo 'export CCACHE_DIR=/ccache_data' >> $BASH_ENV
+        echo 'export CCACHE_SLOPPINESS=include_file_mtime' >> $BASH_ENV
+
+        sudo mkdir -p /tmp/aptcache
+        sudo cp -R /var/cache/apt/archives/* /tmp/aptcache/
+
         echo
         echo System Info:
         cat /etc/issue
         uname -a
         cmake --version
-        $CC --version
-        $CXX --version
+        /usr/local/bin/$CC --version
+        /usr/local/bin/$CXX --version
+  upgrade_boost_pre: &upgrade_boost_pre
+    restore_cache:
+      keys:
+        # Find the most recent cache from any branch
+        - v2_upgrade_boost_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
+  upgrade_boost_post: &upgrade_boost_post
+    save_cache:
+      key: v2_upgrade_boost_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
+      paths:
+      - /tmp/boost_1_56_0
   upgrade_boost: &upgrade_boost
     run:
       name: Upgrade Boost
       command: |
-        set -e
-        whoami
-        pwd
         # Detect number of CPU cores
         export NUMCORES=`nproc`
         echo Using $NUMCORES cores
-        # Install boost
-        wget -O boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
-        if [ $(sha512sum boost.tar.bz2 | awk '{print $1;}') == "1ce9871c3a2519682538a5f1331938b948123432d99aea0ce026958cbebd25d84019a3a28c452724b9693df98a8d1264bb2d93d2fee508453f8b42836e67481e" ]; then
-          echo Correct sha512sum
+        # Download and prepare boost (only if not already present from cache)
+        if [ ! -d "/tmp/boost_1_56_0" ]; then
+          echo "Didn't find boost in cache. Downloading and building."
+          wget -O /tmp/boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.56.0/boost_1_56_0.tar.bz2/download
+          if [ $(sha512sum /tmp/boost.tar.bz2 | awk '{print $1;}') == "1ce9871c3a2519682538a5f1331938b948123432d99aea0ce026958cbebd25d84019a3a28c452724b9693df98a8d1264bb2d93d2fee508453f8b42836e67481e" ]; then
+            echo Correct sha512sum
+          else
+            echo Wrong sha512sum
+            sha512sum boost.tar.bz2
+            exit 1
+          fi
+          echo Extracting...
+          tar -xf /tmp/boost.tar.bz2 -C /tmp
+          rm -rf boost.tar.bz2
+          cd /tmp/boost_1_56_0
+          ./bootstrap.sh --with-toolset=${BUILD_TOOLSET} --with-libraries=filesystem,thread,chrono,program_options
+          cd ..
         else
-          echo Wrong sha512sum
-          sha512sum boost.tar.bz2
-          exit 1
+          echo Found boost in cache. Use cache and build.
         fi
-        echo Extracting...
-        tar -xf boost.tar.bz2
-        cd boost_1_56_0
-        ./bootstrap.sh --with-toolset=${BUILD_TOOLSET} --with-libraries=filesystem,thread,chrono,program_options
+        # Compile and install boost (if cached, this should be fast)
+        cd /tmp/boost_1_56_0
         sudo ./b2 toolset=${BUILD_TOOLSET} link=static cxxflags=-fPIC -d0 -j$NUMCORES install
-        cd ..
-        sudo rm -rf boost.tar.bz2 boost_1_56_0
+  build_pre: &build_pre
+    restore_cache:
+      keys:
+        # Find most recent cache from any revision on the same branch (cache keys are prefix matched)
+        # CIRCLE_PR_NUMBER is only set if this is a pull request.
+        - v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}_{{ .Environment.CIRCLE_PR_NUMBER }}
+        # Fallback to less specific caches if the one above wasn't found
+        - v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}
+        - v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}
+  build_post: &build_post
+    save_cache:
+      key: v3_build_cache_{{ checksum "/tmp/_build_env_vars" }}_{{ arch }}_{{ .Branch }}_{{ .Environment.CIRCLE_PR_NUMBER }}_{{ .Revision }}
+      paths:
+      - /ccache_data
   build: &build
     run:
       name: Build
       command: |
         export NUMCORES=`nproc` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
         echo Using $NUMCORES cores
+        # Use ccache
+        export CXX=/usr/local/bin/$CXX
+        export CC=/usr/local/bin/$CC
+        ccache --max-size=512M
+        ccache --show-stats
+
+        # Build
         mkdir cmake
         cd cmake
         cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
         make -j$NUMCORES
+
+        ccache --show-stats
   test: &test
     run:
       name: Test
@@ -97,10 +171,17 @@ references:
   job_definition: &job_definition
     <<: *container_config
     steps:
+    - <<: *cache_init
+    - <<: *container_setup_pre
     - <<: *container_setup
+    - <<: *container_setup_post
+    - <<: *upgrade_boost_pre
     - <<: *upgrade_boost
+    - <<: *upgrade_boost_post
     - checkout
+    - <<: *build_pre
     - <<: *build
+    - <<: *build_post
     - <<: *test
   enable_for_tags: &enable_for_tags
     filters: