From db1cc7af8aba60acd392ab8ac0debd3437f3d082 Mon Sep 17 00:00:00 2001 From: Sebastian Messmer Date: Sat, 17 Apr 2021 09:31:10 -0700 Subject: [PATCH] Add Linux CI to GitHub Actions --- .../workflows/actions/osx_setup/action.yaml | 16 --- .../workflows/actions/run_build/action.yaml | 6 +- .../workflows/actions/run_tests/action.yaml | 12 +- .github/workflows/main.yaml | 110 ++++++++++++++---- 4 files changed, 96 insertions(+), 48 deletions(-) delete mode 100644 .github/workflows/actions/osx_setup/action.yaml diff --git a/.github/workflows/actions/osx_setup/action.yaml b/.github/workflows/actions/osx_setup/action.yaml deleted file mode 100644 index 0c9f6296..00000000 --- a/.github/workflows/actions/osx_setup/action.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: 'Install OSX dependencies' -description: 'Install OSX dependencies' -inputs: - compiler_homebrew_package: - description: "Which compiler package to install from homebrew" - required: true -runs: - using: "composite" - steps: - - name: Install dependencies - shell: bash - run: | - brew install ninja macfuse libomp ${{inputs.compiler_homebrew_package}} - pip3 install conan - conan profile new default --detect - conan profile update settings.compiler.libcxx=libstdc++11 default diff --git a/.github/workflows/actions/run_build/action.yaml b/.github/workflows/actions/run_build/action.yaml index aa7867d6..6295eee7 100644 --- a/.github/workflows/actions/run_build/action.yaml +++ b/.github/workflows/actions/run_build/action.yaml @@ -26,6 +26,7 @@ runs: echo CXX: ${{inputs.cxx}} ${{inputs.cxx}} --version echo CCache: + ccache --version ccache -s - name: Run cmake shell: bash @@ -40,8 +41,3 @@ runs: set -v cd build ninja - - name: Reduce cache size - shell: bash - run: | - set -v - ccache --evict-older-than 7d diff --git a/.github/workflows/actions/run_tests/action.yaml b/.github/workflows/actions/run_tests/action.yaml index 593ddbe1..aaf8b408 100644 --- a/.github/workflows/actions/run_tests/action.yaml +++ b/.github/workflows/actions/run_tests/action.yaml @@ -6,6 +6,8 @@ runs: - name: Run tests shell: bash run: | + set -v + echo Running on ${{runner.os}} cd build ./test/gitversion/gitversion-test ./test/cpp-utils/cpp-utils-test @@ -14,6 +16,10 @@ runs: ./test/blobstore/blobstore-test ./test/cryfs/cryfs-test - # TODO Also run once fixed - # ./test/fspp/fspp-test - # ./test/cryfs-cli/cryfs-cli-test + # TODO Also run on macOS once fixed + if [[ "${{runner.os}}" == "macOS" ]]; then + echo Skipping some tests because they are not fixed for macOS yet + else + ./test/fspp/fspp-test + ./test/cryfs-cli/cryfs-cli-test + fi diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 81cf4bea..30b51692 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,30 +9,56 @@ jobs: matrix: os: - macos-10.15 + - ubuntu-20.04 compiler: - cxx: g++-7 cc: gcc-7 + macos_cxx: g++-7 + macos_cc: gcc-7 homebrew_package: gcc@7 + apt_package: g++-7 - cxx: g++-8 cc: gcc-8 + macos_cxx: g++-8 + macos_cc: gcc-8 homebrew_package: gcc@8 + apt_package: g++-8 - cxx: g++-9 cc: gcc-9 + macos_cxx: g++-9 + macos_cc: gcc-9 + apt_package: g++-9 homebrew_package: gcc@9 # - cxx: g++-10 # cc: gcc-10 +# macos_cxx: g++-10 +# macos_cc: gcc-10 +# apt_package: g++-10 # homebrew_package: gcc@10 - - cxx: /usr/local/opt/llvm@7/bin/clang++ - cc: /usr/local/opt/llvm@7/bin/clang + - cxx: clang++-7 + cc: clang-7 + macos_cxx: /usr/local/opt/llvm@7/bin/clang++ + macos_cc: /usr/local/opt/llvm@7/bin/clang + apt_package: clang-7 homebrew_package: llvm@7 - - cxx: /usr/local/opt/llvm@8/bin/clang++ - cc: /usr/local/opt/llvm@8/bin/clang + - cxx: clang++-8 + cc: clang-8 + macos_cxx: /usr/local/opt/llvm@8/bin/clang++ + macos_cc: /usr/local/opt/llvm@8/bin/clang + apt_package: clang-8 homebrew_package: llvm@8 - - cxx: /usr/local/opt/llvm@9/bin/clang++ - cc: /usr/local/opt/llvm@9/bin/clang + - cxx: clang++-9 + cc: clang-9 + macos_cxx: /usr/local/opt/llvm@9/bin/clang++ + macos_cc: /usr/local/opt/llvm@9/bin/clang + apt_package: clang-9 homebrew_package: llvm@9 - - cxx: /usr/local/opt/llvm@11/bin/clang++ - cc: /usr/local/opt/llvm@11/bin/clang + # TODO Clang-10 on linux? macos homebrew doesn't seem to have it + - cxx: clang++-11 + cc: clang-11 + macos_cxx: /usr/local/opt/llvm@11/bin/clang++ + macos_cc: /usr/local/opt/llvm@11/bin/clang + apt_package: clang-11 homebrew_package: llvm@11 # Apple Clang # - cxx: clang++ @@ -49,27 +75,49 @@ jobs: steps: - name: Checkout uses: actions/checkout@v1 - #TODO Ideally, the Find pip cache location + Setup pip cache step would be part of the setup action that runs pip, but Github doesn't support nested actions yet, see https://github.com/actions/runner/issues/862 - - name: Find pip cache location - id: pip_cache_dir - run: | - echo "::set-output name=pip_cache_dir::$(pip3 cache dir)" + #TODO Ideally, all the setup actions would be in their own subaction, but Github doesn't support using third party actions (e.g. cache) from nested actions yet, see https://github.com/actions/runner/issues/862 + #TODO I disabled pip cache because Github Actions have a limit for total cached data of 5GB and we're above that. Let's rather keep that space for ccache data. TODO Find a way to increase cache size and re-enable the pip cache. +# - name: Find pip cache location +# id: pip_cache_dir +# run: | +# # We need at least pip 20.1 to get the "pip cache dir" command. Ubuntu doesn't have pip 20.1 by default yet, let's upgrade it +# python3 -m pip install -U pip +# python3 -m pip --version +# echo "::set-output name=pip_cache_dir::$(python3 -m pip cache dir)" +# shell: bash +# - name: Setup pip cache +# uses: actions/cache@v2 +# with: +# path: ${{ steps.pip_cache_dir.outputs.pip_cache_dir }} +# key: v0-${{ runner.os }}-setup-pip-${{ github.run_number }} +# restore-keys: v0-${{ runner.os }}-setup-pip- + - name: Install OSX dependencies + if: ${{ runner.os == 'macOS' }} shell: bash - - name: Setup pip cache - uses: actions/cache@v2 - with: - path: ${{ steps.pip_cache_dir.outputs.pip_cache_dir }} - key: v0-${{ runner.os }}-setup-pip-${{ github.run_number }} - restore-keys: v0-${{ runner.os }}-setup-pip- - - name: Setup - uses: ./.github/workflows/actions/osx_setup - with: - compiler_homebrew_package: ${{ matrix.compiler.homebrew_package }} + run: | + brew install ninja macfuse libomp ${{matrix.compiler.homebrew_package}} + - name: Install Linux dependencies + if: ${{ runner.os == 'Linux' }} + shell: bash + run: | + sudo apt-get install ninja-build libcurl4-openssl-dev libfuse-dev ${{matrix.compiler.apt_package}} + - name: Install Conan + shell: bash + run: | + # Using "python3 -m pip" instead of "pip3" to make sure we get the same pip that we queried the cache dir for the Github Cache action + python3 -m pip install conan + conan profile new default --detect + conan profile update settings.compiler.libcxx=libstdc++11 default #TODO Ideally, the Setup ccache step would be part of the build action, but Github doesn't support nested actions yet, see https://github.com/actions/runner/issues/862 - name: Setup ccache uses: hendrikmuhs/ccache-action@7a464b8f54f1e1b78e7eb9d0272bc83072959235 # ccache-action@v1.0.3 with: key: "v0-${{ runner.os }}-ccache__${{matrix.os}}__${{matrix.compiler.cxx}}__${{matrix.compiler.cc}}__${{matrix.build_type}}__" + # TODO We reduced the max cache size from 500M to 200M because GitHub Actions have a maximum for total cached data of 5GB and we're above that. + # I observed ccache sizes after a clean run to be 130-420MB, so a size of 200MB will only store partial caches for some builds, but it will + # also not waste much space and keep more cache instances alive within the 5GB limit. TODO Find a better way of handling this, e.g. increase + # GitHub Action cache space, or at least use ccache --evict-older-than to shrink the cache to its actual size like we do in the macOS builds. + max-size: 200M - name: Configure ccache shell: bash run: | @@ -82,7 +130,15 @@ jobs: path: ${{ env.CONAN_USER_HOME }} key: "v0-${{ runner.os }}-conancache__${{matrix.os}}__${{matrix.compiler.cxx}}__${{matrix.compiler.cc}}__${{matrix.build_type}}__-${{ github.run_number }}" restore-keys: "v0-${{ runner.os }}-conancache__${{matrix.os}}__${{matrix.compiler.cxx}}__${{matrix.compiler.cc}}__${{matrix.build_type}}__-" - - name: Build + - name: Build (macOS) + if: ${{ runner.os == 'macOS' }} + uses: ./.github/workflows/actions/run_build + with: + cxx: ${{ matrix.compiler.macos_cxx }} + cc: ${{ matrix.compiler.macos_cc }} + build_type: ${{ matrix.build_type }} + - name: Build (Linux) + if: ${{ runner.os == 'Linux' }} uses: ./.github/workflows/actions/run_build with: cxx: ${{ matrix.compiler.cxx }} @@ -90,3 +146,9 @@ jobs: build_type: ${{ matrix.build_type }} - name: Test uses: ./.github/workflows/actions/run_tests + - name: Reduce cache size + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + set -v + ccache --evict-older-than 7d