Allow building with local dependencies instead of using conan

.circleci/config.yml

@@ -8,7 +8,7 @@ references:
     run:
       name: Initialize Cache
       command: |
-        echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}_${BUILD_TYPE}_${CXXFLAGS}" > /tmp/_build_env_vars
+        echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}_${BUILD_TYPE}_${CMAKE_FLAGS}_${CXXFLAGS}" > /tmp/_build_env_vars
         echo Build env vars used for cache keys:
         cat /tmp/_build_env_vars
   container_setup_pre: &container_setup_pre
@@ -117,6 +117,78 @@ references:
         cmake --version
         /usr/local/bin/$CC --version
         /usr/local/bin/$CXX --version
+  install_dependencies_locally: &install_dependencies_locally
+    # This is not required for a build with conan, this is only to be able to test building CryFS
+    # against locally installed dependency libraries.
+    run:
+      name: Install dependencies locally
+      no_output_timeout: 30m
+      command: |
+        # TODO Cache these dependencies for faster runtime
+
+        export NUMCORES=`nproc` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
+        echo Using $NUMCORES cores
+
+        echo Download range-v3
+        cd ~
+        wget https://github.com/ericniebler/range-v3/archive/0.9.1.tar.gz -O range-v3-0.9.1.tar.gz
+        if [ $(sha512sum range-v3-0.9.1.tar.gz | awk '{print $1;}') == "167db645527b769f3d375db63bb0a5c831e9c854bc729581a7d25c571cc1741eafb82812006c5bbe346222a73d5fbbd0b2f17119317038394ac15cf892088aa3" ]; then
+          echo Correct sha512sum
+        else
+          echo Wrong sha512sum
+          sha512sum range-v3-0.9.1.tar.gz
+          exit 1
+        fi
+        tar -xvf range-v3-0.9.1.tar.gz
+        cd range-v3-0.9.1/
+
+        echo Install range-v3
+        mkdir build
+        cd build
+        cmake .. -DRANGES_HAS_WERROR=off -DRANGE_V3_EXAMPLES=off -DRANGE_V3_TESTS=off
+        make -j$NUMCORES
+        sudo make install
+
+        echo Download spdlog
+        cd ~
+        wget https://github.com/gabime/spdlog/archive/v1.4.2.tar.gz -O spdlog.tar.gz
+        if [ $(sha512sum spdlog.tar.gz | awk '{print $1;}') == "886b489138a1bdf363b5eab65a7d973f570a01c399ff5b910fcfee172ad4ff4d42a45dc2ae24e77b07130df527fb13a86b3e55ac992e72c418aebb232e27eabf" ]; then
+          echo Correct sha512sum
+        else
+          echo Wrong sha512sum
+          sha512sum spdlog.tar.gz
+          exit 1
+        fi
+        tar -xvf spdlog.tar.gz
+        rm spdlog.tar.gz
+        cd spdlog-1.4.2
+
+        echo Install spdlog
+        mkdir build
+        cd build
+        cmake ..
+        make -j$NUMCORES
+        sudo make install
+
+        echo Download boost
+        cd ~
+        wget -O boost.tar.bz2 https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download
+        if [ $(sha512sum boost.tar.bz2 | awk '{print $1;}') == "63bbd1743e7c904b2e69fdc2eafd1c2a8a30fd70d960dcd366059d0100f000cb605d56cbd9397bb18421631f1c9552a772c434d0f0caddbe56302273f51fd1f6" ]; then
+          echo Correct sha512sum
+        else
+          echo Wrong sha512sum
+          sha512sum boost.tar.bz2
+          exit 1
+        fi
+        echo Extracting boost
+        tar -xf boost.tar.bz2
+        rm boost.tar.bz2
+        cd boost_1_72_0
+
+        echo Install boost
+        ./bootstrap.sh --with-libraries=filesystem,system,thread,chrono,program_options
+        sudo ./b2 link=shared cxxflags=-fPIC --prefix=/usr -d0 -j$NUMCORES install
+
   build_pre: &build_pre
     restore_cache:
       keys:
@@ -164,14 +236,14 @@ references:
       command: |
         if "${RUN_TESTS}"; then
           cd cmake
-          ./bin/gitversion-test ${GTEST_ARGS}
-          ./bin/cpp-utils-test ${GTEST_ARGS}
-          if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./bin/fspp-test ${GTEST_ARGS} ; fi
-          ./bin/parallelaccessstore-test ${GTEST_ARGS}
-          ./bin/blockstore-test ${GTEST_ARGS}
-          ./bin/blobstore-test ${GTEST_ARGS}
-          ./bin/cryfs-test ${GTEST_ARGS}
-          ./bin/cryfs-cli-test ${GTEST_ARGS}
+          ./test/gitversion/gitversion-test ${GTEST_ARGS}
+          ./test/cpp-utils/cpp-utils-test ${GTEST_ARGS}
+          if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ${GTEST_ARGS} ; fi
+          ./test/parallelaccessstore/parallelaccessstore-test ${GTEST_ARGS}
+          ./test/blockstore/blockstore-test ${GTEST_ARGS}
+          ./test/blobstore/blobstore-test ${GTEST_ARGS}
+          ./test/cryfs/cryfs-test ${GTEST_ARGS}
+          ./test/cryfs-cli/cryfs-cli-test ${GTEST_ARGS}
         fi
   job_definition: &job_definition
     <<: *container_config
@@ -491,6 +563,29 @@ jobs:
       GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
+  local_dependencies:
+    <<: *container_config
+    steps:
+      - <<: *cache_init
+      - <<: *container_setup_pre
+      - <<: *container_setup
+      - <<: *container_setup_post
+      - checkout
+      - <<: *install_dependencies_locally
+      - <<: *build_pre
+      - <<: *build
+      - <<: *build_post
+      - <<: *test
+    environment:
+      CC: clang-9
+      CXX: clang++-9
+      BUILD_TOOLSET: clang
+      APT_COMPILER_PACKAGE: clang-9
+      CXXFLAGS: ""
+      BUILD_TYPE: "RelWithDebInfo"
+      GTEST_ARGS: ""
+      CMAKE_FLAGS: "-DDEPENDENCY_CONFIG=../cmake-utils/DependenciesFromLocalSystem.cmake"
+      RUN_TESTS: true
   address_sanitizer:
     <<: *job_definition
     environment:
@@ -613,6 +708,8 @@ workflows:
         <<: *enable_for_tags
     - no_compatibility:
         <<: *enable_for_tags
+    - local_dependencies:
+        <<: *enable_for_tags
     - address_sanitizer:
         <<: *enable_for_tags
     - ub_sanitizer:

CMakeLists.txt

@@ -26,11 +26,7 @@ option(USE_CLANG_TIDY "build with clang-tidy checks enabled" OFF)
 option(USE_IWYU "build with iwyu checks enabled" OFF)
 option(CLANG_TIDY_WARNINGS_AS_ERRORS "treat clang-tidy warnings as errors" OFF)
 
-if(USE_CLANG_TIDY)
-    # note: for clang-tidy, we need cmake 3.6, or (if the return code should be handled correctly, e.g. on CI), we need 3.8.
-    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
-endif()
-
+set(DEPENDENCY_CONFIG "cmake-utils/DependenciesFromConan.cmake" CACHE FILEPATH "cmake configuration file defining how to get dependencies")
 
 if (MSVC)
     option(DOKAN_PATH "Location of the Dokan library, e.g. C:\\Program Files\\Dokan\\DokanLibrary-1.1.0" "")
@@ -41,13 +37,17 @@ if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE INTERNAL "CMAKE_BUILD_TYPE")
 endif(NOT CMAKE_BUILD_TYPE)
 
+if(USE_CLANG_TIDY)
+    # note: for clang-tidy, we need cmake 3.6, or (if the return code should be handled correctly, e.g. on CI), we need 3.8.
+    cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
+endif()
+
 # The MSVC version on AppVeyor CI needs this
 if(MSVC)
   add_definitions(/bigobj)
 endif()
 
-include (cmake-utils/conan-setup.cmake)
-setup_conan()
+include(${DEPENDENCY_CONFIG})
 
 add_subdirectory(vendor EXCLUDE_FROM_ALL)
 add_subdirectory(src)

ChangeLog.txt

@@ -2,6 +2,7 @@ Version 0.11.0 (unreleased)
 ---------------
 Build changes:
 * Switch to Conan package manager
+* Allow an easy way to modify how the dependencies are found. This is mostly helpful for package maintainers.
 
 Other changes:
 * Now requires CMake 3.6 or later

README.md

@@ -135,6 +135,27 @@ On most systems, CMake should find the libraries automatically. However, that do
         cmake .. -DOpenMP_CXX_FLAGS='-Xpreprocessor -fopenmp -I/path/to/openmp/include' -DOpenMP_CXX_LIB_NAMES=omp -DOpenMP_omp_LIBRARY=/path/to/libomp.dylib
 
 
+Using local dependencies
+-------------------------------
+Starting with CryFS 0.11, Conan is used for dependency management.
+When you build CryFS, Conan downloads the exact version of each dependency library that was also used for development.
+All dependencies are linked statically, so there should be no incompatibility with locally installed libraries.
+This is the recommended way because it has the highest probability of working correctly.
+
+However, some distributions prefer software packages to be built against dependencies dynamically and against locally installed versions of libraries.
+So if you're building a package for such a distribution, you have the option of doing that, at the cost of potential incompatibilities.
+If you follow this workflow, please make sure to extensively test your build of CryFS.
+You're using a setup that wasn't tested by the CryFS developers.
+
+To use local dependencies, you need to tell the CryFS build how to get these dependencies.
+You can do this by writing a small CMake configuration file and passing it to the CryFS build using `-DDEPENDENCY_CONFIG=filename`.
+This configuration file needs to define a cmake target for each of the dependencies.
+
+Here's an [example config file](cmake-utils/DependenciesFromConan.cmake) that gets the dependencies from conan.
+And here's another [example config file](cmake-utils/DependenciesFromLocalSystem.cmake) that works for getting dependencies that are locally installed in Ubuntu.
+You can create your own configuration file to tell the build how to get its dependencies and, for example, mix and match. Get some dependencies from Conan and others from the local system.
+
+
 Creating .deb and .rpm packages
 -------------------------------
 

cmake-utils/DependenciesFromConan.cmake

@@ -0,0 +1,16 @@
+include(cmake-utils/conan.cmake)
+
+conan_cmake_run(
+    CONANFILE conanfile.py
+    BUILD missing)
+
+conan_basic_setup(TARGETS SKIP_STD NO_OUTPUT_DIRS)
+
+add_library(CryfsDependencies_range-v3 INTERFACE)
+target_link_libraries(CryfsDependencies_range-v3 INTERFACE CONAN_PKG::range-v3)
+
+add_library(CryfsDependencies_spdlog INTERFACE)
+target_link_libraries(CryfsDependencies_spdlog INTERFACE CONAN_PKG::spdlog)
+
+add_library(CryfsDependencies_boost INTERFACE)
+target_link_libraries(CryfsDependencies_boost INTERFACE CONAN_PKG::boost)

cmake-utils/DependenciesFromLocalSystem.cmake

@@ -0,0 +1,61 @@
+# This configuration file can be used to build CryFS against local dependencies instead of using Conan.
+#
+# Example:
+# $ mkdir build && cd build && cmake .. -DDEPENDENCY_CONFIG=../cmake-utils/DependenciesFromLocalSystem.cmake
+#
+# Note that this is only provided as an example and not officially supported. Please still open issues
+# on GitHub if it doesn't work though.
+#
+# There's another file in this directory, DependenciesFromConan.cmake, which, well, gets the dependencies from
+# Conan instead of from the local system. This is the default. You can also create your own file to tell the build
+# how to get its dependencies, for example you can mix and match, get some dependencies from Conan and others
+# from the local system. If you mix and match Conan and local dependencies, please call conan_basic_setup()
+# **after** running all find_package() for your local dependencies, otherwise find_package() might also find
+# the versions from Conan.
+#
+# Note that if you use dependencies from the local system, you're very likely using different versions of the
+# dependencies than were used in the development of CryFS. The official version of each dependency required is
+# listed in conanfile.py. Different versions might work but are untested. Please intensively test your CryFS build
+# if you build it with different versions of the dependencies.
+
+
+function(check_target_is_not_from_conan TARGET)
+    get_target_property(INCLUDE_DIRS ${TARGET} INTERFACE_INCLUDE_DIRECTORIES)
+    if("${INCLUDE_DIRS}" MATCHES "conan")
+        message(WARNING "It seems setting up the local ${TARGET} dependency didn't work correctly and it got the version from Conan instead. Please set up cmake so that it sets up conan after all local dependencies are defined.")
+    endif()
+endfunction()
+
+
+
+
+# Setup range-v3 dependency
+find_package(range-v3 REQUIRED)
+check_target_is_not_from_conan(range-v3)
+add_library(CryfsDependencies_range-v3 INTERFACE)
+target_link_libraries(CryfsDependencies_range-v3 INTERFACE range-v3)
+
+
+
+
+# Setup boost dependency
+set(Boost_USE_STATIC_LIBS OFF)
+find_package(Boost 1.65.1
+        REQUIRED
+        COMPONENTS filesystem system thread chrono program_options)
+check_target_is_not_from_conan(Boost::boost)
+add_library(CryfsDependencies_boost INTERFACE)
+target_link_libraries(CryfsDependencies_boost INTERFACE Boost::boost Boost::filesystem Boost::thread Boost::chrono Boost::program_options)
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+    # Also link to rt, because boost thread needs that.
+    target_link_libraries(CryfsDependencies_boost INTERFACE rt)
+endif()
+
+
+
+
+# Setup spdlog dependency
+find_package(spdlog REQUIRED)
+check_target_is_not_from_conan(spdlog::spdlog)
+add_library(CryfsDependencies_spdlog INTERFACE)
+target_link_libraries(CryfsDependencies_spdlog INTERFACE spdlog::spdlog)

cmake-utils/conan-setup.cmake

@@ -1,9 +0,0 @@
-macro(setup_conan)
-    include(cmake-utils/conan.cmake)
-
-    conan_cmake_run(
-        CONANFILE conanfile.py
-        BUILD missing)
-
-    conan_basic_setup(TARGETS SKIP_STD)
-endmacro()

cmake-utils/utils.cmake

@@ -97,7 +97,7 @@ endfunction(target_enable_style_warnings)
 #  target_add_boost(buildtarget system filesystem) # list all libraries to link against in the dependencies
 ##################################################
 function(target_add_boost TARGET)
-    target_link_libraries(${TARGET} PUBLIC CONAN_PKG::boost)
+    target_link_libraries(${TARGET} PUBLIC CryfsDependencies_boost)
     target_compile_definitions(${TARGET} PUBLIC BOOST_THREAD_VERSION=4)
 endfunction(target_add_boost)
 

src/cpp-utils/CMakeLists.txt

@@ -89,7 +89,7 @@ target_link_libraries(${PROJECT_NAME} PUBLIC ${CMAKE_THREAD_LIBS_INIT})
 
 target_link_libraries(${PROJECT_NAME} PUBLIC ${CMAKE_DL_LIBS})
 
-target_link_libraries(${PROJECT_NAME} PUBLIC CONAN_PKG::spdlog cryptopp CONAN_PKG::range-v3)
+target_link_libraries(${PROJECT_NAME} PUBLIC CryfsDependencies_spdlog cryptopp CryfsDependencies_range-v3)
 
 target_add_boost(${PROJECT_NAME} filesystem system thread chrono)
 target_enable_style_warnings(${PROJECT_NAME})