merge

.circleci/config.yml

@@ -173,14 +173,14 @@ references:
       command: |
         if "${RUN_TESTS}"; then
           cd cmake
-          ./test/gitversion/gitversion-test
+          ./test/gitversion/gitversion-test ${GTEST_ARGS}
-          if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cpp-utils/cpp-utils-test ; fi
+          ./test/cpp-utils/cpp-utils-test ${GTEST_ARGS}
-          if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ; fi
+          if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ${GTEST_ARGS} ; fi
-          ./test/parallelaccessstore/parallelaccessstore-test
+          ./test/parallelaccessstore/parallelaccessstore-test ${GTEST_ARGS}
-          ./test/blockstore/blockstore-test
+          ./test/blockstore/blockstore-test ${GTEST_ARGS}
-          ./test/blobstore/blobstore-test
+          ./test/blobstore/blobstore-test ${GTEST_ARGS}
-          ./test/cryfs/cryfs-test
+          ./test/cryfs/cryfs-test ${GTEST_ARGS}
-          if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs-cli/cryfs-cli-test ; fi
+          ./test/cryfs-cli/cryfs-cli-test ${GTEST_ARGS}
         fi
   job_definition: &job_definition
     <<: *container_config
@@ -212,6 +212,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-5"
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_5_release:
@@ -223,6 +224,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-5"
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_6_debug:
@@ -234,6 +236,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-6"
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_6_release:
@@ -245,6 +248,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-6"
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_7_debug:
@@ -256,6 +260,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-7"
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_7_release:
@@ -267,6 +272,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-7"
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_8_debug:
@@ -278,6 +284,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-8"
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   gcc_8_release:
@@ -289,6 +296,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-8"
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_4_debug:
@@ -300,6 +308,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-4.0
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_4_release:
@@ -311,6 +320,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-4.0
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_5_debug:
@@ -322,6 +332,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-5.0
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_5_release:
@@ -333,6 +344,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-5.0
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_6_debug:
@@ -344,6 +356,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-6.0
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_6_release:
@@ -355,6 +368,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-6.0
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_7_debug:
@@ -366,6 +380,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-7
       CXXFLAGS: ""
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_7_release:
@@ -377,6 +392,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-7
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_werror:
@@ -388,6 +404,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-7
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: "-DUSE_WERROR=on"
       RUN_TESTS: false
   gcc_werror:
@@ -399,6 +416,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-8"
       CXXFLAGS: ""
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: "-DUSE_WERROR=on"
       RUN_TESTS: false
   gcc_werror:
@@ -410,6 +428,7 @@ jobs:
       APT_COMPILER_PACKAGE: "g++-8"
       CXXFLAGS: "-Werror"
       BUILD_TYPE: "Release"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: false
   no_compatibility:
@@ -421,6 +440,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-7
       CXXFLAGS: "-DCRYFS_NO_COMPATIBILITY"
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   address_sanitizer:
@@ -434,6 +454,7 @@ jobs:
       BUILD_TYPE: "Debug"
       ASAN_OPTIONS: "detect_leaks=1 check_initialization_order=1 detect_stack_use_after_return=1 detect_invalid_pointer_pairs=1 atexit=1"
       DISABLE_BROKEN_ASAN_TESTS: true
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   ub_sanitizer:
@@ -445,6 +466,7 @@ jobs:
       APT_COMPILER_PACKAGE: clang-7
       CXXFLAGS: "-O2 -fno-sanitize-recover=undefined,nullability,implicit-conversion,unsigned-integer-overflow -fno-omit-frame-pointer -fno-common"
       BUILD_TYPE: "Debug"
+      GTEST_ARGS: ""
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   thread_sanitizer:
@@ -457,7 +479,7 @@ jobs:
       OMP_NUM_THREADS: "1"
       CXXFLAGS: "-O2 -fsanitize=thread -fno-omit-frame-pointer"
       BUILD_TYPE: "Debug"
-      DISABLE_BROKEN_TSAN_TESTS: true
+      GTEST_ARGS: "--gtest_filter=-LoggingTest.LoggingAlsoWorksAfterFork:AssertTest_DebugBuild.*:CliTest_Setup.*:CliTest_IntegrityCheck.*:*/CliTest_WrongEnvironment.*:CliTest_Unmount.*"
       CMAKE_FLAGS: ""
       RUN_TESTS: true
   clang_tidy:

.travisci/install.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+# Workaround homebrew bug, see https://twitter.com/axccl/status/1083393735277363205 and https://github.com/openPMD/openPMD-api/pull/431/files
+brew upgrade --cleanup || brew upgrade --cleanup
+
 # Install newer GCC if we're running on GCC
 if [ "${CXX}" == "g++" ]; then
     # We need to uninstall oclint because it creates a /usr/local/include/c++ symlink that clashes with the gcc5 package

ChangeLog.txt

@@ -12,16 +12,19 @@ New Features & Improvements:
 * New block size options: 4KB and 16KB
 * New default block size: 16KB. This should decrease the size of the ciphertext directory for most users.
 * Increased scrypt hardness to (N=1048576, r=4, p=8) to make it harder to crack the key while allowing cryfs to take advantage of multicore machines.
+* cryfs-unmount tool to unmount filesystems
 
 Fixed bugs:
 * `du` shows correct file system size on Mac OS X.
 * On Mac OS X, Finder shows the correct name for the mount directory
 
 
-Version 0.9.10 (unreleased)
+Version 0.9.10
 --------------
 Fixed bugs:
 * Fixed occasional deadlock (https://github.com/cryfs/cryfs/issues/64)
+* Fix for reading empty files out of bounds
+* Fixed race condition (https://github.com/cryfs/cryfs/issues/224 and https://github.com/cryfs/cryfs/issues/243)
 
 
 Version 0.9.9

appveyor.yml

@@ -12,7 +12,7 @@ platform:
 configuration:
  - Debug
  - RelWithDebInfo
-# - Release
+ - Release
 
 version: '{branch}-{build}'
 
@@ -20,13 +20,11 @@ init:
 - echo %NUMBER_OF_PROCESSORS%
 - echo %PLATFORM%
 - echo %APPVEYOR_BUILD_WORKER_IMAGE%
-- set arch=
+- set arch=32
-- if "%PLATFORM%"=="x64" ( set arch= Win64)
+- if "%PLATFORM%"=="x64" ( set arch=64)
-- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2013" (set generator=Visual Studio 12 2013%arch%)
+- set VisualStudioVersion=2017
-- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" (set generator=Visual Studio 14 2015%arch%)
+- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017 Preview" ( set VisualStudioVersion=Preview)
-- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" (set generator=Visual Studio 15 2017%arch%)
+- cmd: call "C:\Program Files (x86)\Microsoft Visual Studio\%VisualStudioVersion%\Community\VC\Auxiliary\Build\vcvars%arch%.bat"
-- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017 Preview" (set generator=Visual Studio 15 2017%arch%)
-- echo %generator%
 
 install:
   - choco install -y dokany --version 1.1.0.2000 --installargs INSTALLDEVFILES=1
@@ -35,17 +33,17 @@ install:
 build_script:
   - cmd: mkdir build
   - cmd: cd build
-  - cmd: cmake .. -G "%generator%" -DBUILD_TESTING=on -DBOOST_ROOT="C:/Libraries/boost_1_65_1" -DDOKAN_PATH="C:/Program Files/Dokan/DokanLibrary-1.1.0"
+  - cmd: cmake .. -G "Ninja" -DBUILD_TESTING=on -DBOOST_ROOT="C:/Libraries/boost_1_65_1" -DDOKAN_PATH="C:/Program Files/Dokan/DokanLibrary-1.1.0"
-  # TODO Make build parallel
   - cmd: cmake --build . --config %CONFIGURATION%
-  - cmd: .\test\gitversion\%CONFIGURATION%\gitversion-test.exe
+  - cmd: .\test\gitversion\gitversion-test.exe
-  - cmd: cd .\test\cpp-utils\%CONFIGURATION%\ && .\cpp-utils-test.exe && cd ..\..\..
+  # cpp-utils-test disables ThreadDebuggingTest_ThreadName.*_thenIsCorrect because the appveyor image is too old to support the API needed for that
-  #- cmd: .\test\fspp\%CONFIGURATION%\fspp-test.exe
+  - cmd: cd .\test\cpp-utils\ && .\cpp-utils-test.exe --gtest_filter=-ThreadDebuggingTest_ThreadName.*_thenIsCorrect && cd ..\..
-  - cmd: .\test\parallelaccessstore\%CONFIGURATION%\parallelaccessstore-test.exe
+  #- cmd: .\test\fspp\fspp-test.exe
-  - cmd: .\test\blockstore\%CONFIGURATION%\blockstore-test.exe
+  - cmd: .\test\parallelaccessstore\parallelaccessstore-test.exe
-  - cmd: .\test\blobstore\%CONFIGURATION%\blobstore-test.exe
+  - cmd: .\test\blockstore\blockstore-test.exe
-  - cmd: .\test\cryfs\%CONFIGURATION%\cryfs-test.exe
+  - cmd: .\test\blobstore\blobstore-test.exe
-  #- cmd: .\test\cryfs-cli\%CONFIGURATION%\cryfs-cli-test.exe
+  - cmd: .\test\cryfs\cryfs-test.exe
+  #- cmd: .\test\cryfs-cli\cryfs-cli-test.exe
 
   - cmd: cpack -C %CONFIGURATION% --verbose -G WIX
 

src/CMakeLists.txt

@@ -8,4 +8,5 @@ add_subdirectory(blockstore)
 add_subdirectory(blobstore)
 add_subdirectory(cryfs)
 add_subdirectory(cryfs-cli)
+add_subdirectory(cryfs-unmount)
 add_subdirectory(stats)

src/blobstore/CMakeLists.txt

@@ -11,6 +11,7 @@ set(SOURCES
     implementations/onblocks/datanodestore/DataInnerNode.cpp
     implementations/onblocks/datanodestore/DataNodeStore.cpp
     implementations/onblocks/datatreestore/impl/algorithms.cpp
+    implementations/onblocks/datatreestore/impl/CachedValue.cpp
     implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
     implementations/onblocks/datatreestore/LeafHandle.cpp
     implementations/onblocks/datatreestore/DataTree.cpp

src/blobstore/implementations/onblocks/BlobOnBlocks.cpp

@@ -8,13 +8,9 @@
 #include <cpp-utils/assert/assert.h>
 #include "datatreestore/LeafHandle.h"
 
-using std::function;
-using std::unique_lock;
-using std::mutex;
 using cpputils::unique_ref;
 using cpputils::Data;
 using blockstore::BlockId;
-using blobstore::onblocks::datatreestore::LeafHandle;
 
 namespace blobstore {
 namespace onblocks {
@@ -22,128 +18,34 @@ namespace onblocks {
 using parallelaccessdatatreestore::DataTreeRef;
 
 BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
-: _datatree(std::move(datatree)), _sizeCache(boost::none), _mutex() {
+: _datatree(std::move(datatree)) {
 }
 
 BlobOnBlocks::~BlobOnBlocks() {
 } // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
 
 uint64_t BlobOnBlocks::size() const {
-  if (_sizeCache == boost::none) {
+  return _datatree->numBytes();
-    _sizeCache = _datatree->numStoredBytes();
-  }
-  return *_sizeCache;
 }
 
 void BlobOnBlocks::resize(uint64_t numBytes) {
   _datatree->resizeNumBytes(numBytes);
-  _sizeCache = numBytes;
-}
-
-void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
-  unique_lock<mutex> lock(_mutex); // TODO Multiple traverse calls in parallel?
-  uint64_t endByte = beginByte + sizeBytes;
-  uint64_t maxBytesPerLeaf = _datatree->maxBytesPerLeaf();
-  uint32_t firstLeaf = beginByte / maxBytesPerLeaf;
-  uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
-  bool blobIsGrowingFromThisTraversal = false;
-  auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
-      uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
-      ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
-      uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
-      uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
-      // If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
-      if (isRightBorderLeaf) {
-        ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
-        auto leaf = leafHandle.node();
-        if (leaf->numBytes() < dataEnd) {
-          leaf->resize(dataEnd);
-          blobIsGrowingFromThisTraversal = true;
-        }
-      }
-      onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
-  };
-  auto _onCreateLeaf = [&onCreateLeaf, maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex) -> Data {
-      blobIsGrowingFromThisTraversal = true;
-      uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
-      ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
-      uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
-      uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
-      ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
-      ASSERT(leafIndex == endLeaf-1 || dataEnd == maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
-      Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
-      ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
-      // If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
-      if (dataBegin != 0) {
-        Data actualData(dataBegin + data.size());
-        std::memset(actualData.data(), 0, dataBegin);
-        std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
-        data = std::move(actualData);
-      }
-      return data;
-  };
-  _datatree->traverseLeaves(firstLeaf, endLeaf, _onExistingLeaf, _onCreateLeaf);
-  if (blobIsGrowingFromThisTraversal) {
-    ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
-    _sizeCache = endByte;
-  }
 }
 
 Data BlobOnBlocks::readAll() const {
-  //TODO Querying size is inefficient. Is this possible without a call to size()?
+  return _datatree->readAllBytes();
-  uint64_t count = size();
-  Data result(count);
-  _read(result.data(), 0, count);
-  return result;
 }
 
 void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
-  uint64_t _size = size();
+  return _datatree->readBytes(target, offset, count);
-  ASSERT(offset <= _size && offset + count <= _size, "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
-  uint64_t read = tryRead(target, offset, count);
-  ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
 }
 
 uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
-  //TODO Quite inefficient to call size() here, because that has to traverse the tree
+  return _datatree->tryReadBytes(target, offset, count);
-  uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(size())-static_cast<int64_t>(offset)));
-  _read(target, offset, realCount);
-  return realCount;
-}
-
-void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
-  auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
-      ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
-      //TODO Simplify formula, make it easier to understand
-      leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
-  };
-  auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
-      ASSERT(false, "Reading shouldn't create new leaves.");
-  };
-  _traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
 }
 
 void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
-  auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+  _datatree->writeBytes(source, offset, count);
-      ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
-      if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
-        Data leafData(leafDataSize);
-        std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
-        leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
-      } else {
-        //TODO Simplify formula, make it easier to understand
-        leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
-                           leafDataSize);
-      }
-  };
-  auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
-      ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
-      Data result(numBytes);
-      //TODO Simplify formula, make it easier to understand
-      std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
-      return result;
-  };
-  _traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
 }
 
 void BlobOnBlocks::flush() {

src/blobstore/implementations/onblocks/BlobOnBlocks.h

@@ -7,6 +7,7 @@
 
 #include <memory>
 #include <boost/optional.hpp>
+#include <boost/thread/shared_mutex.hpp>
 
 namespace blobstore {
 namespace onblocks {
@@ -38,12 +39,11 @@ public:
 
 private:
 
+  uint64_t _tryRead(void *target, uint64_t offset, uint64_t size) const;
   void _read(void *target, uint64_t offset, uint64_t count) const;
   void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datatreestore::LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
 
   cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
-  mutable boost::optional<uint64_t> _sizeCache;
-  mutable std::mutex _mutex;
 
   DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
 };

src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp

@@ -12,6 +12,7 @@
 #include <cmath>
 #include <cpp-utils/assert/assert.h>
 #include "impl/LeafTraverser.h"
+#include <boost/thread.hpp>
 
 using blockstore::BlockId;
 using blobstore::onblocks::datanodestore::DataNodeStore;
@@ -24,17 +25,21 @@ using boost::shared_mutex;
 using boost::shared_lock;
 using boost::unique_lock;
 using boost::none;
+using boost::optional;
 
 using cpputils::optional_ownership_ptr;
 using cpputils::unique_ref;
 using cpputils::Data;
+using namespace cpputils::logging;
+
+//TODO shared_lock currently not enough for traverse because of root replacement. Can be fixed while keeping shared?
 
 namespace blobstore {
 namespace onblocks {
 namespace datatreestore {
 
 DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
-  : _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _numLeavesCache(none) {
+  : _treeStructureMutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _sizeCache() {
 }
 
 DataTree::~DataTree() {
@@ -45,102 +50,143 @@ const BlockId &DataTree::blockId() const {
 }
 
 void DataTree::flush() const {
-  // By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
+  // By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed.
-  unique_lock<shared_mutex> lock(_mutex);
+  // It's only a shared lock, because this doesn't modify the tree structure.
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
   // We also have to flush the root node
   _rootNode->flush();
 }
 
 unique_ref<DataNode> DataTree::releaseRootNode() {
-  unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
+  // Lock also ensures that the root node is currently set (traversing unsets it temporarily)
+  // It's a unique lock because this "modifies" tree structure by changing _rootNode.
+  unique_lock<shared_mutex> lock(_treeStructureMutex);
   return std::move(_rootNode);
 }
 
 //TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
 uint32_t DataTree::numLeaves() const {
-    shared_lock<shared_mutex> lock(_mutex);
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
-    return _numLeaves();
+
+  return _getOrComputeSizeCache().numLeaves;
 }
 
-uint32_t DataTree::_numLeaves() const {
+uint64_t DataTree::numBytes() const {
-  if (_numLeavesCache == none) {
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
-    _numLeavesCache = _computeNumLeaves(*_rootNode);
+  return _numBytes();
-  }
-  return *_numLeavesCache;
 }
 
-uint32_t DataTree::_forceComputeNumLeaves() const {
+uint64_t DataTree::_numBytes() const {
-  unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
+  return _getOrComputeSizeCache().numBytes;
-  _numLeavesCache = _computeNumLeaves(*_rootNode);
-  return *_numLeavesCache;
 }
 
-uint32_t DataTree::_computeNumLeaves(const DataNode &node) const {
+DataTree::SizeCache DataTree::_getOrComputeSizeCache() const {
+  return _sizeCache.getOrCompute([this] () {
+    return _computeSizeCache(*_rootNode);
+  });
+}
+
+uint32_t DataTree::forceComputeNumLeaves() const {
+  _sizeCache.clear();
+  return numLeaves();
+}
+
+DataTree::SizeCache DataTree::_computeSizeCache(const DataNode &node) const {
   const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
   if (leaf != nullptr) {
-    return 1;
+    return {1, leaf->numBytes()};
   }
 
   const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
-  uint64_t numLeavesInLeftChildren = static_cast<uint64_t>(inner.numChildren()-1) * leavesPerFullChild(inner);
+  uint32_t numLeavesInLeftChildren = static_cast<uint32_t>(inner.numChildren()-1) * _leavesPerFullChild(inner);
+  uint64_t numBytesInLeftChildren = numLeavesInLeftChildren * _nodeStore->layout().maxBytesPerLeaf();
   auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
   ASSERT(lastChild != none, "Couldn't load last child");
-  uint64_t numLeavesInRightChild = _computeNumLeaves(**lastChild);
+  SizeCache sizeInRightChild = _computeSizeCache(**lastChild);
 
-  return numLeavesInLeftChildren + numLeavesInRightChild;
+  return SizeCache {
+    numLeavesInLeftChildren + sizeInRightChild.numLeaves,
+    numBytesInLeftChildren + sizeInRightChild.numBytes
+  };
 }
 
-void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
+void DataTree::_traverseLeavesByLeafIndices(uint32_t beginIndex, uint32_t endIndex, bool readOnlyTraversal,
-  //TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()?
+    function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
-  std::unique_lock<shared_mutex> lock(_mutex);
+    function<Data (uint32_t index)> onCreateLeaf,
-  ASSERT(beginIndex <= endIndex, "Invalid parameters");
+    function<void (DataInnerNode *node)> onBacktrackFromSubtree) const {
+  if(endIndex <= beginIndex) {
+    return;
+  }
 
-  auto onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
+  // TODO no const cast
+  LeafTraverser(_nodeStore, readOnlyTraversal).traverseAndUpdateRoot(&const_cast<DataTree*>(this)->_rootNode, beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+}
 
-  _traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+void DataTree::_traverseLeavesByByteIndices(uint64_t beginByte, uint64_t sizeBytes, bool readOnlyTraversal, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
+  if (sizeBytes == 0) {
+    return;
+  }
 
-  if (_numLeavesCache != none && *_numLeavesCache < endIndex) {
+  uint64_t endByte = beginByte + sizeBytes;
-    _numLeavesCache = endIndex;
+  uint64_t _maxBytesPerLeaf = maxBytesPerLeaf();
+  uint32_t firstLeaf = beginByte / _maxBytesPerLeaf;
+  uint32_t endLeaf = utils::ceilDivision(endByte, _maxBytesPerLeaf);
+  bool blobIsGrowingFromThisTraversal = false;
+  auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, _maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
+    uint64_t indexOfFirstLeafByte = leafIndex * _maxBytesPerLeaf;
+    ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
+    uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
+    uint32_t dataEnd = std::min(_maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
+    // If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
+    if (isRightBorderLeaf) {
+      ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
+      auto leaf = leafHandle.node();
+      if (leaf->numBytes() < dataEnd) {
+        leaf->resize(dataEnd);
+        blobIsGrowingFromThisTraversal = true;
+      }
+    }
+    onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
+  };
+  auto _onCreateLeaf = [&onCreateLeaf, _maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal, readOnlyTraversal] (uint32_t leafIndex) -> Data {
+    ASSERT(!readOnlyTraversal, "Cannot create leaves in a read-only traversal");
+    blobIsGrowingFromThisTraversal = true;
+    uint64_t indexOfFirstLeafByte = leafIndex * _maxBytesPerLeaf;
+    ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
+    uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
+    uint32_t dataEnd = std::min(_maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
+    ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
+    ASSERT(leafIndex == endLeaf-1 || dataEnd == _maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
+    Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
+    ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
+    // If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
+    if (dataBegin != 0) {
+      Data actualData(dataBegin + data.size());
+      std::memset(actualData.data(), 0, dataBegin);
+      std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
+      data = std::move(actualData);
+    }
+    return data;
+  };
+  auto _onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
+
+  _traverseLeavesByLeafIndices(firstLeaf, endLeaf, readOnlyTraversal, _onExistingLeaf, _onCreateLeaf, _onBacktrackFromSubtree);
+
+  ASSERT(!readOnlyTraversal || !blobIsGrowingFromThisTraversal, "Blob grew from traversal that didn't allow growing (i.e. reading)");
+
+  if (blobIsGrowingFromThisTraversal) {
+    _sizeCache.update([endLeaf, endByte] (optional<SizeCache>* cache) {
+        *cache = SizeCache{endLeaf, endByte};
+    });
   }
 }
 
-void DataTree::_traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
+uint32_t DataTree::_leavesPerFullChild(const DataInnerNode &root) const {
-    function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
-    function<Data (uint32_t index)> onCreateLeaf,
-    function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
-  _rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
-}
-
-uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
   return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(root.depth())-1);
 }
 
-uint64_t DataTree::numStoredBytes() const {
-  shared_lock<shared_mutex> lock(_mutex);
-  return _numStoredBytes();
-}
-
-uint64_t DataTree::_numStoredBytes() const {
-  return _numStoredBytes(*_rootNode);
-}
-
-uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
-  const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&root);
-  if (leaf != nullptr) {
-    return leaf->numBytes();
-  }
-
-  const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
-  uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
-  auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
-  ASSERT(lastChild != none, "Couldn't load last child");
-  uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
-
-  return numBytesInLeftChildren + numBytesInRightChild;
-}
-
 void DataTree::resizeNumBytes(uint64_t newNumBytes) {
-  std::unique_lock<shared_mutex> lock(_mutex); // TODO Multiple ones in parallel? Also in parallel with traverseLeaves()?
+  std::unique_lock<shared_mutex> lock(_treeStructureMutex);
 
   uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
   uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf();
@@ -171,8 +217,11 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) {
       }
   };
 
-  _traverseLeaves(newNumLeaves - 1, newNumLeaves, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+  _traverseLeavesByLeafIndices(newNumLeaves - 1, newNumLeaves, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
-  _numLeavesCache = newNumLeaves;
+  _sizeCache.update([newNumLeaves, newNumBytes] (boost::optional<SizeCache>* cache) {
+    *cache = SizeCache{newNumLeaves, newNumBytes};
+  });
+
 }
 
 uint64_t DataTree::maxBytesPerLeaf() const {
@@ -180,9 +229,87 @@ uint64_t DataTree::maxBytesPerLeaf() const {
 }
 
 uint8_t DataTree::depth() const {
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
   return _rootNode->depth();
 }
 
+void DataTree::readBytes(void *target, uint64_t offset, uint64_t count) const {
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
+
+  const uint64_t _size = _numBytes();
+  if(offset > _size || offset + count > _size) {
+    throw std::runtime_error("BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
+  }
+  const uint64_t read = _tryReadBytes(target, offset, count);
+  if (read != count) {
+    throw std::runtime_error("BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
+  }
+}
+
+Data DataTree::readAllBytes() const {
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
+
+  //TODO Querying numBytes can be inefficient. Is this possible without a call to size()?
+  uint64_t count = _numBytes();
+  Data result(count);
+  _doReadBytes(result.data(), 0, count);
+
+  return result;
+}
+
+uint64_t DataTree::tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
+  shared_lock<shared_mutex> lock(_treeStructureMutex);
+  auto result = _tryReadBytes(target, offset, count);
+  return result;
+}
+
+uint64_t DataTree::_tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
+  //TODO Quite inefficient to call size() here, because that has to traverse the tree
+  const uint64_t _size = _numBytes();
+  const uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(_size)-static_cast<int64_t>(offset)));
+  _doReadBytes(target, offset, realCount);
+  return realCount;
+}
+
+void DataTree::_doReadBytes(void *target, uint64_t offset, uint64_t count) const {
+  auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+    ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
+    //TODO Simplify formula, make it easier to understand
+    leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
+  };
+  auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
+    ASSERT(false, "Reading shouldn't create new leaves.");
+  };
+
+  _traverseLeavesByByteIndices(offset, count, true, onExistingLeaf, onCreateLeaf);
+}
+
+void DataTree::writeBytes(const void *source, uint64_t offset, uint64_t count) {
+  unique_lock<shared_mutex> lock(_treeStructureMutex);
+
+  auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
+    ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
+    if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
+      Data leafData(leafDataSize);
+      std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
+      leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
+    } else {
+      //TODO Simplify formula, make it easier to understand
+      leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
+                         leafDataSize);
+    }
+  };
+  auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
+    ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
+    Data result(numBytes);
+    //TODO Simplify formula, make it easier to understand
+    std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
+    return result;
+  };
+
+  _traverseLeavesByByteIndices(offset, count, false, onExistingLeaf, onCreateLeaf);
+}
+
 }
 }
 }

src/blobstore/implementations/onblocks/datatreestore/DataTree.h

@@ -10,6 +10,7 @@
 #include <boost/thread/shared_mutex.hpp>
 #include <blockstore/utils/BlockId.h>
 #include "LeafHandle.h"
+#include "impl/CachedValue.h"
 
 namespace blobstore {
 namespace onblocks {
@@ -31,39 +32,57 @@ public:
   //Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
   uint64_t maxBytesPerLeaf() const;
 
-  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
+  uint64_t tryReadBytes(void *target, uint64_t offset, uint64_t count) const;
+  void readBytes(void *target, uint64_t offset, uint64_t count) const;
+  cpputils::Data readAllBytes() const;
+
+  void writeBytes(const void *source, uint64_t offset, uint64_t count);
+
   void resizeNumBytes(uint64_t newNumBytes);
 
   uint32_t numLeaves() const;
-  uint64_t numStoredBytes() const;
+  uint64_t numBytes() const;
 
   uint8_t depth() const;
 
   // only used by test cases
-  uint32_t _forceComputeNumLeaves() const;
+  uint32_t forceComputeNumLeaves() const;
 
   void flush() const;
 
 private:
-  mutable boost::shared_mutex _mutex;
+  // This mutex must protect the tree structure, i.e. which nodes exist and how they're connected.
+  // Also protects total number of bytes (i.e. number of leaves + size of last leaf).
+  // It also protects the data in leaf nodes, because writing bytes might grow the blob and change the structure.
+  mutable boost::shared_mutex _treeStructureMutex;
+
   datanodestore::DataNodeStore *_nodeStore;
   cpputils::unique_ref<datanodestore::DataNode> _rootNode;
   blockstore::BlockId _blockId; // BlockId is stored in a member variable, since _rootNode is nullptr while traversing, but we still want to be able to return the blockId.
-  mutable boost::optional<uint32_t> _numLeavesCache;
+
+  struct SizeCache final {
+    uint32_t numLeaves;
+    uint64_t numBytes;
+  };
+  mutable CachedValue<SizeCache> _sizeCache;
 
   cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
   friend class DataTreeStore;
 
-  //TODO Use underscore for private methods
+  void _traverseLeavesByLeafIndices(uint32_t beginIndex, uint32_t endIndex, bool readOnlyTraversal,
-  void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
+                                    std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
-                       std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
+                                    std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
-                       std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
+                                    std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree) const;
-                       std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
+  void _traverseLeavesByByteIndices(uint64_t beginByte, uint64_t sizeBytes, bool readOnlyTraversal, std::function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
-  uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
+
-  uint64_t _numStoredBytes() const;
+  uint32_t _leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
-  uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
+
-  uint32_t _numLeaves() const;
+  SizeCache _getOrComputeSizeCache() const;
-  uint32_t _computeNumLeaves(const datanodestore::DataNode &node) const;
+  SizeCache _computeSizeCache(const datanodestore::DataNode &node) const;
+
+  uint64_t _tryReadBytes(void *target, uint64_t offset, uint64_t count) const;
+  void _doReadBytes(void *target, uint64_t offset, uint64_t count) const;
+  uint64_t _numBytes() const;
 
   DISALLOW_COPY_AND_ASSIGN(DataTree);
 };

src/blobstore/implementations/onblocks/datatreestore/impl/CachedValue.cpp

@@ -0,0 +1,2 @@
+#include "CachedValue.h"
+

src/blobstore/implementations/onblocks/datatreestore/impl/CachedValue.h

@@ -0,0 +1,46 @@
+#pragma once
+#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_CACHEDVALUE_H_
+#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_CACHEDVALUE_H_
+
+#include <boost/optional.hpp>
+#include <boost/thread/shared_mutex.hpp>
+#include <functional>
+
+namespace blobstore {
+namespace onblocks {
+
+// TODO Test
+template<class T>
+class CachedValue final {
+public:
+  CachedValue() :_cache(boost::none), _mutex() {}
+
+  T getOrCompute(std::function<T ()> compute) {
+    boost::upgrade_lock<boost::shared_mutex> readLock(_mutex);
+    if (_cache == boost::none) {
+      boost::upgrade_to_unique_lock<boost::shared_mutex> writeLock(readLock);
+      _cache = compute();
+    }
+    return *_cache;
+  }
+
+  void update(std::function<void (boost::optional<T>*)> func) {
+    boost::unique_lock<boost::shared_mutex> writeLock(_mutex);
+    func(&_cache);
+  }
+
+  void clear() {
+    update([] (boost::optional<T>* cache) {
+      *cache = boost::none;
+    });
+  }
+
+private:
+  boost::optional<T> _cache;
+  boost::shared_mutex _mutex;
+};
+
+}
+}
+
+#endif

src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp

@@ -20,15 +20,15 @@ namespace blobstore {
     namespace onblocks {
         namespace datatreestore {
 
-            LeafTraverser::LeafTraverser(DataNodeStore *nodeStore)
+            LeafTraverser::LeafTraverser(DataNodeStore *nodeStore, bool readOnlyTraversal)
-                : _nodeStore(nodeStore) {
+                : _nodeStore(nodeStore), _readOnlyTraversal(readOnlyTraversal) {
             }
 
-            unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+            void LeafTraverser::traverseAndUpdateRoot(unique_ref<DataNode>* root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
-                return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                _traverseAndUpdateRoot(root, beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
             }
 
-            unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+            void LeafTraverser::_traverseAndUpdateRoot(unique_ref<DataNode>* root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
                 ASSERT(beginIndex <= endIndex, "Invalid parameters");
 
                 //TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations:
@@ -36,11 +36,12 @@ namespace blobstore {
                 //     beginIndex<numLeaves<endIndex, beginIndex=numLeaves<endIndex,
                 //     numLeaves<beginIndex<endIndex, numLeaves<beginIndex=endIndex
 
-                uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth());
+                uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth((*root)->depth());
                 bool increaseTreeDepth = endIndex > maxLeavesForDepth;
+                ASSERT(!_readOnlyTraversal || !increaseTreeDepth, "Tried to grow a tree on a read only traversal");
 
-                if (root->depth() == 0) {
+                if ((*root)->depth() == 0) {
-                    DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root.get());
+                    DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root->get());
                     ASSERT(leaf != nullptr, "Depth 0 has to be leaf node");
 
                     if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
@@ -51,7 +52,7 @@ namespace blobstore {
                         onExistingLeaf(0, isRightBorderLeaf, LeafHandle(_nodeStore, leaf));
                     }
                 } else {
-                    DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
+                    DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root->get());
                     ASSERT(inner != nullptr, "Depth != 0 has to be leaf node");
                     _traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth),
                                              std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, !increaseTreeDepth,
@@ -63,17 +64,21 @@ namespace blobstore {
                 // We don't increase to the full needed tree depth in one step, because we want the traversal to go as far as possible
                 // and only then increase the depth - this causes the tree to be in consistent shape (balanced) for longer.
                 if (increaseTreeDepth) {
+                    ASSERT(!_readOnlyTraversal, "Can't increase tree depth in a read-only traversal");
+
                     // TODO Test cases that increase tree depth by 0, 1, 2, ... levels
-                    auto newRoot = _increaseTreeDepth(std::move(root));
+                    *root = _increaseTreeDepth(std::move(*root));
-                    return _traverseAndReturnRoot(std::move(newRoot), std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
+                    _traverseAndUpdateRoot(root, std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
                 } else {
                     // Once we're done growing the tree and done with the traversal, we might have to decrease tree depth,
                     // because the callbacks could have deleted nodes (this happens for example when shrinking the tree using a traversal).
-                    return _whileRootHasOnlyOneChildReplaceRootWithItsChild(std::move(root));
+                    _whileRootHasOnlyOneChildReplaceRootWithItsChild(root);
                 }
             }
 
             unique_ref<DataInnerNode> LeafTraverser::_increaseTreeDepth(unique_ref<DataNode> root) {
+                ASSERT(!_readOnlyTraversal, "Can't increase tree depth in a read-only traversal");
+
                 auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*root);
                 return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot);
             }
@@ -85,6 +90,7 @@ namespace blobstore {
                     LeafHandle leafHandle(_nodeStore, blockId);
                     if (growLastLeaf) {
                         if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
+                            ASSERT(!_readOnlyTraversal, "Can't grow the last leaf in a read-only traversal");
                             leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf());
                         }
                     }
@@ -116,6 +122,7 @@ namespace blobstore {
                 ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function.");
                 uint32_t numChildren = root->numChildren();
                 ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists");
+                ASSERT(!_readOnlyTraversal || endChild <= numChildren, "Can only traverse out of bounds in a read-only traversal");
                 bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren;
 
                 // If we traverse outside of the valid region (i.e. usually would only traverse to new leaves and not to the last leaf),
@@ -146,6 +153,8 @@ namespace blobstore {
 
                 // Traverse new children (including gap children, i.e. children that are created but not traversed because they're to the right of the current size, but to the left of the traversal region)
                 for (uint32_t childIndex = numChildren; childIndex < endChild; ++childIndex) {
+                    ASSERT(!_readOnlyTraversal, "Can't create new children in a read-only traversal");
+
                     uint32_t childOffset = childIndex * leavesPerChild;
                     uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset));
                     uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
@@ -161,6 +170,8 @@ namespace blobstore {
             }
 
             unique_ref<DataNode> LeafTraverser::_createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
+                ASSERT(!_readOnlyTraversal, "Can't create a new subtree in a read-only traversal");
+
                 ASSERT(beginIndex <= endIndex, "Invalid parameters");
                 if (0 == depth) {
                     ASSERT(beginIndex <= 1 && endIndex == 1, "With depth 0, we can only traverse one or zero leaves (i.e. traverse one leaf or traverse a gap leaf).");
@@ -212,25 +223,28 @@ namespace blobstore {
             }
 
             function<Data (uint32_t index)> LeafTraverser::_createMaxSizeLeaf() const {
+                ASSERT(!_readOnlyTraversal, "Can't create a new leaf in a read-only traversal");
+
                 uint64_t maxBytesPerLeaf = _nodeStore->layout().maxBytesPerLeaf();
                 return [maxBytesPerLeaf] (uint32_t /*index*/) -> Data {
                    return Data(maxBytesPerLeaf).FillWithZeroes();
                 };
             }
 
-            unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode> root) {
+            void LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode>* root) {
-                DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
+                DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root->get());
                 if (inner != nullptr && inner->numChildren() == 1) {
+                    ASSERT(!_readOnlyTraversal, "Can't decrease tree depth in a read-only traversal");
+
                     auto newRoot = _whileRootHasOnlyOneChildRemoveRootReturnChild(inner->readChild(0).blockId());
-                    auto result = _nodeStore->overwriteNodeWith(std::move(root), *newRoot);
+                    *root = _nodeStore->overwriteNodeWith(std::move(*root), *newRoot);
                     _nodeStore->remove(std::move(newRoot));
-                    return result;
-                } else {
-                    return root;
                 }
             }
 
             unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId) {
+                ASSERT(!_readOnlyTraversal, "Can't decrease tree depth in a read-only traversal");
+
                 auto current = _nodeStore->load(blockId);
                 ASSERT(current != none, "Node not found");
                 auto inner = dynamic_pointer_move<DataInnerNode>(*current);

src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h

@@ -25,19 +25,20 @@ namespace blobstore {
              */
             class LeafTraverser final {
             public:
-                LeafTraverser(datanodestore::DataNodeStore *nodeStore);
+                LeafTraverser(datanodestore::DataNodeStore *nodeStore, bool readOnlyTraversal);
 
-                cpputils::unique_ref<datanodestore::DataNode> traverseAndReturnRoot(
+                void traverseAndUpdateRoot(
-                      cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
+                      cpputils::unique_ref<datanodestore::DataNode>* root, uint32_t beginIndex, uint32_t endIndex,
                       std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
                       std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
                       std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
 
             private:
                 datanodestore::DataNodeStore *_nodeStore;
+                const bool _readOnlyTraversal;
 
-                cpputils::unique_ref<datanodestore::DataNode> _traverseAndReturnRoot(
+                void _traverseAndUpdateRoot(
-                      cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
+                      cpputils::unique_ref<datanodestore::DataNode>* root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
                       std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
                       std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
                       std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
@@ -55,7 +56,7 @@ namespace blobstore {
                                                                                 std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
                 uint32_t _maxLeavesForTreeDepth(uint8_t depth) const;
                 std::function<cpputils::Data (uint32_t index)> _createMaxSizeLeaf() const;
-                cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode> root);
+                void _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode>* root);
                 cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId);
 
                 DISALLOW_COPY_AND_ASSIGN(LeafTraverser);

src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h

@@ -22,10 +22,6 @@ public:
     return _baseTree->maxBytesPerLeaf();
   }
 
-  void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, datatreestore::LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
-    return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf);
-  }
-
   uint32_t numLeaves() const {
     return _baseTree->numLeaves();
   }
@@ -34,8 +30,24 @@ public:
     return _baseTree->resizeNumBytes(newNumBytes);
   }
 
-  uint64_t numStoredBytes() const {
+  uint64_t numBytes() const {
-    return _baseTree->numStoredBytes();
+    return _baseTree->numBytes();
+  }
+
+  uint64_t tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
+    return _baseTree->tryReadBytes(target, offset, count);
+  }
+
+  void readBytes(void *target, uint64_t offset, uint64_t count) const {
+    return _baseTree->readBytes(target, offset, count);
+  }
+
+  cpputils::Data readAllBytes() const {
+    return _baseTree->readAllBytes();
+  }
+
+  void writeBytes(const void *source, uint64_t offset, uint64_t count) {
+    return _baseTree->writeBytes(source, offset, count);
   }
 
   void flush() {

src/blockstore/implementations/caching/CachingBlockStore2.cpp

@@ -45,7 +45,7 @@ void CachingBlockStore2::CachedBlock::write(Data data) {
 }
 
 CachingBlockStore2::CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore)
-: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache() {
+: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache("blockstore") {
 }
 
 bool CachingBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {

src/blockstore/implementations/caching/cache/Cache.h

@@ -24,7 +24,7 @@ public:
   static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge
   static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time)
 
-  Cache();
+  Cache(const std::string& cacheName);
   ~Cache();
 
   uint32_t size() const;
@@ -56,10 +56,10 @@ template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Ke
 template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::MAX_LIFETIME_SEC;
 
 template<class Key, class Value, uint32_t MAX_ENTRIES>
-Cache<Key, Value, MAX_ENTRIES>::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
+Cache<Key, Value, MAX_ENTRIES>::Cache(const std::string& cacheName): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
   //Don't initialize timeoutFlusher in the initializer list,
   //because it then might already call Cache::popOldEntries() before Cache is done constructing.
-  _timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL);
+  _timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL, "flush_" + cacheName);
 }
 
 template<class Key, class Value, uint32_t MAX_ENTRIES>

src/blockstore/implementations/caching/cache/PeriodicTask.cpp

@@ -7,10 +7,10 @@ using namespace cpputils::logging;
 namespace blockstore {
 namespace caching {
 
-PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
+PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec, std::string threadName) :
         _task(task),
         _interval(static_cast<uint64_t>(UINT64_C(1000000000) * intervalSec)),
-        _thread(std::bind(&PeriodicTask::_loopIteration, this)) {
+        _thread(std::bind(&PeriodicTask::_loopIteration, this), std::move(threadName)) {
     _thread.start();
 }
 

src/blockstore/implementations/caching/cache/PeriodicTask.h

@@ -11,7 +11,7 @@ namespace caching {
 
 class PeriodicTask final {
 public:
-	PeriodicTask(std::function<void ()> task, double intervalSec);
+	PeriodicTask(std::function<void ()> task, double intervalSec, std::string threadName);
 
 private:
   bool _loopIteration();

src/cpp-utils/CMakeLists.txt

@@ -24,6 +24,8 @@ set(SOURCES
         io/pipestream.cpp
         thread/LoopThread.cpp
         thread/ThreadSystem.cpp
+        thread/debugging_nonwindows.cpp
+        thread/debugging_windows.cpp
         random/Random.cpp
         random/RandomGeneratorThread.cpp
         random/OSRandomGenerator.cpp

src/cpp-utils/random/RandomGeneratorThread.cpp

@@ -8,7 +8,7 @@ namespace cpputils {
               _buffer(buffer),
               _minSize(minSize),
               _maxSize(maxSize),
-              _thread(std::bind(&RandomGeneratorThread::_loopIteration, this)) {
+              _thread(std::bind(&RandomGeneratorThread::_loopIteration, this), "RandomGeneratorThread") {
         ASSERT(_maxSize >= _minSize, "Invalid parameters");
     }
 

src/cpp-utils/system/path.h

@@ -0,0 +1,26 @@
+#pragma once
+#ifndef MESSMER_CPPUTILS_SYSTEM_PATH_H
+#define MESSMER_CPPUTILS_SYSTEM_PATH_H
+
+#include <boost/filesystem/path.hpp>
+#include <cpp-utils/macros.h>
+
+namespace cpputils {
+
+#if defined(_MSC_VER)
+
+inline bool path_is_just_drive_letter(const boost::filesystem::path& path) {
+    return path.has_root_path() && !path.has_root_directory() && !path.has_parent_path();
+}
+
+#else
+
+inline constexpr bool path_is_just_drive_letter(const boost::filesystem::path& /*path*/) {
+    return false;
+}
+
+#endif
+
+}
+
+#endif

src/cpp-utils/thread/LoopThread.cpp

@@ -6,7 +6,8 @@ using boost::none;
 
 namespace cpputils {
 
-    LoopThread::LoopThread(function<bool()> loopIteration): _loopIteration(std::move(loopIteration)), _runningHandle(none) {
+    LoopThread::LoopThread(function<bool()> loopIteration, std::string threadName)
+    : _loopIteration(std::move(loopIteration)), _runningHandle(none), _threadName(std::move(threadName)) {
     }
 
     LoopThread::~LoopThread() {
@@ -16,7 +17,7 @@ namespace cpputils {
     }
 
     void LoopThread::start() {
-        _runningHandle = ThreadSystem::singleton().start(_loopIteration);
+        _runningHandle = ThreadSystem::singleton().start(_loopIteration, _threadName);
     }
 
     void LoopThread::stop() {

src/cpp-utils/thread/LoopThread.h

@@ -14,7 +14,7 @@ namespace cpputils {
     class LoopThread final {
     public:
         // The loopIteration callback returns true, if more iterations should be run, and false, if the thread should be terminated.
-        LoopThread(std::function<bool()> loopIteration);
+        LoopThread(std::function<bool()> loopIteration, std::string threadName);
         ~LoopThread();
         void start();
         void stop();
@@ -22,6 +22,7 @@ namespace cpputils {
     private:
         std::function<bool()> _loopIteration;
         boost::optional<ThreadSystem::Handle> _runningHandle;
+        std::string _threadName;
 
         DISALLOW_COPY_AND_ASSIGN(LoopThread);
     };

src/cpp-utils/thread/ThreadSystem.cpp

@@ -1,7 +1,9 @@
 #include "ThreadSystem.h"
 #include "../logging/logging.h"
+#include "debugging.h"
 
 using std::function;
+using std::string;
 using namespace cpputils::logging;
 
 namespace cpputils {
@@ -21,10 +23,10 @@ namespace cpputils {
 #endif
     }
 
-    ThreadSystem::Handle ThreadSystem::start(function<bool()> loopIteration) {
+    ThreadSystem::Handle ThreadSystem::start(function<bool()> loopIteration, string threadName) {
         boost::unique_lock<boost::mutex> lock(_mutex);
-        auto thread = _startThread(loopIteration);
+        auto thread = _startThread(loopIteration, threadName);
-        _runningThreads.push_back(RunningThread{std::move(loopIteration), std::move(thread)});
+        _runningThreads.push_back(RunningThread{std::move(threadName), std::move(loopIteration), std::move(thread)});
         return std::prev(_runningThreads.end());
     }
 
@@ -59,13 +61,14 @@ namespace cpputils {
 
     void ThreadSystem::_restartAllThreads() {
         for (RunningThread &thread : _runningThreads) {
-            thread.thread = _startThread(thread.loopIteration);
+            thread.thread = _startThread(thread.loopIteration, thread.threadName);
         }
         _mutex.unlock(); // Was locked in the before-fork handler
     }
 
-    boost::thread ThreadSystem::_startThread(function<bool()> loopIteration) {
+    boost::thread ThreadSystem::_startThread(function<bool()> loopIteration, const string& threadName) {
-        return boost::thread([loopIteration = std::move(loopIteration)] {
+        return boost::thread([loopIteration = std::move(loopIteration), threadName] {
+            cpputils::set_thread_name(threadName.c_str());
             ThreadSystem::_runThread(loopIteration);
         });
     }

src/cpp-utils/thread/ThreadSystem.h

@@ -13,6 +13,7 @@ namespace cpputils {
     class ThreadSystem final {
     private:
         struct RunningThread {
+            std::string threadName;
             std::function<bool()> loopIteration;  // The loopIteration callback returns true, if more iterations should be run, and false, if the thread should be terminated.
             boost::thread thread;  // boost::thread because we need it to be interruptible.
         };
@@ -21,7 +22,7 @@ namespace cpputils {
 
         static ThreadSystem &singleton();
 
-        Handle start(std::function<bool()> loopIteration);
+        Handle start(std::function<bool()> loopIteration, std::string threadName);
         void stop(Handle handle);
 
     private:
@@ -34,7 +35,7 @@ namespace cpputils {
         //TODO Rename to _doOnBeforeFork and _doAfterFork or similar, because they also handle locking _mutex for fork().
         void _stopAllThreadsForRestart();
         void _restartAllThreads();
-        boost::thread _startThread(std::function<bool()> loopIteration);
+        boost::thread _startThread(std::function<bool()> loopIteration, const std::string& threadName);
 
         std::list<RunningThread> _runningThreads;  // std::list, because we give out iterators as handles
         boost::mutex _mutex;

src/cpp-utils/thread/debugging.h

@@ -0,0 +1,16 @@
+#pragma once
+#ifndef MESSMER_CPPUTILS_DEBUGGING_H
+#define MESSMER_CPPUTILS_DEBUGGING_H
+
+#include <string>
+#include <thread>
+
+namespace cpputils {
+
+void set_thread_name(const char* name);
+std::string get_thread_name();
+std::string get_thread_name(std::thread* thread);
+
+}
+
+#endif

src/cpp-utils/thread/debugging_nonwindows.cpp

@@ -0,0 +1,55 @@
+#if !defined(_MSC_VER)
+
+#include "debugging.h"
+#include <stdexcept>
+#include <thread>
+#include <pthread.h>
+#include <cpp-utils/assert/assert.h>
+
+namespace cpputils {
+
+namespace {
+constexpr size_t MAX_NAME_LEN = 16; // this length includes the terminating null character at the end
+}
+
+void set_thread_name(const char* name) {
+  std::string name_(name);
+  if (name_.size() > MAX_NAME_LEN - 1) {
+    name_.resize(MAX_NAME_LEN - 1);
+  }
+#if defined(__APPLE__)
+  int result = pthread_setname_np(name_.c_str());
+#else
+  int result = pthread_setname_np(pthread_self(), name_.c_str());
+#endif
+  if (0 != result) {
+    throw std::runtime_error("Error setting thread name with pthread_setname_np. Code: " + std::to_string(result));
+  }
+}
+
+namespace {
+std::string get_thread_name(pthread_t thread) {
+  char name[MAX_NAME_LEN];
+  int result = pthread_getname_np(thread, name, MAX_NAME_LEN);
+  if (0 != result) {
+    throw std::runtime_error("Error getting thread name with pthread_getname_np. Code: " + std::to_string(result));
+  }
+  // pthread_getname_np returns a null terminated string with maximum 16 bytes.
+  // but just to be safe against a buggy implementation, let's set the last byte to zero.
+  name[MAX_NAME_LEN - 1] = '\0';
+  return name;
+}
+}
+
+std::string get_thread_name() {
+  return get_thread_name(pthread_self());
+}
+
+std::string get_thread_name(std::thread* thread) {
+  ASSERT(thread->joinable(), "Thread not running");
+  return get_thread_name(thread->native_handle());
+}
+
+}
+
+#endif

src/cpp-utils/thread/debugging_windows.cpp

@@ -0,0 +1,111 @@
+#if defined(_MSC_VER)
+
+#include <Windows.h>
+#include "debugging.h"
+#include <codecvt>
+#include <cpp-utils/assert/assert.h>
+
+using std::string;
+using std::wstring;
+using std::wstring_convert;
+
+namespace cpputils {
+
+namespace {
+struct NameData final {
+  wchar_t *name = nullptr;
+
+  ~NameData() {
+    if (nullptr != LocalFree(name)) {
+      throw std::runtime_error("Error releasing thread description memory. Error code: " + std::to_string(GetLastError()));
+    }
+  }
+};
+
+struct ModuleHandle final {
+	HMODULE module;
+
+	ModuleHandle(const char* dll) {
+		bool success = GetModuleHandleExA(0, dll, &module);
+		if (!success) {
+			throw std::runtime_error(string() + "Error loading dll: " + dll + ". Error code: " + std::to_string(GetLastError()));
+		}
+	}
+
+	~ModuleHandle() {
+		bool success = FreeLibrary(module);
+		if (!success) {
+			throw std::runtime_error("Error unloading dll. Error code: " + std::to_string(GetLastError()));
+		}
+	}
+};
+template<class Fn>
+class APIFunction final {
+private:
+	ModuleHandle module_;
+	Fn func_;
+
+public:
+	APIFunction(const char* dll, const char* function)
+	: module_(dll), func_(reinterpret_cast<Fn>(GetProcAddress(module_.module, function))) {
+	}
+
+	bool valid() const {
+		return func_ != nullptr;
+	}
+
+	Fn func() const {
+		return func_;
+	}
+};
+
+std::string get_thread_name(HANDLE thread) {
+  // The GetThreadDescription API was brought in version 1607 of Windows 10.
+  typedef HRESULT(WINAPI* GetThreadDescriptionFn)(HANDLE hThread, PWSTR* ppszThreadDescription);
+  static APIFunction<GetThreadDescriptionFn> get_thread_description_func("Kernel32.dll", "GetThreadDescription");
+
+  if (get_thread_description_func.valid()) {
+	  NameData name_data;
+
+	  HRESULT status = get_thread_description_func.func()(thread, &name_data.name);
+	  if (FAILED(status)) {
+		throw std::runtime_error("Error getting thread description. Error code: " + std::to_string(status));
+	  }
+	  return wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().to_bytes(name_data.name);
+  }
+  else {
+	  // GetThreadDescription API is not available.
+	  return "";
+  }
+}
+}
+
+void set_thread_name(const char* name) {
+  // The GetThreadDescription API was brought in version 1607 of Windows 10.
+  typedef HRESULT(WINAPI* SetThreadDescriptionFn)(HANDLE hThread, PCWSTR lpThreadDescription);
+  static APIFunction<SetThreadDescriptionFn> set_thread_description_func("Kernel32.dll", "SetThreadDescription");
+
+  if (set_thread_description_func.valid()) {
+	  wstring wname = wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(name);
+	  HRESULT status = set_thread_description_func.func()(GetCurrentThread(), wname.c_str());
+	  if (FAILED(status)) {
+		  throw std::runtime_error("Error setting thread description. Error code: " + std::to_string(status));
+	  }
+  }
+  else {
+	  // intentionally empty. SetThreadDescription API is not available.
+  }
+}
+
+std::string get_thread_name() {
+  return get_thread_name(GetCurrentThread());
+}
+
+std::string get_thread_name(std::thread* thread) {
+  ASSERT(thread->joinable(), "Thread not running");
+  return get_thread_name(static_cast<HANDLE>(thread->native_handle()));
+}
+
+}
+
+#endif

src/cryfs-cli/CallAfterTimeout.h

@@ -1,15 +1,15 @@
 #pragma once
-#ifndef MESSMER_CRYFS_SRC_CLI_CALLAFTERTIMEOUT_H
+#ifndef MESSMER_CRYFSCLI_CALLAFTERTIMEOUT_H
-#define MESSMER_CRYFS_SRC_CLI_CALLAFTERTIMEOUT_H
+#define MESSMER_CRYFSCLI_CALLAFTERTIMEOUT_H
 
 #include <functional>
 #include <mutex>
 #include <cpp-utils/thread/LoopThread.h>
 
-namespace cryfs {
+namespace cryfs_cli {
     class CallAfterTimeout final {
     public:
-        CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback);
+        CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback, const std::string& timeoutName);
         void resetTimer();
     private:
         bool _checkTimeoutThreadIteration();
@@ -25,8 +25,8 @@ namespace cryfs {
         DISALLOW_COPY_AND_ASSIGN(CallAfterTimeout);
     };
 
-    inline CallAfterTimeout::CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback)
+    inline CallAfterTimeout::CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback, const std::string& timeoutName)
-        :_callback(std::move(callback)), _timeout(timeout), _start(), _checkTimeoutThread(std::bind(&CallAfterTimeout::_checkTimeoutThreadIteration, this)) {
+        :_callback(std::move(callback)), _timeout(timeout), _start(), _checkTimeoutThread(std::bind(&CallAfterTimeout::_checkTimeoutThreadIteration, this), "timeout_" + timeoutName) {
         resetTimer();
         _checkTimeoutThread.start();
     }

src/cryfs-cli/Cli.cpp

@@ -26,10 +26,12 @@
 #include <cryfs/localstate/BasedirMetadata.h>
 #include "Environment.h"
 #include <cryfs/CryfsException.h>
+#include <cpp-utils/thread/debugging.h>
 
 //TODO Many functions accessing the ProgramOptions object. Factor out into class that stores it as a member.
 //TODO Factor out class handling askPassword
 
+using namespace cryfs_cli;
 using namespace cryfs;
 namespace bf = boost::filesystem;
 using namespace cpputils::logging;
@@ -66,7 +68,7 @@ using gitversion::VersionCompare;
 //TODO Replace ASSERTs with other error handling when it is not a programming error but an environment influence (e.g. a block is missing)
 //TODO Can we improve performance by setting compiler parameter -maes for scrypt?
 
-namespace cryfs {
+namespace cryfs_cli {
 
     Cli::Cli(RandomGenerator &keyGenerator, const SCryptSettings &scryptSettings, shared_ptr<Console> console):
             _keyGenerator(keyGenerator), _scryptSettings(scryptSettings), _console(), _noninteractive(false), _idleUnmounter(none), _device(none) {
@@ -262,12 +264,9 @@ namespace cryfs {
 
             _initLogfile(options);
 
-#ifdef __APPLE__
+            std::cout << "\nMounting filesystem. To unmount, call:\n$ cryfs-unmount " << options.mountDir() << "\n"
-          std::cout << "\nMounting filesystem. To unmount, call:\n$ umount " << options.mountDir() << "\n" << std::endl;
+                      << std::endl;
-#else
+
-          std::cout << "\nMounting filesystem. To unmount, call:\n$ fusermount -u " << options.mountDir() << "\n"
-                    << std::endl;
-#endif
             if (options.foreground()) {
                 fuse->runInForeground(options.mountDir(), options.fuseOptions());
             } else {
@@ -304,7 +303,7 @@ namespace cryfs {
             return none;
         }
         uint64_t millis = std::llround(60000 * (*minutes));
-        return make_unique_ref<CallAfterTimeout>(milliseconds(millis), callback);
+        return make_unique_ref<CallAfterTimeout>(milliseconds(millis), callback, "idlecallback");
     }
 
     void Cli::_initLogfile(const ProgramOptions &options) {
@@ -320,10 +319,17 @@ namespace cryfs {
         }
     }
 
-    void Cli::_sanityChecks(const ProgramOptions &options) {
+	void Cli::_sanityChecks(const ProgramOptions &options) {
-        _checkDirAccessible(options.baseDir(), "base directory", ErrorCode::InaccessibleBaseDir);
+		_checkDirAccessible(bf::absolute(options.baseDir()), "base directory", ErrorCode::InaccessibleBaseDir);
-        _checkDirAccessible(options.mountDir(), "mount directory", ErrorCode::InaccessibleMountDir);
+
-        _checkMountdirDoesntContainBasedir(options);
+		if (!options.mountDirIsDriveLetter()) {
+			_checkDirAccessible(options.mountDir(), "mount directory", ErrorCode::InaccessibleMountDir);
+			_checkMountdirDoesntContainBasedir(options);
+		} else {
+			if (bf::exists(options.mountDir())) {
+				throw CryfsException("Drive " + options.mountDir().string() + " already exists.", ErrorCode::InaccessibleMountDir);
+			}
+		}
     }
 
     void Cli::_checkDirAccessible(const bf::path &dir, const std::string &name, ErrorCode errorCode) {
@@ -396,6 +402,7 @@ namespace cryfs {
 
     int Cli::main(int argc, const char *argv[], unique_ref<HttpClient> httpClient, std::function<void()> onMounted) {
         cpputils::showBacktraceOnCrash();
+        cpputils::set_thread_name("cryfs");
 
         try {
             _showVersion(std::move(httpClient));

src/cryfs-cli/Cli.h

@@ -1,6 +1,6 @@
 #pragma once
-#ifndef MESSMER_CRYFS_CLI_H
+#ifndef MESSMER_CRYFSCLI_CLI_H
-#define MESSMER_CRYFS_CLI_H
+#define MESSMER_CRYFSCLI_CLI_H
 
 #include "program_options/ProgramOptions.h"
 #include <cryfs/config/CryConfigFile.h>
@@ -14,7 +14,7 @@
 #include <cryfs/config/CryConfigLoader.h>
 #include <cryfs/ErrorCodes.h>
 
-namespace cryfs {
+namespace cryfs_cli {
     class Cli final {
     public:
         Cli(cpputils::RandomGenerator &keyGenerator, const cpputils::SCryptSettings& scryptSettings, std::shared_ptr<cpputils::Console> console);
@@ -23,9 +23,9 @@ namespace cryfs {
     private:
         void _checkForUpdates(cpputils::unique_ref<cpputils::HttpClient> httpClient);
         void _runFilesystem(const program_options::ProgramOptions &options, std::function<void()> onMounted);
-        CryConfigLoader::ConfigLoadResult _loadOrCreateConfig(const program_options::ProgramOptions &options, const LocalStateDir& localStateDir);
+        cryfs::CryConfigLoader::ConfigLoadResult _loadOrCreateConfig(const program_options::ProgramOptions &options, const cryfs::LocalStateDir& localStateDir);
-        void _checkConfigIntegrity(const boost::filesystem::path& basedir, const LocalStateDir& localStateDir, const CryConfigFile& config, bool allowReplacedFilesystem);
+        void _checkConfigIntegrity(const boost::filesystem::path& basedir, const cryfs::LocalStateDir& localStateDir, const cryfs::CryConfigFile& config, bool allowReplacedFilesystem);
-        boost::optional<CryConfigLoader::ConfigLoadResult> _loadOrCreateConfigFile(boost::filesystem::path configFilePath, LocalStateDir localStateDir, const boost::optional<std::string> &cipher, const boost::optional<uint32_t> &blocksizeBytes, bool allowFilesystemUpgrade, const boost::optional<bool> &missingBlockIsIntegrityViolation, bool allowReplacedFilesystem);
+        boost::optional<cryfs::CryConfigLoader::ConfigLoadResult> _loadOrCreateConfigFile(boost::filesystem::path configFilePath, cryfs::LocalStateDir localStateDir, const boost::optional<std::string> &cipher, const boost::optional<uint32_t> &blocksizeBytes, bool allowFilesystemUpgrade, const boost::optional<bool> &missingBlockIsIntegrityViolation, bool allowReplacedFilesystem);
         boost::filesystem::path _determineConfigFile(const program_options::ProgramOptions &options);
         static std::function<std::string()> _askPasswordForExistingFilesystem(std::shared_ptr<cpputils::Console> console);
         static std::function<std::string()> _askPasswordForNewFilesystem(std::shared_ptr<cpputils::Console> console);
@@ -37,11 +37,11 @@ namespace cryfs {
         void _sanityChecks(const program_options::ProgramOptions &options);
         void _checkMountdirDoesntContainBasedir(const program_options::ProgramOptions &options);
         bool _pathContains(const boost::filesystem::path &parent, const boost::filesystem::path &child);
-        void _checkDirAccessible(const boost::filesystem::path &dir, const std::string &name, ErrorCode errorCode);
+        void _checkDirAccessible(const boost::filesystem::path &dir, const std::string &name, cryfs::ErrorCode errorCode);
-        std::shared_ptr<cpputils::TempFile> _checkDirWriteable(const boost::filesystem::path &dir, const std::string &name, ErrorCode errorCode);
+        std::shared_ptr<cpputils::TempFile> _checkDirWriteable(const boost::filesystem::path &dir, const std::string &name, cryfs::ErrorCode errorCode);
-        void _checkDirReadable(const boost::filesystem::path &dir, std::shared_ptr<cpputils::TempFile> tempfile, const std::string &name, ErrorCode errorCode);
+        void _checkDirReadable(const boost::filesystem::path &dir, std::shared_ptr<cpputils::TempFile> tempfile, const std::string &name, cryfs::ErrorCode errorCode);
         boost::optional<cpputils::unique_ref<CallAfterTimeout>> _createIdleCallback(boost::optional<double> minutes, std::function<void()> callback);
-        void _sanityCheckFilesystem(CryDevice *device);
+        void _sanityCheckFilesystem(cryfs::CryDevice *device);
 
 
         cpputils::RandomGenerator &_keyGenerator;
@@ -49,7 +49,7 @@ namespace cryfs {
         std::shared_ptr<cpputils::Console> _console;
         bool _noninteractive;
         boost::optional<cpputils::unique_ref<CallAfterTimeout>> _idleUnmounter;
-        boost::optional<cpputils::unique_ref<CryDevice>> _device;
+        boost::optional<cpputils::unique_ref<cryfs::CryDevice>> _device;
 
         DISALLOW_COPY_AND_ASSIGN(Cli);
     };

src/cryfs-cli/Environment.cpp

@@ -6,7 +6,7 @@
 using std::string;
 namespace bf = boost::filesystem;
 
-namespace cryfs {
+namespace cryfs_cli {
     const string Environment::FRONTEND_KEY = "CRYFS_FRONTEND";
     const string Environment::FRONTEND_NONINTERACTIVE = "noninteractive";
     const string Environment::NOUPDATECHECK_KEY = "CRYFS_NO_UPDATE_CHECK";

src/cryfs-cli/Environment.h

@@ -1,11 +1,11 @@
 #pragma once
-#ifndef MESSMER_CRYFS_CLI_ENVIRONMENT_H
+#ifndef MESSMER_CRYFSCLI_ENVIRONMENT_H
-#define MESSMER_CRYFS_CLI_ENVIRONMENT_H
+#define MESSMER_CRYFSCLI_ENVIRONMENT_H
 
 #include <string>
 #include <boost/filesystem/path.hpp>
 
-namespace cryfs {
+namespace cryfs_cli {
 
     class Environment {
     public:

src/cryfs-cli/VersionChecker.cpp

@@ -13,7 +13,7 @@ using boost::property_tree::ptree;
 using boost::property_tree::json_parser_error;
 using namespace cpputils::logging;
 
-namespace cryfs {
+namespace cryfs_cli {
 
     VersionChecker::VersionChecker(HttpClient* httpClient)
             : _versionInfo(_getVersionInfo(httpClient)) {}

src/cryfs-cli/VersionChecker.h

@@ -1,5 +1,5 @@
-#ifndef MESSMER_CRYFS_SRC_CLI_VERSIONCHECKER_H
+#ifndef MESSMER_CRYFSCLI_VERSIONCHECKER_H
-#define MESSMER_CRYFS_SRC_CLI_VERSIONCHECKER_H
+#define MESSMER_CRYFSCLI_VERSIONCHECKER_H
 
 #include <cpp-utils/macros.h>
 #include <string>
@@ -8,7 +8,7 @@
 #include <cpp-utils/network/HttpClient.h>
 #include <cpp-utils/pointer/unique_ref.h>
 
-namespace cryfs {
+namespace cryfs_cli {
     class VersionChecker final {
     public:
         //TODO Write a cpputils::shared_ref and use it

src/cryfs-cli/main.cpp

@@ -10,7 +10,7 @@
 #include <cpp-utils/network/CurlHttpClient.h>
 #endif
 
-using namespace cryfs;
+using namespace cryfs_cli;
 using cpputils::Random;
 using cpputils::SCrypt;
 using cpputils::IOStreamConsole;
@@ -35,13 +35,13 @@ int main(int argc, const char *argv[]) {
 #endif
         return Cli(keyGenerator, SCrypt::DefaultSettings, make_shared<IOStreamConsole>())
             .main(argc, argv, std::move(httpClient), []{});
-    } catch (const CryfsException &e) {
+    } catch (const cryfs::CryfsException &e) {
         if (e.what() != string()) {
             std::cerr << "Error: " << e.what() << std::endl;
         }
         return exitCode(e.errorCode());
     } catch (const std::exception &e) {
         cerr << "Error: " << e.what();
-        return exitCode(ErrorCode::UnspecifiedError);
+        return exitCode(cryfs::ErrorCode::UnspecifiedError);
     }
 }

src/cryfs-cli/program_options/Parser.cpp

@@ -8,7 +8,7 @@
 
 namespace po = boost::program_options;
 namespace bf = boost::filesystem;
-using namespace cryfs::program_options;
+using namespace cryfs_cli::program_options;
 using cryfs::CryConfigConsole;
 using cryfs::CryfsException;
 using cryfs::ErrorCode;
@@ -49,8 +49,8 @@ ProgramOptions Parser::parse(const vector<string> &supportedCiphers) const {
     if (!vm.count("mount-dir")) {
         _showHelpAndExit("Please specify a mount directory.", ErrorCode::InvalidArguments);
     }
-    bf::path baseDir = bf::absolute(vm["base-dir"].as<string>());
+    bf::path baseDir = vm["base-dir"].as<string>();
-    bf::path mountDir = bf::absolute(vm["mount-dir"].as<string>());
+    bf::path mountDir = vm["mount-dir"].as<string>();
     optional<bf::path> configfile = none;
     if (vm.count("config")) {
         configfile = bf::absolute(vm["config"].as<string>());

src/cryfs-cli/program_options/Parser.h

@@ -1,12 +1,12 @@
 #pragma once
-#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_PARSER_H
+#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_PARSER_H
-#define MESSMER_CRYFS_PROGRAMOPTIONS_PARSER_H
+#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_PARSER_H
 
 #include "ProgramOptions.h"
 #include <boost/program_options.hpp>
 #include <cryfs/ErrorCodes.h>
 
-namespace cryfs {
+namespace cryfs_cli {
     namespace program_options {
         class Parser final {
         public:
@@ -21,7 +21,7 @@ namespace cryfs {
             static void _addPositionalOptionForBaseDir(boost::program_options::options_description *desc,
                                                        boost::program_options::positional_options_description *positional);
             static void _showHelp();
-            [[noreturn]] static void _showHelpAndExit(const std::string& message, ErrorCode errorCode);
+            [[noreturn]] static void _showHelpAndExit(const std::string& message, cryfs::ErrorCode errorCode);
             [[noreturn]] static void _showCiphersAndExit(const std::vector<std::string> &supportedCiphers);
             [[noreturn]] static void _showVersionAndExit();
             static boost::program_options::variables_map _parseOptionsOrShowHelp(const std::vector<std::string> &options, const std::vector<std::string> &supportedCiphers);

src/cryfs-cli/program_options/ProgramOptions.cpp

@@ -1,8 +1,9 @@
 #include "ProgramOptions.h"
 #include <cstring>
 #include <cpp-utils/assert/assert.h>
+#include <cpp-utils/system/path.h>
 
-using namespace cryfs::program_options;
+using namespace cryfs_cli::program_options;
 using std::string;
 using std::vector;
 using boost::optional;
@@ -15,9 +16,15 @@ ProgramOptions::ProgramOptions(bf::path baseDir, bf::path mountDir, optional<bf:
                                bool allowIntegrityViolations,
                                boost::optional<bool> missingBlockIsIntegrityViolation,
                                vector<string> fuseOptions)
-    :_baseDir(std::move(baseDir)), _mountDir(std::move(mountDir)), _configFile(std::move(configFile)), _foreground(foreground), _allowFilesystemUpgrade(allowFilesystemUpgrade), _allowReplacedFilesystem(allowReplacedFilesystem), _allowIntegrityViolations(allowIntegrityViolations),
+    : _configFile(std::move(configFile)), _baseDir(bf::absolute(std::move(baseDir))), _mountDir(std::move(mountDir)),
-     _cipher(std::move(cipher)), _blocksizeBytes(std::move(blocksizeBytes)), _unmountAfterIdleMinutes(std::move(unmountAfterIdleMinutes)),
+      _mountDirIsDriveLetter(cpputils::path_is_just_drive_letter(_mountDir)),
-     _missingBlockIsIntegrityViolation(std::move(missingBlockIsIntegrityViolation)), _logFile(std::move(logFile)), _fuseOptions(std::move(fuseOptions)) {
+	  _foreground(foreground),
+	  _allowFilesystemUpgrade(allowFilesystemUpgrade), _allowReplacedFilesystem(allowReplacedFilesystem), _allowIntegrityViolations(allowIntegrityViolations),
+      _cipher(std::move(cipher)), _blocksizeBytes(std::move(blocksizeBytes)), _unmountAfterIdleMinutes(std::move(unmountAfterIdleMinutes)),
+      _missingBlockIsIntegrityViolation(std::move(missingBlockIsIntegrityViolation)), _logFile(std::move(logFile)), _fuseOptions(std::move(fuseOptions)) {
+	if (!_mountDirIsDriveLetter) {
+		_mountDir = bf::absolute(std::move(_mountDir));
+	}
 }
 
 const bf::path &ProgramOptions::baseDir() const {
@@ -28,6 +35,10 @@ const bf::path &ProgramOptions::mountDir() const {
     return _mountDir;
 }
 
+bool ProgramOptions::mountDirIsDriveLetter() const {
+	return _mountDirIsDriveLetter;
+}
+
 const optional<bf::path> &ProgramOptions::configFile() const {
     return _configFile;
 }

src/cryfs-cli/program_options/ProgramOptions.h

@@ -1,6 +1,6 @@
 #pragma once
-#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_PROGRAMOPTIONS_H
+#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_PROGRAMOPTIONS_H
-#define MESSMER_CRYFS_PROGRAMOPTIONS_PROGRAMOPTIONS_H
+#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_PROGRAMOPTIONS_H
 
 #include <vector>
 #include <string>
@@ -8,7 +8,7 @@
 #include <cpp-utils/macros.h>
 #include <boost/filesystem.hpp>
 
-namespace cryfs {
+namespace cryfs_cli {
     namespace program_options {
         class ProgramOptions final {
         public:
@@ -25,6 +25,7 @@ namespace cryfs {
 
             const boost::filesystem::path &baseDir() const;
             const boost::filesystem::path &mountDir() const;
+			bool mountDirIsDriveLetter() const;
             const boost::optional<boost::filesystem::path> &configFile() const;
             bool foreground() const;
             bool allowFilesystemUpgrade() const;
@@ -38,9 +39,10 @@ namespace cryfs {
             const std::vector<std::string> &fuseOptions() const;
 
         private:
-            boost::filesystem::path _baseDir;
+			boost::optional<boost::filesystem::path> _configFile;
-            boost::filesystem::path _mountDir;
+            boost::filesystem::path _baseDir; // this is always absolute
-            boost::optional<boost::filesystem::path> _configFile;
+            boost::filesystem::path _mountDir; // this is absolute iff !_mountDirIsDriveLetter
+			bool _mountDirIsDriveLetter;
             bool _foreground;
             bool _allowFilesystemUpgrade;
             bool _allowReplacedFilesystem;

src/cryfs-cli/program_options/utils.cpp

@@ -7,7 +7,7 @@ using std::make_pair;
 using std::vector;
 using std::string;
 
-namespace cryfs {
+namespace cryfs_cli {
     namespace program_options {
         pair<vector<string>, vector<string>> splitAtDoubleDash(const vector<string> &options) {
             auto doubleDashIterator = std::find(options.begin(), options.end(), string("--"));

src/cryfs-cli/program_options/utils.h

@@ -1,12 +1,12 @@
 #pragma once
-#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_UTILS_H
+#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_UTILS_H
-#define MESSMER_CRYFS_PROGRAMOPTIONS_UTILS_H
+#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_UTILS_H
 
 #include <utility>
 #include <vector>
 #include <string>
 
-namespace cryfs {
+namespace cryfs_cli {
     namespace program_options {
         /**
          * Splits an array of program options into two arrays of program options, split at a double dash '--' option.

src/cryfs-unmount/CMakeLists.txt

@@ -0,0 +1,24 @@
+project (cryfs-unmount)
+INCLUDE(GNUInstallDirs)
+
+set(SOURCES
+	program_options/ProgramOptions.cpp
+	program_options/Parser.cpp
+	Cli.cpp
+)
+
+add_library(${PROJECT_NAME} ${SOURCES})
+target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils cryfs fspp-fuse)
+target_enable_style_warnings(${PROJECT_NAME})
+target_activate_cpp14(${PROJECT_NAME})
+
+add_executable(${PROJECT_NAME}_bin main_unmount.cpp)
+set_target_properties(${PROJECT_NAME}_bin PROPERTIES OUTPUT_NAME cryfs-unmount)
+target_link_libraries(${PROJECT_NAME}_bin PUBLIC ${PROJECT_NAME})
+target_enable_style_warnings(${PROJECT_NAME}_bin)
+target_activate_cpp14(${PROJECT_NAME}_bin)
+
+install(TARGETS ${PROJECT_NAME}_bin
+		CONFIGURATIONS Debug Release RelWithDebInfo
+		DESTINATION ${CMAKE_INSTALL_BINDIR}
+)

src/cryfs-unmount/Cli.cpp

@@ -0,0 +1,36 @@
+#include "Cli.h"
+#include <fspp/fuse/Fuse.h>
+#include <cryfs-unmount/program_options/Parser.h>
+#include <gitversion/gitversion.h>
+#include <cryfs/CryfsException.h>
+
+#include <iostream>
+
+using fspp::fuse::Fuse;
+using cryfs_unmount::program_options::Parser;
+using cryfs_unmount::program_options::ProgramOptions;
+
+namespace cryfs_unmount {
+
+namespace {
+void _showVersion() {
+    std::cout << "CryFS Version " << gitversion::VersionString() << std::endl;
+}
+}
+
+void Cli::main(int argc, const char* argv[]) {
+    _showVersion();
+    ProgramOptions options = Parser(argc, argv).parse();
+
+    if (!boost::filesystem::exists(options.mountDir())) {
+        throw cryfs::CryfsException("Given mountdir doesn't exist", cryfs::ErrorCode::InaccessibleMountDir);
+    }
+    // TODO This doesn't seem to work with relative paths
+    std::cout << "Unmounting CryFS filesystem at " << options.mountDir() << "." << std::endl;
+    Fuse::unmount(options.mountDir());
+
+    // TODO Wait until it is actually unmounted and then show a better success message?
+    std::cout << "Filesystem is unmounting now." << std::endl;
+}
+
+}

src/cryfs-unmount/Cli.h

@@ -0,0 +1,14 @@
+#pragma once
+#ifndef MESSMER_CRYFSUNMOUNT_CLI_H
+#define MESSMER_CRYFSUNMOUNT_CLI_H
+
+namespace cryfs_unmount {
+
+class Cli final {
+public:
+    void main(int argc, const char* argv[]);
+};
+
+}
+
+#endif

src/cryfs-unmount/main_unmount.cpp

@@ -0,0 +1,38 @@
+#if defined(_MSC_VER)
+#include <Windows.h>
+#include <VersionHelpers.h>
+#endif
+
+#include <iostream>
+#include <cryfs/CryfsException.h>
+#include <cpp-utils/assert/backtrace.h>
+#include "Cli.h"
+
+using std::cerr;
+using cryfs::ErrorCode;
+
+int main(int argc, const char *argv[]) {
+#if defined(_MSC_VER)
+	if (!IsWindows7SP1OrGreater()) {
+		std::cerr << "CryFS is currently only supported on Windows 7 SP1 (or later)." << std::endl;
+		exit(1);
+	}
+#endif
+
+	cpputils::showBacktraceOnCrash();
+
+	try {
+		cryfs_unmount::Cli().main(argc, argv);
+	}
+	catch (const cryfs::CryfsException &e) {
+		if (e.what() != std::string()) {
+			std::cerr << "Error " << static_cast<int>(e.errorCode()) << ": " << e.what() << std::endl;
+		}
+		return exitCode(e.errorCode());
+	}
+	catch (const std::runtime_error &e) {
+		std::cerr << "Error: " << e.what() << std::endl;
+		return exitCode(ErrorCode::UnspecifiedError);
+	}
+	return exitCode(ErrorCode::Success);
+}

src/cryfs-unmount/program_options/Parser.cpp

@@ -0,0 +1,128 @@
+#include "Parser.h"
+#include <iostream>
+#include <boost/optional.hpp>
+#include <cryfs/config/CryConfigConsole.h>
+#include <cryfs/CryfsException.h>
+#include <cryfs-cli/Environment.h>
+
+namespace po = boost::program_options;
+namespace bf = boost::filesystem;
+using namespace cryfs_unmount::program_options;
+using cryfs::CryConfigConsole;
+using cryfs::CryfsException;
+using cryfs::ErrorCode;
+using std::vector;
+using std::cerr;
+using std::endl;
+using std::string;
+using namespace cpputils::logging;
+
+Parser::Parser(int argc, const char *argv[])
+	:_options(_argsToVector(argc, argv)) {
+}
+
+vector<string> Parser::_argsToVector(int argc, const char *argv[]) {
+	vector<string> result;
+	for (int i = 0; i < argc; ++i) {
+		result.push_back(argv[i]);
+	}
+	return result;
+}
+
+ProgramOptions Parser::parse() const {
+	po::variables_map vm = _parseOptionsOrShowHelp(_options);
+
+	if (!vm.count("mount-dir")) {
+		_showHelpAndExit("Please specify a mount directory.", ErrorCode::InvalidArguments);
+	}
+	bf::path mountDir = vm["mount-dir"].as<string>();
+
+	return ProgramOptions(std::move(mountDir));
+}
+
+po::variables_map Parser::_parseOptionsOrShowHelp(const vector<string> &options) {
+	try {
+		return _parseOptions(options);
+	}
+	catch (const CryfsException& e) {
+		// If CryfsException is thrown, we already know what's wrong.
+		// Show usage information and pass through the exception, don't catch it.
+		if (e.errorCode() != ErrorCode::Success) {
+			_showHelp();
+		}
+		throw;
+	}
+	catch (const std::exception &e) {
+		std::cerr << e.what() << std::endl;
+		_showHelpAndExit("Invalid arguments", ErrorCode::InvalidArguments);
+	}
+}
+
+po::variables_map Parser::_parseOptions(const vector<string> &options) {
+	po::options_description desc;
+	po::positional_options_description positional_desc;
+	_addAllowedOptions(&desc);
+	_addPositionalOptionForBaseDir(&desc, &positional_desc);
+
+	po::variables_map vm;
+	vector<const char*> _options = _to_const_char_vector(options);
+	po::store(po::command_line_parser(_options.size(), _options.data())
+		.options(desc).positional(positional_desc).run(), vm);
+	if (vm.count("help")) {
+		_showHelpAndExit("", ErrorCode::Success);
+	}
+	if (vm.count("version")) {
+		_showVersionAndExit();
+	}
+	po::notify(vm);
+
+	return vm;
+}
+
+vector<const char*> Parser::_to_const_char_vector(const vector<string> &options) {
+	vector<const char*> result;
+	result.reserve(options.size());
+	for (const string &option : options) {
+		result.push_back(option.c_str());
+	}
+	return result;
+}
+
+void Parser::_addAllowedOptions(po::options_description *desc) {
+	po::options_description options("Allowed options");
+	string cipher_description = "Cipher to use for encryption. See possible values by calling cryfs with --show-ciphers. Default: ";
+	cipher_description += CryConfigConsole::DEFAULT_CIPHER;
+	string blocksize_description = "The block size used when storing ciphertext blocks (in bytes). Default: ";
+	blocksize_description += std::to_string(CryConfigConsole::DEFAULT_BLOCKSIZE_BYTES);
+	options.add_options()
+		("help,h", "show help message")
+		("version", "Show CryFS version number")
+		;
+	desc->add(options);
+}
+
+void Parser::_addPositionalOptionForBaseDir(po::options_description *desc, po::positional_options_description *positional) {
+	positional->add("mount-dir", 1);
+	po::options_description hidden("Hidden options");
+	hidden.add_options()
+		("mount-dir", po::value<string>(), "Mount directory")
+		;
+	desc->add(hidden);
+}
+
+void Parser::_showHelp() {
+	cerr << "Usage: cryfs-unmount [mountPoint]\n";
+	po::options_description desc;
+	_addAllowedOptions(&desc);
+	cerr << desc << endl;
+}
+
+[[noreturn]] void Parser::_showHelpAndExit(const std::string& message, ErrorCode errorCode) {
+	_showHelp();
+	throw CryfsException(message, errorCode);
+}
+
+[[noreturn]] void Parser::_showVersionAndExit() {
+	// no need to show version because it was already shown in the CryFS header before parsing program options
+	throw CryfsException("", ErrorCode::Success);
+}

src/cryfs-unmount/program_options/Parser.h

@@ -0,0 +1,37 @@
+#pragma once
+#ifndef MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PARSER_H
+#define MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PARSER_H
+
+#include "ProgramOptions.h"
+#include <boost/program_options.hpp>
+#include <cryfs/ErrorCodes.h>
+
+namespace cryfs_unmount {
+	namespace program_options {
+		class Parser final {
+		public:
+			Parser(int argc, const char *argv[]);
+
+			ProgramOptions parse() const;
+
+		private:
+			static std::vector<std::string> _argsToVector(int argc, const char *argv[]);
+			static std::vector<const char*> _to_const_char_vector(const std::vector<std::string> &options);
+			static void _addAllowedOptions(boost::program_options::options_description *desc);
+			static void _addPositionalOptionForBaseDir(boost::program_options::options_description *desc,
+				boost::program_options::positional_options_description *positional);
+			static void _showHelp();
+			[[noreturn]] static void _showHelpAndExit(const std::string& message, cryfs::ErrorCode errorCode);
+			[[noreturn]] static void _showCiphersAndExit(const std::vector<std::string> &supportedCiphers);
+			[[noreturn]] static void _showVersionAndExit();
+			static boost::program_options::variables_map _parseOptionsOrShowHelp(const std::vector<std::string> &options);
+			static boost::program_options::variables_map _parseOptions(const std::vector<std::string> &options);
+
+			std::vector<std::string> _options;
+
+			DISALLOW_COPY_AND_ASSIGN(Parser);
+		};
+	}
+}
+
+#endif

src/cryfs-unmount/program_options/ProgramOptions.cpp

@@ -0,0 +1,25 @@
+#include "ProgramOptions.h"
+#include <cstring>
+#include <cpp-utils/assert/assert.h>
+#include <cpp-utils/system/path.h>
+
+using namespace cryfs_unmount::program_options;
+using std::string;
+namespace bf = boost::filesystem;
+
+ProgramOptions::ProgramOptions(bf::path mountDir)
+	:_mountDir(std::move(mountDir)),
+	 _mountDirIsDriveLetter(cpputils::path_is_just_drive_letter(_mountDir))
+{
+	if (!_mountDirIsDriveLetter) {
+		_mountDir = bf::absolute(std::move(_mountDir));
+	}
+}
+
+const bf::path &ProgramOptions::mountDir() const {
+	return _mountDir;
+}
+
+bool ProgramOptions::mountDirIsDriveLetter() const {
+	return _mountDirIsDriveLetter;
+}

src/cryfs-unmount/program_options/ProgramOptions.h

@@ -0,0 +1,30 @@
+#pragma once
+#ifndef MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PROGRAMOPTIONS_H
+#define MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PROGRAMOPTIONS_H
+
+#include <vector>
+#include <string>
+#include <boost/optional.hpp>
+#include <cpp-utils/macros.h>
+#include <boost/filesystem.hpp>
+
+namespace cryfs_unmount {
+	namespace program_options {
+		class ProgramOptions final {
+		public:
+			ProgramOptions(boost::filesystem::path mountDir);
+			ProgramOptions(ProgramOptions &&rhs) = default;
+
+			const boost::filesystem::path &mountDir() const;
+			bool mountDirIsDriveLetter() const;
+
+		private:
+			boost::filesystem::path _mountDir;
+			bool _mountDirIsDriveLetter;
+
+			DISALLOW_COPY_AND_ASSIGN(ProgramOptions);
+		};
+	}
+}
+
+#endif

src/cryfs/config/CryConfigLoader.cpp

@@ -71,7 +71,7 @@ void CryConfigLoader::_checkVersion(const CryConfig &config, bool allowFilesyste
     }
   }
   if (!allowFilesystemUpgrade && gitversion::VersionCompare::isOlderThan(config.Version(), CryConfig::FilesystemFormatVersion)) {
-    if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It can be migrated, but afterwards couldn't be opened anymore with older versions. Do you want to migrate it?", false)) {
+    if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It can be migrated, but afterwards couldn't be opened anymore with older versions. Please make a backup of your data before attempting a migration. Do you want to migrate it now?", false)) {
       throw CryfsException("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). It has to be migrated.", ErrorCode::TooOldFilesystemFormat);
     }
   }

src/cryfs/filesystem/cachingfsblobstore/CachingFsBlobStore.h

@@ -50,7 +50,7 @@ namespace cryfs {
 
 
         inline CachingFsBlobStore::CachingFsBlobStore(cpputils::unique_ref<fsblobstore::FsBlobStore> baseBlobStore)
-                : _baseBlobStore(std::move(baseBlobStore)), _cache() {
+                : _baseBlobStore(std::move(baseBlobStore)), _cache("fsblobstore") {
         }
 
         inline CachingFsBlobStore::~CachingFsBlobStore() {

src/fspp/fuse/Fuse.cpp

@@ -7,9 +7,16 @@
 #include <iostream>
 #include <cpp-utils/assert/assert.h>
 #include <cpp-utils/logging/logging.h>
+#include <cpp-utils/process/subprocess.h>
+#include <cpp-utils/thread/debugging.h>
 #include <csignal>
 #include "InvalidFilesystem.h"
 
+#if defined(_MSC_VER)
+#include <codecvt>
+#include <dokan/dokan.h>
+#endif
+
 using std::vector;
 using std::string;
 
@@ -17,7 +24,9 @@ namespace bf = boost::filesystem;
 using namespace cpputils::logging;
 using std::make_shared;
 using std::shared_ptr;
+using std::string;
 using namespace fspp::fuse;
+using cpputils::set_thread_name;
 
 namespace {
 bool is_valid_fspp_path(const bf::path& path) {
@@ -26,6 +35,18 @@ bool is_valid_fspp_path(const bf::path& path) {
          && !path.has_root_name()                      // on Windows, it shouldn't have a device specifier (i.e. no "C:")
          && (path.string() == path.generic_string());  // must use portable '/' as directory separator
 }
+
+class ThreadNameForDebugging final {
+public:
+  ThreadNameForDebugging(const string& threadName) {
+    std::string name = "fspp_" + threadName;
+    set_thread_name(name.c_str());
+  }
+
+  ~ThreadNameForDebugging() {
+    set_thread_name("fspp_idle");
+  }
+};
 }
 
 #define FUSE_OBJ (static_cast<Fuse *>(fuse_get_context()->private_data))
@@ -333,18 +354,29 @@ bool Fuse::running() const {
 }
 
 void Fuse::stop() {
+  unmount(_mountdir, false);
+}
+
+void Fuse::unmount(const bf::path& mountdir, bool force) {
   //TODO Find better way to unmount (i.e. don't use external fusermount). Unmounting by kill(getpid(), SIGINT) worked, but left the mount directory transport endpoint as not connected.
-#ifdef __APPLE__
+#if defined(__APPLE__)
-  int ret = system(("umount " + _mountdir.string()).c_str());
+  int returncode = cpputils::Subprocess::call(std::string("umount ") + mountdir.string()).exitcode;
+#elif defined(_MSC_VER)
+  std::wstring mountdir_ = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(mountdir.string());
+  BOOL success = DokanRemoveMountPoint(mountdir_.c_str());
+  int returncode = success ? 0 : -1;
 #else
-  int ret = system(("fusermount -z -u " + _mountdir.string()).c_str()); // "-z" takes care that if the filesystem can't be unmounted right now because something is opened, it will be unmounted as soon as it can be.
+  std::string command = force ? "fusermount -u" : "fusermount -z -u";  // "-z" takes care that if the filesystem can't be unmounted right now because something is opened, it will be unmounted as soon as it can be.
+  int returncode = cpputils::Subprocess::call(
+	  command + " " + mountdir.string()).exitcode;
 #endif
-  if (ret != 0) {
+  if (returncode != 0) {
-    LOG(ERR, "Could not unmount filesystem");
+    throw std::runtime_error("Could not unmount filesystem");
   }
 }
 
 int Fuse::getattr(const bf::path &path, fspp::fuse::STAT *stbuf) {
+  ThreadNameForDebugging _threadName("getattr");
 #ifdef FSPP_LOG
   LOG(DEBUG, "getattr({}, _, _)", path);
 #endif
@@ -367,6 +399,7 @@ int Fuse::getattr(const bf::path &path, fspp::fuse::STAT *stbuf) {
 }
 
 int Fuse::fgetattr(const bf::path &path, fspp::fuse::STAT *stbuf, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("fgetattr");
 #ifdef FSPP_LOG
   LOG(DEBUG, "fgetattr({}, _, _)\n", path);
 #endif
@@ -399,6 +432,7 @@ int Fuse::fgetattr(const bf::path &path, fspp::fuse::STAT *stbuf, fuse_file_info
 }
 
 int Fuse::readlink(const bf::path &path, char *buf, size_t size) {
+  ThreadNameForDebugging _threadName("readlink");
 #ifdef FSPP_LOG
   LOG(DEBUG, "readlink({}, _, {})", path, size);
 #endif
@@ -424,11 +458,13 @@ int Fuse::mknod(const bf::path &path, ::mode_t mode, dev_t rdev) {
   UNUSED(rdev);
   UNUSED(mode);
   UNUSED(path);
+  ThreadNameForDebugging _threadName("mknod");
   LOG(WARN, "Called non-implemented mknod({}, {}, _)", path, mode);
   return ENOSYS;
 }
 
 int Fuse::mkdir(const bf::path &path, ::mode_t mode) {
+  ThreadNameForDebugging _threadName("mkdir");
 #ifdef FSPP_LOG
   LOG(DEBUG, "mkdir({}, {})", path, mode);
 #endif
@@ -452,6 +488,7 @@ int Fuse::mkdir(const bf::path &path, ::mode_t mode) {
 }
 
 int Fuse::unlink(const bf::path &path) {
+  ThreadNameForDebugging _threadName("unlink");
 #ifdef FSPP_LOG
   LOG(DEBUG, "unlink({})", path);
 #endif
@@ -474,6 +511,7 @@ int Fuse::unlink(const bf::path &path) {
 }
 
 int Fuse::rmdir(const bf::path &path) {
+  ThreadNameForDebugging _threadName("rmdir");
 #ifdef FSPP_LOG
   LOG(DEBUG, "rmdir({})", path);
 #endif
@@ -495,14 +533,15 @@ int Fuse::rmdir(const bf::path &path) {
   }
 }
 
-int Fuse::symlink(const bf::path &from, const bf::path &to) {
+int Fuse::symlink(const bf::path &to, const bf::path &from) {
+  ThreadNameForDebugging _threadName("symlink");
 #ifdef FSPP_LOG
-  LOG(DEBUG, "symlink({}, {})", from, to);
+  LOG(DEBUG, "symlink({}, {})", to, from);
 #endif
   try {
     ASSERT(is_valid_fspp_path(from), "has to be an absolute path");
 	auto context = fuse_get_context();
-    _fs->createSymlink(from, to, context->uid, context->gid);
+    _fs->createSymlink(to, from, context->uid, context->gid);
     return 0;
   } catch(const cpputils::AssertFailed &e) {
     LOG(ERR, "AssertFailed in Fuse::symlink: {}", e.what());
@@ -519,6 +558,7 @@ int Fuse::symlink(const bf::path &from, const bf::path &to) {
 }
 
 int Fuse::rename(const bf::path &from, const bf::path &to) {
+  ThreadNameForDebugging _threadName("rename");
 #ifdef FSPP_LOG
   LOG(DEBUG, "rename({}, {})", from, to);
 #endif
@@ -543,6 +583,7 @@ int Fuse::rename(const bf::path &from, const bf::path &to) {
 
 //TODO
 int Fuse::link(const bf::path &from, const bf::path &to) {
+  ThreadNameForDebugging _threadName("link");
   LOG(WARN, "NOT IMPLEMENTED: link({}, {})", from, to);
   //auto real_from = _impl->RootDir() / from;
   //auto real_to = _impl->RootDir() / to;
@@ -552,6 +593,7 @@ int Fuse::link(const bf::path &from, const bf::path &to) {
 }
 
 int Fuse::chmod(const bf::path &path, ::mode_t mode) {
+  ThreadNameForDebugging _threadName("chmod");
 #ifdef FSPP_LOG
   LOG(DEBUG, "chmod({}, {})", path, mode);
 #endif
@@ -574,6 +616,7 @@ int Fuse::chmod(const bf::path &path, ::mode_t mode) {
 }
 
 int Fuse::chown(const bf::path &path, ::uid_t uid, ::gid_t gid) {
+  ThreadNameForDebugging _threadName("chown");
 #ifdef FSPP_LOG
   LOG(DEBUG, "chown({}, {}, {})", path, uid, gid);
 #endif
@@ -596,6 +639,7 @@ int Fuse::chown(const bf::path &path, ::uid_t uid, ::gid_t gid) {
 }
 
 int Fuse::truncate(const bf::path &path, int64_t size) {
+  ThreadNameForDebugging _threadName("truncate");
 #ifdef FSPP_LOG
   LOG(DEBUG, "truncate({}, {})", path, size);
 #endif
@@ -618,6 +662,7 @@ int Fuse::truncate(const bf::path &path, int64_t size) {
 }
 
 int Fuse::ftruncate(const bf::path &path, int64_t size, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("ftruncate");
 #ifdef FSPP_LOG
   LOG(DEBUG, "ftruncate({}, {})", path, size);
 #endif
@@ -640,6 +685,7 @@ int Fuse::ftruncate(const bf::path &path, int64_t size, fuse_file_info *fileinfo
 }
 
 int Fuse::utimens(const bf::path &path, const timespec times[2]) {
+  ThreadNameForDebugging _threadName("utimens");
 #ifdef FSPP_LOG
   LOG(DEBUG, "utimens({}, _)", path);
 #endif
@@ -662,6 +708,7 @@ int Fuse::utimens(const bf::path &path, const timespec times[2]) {
 }
 
 int Fuse::open(const bf::path &path, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("open");
 #ifdef FSPP_LOG
   LOG(DEBUG, "open({}, _)", path);
 #endif
@@ -684,6 +731,7 @@ int Fuse::open(const bf::path &path, fuse_file_info *fileinfo) {
 }
 
 int Fuse::release(const bf::path &path, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("release");
 #ifdef FSPP_LOG
   LOG(DEBUG, "release({}, _)", path);
 #endif
@@ -706,6 +754,7 @@ int Fuse::release(const bf::path &path, fuse_file_info *fileinfo) {
 }
 
 int Fuse::read(const bf::path &path, char *buf, size_t size, int64_t offset, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("read");
 #ifdef FSPP_LOG
   LOG(DEBUG, "read({}, _, {}, {}, _)", path, size, offset);
 #endif
@@ -727,6 +776,7 @@ int Fuse::read(const bf::path &path, char *buf, size_t size, int64_t offset, fus
 }
 
 int Fuse::write(const bf::path &path, const char *buf, size_t size, int64_t offset, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("write");
 #ifdef FSPP_LOG
   LOG(DEBUG, "write({}, _, {}, {}, _)", path, size, offsset);
 #endif
@@ -749,6 +799,7 @@ int Fuse::write(const bf::path &path, const char *buf, size_t size, int64_t offs
 }
 
 int Fuse::statfs(const bf::path &path, struct ::statvfs *fsstat) {
+  ThreadNameForDebugging _threadName("statfs");
 #ifdef FSPP_LOG
   LOG(DEBUG, "statfs({}, _)", path);
 #endif
@@ -772,6 +823,7 @@ int Fuse::statfs(const bf::path &path, struct ::statvfs *fsstat) {
 }
 
 int Fuse::flush(const bf::path &path, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("flush");
 #ifdef FSPP_LOG
   LOG(WARN, "flush({}, _)", path);
 #endif
@@ -794,6 +846,7 @@ int Fuse::flush(const bf::path &path, fuse_file_info *fileinfo) {
 }
 
 int Fuse::fsync(const bf::path &path, int datasync, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("fsync");
 #ifdef FSPP_LOG
   LOG(DEBUG, "fsync({}, {}, _)", path, datasync);
 #endif
@@ -822,12 +875,14 @@ int Fuse::fsync(const bf::path &path, int datasync, fuse_file_info *fileinfo) {
 int Fuse::opendir(const bf::path &path, fuse_file_info *fileinfo) {
   UNUSED(path);
   UNUSED(fileinfo);
+  ThreadNameForDebugging _threadName("opendir");
   //LOG(DEBUG, "opendir({}, _)", path);
   //We don't need opendir, because readdir works directly on the path
   return 0;
 }
 
 int Fuse::readdir(const bf::path &path, void *buf, fuse_fill_dir_t filler, int64_t offset, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("readdir");
 #ifdef FSPP_LOG
   LOG(DEBUG, "readdir({}, _, _, {}, _)", path, offest);
 #endif
@@ -873,6 +928,7 @@ int Fuse::readdir(const bf::path &path, void *buf, fuse_fill_dir_t filler, int64
 int Fuse::releasedir(const bf::path &path, fuse_file_info *fileinfo) {
   UNUSED(path);
   UNUSED(fileinfo);
+  ThreadNameForDebugging _threadName("releasedir");
   //LOG(DEBUG, "releasedir({}, _)", path);
   //We don't need releasedir, because readdir works directly on the path
   return 0;
@@ -883,12 +939,14 @@ int Fuse::fsyncdir(const bf::path &path, int datasync, fuse_file_info *fileinfo)
   UNUSED(fileinfo);
   UNUSED(datasync);
   UNUSED(path);
+  ThreadNameForDebugging _threadName("fsyncdir");
   //LOG(WARN, "Called non-implemented fsyncdir({}, {}, _)", path, datasync);
   return 0;
 }
 
 void Fuse::init(fuse_conn_info *conn) {
   UNUSED(conn);
+  ThreadNameForDebugging _threadName("init");
   _fs = _init(this);
 
   LOG(INFO, "Filesystem started.");
@@ -902,6 +960,7 @@ void Fuse::init(fuse_conn_info *conn) {
 }
 
 void Fuse::destroy() {
+  ThreadNameForDebugging _threadName("destroy");
   _fs = make_shared<InvalidFilesystem>();
   LOG(INFO, "Filesystem stopped.");
   _running = false;
@@ -909,6 +968,7 @@ void Fuse::destroy() {
 }
 
 int Fuse::access(const bf::path &path, int mask) {
+  ThreadNameForDebugging _threadName("access");
 #ifdef FSPP_LOG
   LOG(DEBUG, "access({}, {})", path, mask);
 #endif
@@ -931,6 +991,7 @@ int Fuse::access(const bf::path &path, int mask) {
 }
 
 int Fuse::create(const bf::path &path, ::mode_t mode, fuse_file_info *fileinfo) {
+  ThreadNameForDebugging _threadName("create");
 #ifdef FSPP_LOG
   LOG(DEBUG, "create({}, {}, _)", path, mode);
 #endif

src/fspp/fuse/Fuse.h

@@ -29,6 +29,8 @@ public:
   bool running() const;
   void stop();
 
+  static void unmount(const boost::filesystem::path &mountdir, bool force = false);
+
   int getattr(const boost::filesystem::path &path, fspp::fuse::STAT *stbuf);
   int fgetattr(const boost::filesystem::path &path, fspp::fuse::STAT *stbuf, fuse_file_info *fileinfo);
   int readlink(const boost::filesystem::path &path, char *buf, size_t size);

test/blobstore/CMakeLists.txt

@@ -19,7 +19,7 @@ set(SOURCES
     implementations/onblocks/datatreestore/DataTreeTest_NumStoredBytes.cpp
     implementations/onblocks/datatreestore/DataTreeTest_ResizeNumBytes.cpp
     implementations/onblocks/datatreestore/DataTreeStoreTest.cpp
-    implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp
+    implementations/onblocks/datatreestore/LeafTraverserTest.cpp
     implementations/onblocks/BlobSizeTest.cpp
     implementations/onblocks/BlobReadWriteTest.cpp
     implementations/onblocks/BigBlobsTest.cpp

test/blobstore/implementations/onblocks/BlobReadWriteTest.cpp

@@ -63,6 +63,92 @@ TEST_F(BlobReadWriteTest, WritingCloseTo16ByteLimitDoesntDestroySize) {
   EXPECT_EQ(32780u, blob->size());
 }
 
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenTryReadInFirstLeaf_thenFails) {
+  Data data(5);
+  size_t read = blob->tryRead(data.data(), 3, 5);
+  EXPECT_EQ(0, read);
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenTryReadInLaterLeaf_thenFails) {
+  Data data(5);
+  size_t read = blob->tryRead(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5);
+  EXPECT_EQ(0, read);
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadInFirstLeaf_thenFails) {
+  Data data(5);
+  EXPECT_ANY_THROW(
+          blob->read(data.data(), 3, 5)
+  );
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadInLaterLeaf_thenFails) {
+  Data data(5);
+  EXPECT_ANY_THROW(
+          blob->read(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5)
+  );
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadAll_thenReturnsZeroSizedData) {
+  Data data = blob->readAll();
+  EXPECT_EQ(0, data.size());
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenWrite_thenGrows) {
+  Data data(5);
+  blob->write(data.data(), 4, 5);
+  EXPECT_EQ(9, blob->size());
+}
+
+TEST_F(BlobReadWriteTest, givenEmptyBlob_whenWriteZeroBytes_thenDoesntGrow) {
+  Data data(5);
+  blob->write(data.data(), 4, 0);
+  EXPECT_EQ(0, blob->size());;
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenTryReadInFirstLeaf_thenFails) {
+  Data data(5);
+  size_t read = blob->tryRead(data.data(), 3, 5);
+  EXPECT_EQ(0, read);
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenTryReadInLaterLeaf_thenFails) {
+  Data data(5);
+  size_t read = blob->tryRead(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5);
+  EXPECT_EQ(0, read);
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadInFirstLeaf_thenFails) {
+  Data data(5);
+  EXPECT_ANY_THROW(
+          blob->read(data.data(), 3, 5)
+  );
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadInLaterLeaf_thenFails) {
+  Data data(5);
+  EXPECT_ANY_THROW(
+          blob->read(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5)
+  );
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadAll_thenReturnsZeroSizedData) {
+  Data data = blob->readAll();
+  EXPECT_EQ(0, data.size());
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenWrite_thenGrows) {
+  Data data(5);
+  blob->write(data.data(), 4, 5);
+  EXPECT_EQ(9, blob->size());
+}
+
+TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenWriteZeroBytes_thenDoesntGrow) {
+  Data data(5);
+  blob->write(data.data(), 4, 0);
+  EXPECT_EQ(0, blob->size());
+}
+
 struct DataRange {
   uint64_t blobsize;
   uint64_t offset;

test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_NumStoredBytes.cpp

@@ -13,7 +13,7 @@ public:
 
 TEST_F(DataTreeTest_NumStoredBytes, CreatedTreeIsEmpty) {
   auto tree = treeStore.createNewTree();
-  EXPECT_EQ(0u, tree->numStoredBytes());
+  EXPECT_EQ(0u, tree->numBytes());
 }
 
 class DataTreeTest_NumStoredBytes_P: public DataTreeTest_NumStoredBytes, public WithParamInterface<uint32_t> {};
@@ -24,47 +24,47 @@ INSTANTIATE_TEST_CASE_P(FullLastLeaf, DataTreeTest_NumStoredBytes_P, Values(stat
 TEST_P(DataTreeTest_NumStoredBytes_P, SingleLeaf) {
   BlockId blockId = CreateLeafWithSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, TwoLeafTree) {
   BlockId blockId = CreateTwoLeafWithSecondLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, FullTwolevelTree) {
   BlockId blockId = CreateFullTwoLevelWithLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithOneChild) {
   BlockId blockId = CreateThreeLevelWithOneChildAndLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithTwoChildren) {
   BlockId blockId = CreateThreeLevelWithTwoChildrenAndLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithThreeChildren) {
   BlockId blockId = CreateThreeLevelWithThreeChildrenAndLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(2*nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(2*nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, FullThreeLevelTree) {
   BlockId blockId = CreateFullThreeLevelWithLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numBytes());
 }
 
 TEST_P(DataTreeTest_NumStoredBytes_P, FourLevelMinDataTree) {
   BlockId blockId = CreateFourLevelMinDataWithLastLeafSize(GetParam())->blockId();
   auto tree = treeStore.load(blockId).value();
-  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + GetParam(), tree->numStoredBytes());
+  EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + GetParam(), tree->numBytes());
 }

test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp

@@ -3,14 +3,24 @@
 #include <gmock/gmock.h>
 
 using blobstore::onblocks::datatreestore::DataTree;
-using blobstore::onblocks::datatreestore::LeafHandle;
 using blockstore::BlockId;
 using cpputils::Data;
 
 class DataTreeTest_Performance: public DataTreeTest {
 public:
-    void Traverse(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
+    void TraverseByWriting(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
-        tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, bool /*isRightBorderNode*/, LeafHandle /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();});
+        uint64_t offset = beginIndex * maxBytesPerLeaf;
+        uint64_t count = endIndex * maxBytesPerLeaf - offset;
+        Data data(count);
+        data.FillWithZeroes();
+        tree->writeBytes(data.data(), offset, count);
+    }
+
+    void TraverseByReading(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
+        uint64_t offset = beginIndex * maxBytesPerLeaf;
+        uint64_t count = endIndex * maxBytesPerLeaf - offset;
+        Data data(count);
+        tree->readBytes(data.data(), offset, count);
     }
 
     uint64_t maxChildrenPerInnerNode = nodeStore->layout().maxChildrenPerInnerNode();
@@ -71,84 +81,168 @@ TEST_F(DataTreeTest_Performance, DeletingDoesntLoadLeaves_Threelevel_DeleteByKey
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All_ByWriting) {
     auto blockId = CreateFullTwoLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 0, maxChildrenPerInnerNode);
+    TraverseByWriting(tree.get(), 0, maxChildrenPerInnerNode);
 
-    EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
+    EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Has to load the rightmost leaf once to adapt its size, rest of the leaves aren't loaded but just overwritten
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
-    EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All_ByReading) {
     auto blockId = CreateFullTwoLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 3, 5);
+    TraverseByReading(tree.get(), 0, maxChildrenPerInnerNode);
 
-    EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
+    EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size());  // Has to read the rightmost leaf an additional time in the beginning to determine size.
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
     EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some_ByWriting) {
-    auto blockId = CreateFullThreeLevel()->blockId();
+    auto blockId = CreateFullTwoLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
+    TraverseByWriting(tree.get(), 3, 5);
 
-    EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes. Doesn't load the leaves, but returns the keys of the leaves to the callback
+    EXPECT_EQ(0u, blockStore->loadedBlocks().size());
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some_ByReading) {
+    auto blockId = CreateFullTwoLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByReading(tree.get(), 3, 5);
+
+    EXPECT_EQ(3u, blockStore->loadedBlocks().size());  // reads 2 leaves and the rightmost leaf to determine size
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
     EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All_ByWriting) {
     auto blockId = CreateFullThreeLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 3, 5);
+    TraverseByWriting(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
 
-    EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
+    EXPECT_EQ(maxChildrenPerInnerNode + 1, blockStore->loadedBlocks().size()); // Loads inner nodes and has to load the rightmost leaf once to adapt its size, rest of the leaves aren't loaded but just overwritten.
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(maxChildrenPerInnerNode*maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All_ByReading) {
+    auto blockId = CreateFullThreeLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByReading(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
+
+    EXPECT_EQ(maxChildrenPerInnerNode*maxChildrenPerInnerNode + maxChildrenPerInnerNode + 2, blockStore->loadedBlocks().size()); // Loads inner nodes and leaves. Has to load the rightmost inner node and leaf an additional time at the beginning to compute size
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
     EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner_ByWriting) {
     auto blockId = CreateFullThreeLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 3, 3 + maxChildrenPerInnerNode);
+    TraverseByWriting(tree.get(), 3, 5);
+
+    EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, they're just overwritten.
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner_ByReading) {
+    auto blockId = CreateFullThreeLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByReading(tree.get(), 3, 5);
+
+    EXPECT_EQ(5u, blockStore->loadedBlocks().size());  // reads 2 leaves and the inner node, also has to read the rightmost inner node and leaf additionally at the beginning to determine size
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner_ByWriting) {
+    auto blockId = CreateFullThreeLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByWriting(tree.get(), 3, 3 + maxChildrenPerInnerNode);
 
     EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both inner node
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
-    EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
-TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner) {
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner_ByReading) {
     auto blockId = CreateFullThreeLevel()->blockId();
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
+    TraverseByReading(tree.get(), 3, 3 + maxChildrenPerInnerNode);
 
-    EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
+    EXPECT_EQ(4u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads both inner nodes and the requested leaves. Also has to load rightmost inner node and leaf additionally in the beginning to determine size.
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner_ByWriting) {
+    auto blockId = CreateFullThreeLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByWriting(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
+
+    EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, they're just overwritten.
+    EXPECT_EQ(0u, blockStore->createdBlocks());
+    EXPECT_EQ(0u, blockStore->removedBlocks().size());
+    EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
+    EXPECT_EQ(0u, blockStore->resizedBlocks().size());
+}
+
+TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner_ByReading) {
+    auto blockId = CreateFullThreeLevel()->blockId();
+    auto tree = treeStore.load(blockId).value();
+    blockStore->resetCounters();
+
+    TraverseByReading(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
+
+    EXPECT_EQ(3u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and all requested leaves. Also has to load rightmost inner node and leaf additionally in the beginning to determine size.
     EXPECT_EQ(0u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
     EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@@ -160,12 +254,12 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingInside) {
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 1, 4);
+    TraverseByWriting(tree.get(), 1, 4);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old child (for growing it)
     EXPECT_EQ(2u, blockStore->createdBlocks());
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
-    EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to inner node
+    EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // write the data and add children to inner node
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
@@ -174,7 +268,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingOutside_TwoL
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 4, 5);
+    TraverseByWriting(tree.get(), 4, 5);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
     EXPECT_EQ(3u, blockStore->createdBlocks());
@@ -188,7 +282,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingOutside_Thre
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 2*maxChildrenPerInnerNode+1, 2*maxChildrenPerInnerNode+2);
+    TraverseByWriting(tree.get(), 2*maxChildrenPerInnerNode+1, 2*maxChildrenPerInnerNode+2);
 
     EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads last old leaf (and its inner node) for growing it
     EXPECT_EQ(3u, blockStore->createdBlocks()); // inner node and two leaves
@@ -202,12 +296,12 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingAtBeginOfChi
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode);
+    TraverseByWriting(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode);
 
     EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads inner node and one leaf to check whether we have to grow it. Doesn't load the leaves, but returns the keys of the leaves to the callback.
     EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // Creates an inner node and its leaves
     EXPECT_EQ(0u, blockStore->removedBlocks().size());
-    EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to existing inner node
+    EXPECT_EQ(maxChildrenPerInnerNode + 1u, blockStore->distinctWrittenBlocks().size()); // write data and add children to existing inner node
     EXPECT_EQ(0u, blockStore->resizedBlocks().size());
 }
 
@@ -216,7 +310,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
+    TraverseByWriting(tree.get(), 4, maxChildrenPerInnerNode+2);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
     EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@@ -230,7 +324,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
+    TraverseByWriting(tree.get(), 4, maxChildrenPerInnerNode+2);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
     EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@@ -244,7 +338,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
+    TraverseByWriting(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
     EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@@ -258,7 +352,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
     auto tree = treeStore.load(blockId).value();
     blockStore->resetCounters();
 
-    Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
+    TraverseByWriting(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
 
     EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
     EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves

test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp

@@ -19,7 +19,6 @@ using blobstore::onblocks::datanodestore::DataInnerNode;
 using blobstore::onblocks::datanodestore::DataNode;
 using blobstore::onblocks::datanodestore::DataNodeLayout;
 using blobstore::onblocks::datatreestore::DataTree;
-using blobstore::onblocks::datatreestore::LeafHandle;
 using blobstore::onblocks::utils::ceilDivision;
 using blockstore::BlockId;
 using cpputils::Data;
@@ -109,9 +108,13 @@ public:
     GrowTree(tree.get().get());
   }
 
-  void GrowTree(DataTree *tree, std::function<void (int32_t)> traverse = [] (uint32_t){}) {
+  void GrowTree(DataTree *tree) {
     uint64_t maxBytesPerLeaf = tree->maxBytesPerLeaf();
-    tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [&traverse] (uint32_t index, bool, LeafHandle){traverse(index);}, [maxBytesPerLeaf, &traverse] (uint32_t index) -> Data { traverse(index); return Data(maxBytesPerLeaf).FillWithZeroes();});
+    uint64_t offset = traversalBeginIndex * maxBytesPerLeaf;
+    uint64_t count = newNumberOfLeaves * maxBytesPerLeaf - offset;
+    Data data(count);
+    data.FillWithZeroes();
+    tree->writeBytes(data.data(), offset, count);
     tree->flush();
   }
 
@@ -163,7 +166,6 @@ INSTANTIATE_TEST_CASE_P(DataTreeTest_ResizeByTraversing_P, DataTreeTest_ResizeBy
     ),
     //Decide the traversal begin index
     Values(
-      [] (uint32_t /*oldNumberOfLeaves*/, uint32_t newNumberOfLeaves)     {return newNumberOfLeaves;}, // Don't traverse any leaves, just resize (begin==end)
       [] (uint32_t /*oldNumberOfLeaves*/, uint32_t newNumberOfLeaves)     {return newNumberOfLeaves-1;}, // Traverse last leaf (begin==end-1)
       [] (uint32_t oldNumberOfLeaves,     uint32_t newNumberOfLeaves)     {return (oldNumberOfLeaves+newNumberOfLeaves)/2;}, // Start traversal in middle of new leaves
       [] (uint32_t oldNumberOfLeaves,     uint32_t /*newNumberOfLeaves*/) {return oldNumberOfLeaves-1;}, // Start traversal with last old leaf
@@ -189,9 +191,9 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, NumLeavesIsCorrect_FromCache) {
 
 TEST_P(DataTreeTest_ResizeByTraversing_P, NumLeavesIsCorrect) {
   GrowTree(tree.get());
-  // tree->_forceComputeNumLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
+  // tree->forceComputeNumLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
   // This is what the StructureIsValid test case is for.
-  EXPECT_EQ(newNumberOfLeaves, tree->_forceComputeNumLeaves());
+  EXPECT_EQ(newNumberOfLeaves, tree->forceComputeNumLeaves());
 }
 
 TEST_P(DataTreeTest_ResizeByTraversing_P, DepthFlagsAreCorrect) {
@@ -208,7 +210,8 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, KeyDoesntChange) {
 }
 
 TEST_P(DataTreeTest_ResizeByTraversing_P, DataStaysIntact) {
-  uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numStoredBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
+  uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
+
   TwoLevelDataFixture data(nodeStore, TwoLevelDataFixture::SizePolicy::Unchanged);
   BlockId blockId = tree->blockId();
   cpputils::destruct(std::move(tree));
@@ -216,15 +219,13 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, DataStaysIntact) {
 
   GrowTree(blockId);
 
-  data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), oldNumberOfLeaves, oldLastLeafSize);
+  if (traversalBeginIndex < oldNumberOfLeaves) {
-}
+      // Traversal wrote over part of the pre-existing data, we can only check the data before it.
-
+      if (traversalBeginIndex != 0) {
-TEST_P(DataTreeTest_ResizeByTraversing_P, AllLeavesAreTraversed) {
+          data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), traversalBeginIndex - 1);
-  std::vector<uint32_t> traversedLeaves;
+      }
-  GrowTree(tree.get(), [&traversedLeaves] (uint32_t index) {traversedLeaves.push_back(index);});
+  } else {
-
+      // Here, traversal was entirely outside the preexisting data, we can check all preexisting data.
-  EXPECT_EQ(newNumberOfLeaves-traversalBeginIndex, traversedLeaves.size());
+      data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), oldNumberOfLeaves, oldLastLeafSize);
-  for (uint32_t i = traversalBeginIndex; i < newNumberOfLeaves; ++i) {
-    EXPECT_NE(traversedLeaves.end(), std::find(traversedLeaves.begin(), traversedLeaves.end(), i));
   }
 }

test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeNumBytes.cpp

@@ -171,9 +171,9 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, StructureIsValid) {
 TEST_P(DataTreeTest_ResizeNumBytes_P, NumBytesIsCorrect) {
   tree->resizeNumBytes(newSize);
   tree->flush();
-  // tree->numStoredBytes() only goes down the right border nodes and expects the tree to be a left max data tree.
+  // tree->numBytes() only goes down the right border nodes and expects the tree to be a left max data tree.
   // This is what the StructureIsValid test case is for.
-  EXPECT_EQ(newSize, tree->numStoredBytes());
+  EXPECT_EQ(newSize, tree->numBytes());
 }
 
 TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect) {
@@ -181,7 +181,7 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect) {
   tree->flush();
   // tree->numLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
   // This is what the StructureIsValid test case is for.
-  EXPECT_EQ(newNumberOfLeaves, tree->_forceComputeNumLeaves());
+  EXPECT_EQ(newNumberOfLeaves, tree->forceComputeNumLeaves());
 }
 
 TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect_FromCache) {
@@ -208,7 +208,7 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, KeyDoesntChange) {
 }
 
 TEST_P(DataTreeTest_ResizeNumBytes_P, DataStaysIntact) {
-  uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numStoredBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
+  uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
   TwoLevelDataFixture data(nodeStore, TwoLevelDataFixture::SizePolicy::Unchanged);
   BlockId blockId = tree->blockId();
   cpputils::destruct(std::move(tree));

test/blobstore/implementations/onblocks/datatreestore/LeafTraverserTest.cpp

@@ -1,4 +1,5 @@
 #include "testutils/DataTreeTest.h"
+#include <blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h>
 #include <gmock/gmock.h>
 
 using ::testing::_;
@@ -9,6 +10,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode;
 using blobstore::onblocks::datanodestore::DataInnerNode;
 using blobstore::onblocks::datanodestore::DataNode;
 using blobstore::onblocks::datatreestore::LeafHandle;
+using blobstore::onblocks::datatreestore::LeafTraverser;
 using blockstore::BlockId;
 
 using cpputils::unique_ref;
@@ -26,9 +28,9 @@ MATCHER_P(KeyEq, expected, "node blockId equals") {
   return arg->blockId() == expected;
 }
 
-class DataTreeTest_TraverseLeaves: public DataTreeTest {
+class LeafTraverserTest: public DataTreeTest {
 public:
-  DataTreeTest_TraverseLeaves() :traversor() {}
+  LeafTraverserTest() :traversor() {}
 
   unique_ref<DataInnerNode> CreateThreeLevel() {
     return CreateInner({
@@ -70,166 +72,172 @@ public:
     EXPECT_CALL(traversor, calledCreateLeaf(_)).Times(0);
   }
 
-  void TraverseLeaves(DataNode *root, uint32_t beginIndex, uint32_t endIndex) {
+  void TraverseLeaves(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool expectReadOnly) {
     root->flush();
     auto tree = treeStore.load(root->blockId()).value();
-    tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, bool isRightBorderNode,LeafHandle leaf) {
+    auto* old_root = root.get();
+    LeafTraverser(nodeStore, expectReadOnly).traverseAndUpdateRoot(&root, beginIndex, endIndex, [this] (uint32_t nodeIndex, bool isRightBorderNode,LeafHandle leaf) {
       traversor.calledExistingLeaf(leaf.node(), isRightBorderNode,  nodeIndex);
     }, [this] (uint32_t nodeIndex) -> Data {
         return traversor.calledCreateLeaf(nodeIndex)->copy();
-    });
+    }, [] (auto) {});
+    if (expectReadOnly) {
+      EXPECT_EQ(old_root, root.get());
+    } else {
+      EXPECT_NE(old_root, root.get());
+    }
   }
 
   TraversorMock traversor;
 };
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseSingleLeafTree) {
+TEST_F(LeafTraverserTest, TraverseSingleLeafTree) {
-  auto root = CreateLeaf();
+  unique_ref<DataNode> root = CreateLeaf();
   EXPECT_TRAVERSE_LEAF(root->blockId(), true, 0);
 
-  TraverseLeaves(root.get(), 0, 1);
+  TraverseLeaves(std::move(root), 0, 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInSingleLeafTree1) {
+TEST_F(LeafTraverserTest, TraverseNothingInSingleLeafTree1) {
-  auto root = CreateLeaf();
+  unique_ref<DataNode> root = CreateLeaf();
   EXPECT_DONT_TRAVERSE_ANY_LEAVES();
 
-  TraverseLeaves(root.get(), 0, 0);
+  TraverseLeaves(std::move(root), 0, 0, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInSingleLeafTree2) {
+TEST_F(LeafTraverserTest, TraverseNothingInSingleLeafTree2) {
-  auto root = CreateLeaf();
+  unique_ref<DataNode> root = CreateLeaf();
   EXPECT_DONT_TRAVERSE_ANY_LEAVES();
 
-  TraverseLeaves(root.get(), 1, 1);
+  TraverseLeaves(std::move(root), 1, 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseFirstLeafOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   EXPECT_TRAVERSE_LEAF(root->readChild(0).blockId(), false, 0);
 
-  TraverseLeaves(root.get(), 0, 1);
+  TraverseLeaves(std::move(root), 0, 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseMiddleLeafOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   EXPECT_TRAVERSE_LEAF(root->readChild(5).blockId(), false, 5);
 
-  TraverseLeaves(root.get(), 5, 6);
+  TraverseLeaves(std::move(root), 5, 6, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseLastLeafOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   EXPECT_TRAVERSE_LEAF(root->readChild(nodeStore->layout().maxChildrenPerInnerNode()-1).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode()-1);
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode()-1, nodeStore->layout().maxChildrenPerInnerNode());
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode()-1, nodeStore->layout().maxChildrenPerInnerNode(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInFullTwolevelTree1) {
+TEST_F(LeafTraverserTest, TraverseNothingInFullTwolevelTree1) {
   auto root = CreateFullTwoLevel();
   EXPECT_DONT_TRAVERSE_ANY_LEAVES();
 
-  TraverseLeaves(root.get(), 0, 0);
+  TraverseLeaves(std::move(root), 0, 0, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInFullTwolevelTree2) {
+TEST_F(LeafTraverserTest, TraverseNothingInFullTwolevelTree2) {
   auto root = CreateFullTwoLevel();
   EXPECT_DONT_TRAVERSE_ANY_LEAVES();
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode());
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfThreeLevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseFirstLeafOfThreeLevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(0).blockId(), false, 0);
 
-  TraverseLeaves(root.get(), 0, 1);
+  TraverseLeaves(std::move(root), 0, 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfThreeLevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseMiddleLeafOfThreeLevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(5).blockId(), false, 5);
 
-  TraverseLeaves(root.get(), 5, 6);
+  TraverseLeaves(std::move(root), 5, 6, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfThreeLevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseLastLeafOfThreeLevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode()+1);
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode()+1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseAllLeavesOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   EXPECT_TRAVERSE_ALL_CHILDREN_OF(*root, true, 0);
 
-  TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode());
+  TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseAllLeavesOfThreelevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(root->readChild(0).blockId()), false, 0);
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
 
-  TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode()+1);
+  TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode()+1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstChildOfThreelevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseFirstChildOfThreelevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(root->readChild(0).blockId()), false, 0);
 
-  TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode());
+  TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseFirstPartOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   for (unsigned int i = 0; i < 5; ++i) {
     EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), false, i);
   }
 
-  TraverseLeaves(root.get(), 0, 5);
+  TraverseLeaves(std::move(root), 0, 5, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseInnerPartOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseInnerPartOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   for (unsigned int i = 5; i < 10; ++i) {
     EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), false, i);
   }
 
-  TraverseLeaves(root.get(), 5, 10);
+  TraverseLeaves(std::move(root), 5, 10, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfFullTwolevelTree) {
+TEST_F(LeafTraverserTest, TraverseLastPartOfFullTwolevelTree) {
   auto root = CreateFullTwoLevel();
   for (unsigned int i = 5; i < nodeStore->layout().maxChildrenPerInnerNode(); ++i) {
     EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), i==nodeStore->layout().maxChildrenPerInnerNode()-1, i);
   }
 
-  TraverseLeaves(root.get(), 5, nodeStore->layout().maxChildrenPerInnerNode());
+  TraverseLeaves(std::move(root), 5, nodeStore->layout().maxChildrenPerInnerNode(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseFirstPartOfThreelevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   auto node = LoadInnerNode(root->readChild(0).blockId());
   for (unsigned int i = 0; i < 5; ++i) {
     EXPECT_TRAVERSE_LEAF(node->readChild(i).blockId(), false, i);
   }
 
-  TraverseLeaves(root.get(), 0, 5);
+  TraverseLeaves(std::move(root), 0, 5, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseInnerPartOfThreelevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseInnerPartOfThreelevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   auto node = LoadInnerNode(root->readChild(0).blockId());
   for (unsigned int i = 5; i < 10; ++i) {
     EXPECT_TRAVERSE_LEAF(node->readChild(i).blockId(), false, i);
   }
 
-  TraverseLeaves(root.get(), 5, 10);
+  TraverseLeaves(std::move(root), 5, 10, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelMinDataTree) {
+TEST_F(LeafTraverserTest, TraverseLastPartOfThreelevelMinDataTree) {
   auto root = CreateThreeLevelMinData();
   auto node = LoadInnerNode(root->readChild(0).blockId());
   for (unsigned int i = 5; i < nodeStore->layout().maxChildrenPerInnerNode(); ++i) {
@@ -237,33 +245,33 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelMinDataTree) {
   }
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
 
-  TraverseLeaves(root.get(), 5, nodeStore->layout().maxChildrenPerInnerNode()+1);
+  TraverseLeaves(std::move(root), 5, nodeStore->layout().maxChildrenPerInnerNode()+1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseFirstLeafOfThreelevelTree) {
   auto root = CreateThreeLevel();
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(0).blockId(), false, 0);
 
-  TraverseLeaves(root.get(), 0, 1);
+  TraverseLeaves(std::move(root), 0, 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseLastLeafOfThreelevelTree) {
   auto root = CreateThreeLevel();
   uint32_t numLeaves = nodeStore->layout().maxChildrenPerInnerNode() * 5 + 3;
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readLastChild().blockId())->readLastChild().blockId(), true, numLeaves-1);
 
-  TraverseLeaves(root.get(), numLeaves-1, numLeaves);
+  TraverseLeaves(std::move(root), numLeaves-1, numLeaves, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseMiddleLeafOfThreelevelTree) {
   auto root = CreateThreeLevel();
   uint32_t wantedLeafIndex = nodeStore->layout().maxChildrenPerInnerNode() * 2 + 5;
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(2).blockId())->readChild(5).blockId(), false, wantedLeafIndex);
 
-  TraverseLeaves(root.get(), wantedLeafIndex, wantedLeafIndex+1);
+  TraverseLeaves(std::move(root), wantedLeafIndex, wantedLeafIndex+1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseFirstPartOfThreelevelTree) {
   auto root = CreateThreeLevel();
   //Traverse all leaves in the first two children of the root
   for(unsigned int i = 0; i < 2; ++i) {
@@ -275,10 +283,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelTree) {
     EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 2 * nodeStore->layout().maxChildrenPerInnerNode() + i);
   }
 
-  TraverseLeaves(root.get(), 0, 2 * nodeStore->layout().maxChildrenPerInnerNode() + 5);
+  TraverseLeaves(std::move(root), 0, 2 * nodeStore->layout().maxChildrenPerInnerNode() + 5, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_OnlyFullChildren) {
+TEST_F(LeafTraverserTest, TraverseMiddlePartOfThreelevelTree_OnlyFullChildren) {
   auto root = CreateThreeLevel();
   //Traverse some of the leaves in the second child of the root
   auto child = LoadInnerNode(root->readChild(1).blockId());
@@ -295,10 +303,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_OnlyFullC
     EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 4 * nodeStore->layout().maxChildrenPerInnerNode() + i);
   }
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 4 * nodeStore->layout().maxChildrenPerInnerNode() + 5);
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 4 * nodeStore->layout().maxChildrenPerInnerNode() + 5, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_AlsoLastNonfullChild) {
+TEST_F(LeafTraverserTest, TraverseMiddlePartOfThreelevelTree_AlsoLastNonfullChild) {
   auto root = CreateThreeLevel();
   //Traverse some of the leaves in the second child of the root
   auto child = LoadInnerNode(root->readChild(1).blockId());
@@ -315,10 +323,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_AlsoLastN
     EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
   }
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + 2);
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + 2, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseLastPartOfThreelevelTree) {
   auto root = CreateThreeLevel();
   //Traverse some of the leaves in the second child of the root
   auto child = LoadInnerNode(root->readChild(1).blockId());
@@ -335,10 +343,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelTree) {
     EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), i == child->numChildren()-1, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
   }
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren());
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelTree) {
+TEST_F(LeafTraverserTest, TraverseAllLeavesOfThreelevelTree) {
   auto root = CreateThreeLevel();
   //Traverse all leaves in the third, fourth and fifth child of the root
   for(unsigned int i = 0; i < 5; ++i) {
@@ -350,10 +358,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelTree) {
     EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), i==child->numChildren()-1, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
   }
 
-  TraverseLeaves(root.get(), 0, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren());
+  TraverseLeaves(std::move(root), 0, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren(), true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFourLevelTree) {
+TEST_F(LeafTraverserTest, TraverseAllLeavesOfFourLevelTree) {
   auto root = CreateFourLevel();
   //Traverse all leaves of the full threelevel tree in the first child
   auto firstChild = LoadInnerNode(root->readChild(0).blockId());
@@ -370,10 +378,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFourLevelTree) {
   EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(thirdChild->readChild(0).blockId()), false, 2 * nodeStore->layout().maxChildrenPerInnerNode() * nodeStore->layout().maxChildrenPerInnerNode());
   EXPECT_TRAVERSE_LEAF(LoadInnerNode(thirdChild->readChild(1).blockId())->readChild(0).blockId(), true, 2 * nodeStore->layout().maxChildrenPerInnerNode() * nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode());
 
-  TraverseLeaves(root.get(), 0, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() + 1);
+  TraverseLeaves(std::move(root), 0, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() + 1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfFourLevelTree) {
+TEST_F(LeafTraverserTest, TraverseMiddlePartOfFourLevelTree) {
   auto root = CreateFourLevel();
   //Traverse some leaves of the full threelevel tree in the first child
   auto firstChild = LoadInnerNode(root->readChild(0).blockId());
@@ -396,14 +404,15 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfFourLevelTree) {
     EXPECT_TRAVERSE_LEAF(firstChildOfThirdChild->readChild(i).blockId(), false, 2 * nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode()+i);
   }
 
-  TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode()+5, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() -1);
+  TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode()+5, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() -1, true);
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) {
+TEST_F(LeafTraverserTest, LastLeafIsAlreadyResizedInCallback) {
-  auto root = CreateLeaf();
+  unique_ref<DataNode> root = CreateLeaf();
   root->flush();
+  auto* old_root = root.get();
   auto tree = treeStore.load(root->blockId()).value();
-  tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, bool /*isRightBorderNode*/, LeafHandle leaf) {
+  LeafTraverser(nodeStore, false).traverseAndUpdateRoot(&root, 0, 2, [this] (uint32_t leafIndex, bool /*isRightBorderNode*/, LeafHandle leaf) {
       if (leafIndex == 0) {
         EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
       } else {
@@ -411,28 +420,31 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) {
       }
   }, [] (uint32_t /*nodeIndex*/) -> Data {
       return Data(1);
-  });
+  }, [] (auto) {});
+  EXPECT_NE(old_root, root.get()); // expect that we grew the tree
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback_TwoLevel) {
+TEST_F(LeafTraverserTest, LastLeafIsAlreadyResizedInCallback_TwoLevel) {
-  auto root = CreateFullTwoLevelWithLastLeafSize(5);
+  unique_ref<DataNode> root = CreateFullTwoLevelWithLastLeafSize(5);
   root->flush();
+  auto* old_root = root.get();
   auto tree = treeStore.load(root->blockId()).value();
-  tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, bool /*isRightBorderNode*/, LeafHandle leaf) {
+  LeafTraverser(nodeStore, false).traverseAndUpdateRoot(&root, 0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, bool /*isRightBorderNode*/, LeafHandle leaf) {
       EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
   }, [] (uint32_t /*nodeIndex*/) -> Data {
       return Data(1);
-  });
+  }, [] (auto) {});
+  EXPECT_NE(old_root, root.get()); // expect that we grew the tree
 }
 
-TEST_F(DataTreeTest_TraverseLeaves, ResizeFromOneLeafToMultipleLeaves) {
+TEST_F(LeafTraverserTest, ResizeFromOneLeafToMultipleLeaves) {
   auto root = CreateLeaf();
   EXPECT_TRAVERSE_LEAF(root->blockId(), false, 0);
   //EXPECT_CALL(traversor, calledExistingLeaf(_, false, 0)).Times(1);
   for (uint32_t i = 1; i < 10; ++i) {
     EXPECT_CREATE_LEAF(i);
   }
-  TraverseLeaves(root.get(), 0, 10);
+  TraverseLeaves(std::move(root), 0, 10, false);
 }
 
-//TODO Refactor the test cases that are too long
+////TODO Refactor the test cases that are too long

test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp

@@ -13,7 +13,7 @@ using ::testing::Test;
 //Test that Cache uses a move constructor for Value if possible
 class CacheTest_MoveConstructor: public Test {
 public:
-  CacheTest_MoveConstructor(): cache(make_unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>>()) {
+  CacheTest_MoveConstructor(): cache(make_unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>>("test")) {
     CopyableMovableValueType::numCopyConstructorCalled = 0;
   }
   unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>> cache;

test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp

@@ -36,7 +36,7 @@ private:
 
 class CacheTest_RaceCondition: public ::testing::Test {
 public:
-    CacheTest_RaceCondition(): cache(), destructorStarted(), destructorFinished(false) {}
+    CacheTest_RaceCondition(): cache("test"), destructorStarted(), destructorFinished(false) {}
 
     static constexpr unsigned int MAX_ENTRIES = 100;
 

test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp

@@ -37,7 +37,7 @@ class PeriodicTaskTest: public Test {
 };
 
 TEST_F(PeriodicTaskTest, DoesntDeadlockInDestructorWhenDestructedImmediately) {
-  PeriodicTask task([](){}, 1);
+  PeriodicTask task([](){}, 1, "test");
 }
 
 TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) {
@@ -45,7 +45,7 @@ TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) {
 
   PeriodicTask task([&counter](){
     counter.decrease();
-  }, 0.001);
+  }, 0.001, "test");
 
   counter.waitForZero();
 }
@@ -55,7 +55,7 @@ TEST_F(PeriodicTaskTest, DoesntCallCallbackAfterDestruction) {
   {
     PeriodicTask task([&callCount](){
       callCount += 1;
-    }, 0.001);
+    }, 0.001, "test");
   }
   int callCountDirectlyAfterDestruction = callCount;
   boost::this_thread::sleep_for(boost::chrono::seconds(1));

test/blockstore/implementations/caching/cache/testutils/CacheTest.h

@@ -13,7 +13,7 @@
 // Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values).
 class CacheTest: public ::testing::Test {
 public:
-  CacheTest(): _cache() {}
+  CacheTest(): _cache("test") {}
 
   void push(int key, int value);
   boost::optional<int> pop(int key);

test/cpp-utils/CMakeLists.txt

@@ -49,10 +49,12 @@ set(SOURCES
     assert/assert_debug_test.cpp
     system/GetTotalMemoryTest.cpp
     system/TimeTest.cpp
+	system/PathTest.cpp
 	system/FiletimeTest.cpp
     system/MemoryTest.cpp
     system/HomedirTest.cpp
 	system/EnvTest.cpp
+	thread/debugging_test.cpp
     value_type/ValueTypeTest.cpp
 	either_test.cpp
 )

test/cpp-utils/system/PathTest.cpp

@@ -0,0 +1,32 @@
+#include <gtest/gtest.h>
+#include <cpp-utils/system/path.h>
+
+using cpputils::path_is_just_drive_letter;
+
+#if defined(_MSC_VER)
+
+TEST(PathTest, pathIsJustDriveLetter) {
+    EXPECT_FALSE(path_is_just_drive_letter("C"));
+    EXPECT_TRUE(path_is_just_drive_letter("C:"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:/"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\test"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\test\\"));
+    EXPECT_FALSE(path_is_just_drive_letter("/"));
+    EXPECT_FALSE(path_is_just_drive_letter(""));
+}
+
+#else
+
+TEST(PathTest, onNonWindowsWeDontHaveDriveLetterPaths) {
+    EXPECT_FALSE(path_is_just_drive_letter("C"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:/"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\test"));
+    EXPECT_FALSE(path_is_just_drive_letter("C:\\test\\"));
+    EXPECT_FALSE(path_is_just_drive_letter("/"));
+    EXPECT_FALSE(path_is_just_drive_letter(""));
+}
+
+#endif

test/cpp-utils/thread/debugging_test.cpp

@@ -0,0 +1,62 @@
+#include <cpp-utils/thread/debugging.h>
+#include <cpp-utils/assert/assert.h>
+#include <cpp-utils/lock/ConditionBarrier.h>
+#include <gtest/gtest.h>
+
+using namespace cpputils;
+using std::string;
+
+TEST(ThreadDebuggingTest_ThreadName, givenMainThread_whenSettingAndGetting_thenDoesntCrash) {
+	set_thread_name("my_thread_name");
+	get_thread_name();
+}
+
+TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenSettingAndGetting_thenDoesntCrash) {
+    ConditionBarrier nameIsChecked;
+
+	bool child_didnt_crash = false;
+	std::thread child([&] {
+		set_thread_name("my_thread_name");
+		get_thread_name();
+		child_didnt_crash = true;
+		nameIsChecked.wait();
+	});
+	get_thread_name(&child);
+	nameIsChecked.release(); // getting the name of a not-running thread would cause errors, so let's make sure we only exit after getting the name
+	child.join();
+	EXPECT_TRUE(child_didnt_crash);
+}
+
+TEST(ThreadDebuggingTest_ThreadName, givenMainThread_whenGettingFromInside_thenIsCorrect) {
+    set_thread_name("my_thread_name");
+    string name = get_thread_name();
+    EXPECT_EQ("my_thread_name", name);
+}
+
+TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenGettingFromInside_thenIsCorrect) {
+  std::thread child([] {
+    set_thread_name("my_thread_name");
+    string name = get_thread_name();
+    EXPECT_EQ("my_thread_name", name);
+  });
+  child.join();
+}
+
+TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenGettingFromOutside_thenIsCorrect) {
+  ConditionBarrier nameIsSet;
+  ConditionBarrier nameIsChecked;
+
+  std::thread child([&] {
+    set_thread_name("my_thread_name");
+    nameIsSet.release();
+    nameIsChecked.wait();
+  });
+
+  nameIsSet.wait();
+  string name = get_thread_name(&child);
+  EXPECT_EQ("my_thread_name", name);
+
+  nameIsChecked.release();
+  child.join();
+}
+

test/cryfs-cli/CMakeLists.txt

@@ -12,12 +12,12 @@ set(SOURCES
     EnvironmentTest.cpp
     VersionCheckerTest.cpp
     CliTest_IntegrityCheck.cpp
+    CryfsUnmountTest.cpp
 )
 
 add_executable(${PROJECT_NAME} ${SOURCES})
-target_link_libraries(${PROJECT_NAME} googletest cryfs-cli)
+target_link_libraries(${PROJECT_NAME} googletest cryfs-cli cryfs-unmount fspp-fuse)
 add_test(${PROJECT_NAME} ${PROJECT_NAME})
 
 target_enable_style_warnings(${PROJECT_NAME})
 target_activate_cpp14(${PROJECT_NAME})
-

test/cryfs-cli/CallAfterTimeoutTest.cpp

@@ -8,14 +8,14 @@ using cpputils::make_unique_ref;
 using boost::chrono::milliseconds;
 using boost::chrono::minutes;
 using boost::this_thread::sleep_for;
-using namespace cryfs;
+using namespace cryfs_cli;
 
 class CallAfterTimeoutTest : public ::testing::Test {
 public:
     CallAfterTimeoutTest(): called(false) {}
 
     unique_ref<CallAfterTimeout> callAfterTimeout(milliseconds timeout) {
-        return make_unique_ref<CallAfterTimeout>(timeout, [this] {called = true;});
+        return make_unique_ref<CallAfterTimeout>(timeout, [this] {called = true;}, "test");
     }
 
     std::atomic<bool> called;

test/cryfs-cli/CryfsUnmountTest.cpp

@@ -0,0 +1,25 @@
+#include "testutils/CliTest.h"
+#include <cryfs-cli/Cli.h>
+#include <cryfs-unmount/Cli.h>
+
+using CliTest_Unmount = CliTest;
+namespace bf = boost::filesystem;
+
+namespace {
+void unmount(const bf::path& mountdir) {
+    std::vector<const char*> _args = {"cryfs-unmount", mountdir.string().c_str()};
+    cryfs_unmount::Cli().main(2, _args.data());
+}
+
+TEST_F(CliTest_Unmount, givenMountedFilesystem_whenUnmounting_thenSucceeds) {
+    // we're passing in boost::none as mountdir so EXPECT_RUN_SUCCESS doesn't unmount itself.
+    // if the unmount we're calling here in the onMounted callback wouldn't work, EXPECT_RUN_SUCCESS
+    // would never return and this would be a deadlock.
+    EXPECT_RUN_SUCCESS({basedir.string().c_str(), mountdir.string().c_str(), "-f"}, boost::none, [this] () {
+        unmount(mountdir);
+    });
+}
+
+// TODO Test calling with invalid args, valid '--version' or '--help' args, with a non-mounted mountdir and a nonexisting mountdir.
+
+}

test/cryfs-cli/EnvironmentTest.cpp

@@ -4,7 +4,7 @@
 #include <boost/filesystem.hpp>
 #include <cpp-utils/system/env.h>
 
-using namespace cryfs;
+using namespace cryfs_cli;
 using std::string;
 using boost::optional;
 using boost::none;

test/cryfs-cli/VersionCheckerTest.cpp

@@ -8,7 +8,7 @@ using cpputils::FakeHttpClient;
 using cpputils::unique_ref;
 using cpputils::make_unique_ref;
 using boost::none;
-using namespace cryfs;
+using namespace cryfs_cli;
 
 class VersionCheckerTest: public ::testing::Test {
 public:

test/cryfs-cli/program_options/ParserTest.cpp

@@ -7,7 +7,7 @@
 #include <cpp-utils/testutils/CaptureStderrRAII.h>
 
 using namespace cryfs;
-using namespace cryfs::program_options;
+using namespace cryfs_cli::program_options;
 using std::vector;
 using std::string;
 using boost::none;

test/cryfs-cli/program_options/ProgramOptionsTest.cpp

@@ -2,7 +2,7 @@
 #include <cryfs-cli/program_options/ProgramOptions.h>
 #include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
 
-using namespace cryfs::program_options;
+using namespace cryfs_cli::program_options;
 using boost::none;
 using boost::optional;
 using std::ostream;

test/cryfs-cli/program_options/UtilsTest.cpp

@@ -1,7 +1,7 @@
 #include "testutils/ProgramOptionsTestBase.h"
 #include <cryfs-cli/program_options/utils.h>
 
-using namespace cryfs::program_options;
+using namespace cryfs_cli::program_options;
 using std::pair;
 using std::vector;
 using std::string;

test/cryfs-cli/testutils/CliTest.h

@@ -19,6 +19,7 @@
 #include <cpp-utils/lock/ConditionBarrier.h>
 #include "../../cryfs/testutils/MockConsole.h"
 #include "../../cryfs/testutils/TestWithFakeHomeDirectory.h"
+#include <fspp/fuse/Fuse.h>
 #include <cryfs/ErrorCodes.h>
 #include <cpp-utils/testutils/CaptureStderrRAII.h>
 #include <regex>
@@ -45,7 +46,7 @@ public:
     int run(const std::vector<std::string>& args, std::function<void()> onMounted) {
         std::vector<const char*> _args;
         _args.reserve(args.size() + 1);
-            _args.emplace_back("cryfs");
+        _args.emplace_back("cryfs");
         for (const std::string& arg : args) {
             _args.emplace_back(arg.c_str());
         }
@@ -53,7 +54,7 @@ public:
         ON_CALL(*console, askPassword(testing::StrEq("Password: "))).WillByDefault(testing::Return("pass"));
         ON_CALL(*console, askPassword(testing::StrEq("Confirm Password: "))).WillByDefault(testing::Return("pass"));
         // Run Cryfs
-        return cryfs::Cli(keyGenerator, cpputils::SCrypt::TestSettings, console).main(_args.size(), _args.data(), _httpClient(), std::move(onMounted));
+        return cryfs_cli::Cli(keyGenerator, cpputils::SCrypt::TestSettings, console).main(_args.size(), _args.data(), _httpClient(), std::move(onMounted));
     }
 
     void EXPECT_EXIT_WITH_HELP_MESSAGE(const std::vector<std::string>& args, const std::string &message, cryfs::ErrorCode errorCode) {
@@ -90,18 +91,7 @@ public:
     };
 
     static void _unmount(const boost::filesystem::path &mountDir) {
-        int returncode = -1;
+        fspp::fuse::Fuse::unmount(mountDir, true);
-#if defined(__APPLE__)
-        returncode = cpputils::Subprocess::call(std::string("umount ") + mountDir.string().c_str() + " 2>/dev/null").exitcode;
-#elif defined(_MSC_VER)
-        std::wstring mountDir_ = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(mountDir.string());
-        BOOL success = DokanRemoveMountPoint(mountDir_.c_str());
-        returncode = success ? 0 : -1;
-#else
-        returncode = cpputils::Subprocess::call(
-        std::string("fusermount -u ") + mountDir.string().c_str() + " 2>/dev/null").exitcode;
-#endif
-        EXPECT_EQ(0, returncode);
     }
 
     FilesystemOutput run_filesystem(const std::vector<std::string>& args, boost::optional<boost::filesystem::path> mountDirForUnmounting, std::function<void()> onMounted) {

test/cryfs/config/CryConfigLoaderTest.cpp

@@ -310,7 +310,7 @@ TEST_F(CryConfigLoaderTest, AsksWhenLoadingNewerFilesystem_AnswerNo) {
 }
 
 TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), false)).Times(1).WillOnce(Return(true));
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(true));
 
     string version = olderVersion();
     CreateWithVersion(version, version);
@@ -318,14 +318,14 @@ TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
 }
 
 TEST_F(CryConfigLoaderTest, DoesNotAskForMigrationWhenCorrectVersion) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), _)).Times(0);
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), _)).Times(0);
 
     CreateWithVersion(gitversion::VersionString(), CryConfig::FilesystemFormatVersion);
     EXPECT_NE(boost::none, Load());
 }
 
 TEST_F(CryConfigLoaderTest, DontMigrateWhenAnsweredNo) {
-    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), false)).Times(1).WillOnce(Return(false));
+    EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(false));
 
     string version = olderVersion();
     CreateWithVersion(version, version);

vendor/README

@@ -2,6 +2,6 @@ This directory contains external projects, taken from the following locations:
 googletest: https://github.com/google/googletest/tree/4e4df226fc197c0dda6e37f5c8c3845ca1e73a49
  - changed: added NOLINT comment as workaround for clang-tidy warning https://github.com/google/googletest/issues/853
 spdlog: https://github.com/gabime/spdlog/tree/v0.16.3/include/spdlog
-cryptopp: https://github.com/weidai11/cryptopp/tree/CRYPTOPP_7_0_0
+cryptopp: https://github.com/weidai11/cryptopp/tree/CRYPTOPP_8_0_0
- - changed: added CMakeLists.txt and cryptopp-config.cmake from https://github.com/noloader/cryptopp-cmake/tree/7f3b3540740b365bed673936852d70795c0104f3
+ - changed: added CMakeLists.txt and cryptopp-config.cmake from https://github.com/noloader/cryptopp-cmake/tree/CRYPTOPP_8_0_0
- - changed: removed cmake warning from CMakeLists.txt
+ - changed: commented out line including winapifamily.h in CMakeLists.txt

vendor/cryptopp/CMakeLists.txt

@@ -88,6 +88,7 @@ set(BUILD_TESTING OFF CACHE BOOL "")
 set(BUILD_DOCUMENTATION OFF CACHE BOOL "")
 set(BUILD_SHARED OFF CACHE BOOL "")
 set(BUILD_STATIC ON CACHE BOOL "")
+set(cryptocpp_DISPLAY_CMAKE_SUPPORT_WARNING OFF CACHE BOOL "")
 add_subdirectory(vendor_cryptopp EXCLUDE_FROM_ALL)
 
 target_link_libraries(cryptopp PRIVATE cryptopp-static)

vendor/cryptopp/vendor_cryptopp/Doxyfile

@@ -41,7 +41,7 @@ PROJECT_NAME           = Crypto++
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 7.0
+PROJECT_NUMBER         = 8.0
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -1225,8 +1225,7 @@ DOCSET_PUBLISHER_NAME  = Crypto++
 # compressed HTML files.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
-
+GENERATE_HTMLHELP      = NO
-GENERATE_HTMLHELP      = YES
 
 # The CHM_FILE tag can be used to specify the file name of the resulting .chm
 # file. You can add a path in front of the file if the result should not be
@@ -1494,7 +1493,10 @@ MATHJAX_CODEFILE       =
 # The default value is: YES.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
-SEARCHENGINE           = NO
+# JW: Do _not_ enable server-side search. It consumes too many resources
+# and results in OOM Kills on MySQL, which corrupts the database.
+# https://github.com/weidai11/cryptopp/issues/720
+SEARCHENGINE		 = NO
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
 # implemented using a web server instead of a web client using Javascript. There
@@ -1506,7 +1508,10 @@ SEARCHENGINE           = NO
 # The default value is: NO.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
-SERVER_BASED_SEARCH    = NO
+# JW: Do _not_ enable server-side search. It consumes too many resources
+# and results in OOM Kills on MySQL, which corrupts the database.
+# https://github.com/weidai11/cryptopp/issues/720
+SERVER_BASED_SEARCH	 = NO
 
 # When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
 # script for searching. Instead the search results are written to an XML file

vendor/cryptopp/vendor_cryptopp/Filelist.txt

@@ -1,10 +1,12 @@
 3way.cpp
 3way.h
 adhoc.cpp.proto
-adv-simd.h
+adv_simd.h
 adler32.cpp
 adler32.h
 aes.h
+aes_armv4.h
+aes_armv4.S
 algebra.cpp
 algebra.h
 algparam.cpp
@@ -13,7 +15,7 @@ arc4.cpp
 arc4.h
 ariatab.cpp
 aria.cpp
-aria-simd.cpp
+aria_simd.cpp
 aria.h
 argnames.h
 asn.cpp
@@ -30,9 +32,11 @@ bench.h
 bds10.zip
 bench1.cpp
 bench2.cpp
+bench3.cpp
 bfinit.cpp
 blake2.cpp
-blake2-simd.cpp
+blake2s_simd.cpp
+blake2b_simd.cpp
 blake2.h
 blowfish.cpp
 blowfish.h
@@ -48,7 +52,12 @@ cbcmac.h
 ccm.cpp
 ccm.h
 chacha.cpp
+chacha_avx.cpp
+chacha_simd.cpp
 chacha.h
+cham.cpp
+cham_simd.cpp
+cham.h
 channels.cpp
 channels.h
 cmac.cpp
@@ -57,7 +66,7 @@ config.h
 cpu.cpp
 cpu.h
 crc.cpp
-crc-simd.cpp
+crc_simd.cpp
 crc.h
 cryptdll.vcxproj
 cryptdll.vcxproj.filters
@@ -73,6 +82,8 @@ cryptlib.h
 cryptlib.vcxproj
 cryptlib.vcxproj.filters
 cryptopp.rc
+darn.cpp
+darn.h
 datatest.cpp
 default.cpp
 default.h
@@ -90,6 +101,13 @@ dlltest.vcxproj
 dlltest.vcxproj.filters
 dmac.h
 drbg.h
+donna.h
+donna_32.h
+donna_64.h
+donna_sse.h
+donna_32.cpp
+donna_64.cpp
+donna_sse.cpp
 dsa.cpp
 dsa.h
 eax.cpp
@@ -121,7 +139,7 @@ fips140.h
 fipsalgt.cpp
 fipstest.cpp
 fltrimpl.h
-gcm-simd.cpp
+gcm_simd.cpp
 gcm.cpp
 gcm.h
 gf256.cpp
@@ -137,8 +155,14 @@ gost.h
 gzip.cpp
 gzip.h
 hashfwd.h
+hc128.cpp
+hc128.h
+hc256.cpp
+hc256.h
 hex.cpp
 hex.h
+hight.h
+hight.cpp
 hkdf.h
 hmac.cpp
 hmac.h
@@ -159,7 +183,12 @@ kalyna.cpp
 kalyna.h
 keccak.cpp
 keccak.h
+keccakc.cpp
+keccakc.h
 lubyrack.h
+lea.cpp
+lea_simd.cpp
+lea.h
 luc.cpp
 luc.h
 mars.cpp
@@ -186,9 +215,7 @@ mqv.h
 naclite.h
 nbtheory.cpp
 nbtheory.h
-neon-simd.cpp
+neon_simd.cpp
-network.cpp
-network.h
 nr.h
 oaep.cpp
 oaep.h
@@ -208,8 +235,11 @@ poly1305.cpp
 poly1305.h
 polynomi.cpp
 polynomi.h
-ppc-simd.h
+ppc_power7.cpp
-ppc-simd.cpp
+ppc_power8.cpp
+ppc_power9.cpp
+ppc_simd.cpp
+ppc_simd.h
 pssr.cpp
 pssr.h
 pubkey.cpp
@@ -221,6 +251,8 @@ rabin.cpp
 rabin.h
 randpool.cpp
 randpool.h
+rabbit.cpp
+rabbit.h
 rc2.cpp
 rc2.h
 rc5.cpp
@@ -237,9 +269,10 @@ rdtables.cpp
 regtest1.cpp
 regtest2.cpp
 regtest3.cpp
+regtest4.cpp
 resource.h
 rijndael.cpp
-rijndael-simd.cpp
+rijndael_simd.cpp
 rijndael.h
 ripemd.cpp
 ripemd.h
@@ -265,12 +298,12 @@ serpent.cpp
 serpent.h
 serpentp.h
 sha.cpp
-sha-simd.cpp
+sha_simd.cpp
 sha.h
 sha3.cpp
 sha3.h
 shacal2.cpp
-shacal2-simd.cpp
+shacal2_simd.cpp
 shacal2.h
 shark.cpp
 shark.h
@@ -278,27 +311,31 @@ sharkbox.cpp
 simple.cpp
 simple.h
 siphash.h
+simeck.cpp
+simeck_simd.cpp
+simeck.h
 simon.cpp
-simon-simd.cpp
+simon64_simd.cpp
+simon128_simd.cpp
 simon.h
 skipjack.cpp
 skipjack.h
 sm3.cpp
 sm3.h
 sm4.cpp
+sm4_simd.cpp
 sm4.h
 smartptr.h
-socketft.cpp
-socketft.h
 sosemanuk.cpp
 sosemanuk.h
 speck.cpp
-speck-simd.cpp
+speck64_simd.cpp
+speck128_simd.cpp
 speck.h
 square.cpp
 square.h
 squaretb.cpp
-sse-simd.cpp
+sse_simd.cpp
 stdcpp.h
 strciphr.cpp
 strciphr.h
@@ -312,8 +349,6 @@ threefish.h
 tiger.cpp
 tiger.h
 tigertab.cpp
-trdlocal.cpp
-trdlocal.h
 trunhash.h
 ttmac.cpp
 ttmac.h
@@ -326,21 +361,25 @@ validat1.cpp
 validat2.cpp
 validat3.cpp
 validat4.cpp
+validat5.cpp
+validat6.cpp
+validat7.cpp
+validat8.cpp
+validat9.cpp
+validat10.cpp
 validate.h
 vmac.cpp
 vmac.h
 vs2005.zip
-wait.cpp
-wait.h
 wake.cpp
 wake.h
 whrlpool.cpp
 whrlpool.h
-winpipes.cpp
-winpipes.h
 words.h
 x64dll.asm
 x64masm.asm
+xed25519.h
+xed25519.cpp
 xtr.cpp
 xtr.h
 xtrcrypt.cpp
@@ -366,6 +405,8 @@ TestData/aria.dat
 TestData/camellia.dat
 TestData/cast128v.dat
 TestData/cast256v.dat
+TestData/defdmac1.bin
+TestData/defdmac2.bin
 TestData/descert.dat
 TestData/dh1024.dat
 TestData/dh2048.dat
@@ -374,6 +415,10 @@ TestData/dlie2048.dat
 TestData/dsa1024.dat
 TestData/dsa1024b.dat
 TestData/dsa512.dat
+TestData/ed25519.dat
+TestData/ed25519_ver.dat
+TestData/ed25519v0.dat
+TestData/ed25519v1.dat
 TestData/elgc1024.dat
 TestData/esig1023.dat
 TestData/esig1536.dat
@@ -409,10 +454,10 @@ TestData/rc6val.dat
 TestData/rijndael.dat
 TestData/rsa1024.dat
 TestData/rsa2048.dat
+TestData/rsa2048a.dat
 TestData/rsa400pb.dat
 TestData/rsa400pv.dat
 TestData/rsa512a.dat
-TestData/rsa2048a.dat
 TestData/rw1024.dat
 TestData/rw2048.dat
 TestData/saferval.dat
@@ -423,57 +468,68 @@ TestData/skipjack.dat
 TestData/squareva.dat
 TestData/twofishv.dat
 TestData/usage.dat
+TestData/x25519.dat
+TestData/x25519v0.dat
+TestData/x25519v1.dat
 TestData/xtrdh171.dat
 TestData/xtrdh342.dat
-TestVectors/Readme.txt
 TestVectors/aes.txt
 TestVectors/all.txt
 TestVectors/aria.txt
 TestVectors/blake2.txt
 TestVectors/blake2b.txt
 TestVectors/blake2s.txt
-TestVectors/aria.txt
 TestVectors/camellia.txt
 TestVectors/ccm.txt
 TestVectors/chacha.txt
+TestVectors/chacha_tls.txt
+TestVectors/cham.txt
 TestVectors/cmac.txt
 TestVectors/dlies.txt
 TestVectors/dsa.txt
 TestVectors/dsa_1363.txt
+TestVectors/dsa_rfc6979.txt
 TestVectors/eax.txt
 TestVectors/esign.txt
 TestVectors/gcm.txt
+TestVectors/hc128.txt
+TestVectors/hc256.txt
+TestVectors/hight.txt
 TestVectors/hkdf.txt
 TestVectors/hmac.txt
 TestVectors/kalyna.txt
 TestVectors/keccak.txt
+TestVectors/lea.txt
 TestVectors/mars.txt
 TestVectors/nr.txt
+TestVectors/ocb.txt
 TestVectors/panama.txt
+TestVectors/rabbit.txt
+TestVectors/Readme.txt
 TestVectors/rsa_oaep.txt
 TestVectors/rsa_pkcs1_1_5.txt
 TestVectors/rsa_pss.txt
-TestVectors/dsa_rfc6979.txt
 TestVectors/rw.txt
 TestVectors/salsa.txt
 TestVectors/seal.txt
 TestVectors/seed.txt
 TestVectors/sha.txt
-TestVectors/sha2.txt
-TestVectors/sha3.txt
-TestVectors/sha1_fips_180.txt
 TestVectors/sha1_160_fips_180.txt
-TestVectors/sha2_fips_180.txt
+TestVectors/sha1_fips_180.txt
+TestVectors/sha2.txt
 TestVectors/sha2_224_fips_180.txt
 TestVectors/sha2_256_fips_180.txt
 TestVectors/sha2_384_fips_180.txt
 TestVectors/sha2_512_fips_180.txt
-TestVectors/sha3_fips_202.txt
+TestVectors/sha2_fips_180.txt
+TestVectors/sha3.txt
 TestVectors/sha3_224_fips_202.txt
 TestVectors/sha3_256_fips_202.txt
 TestVectors/sha3_384_fips_202.txt
 TestVectors/sha3_512_fips_202.txt
+TestVectors/sha3_fips_202.txt
 TestVectors/shacal2.txt
+TestVectors/simeck.txt
 TestVectors/simon.txt
 TestVectors/siphash.txt
 TestVectors/sm3.txt
@@ -486,3 +542,39 @@ TestVectors/ttmac.txt
 TestVectors/vmac.txt
 TestVectors/wake.txt
 TestVectors/whrlpool.txt
+TestPrograms/test_32bit.cxx
+TestPrograms/test_64bit.cxx
+TestPrograms/test_arm_acle.cxx
+TestPrograms/test_arm_aes.cxx
+TestPrograms/test_arm_asimd.cxx
+TestPrograms/test_arm_crc.cxx
+TestPrograms/test_arm_neon.cxx
+TestPrograms/test_arm_pmull.cxx
+TestPrograms/test_arm_sha.cxx
+TestPrograms/test_arm_sha3.cxx
+TestPrograms/test_arm_sha512.cxx
+TestPrograms/test_arm_sm3.cxx
+TestPrograms/test_arm_sm4.cxx
+TestPrograms/test_cxx.cxx
+TestPrograms/test_mixed_asm.cxx
+TestPrograms/test_newlib.cxx
+TestPrograms/test_ppc_aes.cxx
+TestPrograms/test_ppc_altivec.cxx
+TestPrograms/test_ppc_power7.cxx
+TestPrograms/test_ppc_power8.cxx
+TestPrograms/test_ppc_power9.cxx
+TestPrograms/test_ppc_sha.cxx
+TestPrograms/test_ppc_vmull.cxx
+TestPrograms/test_pthreads.cxx
+TestPrograms/test_x86_aes.cxx
+TestPrograms/test_x86_avx.cxx
+TestPrograms/test_x86_avx2.cxx
+TestPrograms/test_x86_avx512.cxx
+TestPrograms/test_x86_clmul.cxx
+TestPrograms/test_x86_cpuid.cxx
+TestPrograms/test_x86_sha.cxx
+TestPrograms/test_x86_sse2.cxx
+TestPrograms/test_x86_sse3.cxx
+TestPrograms/test_x86_sse41.cxx
+TestPrograms/test_x86_sse42.cxx
+TestPrograms/test_x86_ssse3.cxx

vendor/cryptopp/vendor_cryptopp/GNUmakefile-cross

@@ -3,6 +3,14 @@
 
 SHELL = /bin/sh
 
+# If needed
+TMPDIR ?= /tmp
+# Used for ARMv7 and NEON.
+FP_ABI ?= hard
+# Used for feature tests
+TOUT ?= a.out
+TOUT := $(strip $(TOUT))
+
 # Default CXXFLAGS if none were provided
 CXXFLAGS ?= -DNDEBUG -g2 -O3 -fPIC -pipe
 
@@ -22,35 +30,25 @@ INSTALL = install
 INSTALL_PROGRAM = $(INSTALL)
 INSTALL_DATA = $(INSTALL) -m 644
 
-# Attempt to determine host machine, fallback to "this" machine.
-#   The host machine is the one the package runs on. Most people
-#   call this the "target", but not Autotools.
-HOSTX := $(shell $(CXX) $(CXXFLAGS) -dumpmachine 2>/dev/null | cut -f 1 -d '-')
-ifeq ($(HOSTX),)
-  HOSTX := $(shell uname -m 2>/dev/null)
-endif
-
-IS_i686 := $(shell echo "$HOSTX" | $(EGREP) -v 64 | $(EGREP) -i -c 'i.86')
-IS_x86_64 := $(shell echo "$HOSTX" | $(EGREP) -i -c 'x86_64|amd64')
-IS_ARM := $(shell echo "$HOSTX" | $(EGREP) -i -c 'arm')
-IS_ARMv8 := $(shell echo "$HOSTX" | $(EGREP) -i -c 'aarch32|aarch64')
-
-CLANG_COMPILER := $(shell $(CXX) --version 2>&1 | $(EGREP) -i -c "clang")
-
 IS_IOS ?= 0
 IS_ANDROID ?= 0
 IS_ARM_EMBEDDED ?= 0
 IS_NEON ?= 0
 
-# Fixup ARM
-ifeq ($(IS_ARMv8),1)
-  IS_ARM := 0
-endif
-
 # Can be used by Android and Embeeded cross-compiles. Disable by default because
 #   Android and embedded users typically don't run this configuration.
 HAS_SOLIB_VERSION ?= 0
 
+# Formely adhoc.cpp was created from adhoc.cpp.proto when needed.
+# This is now needed because ISA tests are performed using adhoc.cpp.
+ifeq ($(wildcard adhoc.cpp),)
+$(shell cp adhoc.cpp.proto adhoc.cpp)
+endif
+
+###########################################################
+#####                General Variables                #####
+###########################################################
+
 # Default prefix for make install
 ifeq ($(PREFIX),)
 PREFIX = /usr/local
@@ -72,29 +70,37 @@ endif
 
 # We honor ARFLAGS, but the "v" option used by default causes a noisy make
 ifeq ($(ARFLAGS),rv)
-ARFLAGS = r
+  ARFLAGS = r
 endif
 
 # Sadly, we can't actually use GCC_PRAGMA_AWARE because of GCC bug 53431.
 # Its a shame because GCC has so much to offer by the way of analysis.
 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53431
 ifneq ($(CLANG_COMPILER),0)
-CXXFLAGS += -Wall
+  CXXFLAGS += -Wall
 endif
 
+###########################################################
+#####                       iOS                       #####
+###########################################################
+
 # iOS cross-compile configuration.
 #   See http://www.cryptopp.com/wiki/iOS_(Command_Line).
 ifeq ($(IS_IOS),1)
   CXX = clang++
 
   CXXFLAGS += $(IOS_FLAGS) -arch $(IOS_ARCH)
-  CXXFLAGS += -isysroot $(IOS_SYSROOT) -stdlib=libc++
+  CXXFLAGS += -isysroot "$(IOS_SYSROOT)" -stdlib=libc++
 
   AR = libtool
   ARFLAGS = -static -o
   RANLIB = ranlib
 endif
 
+###########################################################
+#####                     Android                     #####
+###########################################################
+
 # Android cross-compile configuration.
 #   See http://www.cryptopp.com/wiki/Android_(Command_Line).
 ifeq ($(IS_ANDROID),1)
@@ -116,6 +122,10 @@ ifeq ($(IS_ANDROID),1)
   AOSP_CPU_OBJ = cpu-features.o
 endif
 
+###########################################################
+#####                    Embedded                     #####
+###########################################################
+
 # ARM embedded cross-compile configuration.
 #   See http://www.cryptopp.com/wiki/ARM_Embedded_(Command_Line)
 #   and http://www.cryptopp.com/wiki/ARM_Embedded_(Bare Metal).
@@ -124,6 +134,364 @@ ifeq ($(IS_ARM_EMBEDDED),1)
   CXXFLAGS += $(ARM_EMBEDDED_FLAGS) --sysroot=$(ARM_EMBEDDED_SYSROOT)
 endif
 
+###########################################################
+#####              Compiler and Platform              #####
+###########################################################
+
+# Wait until CXXFLAGS have been set by setenv scripts.
+
+GCC_COMPILER := $(shell $(CXX) --version 2>/dev/null | $(EGREP) -v -E 'llvm|clang' | $(EGREP) -i -c -E '(gcc|g\+\+)')
+CLANG_COMPILER := $(shell $(CXX) --version 2>/dev/null | $(EGREP) -i -c -E 'llvm|clang')
+
+HOSTX := $(shell $(CXX) $(CXXFLAGS) -dumpmachine 2>/dev/null | cut -f 1 -d '-')
+ifeq ($(HOSTX),)
+  HOSTX := $(shell uname -m 2>/dev/null)
+endif
+
+# This dance is because Clang reports the host architecture instead
+# of the target architecture. Running Clang on an x86_64 machine with
+# -arch arm64 yields x86_64 instead of aarch64 or arm64.
+
+ifeq ($(CLANG_COMPILER),1)
+  IS_X86 := $(shell echo $(CXXFLAGS) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'i.86')
+  IS_X64 := $(shell echo $(CXXFLAGS) | $(EGREP) -i -c -E 'x86_64|amd64')
+  IS_ARM32 := $(shell echo $(CXXFLAGS) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'arm|armhf|arm7l|eabihf')
+  IS_ARMV8 := $(shell echo $(CXXFLAGS) | $(EGREP) -i -c -E 'aarch32|aarch64|arm64|armv8')
+else
+  IS_X86 := $(shell echo $(HOSTX) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'i.86')
+  IS_X64 := $(shell echo $(HOSTX) | $(EGREP) -i -c -E 'x86_64|amd64')
+  IS_ARM32 := $(shell echo $(HOSTX) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'arm|armhf|arm7l|eabihf')
+  IS_ARMV8 := $(shell echo $(HOSTX) | $(EGREP) -i -c -E 'aarch32|aarch64|arm64|armv8')
+endif
+
+$(info Here's what we found... IS_X86: $(IS_X86), IS_X64: $(IS_X64), IS_ARM32: $(IS_ARM32), IS_ARMV8: $(IS_ARMV8))
+
+###########################################################
+#####                  Test Program                   #####
+###########################################################
+
+# Hack to skip CPU feature tests for some recipes
+DETECT_FEATURES ?= 1
+ifeq ($(findstring -DCRYPTOPP_DISABLE_ASM,$(CXXFLAGS)),-DCRYPTOPP_DISABLE_ASM)
+  DETECT_FEATURES := 0
+else ifeq ($(findstring clean,$(MAKECMDGOALS)),clean)
+  DETECT_FEATURES := 0
+else ifeq ($(findstring distclean,$(MAKECMDGOALS)),distclean)
+  DETECT_FEATURES := 0
+else ifeq ($(findstring distclean,$(MAKECMDGOALS)),trim)
+  DETECT_FEATURES := 0
+endif
+
+# Strip out -Wall, -Wextra and friends for feature testing
+ifeq ($(DETECT_FEATURES),1)
+  TCXXFLAGS := $(filter-out -Wall -Wextra -Werror% -Wunused -Wconversion -Wp%, $(CXXFLAGS))
+  ifneq ($(strip $(TCXXFLAGS)),)
+    $(info Using testing flags: $(TCXXFLAGS))
+  endif
+endif
+
+# For the previous messages
+$(info )
+
+###########################################################
+#####               X86/X32/X64 Options               #####
+###########################################################
+
+ifneq ($(IS_X86)$(IS_X64),00)
+ifeq ($(DETECT_FEATURES),1)
+
+  SSE2_FLAG = -msse2
+  SSE3_FLAG = -msse3
+  SSSE3_FLAG = -mssse3
+  SSE41_FLAG = -msse4.1
+  SSE42_FLAG = -msse4.2
+  CLMUL_FLAG = -mpclmul
+  AESNI_FLAG = -maes
+  AVX_FLAG = -mavx
+  AVX2_FLAG = -mavx2
+  SHANI_FLAG = -msha
+
+  TPROG = TestPrograms/test_x86_sse2.cxx
+  TOPT = $(SSE2_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    CHACHA_FLAG = $(SSE2_FLAG)
+  else
+    SSE2_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_ssse3.cxx
+  TOPT = $(SSSE3_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    ARIA_FLAG = $(SSSE3_FLAG)
+    CHAM_FLAG = $(SSSE3_FLAG)
+    LEA_FLAG = $(SSSE3_FLAG)
+    SIMECK_FLAG = $(SSSE3_FLAG)
+    SIMON64_FLAG = $(SSSE3_FLAG)
+    SIMON128_FLAG = $(SSSE3_FLAG)
+    SPECK64_FLAG = $(SSSE3_FLAG)
+    SPECK128_FLAG = $(SSSE3_FLAG)
+  else
+    SSSE3_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_sse41.cxx
+  TOPT = $(SSE41_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    BLAKE2B_FLAG = $(SSE41_FLAG)
+    BLAKE2S_FLAG = $(SSE41_FLAG)
+    SIMON64_FLAG = $(SSE41_FLAG)
+    SPECK64_FLAG = $(SSE41_FLAG)
+  else
+    SSE41_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_sse42.cxx
+  TOPT = $(SSE42_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    CRC_FLAG = $(SSE42_FLAG)
+  else
+    SSE42_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_clmul.cxx
+  TOPT = $(CLMUL_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    GCM_FLAG = $(SSSE3_FLAG) $(CLMUL_FLAG)
+  else
+    CLMUL_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_aes.cxx
+  TOPT = $(AESNI_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    AES_FLAG = $(SSE41_FLAG) $(AESNI_FLAG)
+    SM4_FLAG = $(SSSE3_FLAG) $(AESNI_FLAG)
+  else
+    AESNI_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_avx.cxx
+  TOPT = $(AVX_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    # XXX_FLAG = $(AVX_FLAG)
+  else
+    AVX_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_avx2.cxx
+  TOPT = $(AVX2_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    CHACHA_AVX2_FLAG = $(AVX2_FLAG)
+  else
+    AVX2_FLAG =
+  endif
+
+  TPROG = TestPrograms/test_x86_sha.cxx
+  TOPT = $(SHANI_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    SHA_FLAG = $(SSE42_FLAG) $(SHANI_FLAG)
+  else
+    SHANI_FLAG =
+  endif
+
+  ifeq ($(SSE2_FLAG),)
+    CXXFLAGS += -DCRYPTOPP_DISABLE_ASM
+  else ifeq ($(SSE3_FLAG),)
+    CXXFLAGS += -DCRYPTOPP_DISABLE_SSE3
+  else ifeq ($(SSSE3_FLAG),)
+    CXXFLAGS += -DCRYPTOPP_DISABLE_SSSE3
+  else ifeq ($(SSE41_FLAG),)
+    CXXFLAGS += -DCRYPTOPP_DISABLE_SSE4
+  else ifeq ($(SSE42_FLAG),)
+    CXXFLAGS += -DCRYPTOPP_DISABLE_SSE4
+  endif
+
+  ifneq ($(SSE42_FLAG),)
+
+    # Unusual GCC/Clang on Macports. It assembles AES, but not CLMUL.
+    # test_x86_clmul.s:15: no such instruction: 'pclmulqdq $0, %xmm1,%xmm0'
+    ifeq ($(CLMUL_FLAG),)
+      CXXFLAGS += -DCRYPTOPP_DISABLE_CLMUL
+    endif
+    ifeq ($(AESNI_FLAG),)
+      CXXFLAGS += -DCRYPTOPP_DISABLE_AESNI
+    endif
+
+    ifeq ($(AVX_FLAG),)
+      CXXFLAGS += -DCRYPTOPP_DISABLE_AVX
+    else ifeq ($(AVX2_FLAG),)
+      CXXFLAGS += -DCRYPTOPP_DISABLE_AVX2
+    else ifeq ($(SHANI_FLAG),)
+      CXXFLAGS += -DCRYPTOPP_DISABLE_SHANI
+    endif
+  endif
+
+# DETECT_FEATURES
+endif
+
+# IS_X86, IS_X32 and IS_X64
+endif
+
+###########################################################
+#####            ARM A-32, Aach64 and NEON            #####
+###########################################################
+
+ifneq ($(IS_ARM32)$(IS_ARMV8),00)
+ifeq ($(DETECT_FEATURES),1)
+
+ifeq ($(IS_ARM32),1)
+
+  ifneq ($(IS_IOS)$(IS_ANDROID),00)
+    NEON_FLAG =
+  else
+    NEON_FLAG = -march=armv7-a -mfloat-abi=$(FP_ABI) -mfpu=neon
+  endif
+
+  TPROG = TestPrograms/test_arm_neon.cxx
+  TOPT = $(NEON_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    ARIA_FLAG = $(NEON_FLAG)
+    AES_FLAG = $(NEON_FLAG)
+    CRC_FLAG = $(NEON_FLAG)
+    GCM_FLAG = $(NEON_FLAG)
+    BLAKE2B_FLAG = $(NEON_FLAG)
+    BLAKE2S_FLAG = $(NEON_FLAG)
+    CHACHA_FLAG = $(NEON_FLAG)
+    CHAM_FLAG = $(NEON_FLAG)
+    LEA_FLAG = $(NEON_FLAG)
+    SHA_FLAG = $(NEON_FLAG)
+    SIMECK_FLAG = $(NEON_FLAG)
+    SIMON64_FLAG = $(NEON_FLAG)
+    SIMON128_FLAG = $(NEON_FLAG)
+    SPECK64_FLAG = $(NEON_FLAG)
+    SPECK128_FLAG = $(NEON_FLAG)
+    SM4_FLAG = $(NEON_FLAG)
+  else
+    NEON_FLAG =
+    CXXFLAGS += -DCRYPTOPP_DISABLE_NEON
+  endif
+
+# IS_NEON
+endif
+
+ifeq ($(IS_ARMV8),1)
+
+  ifeq ($(IS_IOS),1)
+    ASIMD_FLAG =
+    CRC_FLAG =
+    AES_FLAG =
+    PMUL_FLAG =
+    SHA_FLAG =
+  else
+    ASIMD_FLAG = -march=armv8-a
+    CRC_FLAG = -march=armv8-a+crc
+    AES_FLAG = -march=armv8-a+crypto
+    PMULL_FLAG = -march=armv8-a+crypto
+    SHA_FLAG = -march=armv8-a+crypto
+  endif
+
+  TPROG = TestPrograms/test_arm_acle.cxx
+  TOPT = $(ASIMD_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    ACLE_FLAG += -DCRYPTOPP_ARM_ACLE_AVAILABLE=1
+  else
+    CXXFLAGS += -DCRYPTOPP_ARM_ACLE_AVAILABLE=0
+  endif
+
+  TPROG = TestPrograms/test_arm_asimd.cxx
+  TOPT = $(ASIMD_FLAG)
+  HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+  ifeq ($(strip $(HAVE_OPT)),0)
+    ARIA_FLAG = $(ASIMD_FLAG)
+    BLAKE2B_FLAG = $(ASIMD_FLAG)
+    BLAKE2S_FLAG = $(ASIMD_FLAG)
+    CHACHA_FLAG = $(ASIMD_FLAG)
+    CHAM_FLAG = $(ASIMD_FLAG)
+    LEA_FLAG = $(ASIMD_FLAG)
+    NEON_FLAG = $(ASIMD_FLAG)
+    SIMECK_FLAG = $(ASIMD_FLAG)
+    SIMON64_FLAG = $(ASIMD_FLAG)
+    SIMON128_FLAG = $(ASIMD_FLAG)
+    SPECK64_FLAG = $(ASIMD_FLAG)
+    SPECK128_FLAG = $(ASIMD_FLAG)
+    SM4_FLAG = $(ASIMD_FLAG)
+  else
+    ASIMD_FLAG =
+    CXXFLAGS += -DCRYPTOPP_DISABLE_ASM
+  endif
+
+    TPROG = TestPrograms/test_arm_crc.cxx
+    TOPT = $(CRC_FLAG)
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifneq ($(strip $(HAVE_OPT)),0)
+      CRC_FLAG =
+      CXXFLAGS += -DCRYPTOPP_ARM_CRC32_AVAILABLE=0
+    endif
+
+    TPROG = TestPrograms/test_arm_aes.cxx
+    TOPT = $(AES_FLAG)
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifneq ($(strip $(HAVE_OPT)),0)
+      AES_FLAG =
+      CXXFLAGS += -DCRYPTOPP_ARM_AES_AVAILABLE=0
+    endif
+
+    TPROG = TestPrograms/test_arm_pmull.cxx
+    TOPT = $(PMULL_FLAG)
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifneq ($(strip $(HAVE_OPT)),0)
+      PMULL_FLAG =
+      CXXFLAGS += -DCRYPTOPP_ARM_PMULL_AVAILABLE=0
+    endif
+
+    TPROG = TestPrograms/test_arm_sha.cxx
+    TOPT = $(SHA_FLAG)
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifneq ($(strip $(HAVE_OPT)),0)
+      SHA_FLAG =
+      CXXFLAGS += -DCRYPTOPP_ARM_SHA_AVAILABLE=0
+    endif
+
+    TPROG = TestPrograms/test_arm_sm3.cxx
+    TOPT = -march=armv8.4-a+crypto
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifeq ($(strip $(HAVE_OPT)),0)
+      SM3_FLAG = -march=armv8.4-a+crypto
+      SM4_FLAG = -march=armv8.4-a+crypto
+    endif
+
+    TPROG = TestPrograms/test_arm_sha3.cxx
+    TOPT = -march=armv8.4-a+crypto
+    HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
+    ifeq ($(strip $(HAVE_OPT)),0)
+      SHA3_FLAG = -march=armv8.4-a+crypto
+      SHA512_FLAG = -march=armv8.4-a+crypto
+    endif
+
+# IS_ARMV8
+endif
+
+# DETECT_FEATURES
+endif
+
+# IS_ARM32, IS_ARMV8, IS_NEON
+endif
+
+###########################################################
+#####                      Common                     #####
+###########################################################
+
 # No ASM for Travis testing
 ifeq ($(findstring no-asm,$(MAKECMDGOALS)),no-asm)
   ifeq ($(findstring -DCRYPTOPP_DISABLE_ASM,$(CXXFLAGS)),)
@@ -178,7 +546,7 @@ endif # Valgrind
 # Debug testing on GNU systems. Triggered by -DDEBUG.
 #   Newlib test due to http://sourceware.org/bugzilla/show_bug.cgi?id=20268
 ifneq ($(filter -DDEBUG -DDEBUG=1,$(CXXFLAGS)),)
-  USING_GLIBCXX := $(shell $(CXX) -x c++ $(CXXFLAGS) -E adhoc.cpp.proto 2>&1 | $(EGREP) -i -c "__GLIBCXX__")
+  USING_GLIBCXX := $(shell $(CXX) $(CXXFLAGS) -E pch.cpp 2>&1 | $(EGREP) -i -c "__GLIBCXX__")
   ifneq ($(USING_GLIBCXX),0)
     ifeq ($(HAS_NEWLIB),0)
       ifeq ($(findstring -D_GLIBCXX_DEBUG,$(CXXFLAGS)),)
@@ -207,115 +575,34 @@ ifeq ($(findstring lean,$(MAKECMDGOALS)),lean)
   endif # MAKECMDGOALS
 endif # Dead code stripping
 
-# Pickup ARMv7 and NEON. Do it after Android, iOS and Embedded flags have been set.
+###########################################################
-ifeq ($(IS_ARM),1)
+#####              Source and object files            #####
-  IS_ARMv7 := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -dM -E - 2>/dev/null | $(EGREP) -i -c '__ARM_ARCH 7')
+###########################################################
-  ifeq ($(IS_ARMv7),1)
-    IS_NEON := 1
-  else
-    IS_NEON := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -dM -E - 2>/dev/null | $(EGREP) -i -c -E '\<__ARM_NEON\>')
-  endif
-endif
-
-# ARMv7-a
-ifeq ($(IS_ARMv7),1)
-  ifeq ($(findstring -march=armv7-a,$(CXXFLAGS)),)
-    NEON_FLAG = -march=armv7-a
-    GCM_FLAG = -march=armv7-a
-    ARIA_FLAG = -march=armv7-a
-    BLAKE2_FLAG = -march=armv7-a
-  endif
-endif
-
-# NEON
-ifeq ($(IS_NEON),1)
-  ifeq ($(findstring -mfpu=neon,$(CXXFLAGS)),)
-    NEON_FLAG += -mfpu=neon
-    GCM_FLAG += -mfpu=neon
-    ARIA_FLAG += -mfpu=neon
-    BLAKE2_FLAG += -mfpu=neon
-    SIMON_FLAG += -mfpu=neon
-    SPECK_FLAG += -mfpu=neon
-    ifeq ($(IS_ANDROID),1)
-      ifeq ($(findstring -mfloat-abi=softfp,$(CXXFLAGS)),)
-        NEON_FLAG += -mfloat-abi=softfp
-        GCM_FLAG += -mfloat-abi=softfp
-        ARIA_FLAG += -mfloat-abi=softfp
-        BLAKE2_FLAG += -mfloat-abi=softfp
-        SIMON_FLAG += -mfloat-abi=softfp
-        SPECK_FLAG += -mfloat-abi=softfp
-      endif
-    endif
-  endif
-endif
-
-# ARMv8-a
-ifneq ($(IS_ARMv8),0)
-  IS_NEON := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_NEON)
-  ifeq ($(IS_NEON),1)
-    ARIA_FLAG = -march=armv8-a
-    BLAKE2_FLAG = -march=armv8-a
-    NEON_FLAG = -march=armv8-a
-    SIMON_FLAG = -march=armv8-a
-    SPECK_FLAG = -march=armv8-a
-  endif
-  HAVE_CRC := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a+crc -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_FEATURE_CRC32)
-  ifeq ($(HAVE_CRC),1)
-    CRC_FLAG = -march=armv8-a+crc
-  endif
-  HAVE_CRYPTO := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a+crypto -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_FEATURE_CRYPTO)
-  ifeq ($(HAVE_CRYPTO),1)
-    AES_FLAG = -march=armv8-a+crypto
-    GCM_FLAG = -march=armv8-a+crypto
-    SHA_FLAG = -march=armv8-a+crypto
-  endif
-endif
-
-# i686 and x86_64
-ifneq ($(IS_i686)$(IS_x86_64),00)
-  HAVE_SSSE3 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -mssse3 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSSE3__)
-  ifeq ($(HAVE_SSSE3),1)
-    ARIA_FLAG = -mssse3
-    SSSE3_FLAG = -mssse3
-    SIMON_FLAG = -mssse3
-    SPECK_FLAG = -mssse3
-  endif
-  HAVE_SSE4 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.1 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSE4_1__)
-  ifeq ($(HAVE_SSE4),1)
-    SIMON_FLAG = -msse4.1
-    SPECK_FLAG = -msse4.1
-  endif
-  HAVE_SSE4 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.2 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSE4_2__)
-  ifeq ($(HAVE_SSE4),1)
-    BLAKE2_FLAG = -msse4.2
-    CRC_FLAG = -msse4.2
-  endif
-  HAVE_CLMUL = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -mssse3 -mpclmul -dM -E - 2>/dev/null | $(EGREP) -i -c __PCLMUL__ )
-  ifeq ($(HAVE_CLMUL),1)
-    GCM_FLAG = -mssse3 -mpclmul
-  endif
-  HAVE_AES = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.1 -maes -dM -E - 2>/dev/null | $(EGREP) -i -c __AES__)
-  ifeq ($(HAVE_AES),1)
-    AES_FLAG = -msse4.1 -maes
-  endif
-  HAVE_SHA = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.2 -msha -dM -E - 2>/dev/null | $(EGREP) -i -c __SHA__)
-  ifeq ($(HAVE_SHA),1)
-    SHA_FLAG = -msse4.2 -msha
-  endif
-endif
 
 # List cryptlib.cpp first, then cpu.cpp, then integer.cpp to tame C++ static initialization problems.
-SRCS := cryptlib.cpp cpu.cpp integer.cpp $(filter-out cryptlib.cpp cpu.cpp integer.cpp pch.cpp simple.cpp winpipes.cpp cryptlib_bds.cpp,$(sort $(wildcard *.cpp)))
+SRCS := cryptlib.cpp cpu.cpp integer.cpp $(filter-out cryptlib.cpp cpu.cpp integer.cpp pch.cpp simple.cpp cryptlib_bds.cpp,$(sort $(wildcard *.cpp)))
-
 # For Makefile.am; resource.h is Windows
 INCL := $(filter-out resource.h,$(sort $(wildcard *.h)))
 
+# Cryptogams AES for ARMv4 and above. We couple to ARMv7.
+# Avoid iOS. It cannot consume the assembly.
+ifeq ($(IS_ARM32),1)
+  ifneq ($(IS_IOS),1)
+    CRYPTOGAMS_AES_FLAG = -march=armv7-a
+    CRYPTOGAMS_AES_FLAG += -Wa,--noexecstack
+    SRCS += aes_armv4.S
+  endif
+endif
+
 # List cryptlib.cpp first, then cpu.cpp, then integer.cpp to tame C++ static initialization problems.
 OBJS := $(SRCS:.cpp=.o)
+OBJS := $(OBJS:.S=.o)
 
 # List test.cpp first to tame C++ static initialization problems.
-TESTSRCS := adhoc.cpp test.cpp bench1.cpp bench2.cpp validat0.cpp validat1.cpp validat2.cpp validat3.cpp validat4.cpp datatest.cpp regtest1.cpp regtest2.cpp regtest3.cpp fipsalgt.cpp dlltest.cpp
+TESTSRCS := adhoc.cpp test.cpp bench1.cpp bench2.cpp bench3.cpp datatest.cpp dlltest.cpp fipsalgt.cpp validat0.cpp validat1.cpp validat2.cpp validat3.cpp validat4.cpp validat5.cpp validat6.cpp validat7.cpp validat8.cpp validat9.cpp validat10.cpp regtest1.cpp regtest2.cpp regtest3.cpp regtest4.cpp
 TESTINCL := bench.h factory.h validate.h
+
+# Test objects
 TESTOBJS := $(TESTSRCS:.cpp=.o)
 LIBOBJS := $(filter-out $(TESTOBJS),$(OBJS))
 
@@ -337,11 +624,15 @@ SOLIB_COMPAT_SUFFIX=.$(LIB_MAJOR)
 SOLIB_FLAGS=-Wl,-soname,libcryptopp.so$(SOLIB_COMPAT_SUFFIX)
 endif # HAS_SOLIB_VERSION
 
+###########################################################
+#####                Targets and Recipes              #####
+###########################################################
+
 # Default builds program with static library only
 .PHONY: default
 default: cryptest.exe
 
-.PHONY: all
+.PHONY: all static dynamic
 all: static dynamic cryptest.exe
 
 ifneq ($(IS_IOS),0)
@@ -363,10 +654,10 @@ lean: static dynamic cryptest.exe
 
 .PHONY: clean
 clean:
-	-$(RM) adhoc.cpp.o adhoc.cpp.proto.o $(LIBOBJS) $(AOSP_CPU_OBJ) $(TESTOBJS) $(DLLOBJS) $(LIBIMPORTOBJS) $(TESTIMPORTOBJS) $(DLLTESTOBJS)
+	-$(RM) adhoc.cpp.o adhoc.cpp.proto.o $(LIBOBJS) $(AOSP_CPU_OBJ) rdrand-*.o $(TESTOBJS) $(DLLOBJS) $(LIBIMPORTOBJS) $(TESTIMPORTOBJS) $(DLLTESTOBJS)
 	@-$(RM) libcryptopp.a libcryptopp.dylib cryptopp.dll libcryptopp.dll.a libcryptopp.import.a
 	@-$(RM) libcryptopp.so libcryptopp.so$(SOLIB_COMPAT_SUFFIX) libcryptopp.so$(SOLIB_VERSION_SUFFIX)
-	@-$(RM) cryptest.exe dlltest.exe cryptest.import.exe cryptest.info ct rdrand-???.o
+	@-$(RM) cryptest.exe dlltest.exe cryptest.import.exe cryptest.info ct et
 	@-$(RM) *.la *.lo *.gcov *.gcno *.gcda *.stackdump core core-*
 	@-$(RM) /tmp/adhoc.exe
 	@-$(RM) -r /tmp/cryptopp_test/
@@ -374,31 +665,46 @@ clean:
 	@-$(RM) -r *.dylib.dSYM/
 	@-$(RM) -r cov-int/
 
-.PHONY: distclean
+.PHONY: autotools-clean
-distclean: clean
+autotools-clean:
-	-$(RM) adhoc.cpp adhoc.cpp.copied GNUmakefile.deps benchmarks.html cryptest.txt cryptest-*.txt
-	@-$(RM) cryptopp.tgz *.o *.bc *.ii *~
-	@-$(RM) -r $(SRCS:.cpp=.obj) cryptlib.lib cryptest.exe *.suo *.sdf *.pdb Win32/ x64/ ipch/
 	@-$(RM) -f configure.ac configure configure.in Makefile.am Makefile.in Makefile
-	@-$(RM) -f config.guess config.status config.sub depcomp install-sh compile
+	@-$(RM) -f config.guess config.status config.sub config.h.in compile depcomp
-	@-$(RM) -f stamp-h1 ar-lib *.m4 local.* lt*.sh missing libtool* libcryptopp.pc*
+	@-$(RM) -f install-sh stamp-h1 ar-lib *.lo *.la *.m4 local.* lt*.sh missing
+	@-$(RM) -f cryptest cryptestcwd libtool* libcryptopp.la libcryptopp.pc*
 	@-$(RM) -rf m4/ auto*.cache/ .deps/ .libs/
-	@-$(RM) -r TestCoverage/
-	@-$(RM) cryptopp$(LIB_VER)\.*
-	@-$(RM) CryptoPPRef.zip
 
+.PHONY: cmake-clean
+cmake-clean:
+	@-$(RM) -f cryptopp-config.cmake CMakeLists.txt
+	@-$(RM) -rf cmake_build/
+
+.PHONY: distclean
+distclean: clean autotools-clean cmake-clean
+	-$(RM) adhoc.cpp adhoc.cpp.copied GNUmakefile.deps benchmarks.html cryptest.txt
+	@-$(RM) cryptest-*.txt cryptopp.tgz libcryptopp.pc *.o *.bc *.ii *~
+	@-$(RM) -r cryptlib.lib cryptest.exe *.suo *.sdf *.pdb Win32/ x64/ ipch/
+	@-$(RM) -r $(LIBOBJS:.o=.obj) $(TESTOBJS:.o=.obj)
+	@-$(RM) -r $(LIBOBJS:.o=.lst) $(TESTOBJS:.o=.lst)
+	@-$(RM) -r TestCoverage/ ref*/
+	@-$(RM) cryptopp$(LIB_VER)\.* CryptoPPRef.zip
+
+# Install cryptest.exe, libcryptopp.a and libcryptopp.so.
+# The library install was broken-out into its own recipe at GH #653.
 .PHONY: install
-install:
+install: cryptest.exe install-lib
-	@-$(MKDIR) $(DESTDIR)$(INCLUDEDIR)/cryptopp
-	$(INSTALL_DATA) *.h $(DESTDIR)$(INCLUDEDIR)/cryptopp
-ifneq ($(wildcard cryptest.exe),)
 	@-$(MKDIR) $(DESTDIR)$(BINDIR)
 	$(INSTALL_PROGRAM) cryptest.exe $(DESTDIR)$(BINDIR)
 	@-$(MKDIR) $(DESTDIR)$(DATADIR)/cryptopp/TestData
 	@-$(MKDIR) $(DESTDIR)$(DATADIR)/cryptopp/TestVectors
 	$(INSTALL_DATA) TestData/*.dat $(DESTDIR)$(DATADIR)/cryptopp/TestData
 	$(INSTALL_DATA) TestVectors/*.txt $(DESTDIR)$(DATADIR)/cryptopp/TestVectors
-endif
+
+# A recipe to install only the library, and not cryptest.exe. Also
+# see https://github.com/weidai11/cryptopp/issues/653.
+.PHONY: install-lib
+install-lib:
+	@-$(MKDIR) $(DESTDIR)$(INCLUDEDIR)/cryptopp
+	$(INSTALL_DATA) *.h $(DESTDIR)$(INCLUDEDIR)/cryptopp
 ifneq ($(wildcard libcryptopp.a),)
 	@-$(MKDIR) $(DESTDIR)$(LIBDIR)
 	$(INSTALL_DATA) libcryptopp.a $(DESTDIR)$(LIBDIR)
@@ -435,15 +741,15 @@ ifeq ($(HAS_SOLIB_VERSION),1)
 libcryptopp.so: libcryptopp.so$(SOLIB_VERSION_SUFFIX)
 endif
 
-libcryptopp.so$(SOLIB_VERSION_SUFFIX): $(LIBOBJS)
+libcryptopp.so$(SOLIB_VERSION_SUFFIX): $(LIBOBJS) $(AOSP_CPU_OBJ)
-	$(CXX) -shared $(SOLIB_FLAGS) -o $@ $(strip $(CXXFLAGS)) -Wl,--exclude-libs,ALL $(LIBOBJS) $(LDFLAGS) $(LDLIBS)
+	$(CXX) -shared $(SOLIB_FLAGS) -o $@ $(strip $(CXXFLAGS)) -Wl,--exclude-libs,ALL $(LIBOBJS) $(AOSP_CPU_OBJ) $(LDFLAGS) $(LDLIBS)
 ifeq ($(HAS_SOLIB_VERSION),1)
 	-$(LN) libcryptopp.so$(SOLIB_VERSION_SUFFIX) libcryptopp.so
 	-$(LN) libcryptopp.so$(SOLIB_VERSION_SUFFIX) libcryptopp.so$(SOLIB_COMPAT_SUFFIX)
 endif
 
-libcryptopp.dylib: $(LIBOBJS)
+libcryptopp.dylib: $(LIBOBJS) $(AOSP_CPU_OBJ)
-	$(CXX) -dynamiclib -o $@ $(strip $(CXXFLAGS)) -install_name "$@" -current_version "$(LIB_MAJOR).$(LIB_MINOR).$(LIB_PATCH)" -compatibility_version "$(LIB_MAJOR).$(LIB_MINOR)" -headerpad_max_install_names $(LDFLAGS) $(LIBOBJS)
+	$(CXX) -dynamiclib -o $@ $(strip $(CXXFLAGS)) -install_name "$@" -current_version "$(LIB_MAJOR).$(LIB_MINOR).$(LIB_PATCH)" -compatibility_version "$(LIB_MAJOR).$(LIB_MINOR)" -headerpad_max_install_names $(LDFLAGS) $(LIBOBJS) $(AOSP_CPU_OBJ)
 
 cryptest.exe: libcryptopp.a $(TESTOBJS)
 	$(CXX) -o $@ $(strip $(CXXFLAGS)) $(TESTOBJS) ./libcryptopp.a $(LDFLAGS) $(LDLIBS)
@@ -451,13 +757,17 @@ cryptest.exe: libcryptopp.a $(TESTOBJS)
 # Used to generate list of source files for Autotools, CMakeList and Android.mk
 .PHONY: sources
 sources:
-	$(info Library sources: $(filter-out $(TESTSRCS),$(SRCS)))
+	$(info ***** Library sources *****)
+	$(info $(filter-out $(TESTSRCS),$(SRCS)))
 	$(info )
-	$(info Library headers: $(filter-out $(TESTINCL),$(INCL)))
+	$(info ***** Library headers *****)
+	$(info $(filter-out $(TESTINCL),$(INCL)))
 	$(info )
-	$(info Test sources: $(TESTSRCS))
+	$(info ***** Test sources *****)
+	$(info $(TESTSRCS))
 	$(info )
-	$(info Test headers: $(TESTINCL))
+	$(info ***** Test headers *****)
+	$(info $(TESTINCL))
 
 adhoc.cpp: adhoc.cpp.proto
 ifeq ($(wildcard adhoc.cpp),)
@@ -471,55 +781,108 @@ ifeq ($(wildcard GNUmakefile.deps),GNUmakefile.deps)
 -include GNUmakefile.deps
 endif # Dependencies
 
+# Cryptogams ARM asm implementation.
+aes_armv4.o : aes_armv4.S
+	$(CXX) $(strip $(CXXFLAGS) -fpermissive $(CRYPTOGAMS_AES_FLAG) -c) $<
+
 cpu-features.o: cpu-features.h cpu-features.c
 	$(CXX) $(strip $(CXXFLAGS) -fpermissive -c) cpu-features.c
 
-# SSE4.2 or NEON available
+# SSSE3 or NEON available
-aria-simd.o : aria-simd.cpp
+aria_simd.o : aria_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(ARIA_FLAG) -c) $<
 
-# SSE4.2 or ARMv8a available
+# SSE, NEON or POWER7 available
-blake2-simd.o : blake2-simd.cpp
+blake2s_simd.o : blake2s_simd.cpp
-	$(CXX) $(strip $(CXXFLAGS) $(BLAKE2_FLAG) -c) $<
+	$(CXX) $(strip $(CXXFLAGS) $(BLAKE2S_FLAG) -c) $<
 
-# SSE2 on i586
+# SSE, NEON or POWER8 available
-cpu.o : cpu.cpp
+blake2b_simd.o : blake2b_simd.cpp
-	$(CXX) $(strip $(CXXFLAGS) $(CPU_FLAG) -c) $<
+	$(CXX) $(strip $(CXXFLAGS) $(BLAKE2B_FLAG) -c) $<
+
+# SSE2 or NEON available
+chacha_simd.o : chacha_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(CHACHA_FLAG) -c) $<
+
+# AVX2 available
+chacha_avx.o : chacha_avx.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(CHACHA_AVX2_FLAG) -c) $<
+
+# SSSE3 available
+cham_simd.o : cham_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(CHAM_FLAG) -c) $<
+
+# Power9 available
+darn.o : darn.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(DARN_FLAG) -c) $<
+
+# SSE2 on i686
+donna_sse.o : donna_sse.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SSE2_FLAG) -c) $<
+
+# SSE2 on i686
+sse_simd.o : sse_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SSE2_FLAG) -c) $<
 
 # SSE4.2 or ARMv8a available
-crc-simd.o : crc-simd.cpp
+crc_simd.o : crc_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(CRC_FLAG) -c) $<
 
 # PCLMUL or ARMv7a/ARMv8a available
-gcm-simd.o : gcm-simd.cpp
+gcm_simd.o : gcm_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(GCM_FLAG) -c) $<
 
+# SSSE3 available
+lea_simd.o : lea_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(LEA_FLAG) -c) $<
+
 # NEON available
-neon-simd.o : neon-simd.cpp
+neon_simd.o : neon_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(NEON_FLAG) -c) $<
 
 # AESNI or ARMv7a/ARMv8a available
-rijndael-simd.o : rijndael-simd.cpp
+rijndael_simd.o : rijndael_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(AES_FLAG) -c) $<
 
 # SSE4.2/SHA-NI or ARMv8a available
-sha-simd.o : sha-simd.cpp
+sha_simd.o : sha_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(SHA_FLAG) -c) $<
 
 # SSE4.2/SHA-NI or ARMv8a available
-shacal2-simd.o : shacal2-simd.cpp
+shacal2_simd.o : shacal2_simd.cpp
 	$(CXX) $(strip $(CXXFLAGS) $(SHA_FLAG) -c) $<
 
 # SSSE3 or NEON available
-simon-simd.o : simon-simd.cpp
+simeck_simd.o : simeck_simd.cpp
-	$(CXX) $(strip $(CXXFLAGS) $(SIMON_FLAG) -c) $<
+	$(CXX) $(strip $(CXXFLAGS) $(SIMECK_FLAG) -c) $<
 
-# SSSE3 or NEON available
+# SSE4.1, NEON or POWER7 available
-speck-simd.o : speck-simd.cpp
+simon64_simd.o : simon64_simd.cpp
-	$(CXX) $(strip $(CXXFLAGS) $(SPECK_FLAG) -c) $<
+	$(CXX) $(strip $(CXXFLAGS) $(SIMON64_FLAG) -c) $<
+
+# SSSE3, NEON or POWER8 available
+simon128_simd.o : simon128_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SIMON128_FLAG) -c) $<
+
+# SSE4.1, NEON or POWER7 available
+speck64_simd.o : speck64_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SPECK64_FLAG) -c) $<
+
+# SSSE3, NEON or POWER8 available
+speck128_simd.o : speck128_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SPECK128_FLAG) -c) $<
+
+# ARMv8.4 available
+sm3_simd.o : sm3_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SM3_FLAG) -c) $<
+
+# AESNI available
+sm4_simd.o : sm4_simd.cpp
+	$(CXX) $(strip $(CXXFLAGS) $(SM4_FLAG) -c) $<
 
 %.o : %.cpp
 	$(CXX) $(strip $(CXXFLAGS) -c) $<
 
-GNUmakefile.deps:
+.PHONY: dep deps depend
-	$(CXX) $(strip $(CXXFLAGS) -MM) *.cpp > GNUmakefile.deps
+dep deps depend GNUmakefile.deps:
+	$(CXX) $(strip $(CXXFLAGS) -DCRYPTOPP_DISABLE_ASM) -MM *.cpp > GNUmakefile.deps

vendor/cryptopp/vendor_cryptopp/History.txt

@@ -4,7 +4,8 @@ The History file contains the items that comprise the release notes. The
 items in the list below used to be in Readme.txt. Readme.txt now contans the
 last several releases.
 
-1.0 - First public release.  Withdrawn at the request of RSA DSI.
+1.0 - First public release
+    - Withdrawn at the request of RSA DSI over patent claims
     - included Blowfish, BBS, DES, DH, Diamond, DSA, ElGamal, IDEA,
       MD5, RC4, RC5, RSA, SHA, WAKE, secret sharing, DEFLATE compression
     - had a serious bug in the RSA key generation code.
@@ -432,3 +433,22 @@ last several releases.
       - remove s_nullNameValuePairs from unnamed namespace
       - ported to MSVC 2017, Xcode 9.3, Sun Studio 12.5, GCC 8.0.1,
         MacPorts GCC 7.0, Clang 4.0, Intel C++ 17.00, IBM XL C/C++ 13.1
+
+8.0.0 - December 28, 2018
+
+      - major release, recompile of programs required
+      - expanded community input and support
+         * 54 unique contributors as of this release
+      - add x25519 key exchange and ed25519 signature scheme
+      - add limited Asymmetric Key Package support from RFC 5958
+      - add Power9 DARN random number generator support
+      - add CHAM, HC-128, HC-256, Hight, LEA, Rabbit, Simeck
+      - fix FixedSizeAllocatorWithCleanup may be unaligned on some platforms
+      - cutover to GNU Make-based cpu feature tests
+      - rename files with dashes to underscores
+      - fix LegacyDecryptor and LegacyDecryptorWithMAC use wrong MAC
+      - fix incorrect AES/CBC decryption on Windows
+      - avoid Singleton<T> when possible, avoid std::call_once completely
+      - fix SPARC alignment problems due to GetAlignmentOf<T>() on word64
+      - add ARM AES asm implementation from Cryptogams
+      - remove CRYPTOPP_ALLOW_UNALIGNED_DATA_ACCESS support

vendor/cryptopp/vendor_cryptopp/Readme.txt

@@ -1,5 +1,5 @@
 Crypto++: free C++ Class Library of Cryptographic Schemes
-Version 7.0 - APR/08/2018
+Version 8.0 - DEC/28/2018
 
 Crypto++ Library is a free C++ class library of cryptographic schemes.
 Currently the library contains the following algorithms:
@@ -39,11 +39,12 @@ Currently the library contains the following algorithms:
    padding schemes for public-key  PKCS#1 v2.0, OAEP, PSS, PSSR, IEEE P1363
                           systems  EMSA2 and EMSA5
 
-                                   Diffie-Hellman (DH), Unified Diffie-Hellman
+                                   Diffie-Hellman (DH), Unified Diffie-Hellman (DH2),
-            key agreement schemes  (DH2), Menezes-Qu-Vanstone (MQV), Hashed MQV (HMQV),
+            key agreement schemes  Menezes-Qu-Vanstone (MQV), Hashed MQV (HMQV),
                                    Fully Hashed MQV (FHMQV), LUCDIF, XTR-DH
 
-      elliptic curve cryptography  ECDSA, Determinsitic ECDSA, ECNR, ECIES, ECDH, ECMQV
+      elliptic curve cryptography  ECDSA, Determinsitic ECDSA, ed25519, ECNR, ECIES,
+                                   ECDH, ECMQV, x25519
 
           insecure or obsolescent  MD2, MD4, MD5, Panama Hash, DES, ARC4, SEAL
 algorithms retained for backwards  3.0, WAKE-OFB, DESX (DES-XEX3), RC2,
@@ -53,7 +54,7 @@ algorithms retained for backwards  3.0, WAKE-OFB, DESX (DES-XEX3), RC2,
 Other features include:
 
   * pseudo random number generators (PRNG): ANSI X9.17 appendix C, RandomPool,
-    VIA Padlock, RDRAND, RDSEED, NIST Hash and HMAC DRBGs
+    DARN, VIA Padlock, RDRAND, RDSEED, NIST Hash and HMAC DRBGs
   * password based key derivation functions: PBKDF1 and PBKDF2 from PKCS #5,
     PBKDF from PKCS #12 appendix B, HKDF from RFC 5869, Scrypt from RFC 7914
   * Shamir's secret sharing scheme and Rabin's information dispersal algorithm
@@ -91,13 +92,13 @@ The following compilers are supported for this release. Please visit
 http://www.cryptopp.com the most up to date build instructions and porting notes.
 
   * Visual Studio 2003 - 2017
-  * GCC 3.3 - 8.0
+  * GCC 3.3 - 9.0
   * Apple Clang 4.3 - 9.3
-  * LLVM Clang 2.9 - 4.0
+  * LLVM Clang 2.9 - 7.0
-  * C++Builder 2010
+  * C++Builder 2015
   * Intel C++ Compiler 9 - 16.0
-  * Sun Studio 12u1 - 12.5
+  * Sun Studio 12u1 - 12.6
-  * IBM XL C/C++ 10.0 - 13.1
+  * IBM XL C/C++ 10.0 - 13.3
 
 *** Important Usage Notes ***
 
@@ -282,6 +283,25 @@ documentation is one of the highest returns on investment.
 The items in this section comprise the most recent history. Please see History.txt
 for the record back to Crypto++ 1.0.
 
+8.0.0 - December 28, 2018
+
+      - major release, recompile of programs required
+      - expanded community input and support
+         * 54 unique contributors as of this release
+      - add x25519 key exchange and ed25519 signature scheme
+      - add limited Asymmetric Key Package support from RFC 5958
+      - add Power9 DARN random number generator support
+      - add CHAM, HC-128, HC-256, Hight, LEA, Rabbit, Simeck
+      - fix FixedSizeAllocatorWithCleanup may be unaligned on some platforms
+      - cutover to GNU Make-based cpu feature tests
+      - rename files with dashes to underscores
+      - fix LegacyDecryptor and LegacyDecryptorWithMAC use wrong MAC
+      - fix incorrect AES/CBC decryption on Windows
+      - avoid Singleton<T> when possible, avoid std::call_once completely
+      - fix SPARC alignment problems due to GetAlignmentOf<T>() on word64
+      - add ARM AES asm implementation from Cryptogams
+      - remove CRYPTOPP_ALLOW_UNALIGNED_DATA_ACCESS support
+
 7.0.0 - April 8, 2018
       - major release, recompile of programs required
       - expanded community input and support
@@ -340,67 +360,6 @@ for the record back to Crypto++ 1.0.
       - ported to MSVC 2017, Xcode 8.1, Sun Studio 12.5, GCC 7.3,
         MacPorts GCC 7.0, Clang 4.0, Intel C++ 17.00, IBM XL C/C++ 13.1
 
-5.6.5 - October 11, 2016
-      - maintenance release, recompile of programs recommended
-      - expanded community input and support
-         * 25 unique contributors as of this release
-      - fixed CVE-2016-7420 (Issue 277, document NDEBUG for production/release)
-      - fixed CVE-2016-7544 (Issue 302, avoid _malloca and _freea)
-      - shipped library in recommended state
-         * backwards compatibility achieved with <config.compat>
-      - Visual Studio project file cleanup
-         * improved X86 and X64 MSBuild support
-         * added ARM-based MSBuild awareness
-      - improved Testing and QA
-         * expanded platforms and compilers
-         * expanded Coverity into OS X and Windows platforms
-         * added Windows test scripts using Strawberry Perl
-      - ported to MSVC 2015 SP3, Xcode 7.3, Sun Studio 12.5, GCC 7.0,
-        MacPorts GCC 7.0, Clang 3.8, Intel C++ 17.00
-
-5.6.4 - September 11, 2016
-      - maintenance release, honored API/ABI/Versioning requirements
-      - expanded community input and support
-         * 22 unique contributors for this release
-      - fixed CVE-2016-3995
-      - changed SHA3 to FIPS 202 (F1600, XOF d=0x06)
-      - added Keccak (F1600, XOF d=0x01)
-      - added ChaCha (ChaCha8/12/20)
-      - added HMQV and FHMQV
-         * Hashed and Fully Hashed MQV
-      - added BLAKE2 (BLAKE2s and BLAKE2b)
-         * C++, SSE2, SSE4, ARM NEON and ARMv8 ASIMD
-      - added CRC32-C
-         * C/C++, Amd64 CRC, and ARMv8 CRC
-      - improved Rabin-William signatures
-         * Tweaked roots <em>e</em> and <em>f</em>
-      - improved C++11 support
-         * atomics, threads and fences
-         * alginof, alignas
-         * constexpr
-         * noexcept
-      - improved GCM mode
-         * ARM NEON and ARMv8 ASIMD
-         * ARMv8 carry-less multiply
-      - improved Windows 8 and 10 support
-         * Windows Phone, Universal Windows Platform, Windows Store
-      - improved MIPS, ARMv7 and ARMv8 support
-         * added scripts setenv-{android|embedded|ios}.sh for GNUmakefile-cross
-         * aggressive use of -march=<arch> and -mfpu=<fpu> in cryptest.sh
-      - improved build systems
-         * Visual Studio 2010 default
-         * added CMake support (lacks FindCryptopp.cmake)
-         * archived VC++ 5/0/6.0 project files (vc60.zip)
-         * archived VS2005 project files (vs2005.zip)
-         * archived Borland project files (bds10.zip)
-      - improved Testing and QA
-         * expanded platforms and compilers
-         * added code generation tests based on CPU features
-         * added C++03, C++11, C++14, C++17 testing
-         * added -O3, -O5, -Ofast and -Os testing
-      - ported to MSVC 2015 SP3, Xcode 9.0, Sun Studio 12.5, GCC 7.0,
-        MacPorts GCC 7.0, Clang 3.8, Intel C++ 17.00
-
 June 2015 - Changing of the guard. Wei Dai turned the library over to the
         community. The first community release was Crypto++ 5.6.3. Wei is
         no longer involved with the daily operations of the project. Wei

vendor/cryptopp/vendor_cryptopp/TestData/defdmac1.bin

@@ -0,0 +1 @@
+nCBţpIbź_¤_ĺ|źâ+"w˙¶¶†Ž˝í#Aýíż‡„nŢmÖéŠČ‡ű%±––kXřťąRňłüw`Ză‘^s.+ÚÚµ<C39A>ř„Š`ŠÝBf]z
ó•}¦R`}\ŘdiśbfttÉ^ř˛

vendor/cryptopp/vendor_cryptopp/TestData/ed25519.dat

@@ -0,0 +1 @@
+302E020100300506032B65700422042098C59D3F097FB23D44BA90791281B453258D691A55AF5CE4F1EE712FDF91AE6D

vendor/cryptopp/vendor_cryptopp/TestData/ed25519v0.dat

@@ -0,0 +1 @@
+302E020100300506032B65700422042030BF776A497D7F1E0E26AC4FB03F5BE7E187DDFEFB914CD292A6FEDB7F70CE6B

vendor/cryptopp/vendor_cryptopp/TestData/ed25519v1.dat

@@ -0,0 +1 @@
+3053020101300506032B6570042204206861FD53C7643DABDCDF4C3969CE44A156BAC261242A5AAEC140EDE510071C6CA12303210029CF90E6C1CF1ADC7105720303B2EE303412D2B682C6FEEF3D8736A286B2E27F

vendor/cryptopp/vendor_cryptopp/TestData/x25519.dat

@@ -0,0 +1 @@
+302E020100300506032B656E0422042030D407BB0CC97D0EC493BDB00A4A8EFA06A50D2388F5BA62947030E7D9873F49

vendor/cryptopp/vendor_cryptopp/TestData/x25519v0.dat

@@ -0,0 +1 @@
+302E020100300506032B656E042204202818E54DE6B88EEF3E99E25042CB98E69373B222E4C1E8B3FB10AC9B26C1007B

vendor/cryptopp/vendor_cryptopp/TestData/x25519v1.dat

@@ -0,0 +1 @@
+3053020101300506032B656E04220420A00FADD6D29BE764B851F64F7620E80B700DF65914BED31E486362281BB5D061A123032100E9AD4CC54DAA36F312D98B253854F0076E2BC26FCE5802B3AC79A5B59B3D2C4F

vendor/cryptopp/vendor_cryptopp/TestPrograms/test_32bit.cxx

@@ -0,0 +1,7 @@
+#include <cstddef>
+int main(int argc, char* argv[])
+{
+	enum {N = (sizeof(std::size_t) == 4 ? 4 : -1)};
+	int x[N];
+	return 0;
+}

vendor/cryptopp/vendor_cryptopp/TestPrograms/test_64bit.cxx

@@ -0,0 +1,7 @@
+#include <cstddef>
+int main(int argc, char* argv[])
+{
+	enum {N = (sizeof(std::size_t) == 8 ? 8 : -1)};
+	int x[N];
+	return 0;
+}

vendor/cryptopp/vendor_cryptopp/TestPrograms/test_arm_acle.cxx

@@ -0,0 +1,5 @@
+#include <arm_acle.h>
+int main(int argc, char* argv[])
+{
+	return 0;
+}

vendor/cryptopp/vendor_cryptopp/TestPrograms/test_arm_aes.cxx

@@ -0,0 +1,19 @@
+#include <arm_neon.h>
+#include <stdint.h>
+
+// test_acle.h determines if this is available. Then,
+// -DCRYPTOPP_ARM_ACLE_AVAILABLE=0 is added to CXXFLAGS
+// if the ACLE header is not available.
+#if (CRYPTOPP_ARM_ACLE_AVAILABLE)
+#  include <arm_acle.h>
+#endif
+
+int main(int argc, char* argv[])
+{
+	uint8x16_t x={0};
+	x=vaeseq_u8(x,x);
+	x=vaesmcq_u8(x);
+	x=vaesdq_u8(x,x);
+	x=vaesimcq_u8(x);
+	return 0;
+}