hardcoresushi
/
libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge branch 'develop' into feature/library_intermediate

This commit is contained in:
Sebastian Messmer 2019-01-20 20:21:54 -08:00
parent 0d54e2cd92 ee8be26c83
commit b526c3fd89
375 changed files with 57909 additions and 26506 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
.circleci/config.yml
View File

@ -173,14 +173,14 @@ references:
command: |
if "${RUN_TESTS}"; then
cd cmake
./test/gitversion/gitversion-test
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cpp-utils/cpp-utils-test ; fi
if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ; fi
./test/parallelaccessstore/parallelaccessstore-test
./test/blockstore/blockstore-test
./test/blobstore/blobstore-test
./test/cryfs/cryfs-test
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs-cli/cryfs-cli-test ; fi
./test/gitversion/gitversion-test ${GTEST_ARGS}
./test/cpp-utils/cpp-utils-test ${GTEST_ARGS}
if [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ${GTEST_ARGS} ; fi
./test/parallelaccessstore/parallelaccessstore-test ${GTEST_ARGS}
./test/blockstore/blockstore-test ${GTEST_ARGS}
./test/blobstore/blobstore-test ${GTEST_ARGS}
./test/cryfs/cryfs-test ${GTEST_ARGS}
./test/cryfs-cli/cryfs-cli-test ${GTEST_ARGS}
fi
job_definition: &job_definition
<<: *container_config
@ -212,6 +212,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-5"
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_5_release:
@ -223,6 +224,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-5"
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_6_debug:
@ -234,6 +236,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-6"
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_6_release:
@ -245,6 +248,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-6"
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_7_debug:
@ -256,6 +260,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-7"
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_7_release:
@ -267,6 +272,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-7"
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_8_debug:
@ -278,6 +284,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-8"
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
gcc_8_release:
@ -289,6 +296,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-8"
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_4_debug:
@ -300,6 +308,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-4.0
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_4_release:
@ -311,6 +320,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-4.0
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_5_debug:
@ -322,6 +332,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_5_release:
@ -333,6 +344,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_6_debug:
@ -344,6 +356,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-6.0
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_6_release:
@ -355,6 +368,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-6.0
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_7_debug:
@ -366,6 +380,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-7
CXXFLAGS: ""
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_7_release:
@ -377,6 +392,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-7
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_werror:
@ -388,6 +404,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-7
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: "-DUSE_WERROR=on"
RUN_TESTS: false
gcc_werror:
@ -399,6 +416,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-8"
CXXFLAGS: ""
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: "-DUSE_WERROR=on"
RUN_TESTS: false
gcc_werror:
@ -410,6 +428,7 @@ jobs:
APT_COMPILER_PACKAGE: "g++-8"
CXXFLAGS: "-Werror"
BUILD_TYPE: "Release"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: false
no_compatibility:
@ -421,6 +440,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-7
CXXFLAGS: "-DCRYFS_NO_COMPATIBILITY"
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
address_sanitizer:
@ -434,6 +454,7 @@ jobs:
BUILD_TYPE: "Debug"
ASAN_OPTIONS: "detect_leaks=1 check_initialization_order=1 detect_stack_use_after_return=1 detect_invalid_pointer_pairs=1 atexit=1"
DISABLE_BROKEN_ASAN_TESTS: true
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
ub_sanitizer:
@ -445,6 +466,7 @@ jobs:
APT_COMPILER_PACKAGE: clang-7
CXXFLAGS: "-O2 -fno-sanitize-recover=undefined,nullability,implicit-conversion,unsigned-integer-overflow -fno-omit-frame-pointer -fno-common"
BUILD_TYPE: "Debug"
GTEST_ARGS: ""
CMAKE_FLAGS: ""
RUN_TESTS: true
thread_sanitizer:
@ -457,7 +479,7 @@ jobs:
OMP_NUM_THREADS: "1"
CXXFLAGS: "-O2 -fsanitize=thread -fno-omit-frame-pointer"
BUILD_TYPE: "Debug"
DISABLE_BROKEN_TSAN_TESTS: true
GTEST_ARGS: "--gtest_filter=-LoggingTest.LoggingAlsoWorksAfterFork:AssertTest_DebugBuild.*:CliTest_Setup.*:CliTest_IntegrityCheck.*:*/CliTest_WrongEnvironment.*:CliTest_Unmount.*"
CMAKE_FLAGS: ""
RUN_TESTS: true
clang_tidy:

3
.travisci/install.sh
View File

@ -2,6 +2,9 @@
set -e
# Workaround homebrew bug, see https://twitter.com/axccl/status/1083393735277363205 and https://github.com/openPMD/openPMD-api/pull/431/files
brew upgrade --cleanup || brew upgrade --cleanup
# Install newer GCC if we're running on GCC
if [ "${CXX}" == "g++" ]; then
# We need to uninstall oclint because it creates a /usr/local/include/c++ symlink that clashes with the gcc5 package

5
ChangeLog.txt
View File

@ -12,16 +12,19 @@ New Features & Improvements:
* New block size options: 4KB and 16KB
* New default block size: 16KB. This should decrease the size of the ciphertext directory for most users.
* Increased scrypt hardness to (N=1048576, r=4, p=8) to make it harder to crack the key while allowing cryfs to take advantage of multicore machines.
* cryfs-unmount tool to unmount filesystems
Fixed bugs:
* `du` shows correct file system size on Mac OS X.
* On Mac OS X, Finder shows the correct name for the mount directory
Version 0.9.10 (unreleased)
Version 0.9.10
--------------
Fixed bugs:
* Fixed occasional deadlock (https://github.com/cryfs/cryfs/issues/64)
* Fix for reading empty files out of bounds
* Fixed race condition (https://github.com/cryfs/cryfs/issues/224 and https://github.com/cryfs/cryfs/issues/243)
Version 0.9.9

34
appveyor.yml
View File

@ -12,7 +12,7 @@ platform:
configuration:
- Debug
- RelWithDebInfo
# - Release
- Release
version: '{branch}-{build}'
@ -20,13 +20,11 @@ init:
- echo %NUMBER_OF_PROCESSORS%
- echo %PLATFORM%
- echo %APPVEYOR_BUILD_WORKER_IMAGE%
- set arch=
- if "%PLATFORM%"=="x64" ( set arch= Win64)
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2013" (set generator=Visual Studio 12 2013%arch%)
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" (set generator=Visual Studio 14 2015%arch%)
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" (set generator=Visual Studio 15 2017%arch%)
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017 Preview" (set generator=Visual Studio 15 2017%arch%)
- echo %generator%
- set arch=32
- if "%PLATFORM%"=="x64" ( set arch=64)
- set VisualStudioVersion=2017
- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017 Preview" ( set VisualStudioVersion=Preview)
- cmd: call "C:\Program Files (x86)\Microsoft Visual Studio\%VisualStudioVersion%\Community\VC\Auxiliary\Build\vcvars%arch%.bat"
install:
- choco install -y dokany --version 1.1.0.2000 --installargs INSTALLDEVFILES=1
@ -35,17 +33,17 @@ install:
build_script:
- cmd: mkdir build
- cmd: cd build
- cmd: cmake .. -G "%generator%" -DBUILD_TESTING=on -DBOOST_ROOT="C:/Libraries/boost_1_65_1" -DDOKAN_PATH="C:/Program Files/Dokan/DokanLibrary-1.1.0"
# TODO Make build parallel
- cmd: cmake .. -G "Ninja" -DBUILD_TESTING=on -DBOOST_ROOT="C:/Libraries/boost_1_65_1" -DDOKAN_PATH="C:/Program Files/Dokan/DokanLibrary-1.1.0"
- cmd: cmake --build . --config %CONFIGURATION%
- cmd: .\test\gitversion\%CONFIGURATION%\gitversion-test.exe
- cmd: cd .\test\cpp-utils\%CONFIGURATION%\ && .\cpp-utils-test.exe && cd ..\..\..
#- cmd: .\test\fspp\%CONFIGURATION%\fspp-test.exe
- cmd: .\test\parallelaccessstore\%CONFIGURATION%\parallelaccessstore-test.exe
- cmd: .\test\blockstore\%CONFIGURATION%\blockstore-test.exe
- cmd: .\test\blobstore\%CONFIGURATION%\blobstore-test.exe
- cmd: .\test\cryfs\%CONFIGURATION%\cryfs-test.exe
#- cmd: .\test\cryfs-cli\%CONFIGURATION%\cryfs-cli-test.exe
- cmd: .\test\gitversion\gitversion-test.exe
# cpp-utils-test disables ThreadDebuggingTest_ThreadName.*_thenIsCorrect because the appveyor image is too old to support the API needed for that
- cmd: cd .\test\cpp-utils\ && .\cpp-utils-test.exe --gtest_filter=-ThreadDebuggingTest_ThreadName.*_thenIsCorrect && cd ..\..
#- cmd: .\test\fspp\fspp-test.exe
- cmd: .\test\parallelaccessstore\parallelaccessstore-test.exe
- cmd: .\test\blockstore\blockstore-test.exe
- cmd: .\test\blobstore\blobstore-test.exe
- cmd: .\test\cryfs\cryfs-test.exe
#- cmd: .\test\cryfs-cli\cryfs-cli-test.exe
- cmd: cpack -C %CONFIGURATION% --verbose -G WIX

1
src/CMakeLists.txt
View File

@ -8,4 +8,5 @@ add_subdirectory(blockstore)
add_subdirectory(blobstore)
add_subdirectory(cryfs)
add_subdirectory(cryfs-cli)
add_subdirectory(cryfs-unmount)
add_subdirectory(stats)

1
src/blobstore/CMakeLists.txt
View File

@ -11,6 +11,7 @@ set(SOURCES
implementations/onblocks/datanodestore/DataInnerNode.cpp
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/impl/CachedValue.cpp
implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
implementations/onblocks/datatreestore/LeafHandle.cpp
implementations/onblocks/datatreestore/DataTree.cpp

110
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
View File

@ -8,13 +8,9 @@
#include <cpp-utils/assert/assert.h>
#include "datatreestore/LeafHandle.h"
using std::function;
using std::unique_lock;
using std::mutex;
using cpputils::unique_ref;
using cpputils::Data;
using blockstore::BlockId;
using blobstore::onblocks::datatreestore::LeafHandle;
namespace blobstore {
namespace onblocks {
@ -22,128 +18,34 @@ namespace onblocks {
using parallelaccessdatatreestore::DataTreeRef;
BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
: _datatree(std::move(datatree)), _sizeCache(boost::none), _mutex() {
: _datatree(std::move(datatree)) {
}
BlobOnBlocks::~BlobOnBlocks() {
} // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
uint64_t BlobOnBlocks::size() const {
if (_sizeCache == boost::none) {
_sizeCache = _datatree->numStoredBytes();
}
return *_sizeCache;
return _datatree->numBytes();
}
void BlobOnBlocks::resize(uint64_t numBytes) {
_datatree->resizeNumBytes(numBytes);
_sizeCache = numBytes;
}
void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
unique_lock<mutex> lock(_mutex); // TODO Multiple traverse calls in parallel?
uint64_t endByte = beginByte + sizeBytes;
uint64_t maxBytesPerLeaf = _datatree->maxBytesPerLeaf();
uint32_t firstLeaf = beginByte / maxBytesPerLeaf;
uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
bool blobIsGrowingFromThisTraversal = false;
auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
// If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
if (isRightBorderLeaf) {
ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
auto leaf = leafHandle.node();
if (leaf->numBytes() < dataEnd) {
leaf->resize(dataEnd);
blobIsGrowingFromThisTraversal = true;
}
}
onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
};
auto _onCreateLeaf = [&onCreateLeaf, maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex) -> Data {
blobIsGrowingFromThisTraversal = true;
uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
ASSERT(leafIndex == endLeaf-1 || dataEnd == maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
// If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
if (dataBegin != 0) {
Data actualData(dataBegin + data.size());
std::memset(actualData.data(), 0, dataBegin);
std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
data = std::move(actualData);
}
return data;
};
_datatree->traverseLeaves(firstLeaf, endLeaf, _onExistingLeaf, _onCreateLeaf);
if (blobIsGrowingFromThisTraversal) {
ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
_sizeCache = endByte;
}
}
Data BlobOnBlocks::readAll() const {
//TODO Querying size is inefficient. Is this possible without a call to size()?
uint64_t count = size();
Data result(count);
_read(result.data(), 0, count);
return result;
return _datatree->readAllBytes();
}
void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
uint64_t _size = size();
ASSERT(offset <= _size && offset + count <= _size, "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t read = tryRead(target, offset, count);
ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
return _datatree->readBytes(target, offset, count);
}
uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(size())-static_cast<int64_t>(offset)));
_read(target, offset, realCount);
return realCount;
}
void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
//TODO Simplify formula, make it easier to understand
leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
};
auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
ASSERT(false, "Reading shouldn't create new leaves.");
};
_traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
return _datatree->tryReadBytes(target, offset, count);
}
void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
Data leafData(leafDataSize);
std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
} else {
//TODO Simplify formula, make it easier to understand
leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
leafDataSize);
}
};
auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
Data result(numBytes);
//TODO Simplify formula, make it easier to understand
std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
return result;
};
_traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
_datatree->writeBytes(source, offset, count);
}
void BlobOnBlocks::flush() {

4
src/blobstore/implementations/onblocks/BlobOnBlocks.h
View File

@ -7,6 +7,7 @@
#include <memory>
#include <boost/optional.hpp>
#include <boost/thread/shared_mutex.hpp>
namespace blobstore {
namespace onblocks {
@ -38,12 +39,11 @@ public:
private:
uint64_t _tryRead(void *target, uint64_t offset, uint64_t size) const;
void _read(void *target, uint64_t offset, uint64_t count) const;
void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datatreestore::LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
mutable boost::optional<uint64_t> _sizeCache;
mutable std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
};

253
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
View File

@ -12,6 +12,7 @@
#include <cmath>
#include <cpp-utils/assert/assert.h>
#include "impl/LeafTraverser.h"
#include <boost/thread.hpp>
using blockstore::BlockId;
using blobstore::onblocks::datanodestore::DataNodeStore;
@ -24,17 +25,21 @@ using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
using boost::none;
using boost::optional;
using cpputils::optional_ownership_ptr;
using cpputils::unique_ref;
using cpputils::Data;
using namespace cpputils::logging;
//TODO shared_lock currently not enough for traverse because of root replacement. Can be fixed while keeping shared?
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _numLeavesCache(none) {
: _treeStructureMutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _sizeCache() {
}
DataTree::~DataTree() {
@ -45,102 +50,143 @@ const BlockId &DataTree::blockId() const {
}
void DataTree::flush() const {
// By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed
unique_lock<shared_mutex> lock(_mutex);
// By grabbing a lock, we ensure that all modifying functions don't run currently and are therefore flushed.
// It's only a shared lock, because this doesn't modify the tree structure.
shared_lock<shared_mutex> lock(_treeStructureMutex);
// We also have to flush the root node
_rootNode->flush();
}
unique_ref<DataNode> DataTree::releaseRootNode() {
unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
// Lock also ensures that the root node is currently set (traversing unsets it temporarily)
// It's a unique lock because this "modifies" tree structure by changing _rootNode.
unique_lock<shared_mutex> lock(_treeStructureMutex);
return std::move(_rootNode);
}
//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
uint32_t DataTree::numLeaves() const {
shared_lock<shared_mutex> lock(_mutex);
return _numLeaves();
shared_lock<shared_mutex> lock(_treeStructureMutex);
return _getOrComputeSizeCache().numLeaves;
}
uint32_t DataTree::_numLeaves() const {
if (_numLeavesCache == none) {
_numLeavesCache = _computeNumLeaves(*_rootNode);
}
return *_numLeavesCache;
uint64_t DataTree::numBytes() const {
shared_lock<shared_mutex> lock(_treeStructureMutex);
return _numBytes();
}
uint32_t DataTree::_forceComputeNumLeaves() const {
unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
_numLeavesCache = _computeNumLeaves(*_rootNode);
return *_numLeavesCache;
uint64_t DataTree::_numBytes() const {
return _getOrComputeSizeCache().numBytes;
}
uint32_t DataTree::_computeNumLeaves(const DataNode &node) const {
DataTree::SizeCache DataTree::_getOrComputeSizeCache() const {
return _sizeCache.getOrCompute([this] () {
return _computeSizeCache(*_rootNode);
});
}
uint32_t DataTree::forceComputeNumLeaves() const {
_sizeCache.clear();
return numLeaves();
}
DataTree::SizeCache DataTree::_computeSizeCache(const DataNode &node) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
if (leaf != nullptr) {
return 1;
return {1, leaf->numBytes()};
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
uint64_t numLeavesInLeftChildren = static_cast<uint64_t>(inner.numChildren()-1) * leavesPerFullChild(inner);
uint32_t numLeavesInLeftChildren = static_cast<uint32_t>(inner.numChildren()-1) * _leavesPerFullChild(inner);
uint64_t numBytesInLeftChildren = numLeavesInLeftChildren * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numLeavesInRightChild = _computeNumLeaves(**lastChild);
SizeCache sizeInRightChild = _computeSizeCache(**lastChild);
return numLeavesInLeftChildren + numLeavesInRightChild;
return SizeCache {
numLeavesInLeftChildren + sizeInRightChild.numLeaves,
numBytesInLeftChildren + sizeInRightChild.numBytes
};
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
//TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()?
std::unique_lock<shared_mutex> lock(_mutex);
ASSERT(beginIndex <= endIndex, "Invalid parameters");
void DataTree::_traverseLeavesByLeafIndices(uint32_t beginIndex, uint32_t endIndex, bool readOnlyTraversal,
function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
function<Data (uint32_t index)> onCreateLeaf,
function<void (DataInnerNode *node)> onBacktrackFromSubtree) const {
if(endIndex <= beginIndex) {
return;
}
auto onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
// TODO no const cast
LeafTraverser(_nodeStore, readOnlyTraversal).traverseAndUpdateRoot(&const_cast<DataTree*>(this)->_rootNode, beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
_traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
void DataTree::_traverseLeavesByByteIndices(uint64_t beginByte, uint64_t sizeBytes, bool readOnlyTraversal, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
if (sizeBytes == 0) {
return;
}
if (_numLeavesCache != none && *_numLeavesCache < endIndex) {
_numLeavesCache = endIndex;
uint64_t endByte = beginByte + sizeBytes;
uint64_t _maxBytesPerLeaf = maxBytesPerLeaf();
uint32_t firstLeaf = beginByte / _maxBytesPerLeaf;
uint32_t endLeaf = utils::ceilDivision(endByte, _maxBytesPerLeaf);
bool blobIsGrowingFromThisTraversal = false;
auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, _maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
uint64_t indexOfFirstLeafByte = leafIndex * _maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(_maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
// If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
if (isRightBorderLeaf) {
ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
auto leaf = leafHandle.node();
if (leaf->numBytes() < dataEnd) {
leaf->resize(dataEnd);
blobIsGrowingFromThisTraversal = true;
}
}
onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
};
auto _onCreateLeaf = [&onCreateLeaf, _maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal, readOnlyTraversal] (uint32_t leafIndex) -> Data {
ASSERT(!readOnlyTraversal, "Cannot create leaves in a read-only traversal");
blobIsGrowingFromThisTraversal = true;
uint64_t indexOfFirstLeafByte = leafIndex * _maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(_maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
ASSERT(leafIndex == endLeaf-1 || dataEnd == _maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
// If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
if (dataBegin != 0) {
Data actualData(dataBegin + data.size());
std::memset(actualData.data(), 0, dataBegin);
std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
data = std::move(actualData);
}
return data;
};
auto _onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
_traverseLeavesByLeafIndices(firstLeaf, endLeaf, readOnlyTraversal, _onExistingLeaf, _onCreateLeaf, _onBacktrackFromSubtree);
ASSERT(!readOnlyTraversal || !blobIsGrowingFromThisTraversal, "Blob grew from traversal that didn't allow growing (i.e. reading)");
if (blobIsGrowingFromThisTraversal) {
_sizeCache.update([endLeaf, endByte] (optional<SizeCache>* cache) {
*cache = SizeCache{endLeaf, endByte};
});
}
}
void DataTree::_traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
function<Data (uint32_t index)> onCreateLeaf,
function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
_rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
uint32_t DataTree::_leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(root.depth())-1);
}
uint64_t DataTree::numStoredBytes() const {
shared_lock<shared_mutex> lock(_mutex);
return _numStoredBytes();
}
uint64_t DataTree::_numStoredBytes() const {
return _numStoredBytes(*_rootNode);
}
uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&root);
if (leaf != nullptr) {
return leaf->numBytes();
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
return numBytesInLeftChildren + numBytesInRightChild;
}
void DataTree::resizeNumBytes(uint64_t newNumBytes) {
std::unique_lock<shared_mutex> lock(_mutex); // TODO Multiple ones in parallel? Also in parallel with traverseLeaves()?
std::unique_lock<shared_mutex> lock(_treeStructureMutex);
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf();
@ -171,8 +217,11 @@ void DataTree::resizeNumBytes(uint64_t newNumBytes) {
}
};
_traverseLeaves(newNumLeaves - 1, newNumLeaves, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
_numLeavesCache = newNumLeaves;
_traverseLeavesByLeafIndices(newNumLeaves - 1, newNumLeaves, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
_sizeCache.update([newNumLeaves, newNumBytes] (boost::optional<SizeCache>* cache) {
*cache = SizeCache{newNumLeaves, newNumBytes};
});
}
uint64_t DataTree::maxBytesPerLeaf() const {
@ -180,9 +229,87 @@ uint64_t DataTree::maxBytesPerLeaf() const {
}
uint8_t DataTree::depth() const {
shared_lock<shared_mutex> lock(_treeStructureMutex);
return _rootNode->depth();
}
void DataTree::readBytes(void *target, uint64_t offset, uint64_t count) const {
shared_lock<shared_mutex> lock(_treeStructureMutex);
const uint64_t _size = _numBytes();
if(offset > _size || offset + count > _size) {
throw std::runtime_error("BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
const uint64_t read = _tryReadBytes(target, offset, count);
if (read != count) {
throw std::runtime_error("BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
}
Data DataTree::readAllBytes() const {
shared_lock<shared_mutex> lock(_treeStructureMutex);
//TODO Querying numBytes can be inefficient. Is this possible without a call to size()?
uint64_t count = _numBytes();
Data result(count);
_doReadBytes(result.data(), 0, count);
return result;
}
uint64_t DataTree::tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
shared_lock<shared_mutex> lock(_treeStructureMutex);
auto result = _tryReadBytes(target, offset, count);
return result;
}
uint64_t DataTree::_tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
const uint64_t _size = _numBytes();
const uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(_size)-static_cast<int64_t>(offset)));
_doReadBytes(target, offset, realCount);
return realCount;
}
void DataTree::_doReadBytes(void *target, uint64_t offset, uint64_t count) const {
auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
//TODO Simplify formula, make it easier to understand
leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
};
auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
ASSERT(false, "Reading shouldn't create new leaves.");
};
_traverseLeavesByByteIndices(offset, count, true, onExistingLeaf, onCreateLeaf);
}
void DataTree::writeBytes(const void *source, uint64_t offset, uint64_t count) {
unique_lock<shared_mutex> lock(_treeStructureMutex);
auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
Data leafData(leafDataSize);
std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
} else {
//TODO Simplify formula, make it easier to understand
leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
leafDataSize);
}
};
auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
Data result(numBytes);
//TODO Simplify formula, make it easier to understand
std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
return result;
};
_traverseLeavesByByteIndices(offset, count, false, onExistingLeaf, onCreateLeaf);
}
}
}
}

49
src/blobstore/implementations/onblocks/datatreestore/DataTree.h
View File

@ -10,6 +10,7 @@
#include <boost/thread/shared_mutex.hpp>
#include <blockstore/utils/BlockId.h>
#include "LeafHandle.h"
#include "impl/CachedValue.h"
namespace blobstore {
namespace onblocks {
@ -31,39 +32,57 @@ public:
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
uint64_t tryReadBytes(void *target, uint64_t offset, uint64_t count) const;
void readBytes(void *target, uint64_t offset, uint64_t count) const;
cpputils::Data readAllBytes() const;
void writeBytes(const void *source, uint64_t offset, uint64_t count);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
uint64_t numBytes() const;
uint8_t depth() const;
// only used by test cases
uint32_t _forceComputeNumLeaves() const;
uint32_t forceComputeNumLeaves() const;
void flush() const;
private:
mutable boost::shared_mutex _mutex;
// This mutex must protect the tree structure, i.e. which nodes exist and how they're connected.
// Also protects total number of bytes (i.e. number of leaves + size of last leaf).
// It also protects the data in leaf nodes, because writing bytes might grow the blob and change the structure.
mutable boost::shared_mutex _treeStructureMutex;
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _rootNode;
blockstore::BlockId _blockId; // BlockId is stored in a member variable, since _rootNode is nullptr while traversing, but we still want to be able to return the blockId.
mutable boost::optional<uint32_t> _numLeavesCache;
struct SizeCache final {
uint32_t numLeaves;
uint64_t numBytes;
};
mutable CachedValue<SizeCache> _sizeCache;
cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
friend class DataTreeStore;
//TODO Use underscore for private methods
void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
uint64_t _numStoredBytes() const;
uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
uint32_t _numLeaves() const;
uint32_t _computeNumLeaves(const datanodestore::DataNode &node) const;
void _traverseLeavesByLeafIndices(uint32_t beginIndex, uint32_t endIndex, bool readOnlyTraversal,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree) const;
void _traverseLeavesByByteIndices(uint64_t beginByte, uint64_t sizeBytes, bool readOnlyTraversal, std::function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
uint32_t _leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
SizeCache _getOrComputeSizeCache() const;
SizeCache _computeSizeCache(const datanodestore::DataNode &node) const;
uint64_t _tryReadBytes(void *target, uint64_t offset, uint64_t count) const;
void _doReadBytes(void *target, uint64_t offset, uint64_t count) const;
uint64_t _numBytes() const;
DISALLOW_COPY_AND_ASSIGN(DataTree);
};

2
src/blobstore/implementations/onblocks/datatreestore/impl/CachedValue.cpp Normal file
View File

@ -0,0 +1,2 @@
#include "CachedValue.h"

46
src/blobstore/implementations/onblocks/datatreestore/impl/CachedValue.h Normal file
View File

@ -0,0 +1,46 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_CACHEDVALUE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_CACHEDVALUE_H_
#include <boost/optional.hpp>
#include <boost/thread/shared_mutex.hpp>
#include <functional>
namespace blobstore {
namespace onblocks {
// TODO Test
template<class T>
class CachedValue final {
public:
CachedValue() :_cache(boost::none), _mutex() {}
T getOrCompute(std::function<T ()> compute) {
boost::upgrade_lock<boost::shared_mutex> readLock(_mutex);
if (_cache == boost::none) {
boost::upgrade_to_unique_lock<boost::shared_mutex> writeLock(readLock);
_cache = compute();
}
return *_cache;
}
void update(std::function<void (boost::optional<T>*)> func) {
boost::unique_lock<boost::shared_mutex> writeLock(_mutex);
func(&_cache);
}
void clear() {
update([] (boost::optional<T>* cache) {
*cache = boost::none;
});
}
private:
boost::optional<T> _cache;
boost::shared_mutex _mutex;
};
}
}
#endif

50
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
View File

@ -20,15 +20,15 @@ namespace blobstore {
namespace onblocks {
namespace datatreestore {
LeafTraverser::LeafTraverser(DataNodeStore *nodeStore)
: _nodeStore(nodeStore) {
LeafTraverser::LeafTraverser(DataNodeStore *nodeStore, bool readOnlyTraversal)
: _nodeStore(nodeStore), _readOnlyTraversal(readOnlyTraversal) {
}
unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
void LeafTraverser::traverseAndUpdateRoot(unique_ref<DataNode>* root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
_traverseAndUpdateRoot(root, beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
void LeafTraverser::_traverseAndUpdateRoot(unique_ref<DataNode>* root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
//TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations:
@ -36,11 +36,12 @@ namespace blobstore {
// beginIndex<numLeaves<endIndex, beginIndex=numLeaves<endIndex,
// numLeaves<beginIndex<endIndex, numLeaves<beginIndex=endIndex
uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth());
uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth((*root)->depth());
bool increaseTreeDepth = endIndex > maxLeavesForDepth;
ASSERT(!_readOnlyTraversal || !increaseTreeDepth, "Tried to grow a tree on a read only traversal");
if (root->depth() == 0) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root.get());
if ((*root)->depth() == 0) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root->get());
ASSERT(leaf != nullptr, "Depth 0 has to be leaf node");
if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
@ -51,7 +52,7 @@ namespace blobstore {
onExistingLeaf(0, isRightBorderLeaf, LeafHandle(_nodeStore, leaf));
}
} else {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root->get());
ASSERT(inner != nullptr, "Depth != 0 has to be leaf node");
_traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth),
std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, !increaseTreeDepth,
@ -63,17 +64,21 @@ namespace blobstore {
// We don't increase to the full needed tree depth in one step, because we want the traversal to go as far as possible
// and only then increase the depth - this causes the tree to be in consistent shape (balanced) for longer.
if (increaseTreeDepth) {
ASSERT(!_readOnlyTraversal, "Can't increase tree depth in a read-only traversal");
// TODO Test cases that increase tree depth by 0, 1, 2, ... levels
auto newRoot = _increaseTreeDepth(std::move(root));
return _traverseAndReturnRoot(std::move(newRoot), std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
*root = _increaseTreeDepth(std::move(*root));
_traverseAndUpdateRoot(root, std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
} else {
// Once we're done growing the tree and done with the traversal, we might have to decrease tree depth,
// because the callbacks could have deleted nodes (this happens for example when shrinking the tree using a traversal).
return _whileRootHasOnlyOneChildReplaceRootWithItsChild(std::move(root));
_whileRootHasOnlyOneChildReplaceRootWithItsChild(root);
}
}
unique_ref<DataInnerNode> LeafTraverser::_increaseTreeDepth(unique_ref<DataNode> root) {
ASSERT(!_readOnlyTraversal, "Can't increase tree depth in a read-only traversal");
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*root);
return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot);
}
@ -85,6 +90,7 @@ namespace blobstore {
LeafHandle leafHandle(_nodeStore, blockId);
if (growLastLeaf) {
if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
ASSERT(!_readOnlyTraversal, "Can't grow the last leaf in a read-only traversal");
leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
}
@ -116,6 +122,7 @@ namespace blobstore {
ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function.");
uint32_t numChildren = root->numChildren();
ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists");
ASSERT(!_readOnlyTraversal || endChild <= numChildren, "Can only traverse out of bounds in a read-only traversal");
bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren;
// If we traverse outside of the valid region (i.e. usually would only traverse to new leaves and not to the last leaf),
@ -146,6 +153,8 @@ namespace blobstore {
// Traverse new children (including gap children, i.e. children that are created but not traversed because they're to the right of the current size, but to the left of the traversal region)
for (uint32_t childIndex = numChildren; childIndex < endChild; ++childIndex) {
ASSERT(!_readOnlyTraversal, "Can't create new children in a read-only traversal");
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset));
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
@ -161,6 +170,8 @@ namespace blobstore {
}
unique_ref<DataNode> LeafTraverser::_createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(!_readOnlyTraversal, "Can't create a new subtree in a read-only traversal");
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == depth) {
ASSERT(beginIndex <= 1 && endIndex == 1, "With depth 0, we can only traverse one or zero leaves (i.e. traverse one leaf or traverse a gap leaf).");
@ -212,25 +223,28 @@ namespace blobstore {
}
function<Data (uint32_t index)> LeafTraverser::_createMaxSizeLeaf() const {
ASSERT(!_readOnlyTraversal, "Can't create a new leaf in a read-only traversal");
uint64_t maxBytesPerLeaf = _nodeStore->layout().maxBytesPerLeaf();
return [maxBytesPerLeaf] (uint32_t /*index*/) -> Data {
return Data(maxBytesPerLeaf).FillWithZeroes();
};
}
unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode> root) {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
void LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode>* root) {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root->get());
if (inner != nullptr && inner->numChildren() == 1) {
ASSERT(!_readOnlyTraversal, "Can't decrease tree depth in a read-only traversal");
auto newRoot = _whileRootHasOnlyOneChildRemoveRootReturnChild(inner->readChild(0).blockId());
auto result = _nodeStore->overwriteNodeWith(std::move(root), *newRoot);
*root = _nodeStore->overwriteNodeWith(std::move(*root), *newRoot);
_nodeStore->remove(std::move(newRoot));
return result;
} else {
return root;
}
}
unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId) {
ASSERT(!_readOnlyTraversal, "Can't decrease tree depth in a read-only traversal");
auto current = _nodeStore->load(blockId);
ASSERT(current != none, "Node not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*current);

13
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h
View File

@ -25,19 +25,20 @@ namespace blobstore {
*/
class LeafTraverser final {
public:
LeafTraverser(datanodestore::DataNodeStore *nodeStore);
LeafTraverser(datanodestore::DataNodeStore *nodeStore, bool readOnlyTraversal);
cpputils::unique_ref<datanodestore::DataNode> traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
void traverseAndUpdateRoot(
cpputils::unique_ref<datanodestore::DataNode>* root, uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
private:
datanodestore::DataNodeStore *_nodeStore;
const bool _readOnlyTraversal;
cpputils::unique_ref<datanodestore::DataNode> _traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
void _traverseAndUpdateRoot(
cpputils::unique_ref<datanodestore::DataNode>* root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
@ -55,7 +56,7 @@ namespace blobstore {
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
uint32_t _maxLeavesForTreeDepth(uint8_t depth) const;
std::function<cpputils::Data (uint32_t index)> _createMaxSizeLeaf() const;
cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode> root);
void _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode>* root);
cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId);
DISALLOW_COPY_AND_ASSIGN(LeafTraverser);

24
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h
View File

@ -22,10 +22,6 @@ public:
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, datatreestore::LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf);
}
uint32_t numLeaves() const {
return _baseTree->numLeaves();
}
@ -34,8 +30,24 @@ public:
return _baseTree->resizeNumBytes(newNumBytes);
}
uint64_t numStoredBytes() const {
return _baseTree->numStoredBytes();
uint64_t numBytes() const {
return _baseTree->numBytes();
}
uint64_t tryReadBytes(void *target, uint64_t offset, uint64_t count) const {
return _baseTree->tryReadBytes(target, offset, count);
}
void readBytes(void *target, uint64_t offset, uint64_t count) const {
return _baseTree->readBytes(target, offset, count);
}
cpputils::Data readAllBytes() const {
return _baseTree->readAllBytes();
}
void writeBytes(const void *source, uint64_t offset, uint64_t count) {
return _baseTree->writeBytes(source, offset, count);
}
void flush() {

2
src/blockstore/implementations/caching/CachingBlockStore2.cpp
View File

@ -45,7 +45,7 @@ void CachingBlockStore2::CachedBlock::write(Data data) {
}
CachingBlockStore2::CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache() {
: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache("blockstore") {
}
bool CachingBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {

6
src/blockstore/implementations/caching/cache/Cache.h vendored
View File

@ -24,7 +24,7 @@ public:
static constexpr double PURGE_INTERVAL = 0.5; // With this interval, we check for entries to purge
static constexpr double MAX_LIFETIME_SEC = PURGE_LIFETIME_SEC + PURGE_INTERVAL; // This is the oldest age an entry can reach (given purging works in an ideal world, i.e. with the ideal interval and in zero time)
Cache();
Cache(const std::string& cacheName);
~Cache();
uint32_t size() const;
@ -56,10 +56,10 @@ template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Ke
template<class Key, class Value, uint32_t MAX_ENTRIES> constexpr double Cache<Key, Value, MAX_ENTRIES>::MAX_LIFETIME_SEC;
template<class Key, class Value, uint32_t MAX_ENTRIES>
Cache<Key, Value, MAX_ENTRIES>::Cache(): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
Cache<Key, Value, MAX_ENTRIES>::Cache(const std::string& cacheName): _mutex(), _currentlyFlushingEntries(), _cachedBlocks(), _timeoutFlusher(nullptr) {
//Don't initialize timeoutFlusher in the initializer list,
//because it then might already call Cache::popOldEntries() before Cache is done constructing.
_timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL);
_timeoutFlusher = std::make_unique<PeriodicTask>(std::bind(&Cache::_deleteOldEntriesParallel, this), PURGE_INTERVAL, "flush_" + cacheName);
}
template<class Key, class Value, uint32_t MAX_ENTRIES>

4
src/blockstore/implementations/caching/cache/PeriodicTask.cpp vendored
View File

@ -7,10 +7,10 @@ using namespace cpputils::logging;
namespace blockstore {
namespace caching {
PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec, std::string threadName) :
_task(task),
_interval(static_cast<uint64_t>(UINT64_C(1000000000) * intervalSec)),
_thread(std::bind(&PeriodicTask::_loopIteration, this)) {
_thread(std::bind(&PeriodicTask::_loopIteration, this), std::move(threadName)) {
_thread.start();
}

2
src/blockstore/implementations/caching/cache/PeriodicTask.h vendored
View File

@ -11,7 +11,7 @@ namespace caching {
class PeriodicTask final {
public:
PeriodicTask(std::function<void ()> task, double intervalSec);
PeriodicTask(std::function<void ()> task, double intervalSec, std::string threadName);
private:
bool _loopIteration();

2
src/cpp-utils/CMakeLists.txt
View File

@ -24,6 +24,8 @@ set(SOURCES
io/pipestream.cpp
thread/LoopThread.cpp
thread/ThreadSystem.cpp
thread/debugging_nonwindows.cpp
thread/debugging_windows.cpp
random/Random.cpp
random/RandomGeneratorThread.cpp
random/OSRandomGenerator.cpp

2
src/cpp-utils/random/RandomGeneratorThread.cpp
View File

@ -8,7 +8,7 @@ namespace cpputils {
_buffer(buffer),
_minSize(minSize),
_maxSize(maxSize),
_thread(std::bind(&RandomGeneratorThread::_loopIteration, this)) {
_thread(std::bind(&RandomGeneratorThread::_loopIteration, this), "RandomGeneratorThread") {
ASSERT(_maxSize >= _minSize, "Invalid parameters");
}

26
src/cpp-utils/system/path.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#ifndef MESSMER_CPPUTILS_SYSTEM_PATH_H
#define MESSMER_CPPUTILS_SYSTEM_PATH_H
#include <boost/filesystem/path.hpp>
#include <cpp-utils/macros.h>
namespace cpputils {
#if defined(_MSC_VER)
inline bool path_is_just_drive_letter(const boost::filesystem::path& path) {
return path.has_root_path() && !path.has_root_directory() && !path.has_parent_path();
}
#else
inline constexpr bool path_is_just_drive_letter(const boost::filesystem::path& /*path*/) {
return false;
}
#endif
}
#endif

5
src/cpp-utils/thread/LoopThread.cpp
View File

@ -6,7 +6,8 @@ using boost::none;
namespace cpputils {
LoopThread::LoopThread(function<bool()> loopIteration): _loopIteration(std::move(loopIteration)), _runningHandle(none) {
LoopThread::LoopThread(function<bool()> loopIteration, std::string threadName)
: _loopIteration(std::move(loopIteration)), _runningHandle(none), _threadName(std::move(threadName)) {
}
LoopThread::~LoopThread() {
@ -16,7 +17,7 @@ namespace cpputils {
}
void LoopThread::start() {
_runningHandle = ThreadSystem::singleton().start(_loopIteration);
_runningHandle = ThreadSystem::singleton().start(_loopIteration, _threadName);
}
void LoopThread::stop() {

3
src/cpp-utils/thread/LoopThread.h
View File

@ -14,7 +14,7 @@ namespace cpputils {
class LoopThread final {
public:
// The loopIteration callback returns true, if more iterations should be run, and false, if the thread should be terminated.
LoopThread(std::function<bool()> loopIteration);
LoopThread(std::function<bool()> loopIteration, std::string threadName);
~LoopThread();
void start();
void stop();
@ -22,6 +22,7 @@ namespace cpputils {
private:
std::function<bool()> _loopIteration;
boost::optional<ThreadSystem::Handle> _runningHandle;
std::string _threadName;
DISALLOW_COPY_AND_ASSIGN(LoopThread);
};

15
src/cpp-utils/thread/ThreadSystem.cpp
View File

@ -1,7 +1,9 @@
#include "ThreadSystem.h"
#include "../logging/logging.h"
#include "debugging.h"
using std::function;
using std::string;
using namespace cpputils::logging;
namespace cpputils {
@ -21,10 +23,10 @@ namespace cpputils {
#endif
}
ThreadSystem::Handle ThreadSystem::start(function<bool()> loopIteration) {
ThreadSystem::Handle ThreadSystem::start(function<bool()> loopIteration, string threadName) {
boost::unique_lock<boost::mutex> lock(_mutex);
auto thread = _startThread(loopIteration);
_runningThreads.push_back(RunningThread{std::move(loopIteration), std::move(thread)});
auto thread = _startThread(loopIteration, threadName);
_runningThreads.push_back(RunningThread{std::move(threadName), std::move(loopIteration), std::move(thread)});
return std::prev(_runningThreads.end());
}
@ -59,13 +61,14 @@ namespace cpputils {
void ThreadSystem::_restartAllThreads() {
for (RunningThread &thread : _runningThreads) {
thread.thread = _startThread(thread.loopIteration);
thread.thread = _startThread(thread.loopIteration, thread.threadName);
}
_mutex.unlock(); // Was locked in the before-fork handler
}
boost::thread ThreadSystem::_startThread(function<bool()> loopIteration) {
return boost::thread([loopIteration = std::move(loopIteration)] {
boost::thread ThreadSystem::_startThread(function<bool()> loopIteration, const string& threadName) {
return boost::thread([loopIteration = std::move(loopIteration), threadName] {
cpputils::set_thread_name(threadName.c_str());
ThreadSystem::_runThread(loopIteration);
});
}

5
src/cpp-utils/thread/ThreadSystem.h
View File

@ -13,6 +13,7 @@ namespace cpputils {
class ThreadSystem final {
private:
struct RunningThread {
std::string threadName;
std::function<bool()> loopIteration; // The loopIteration callback returns true, if more iterations should be run, and false, if the thread should be terminated.
boost::thread thread; // boost::thread because we need it to be interruptible.
};
@ -21,7 +22,7 @@ namespace cpputils {
static ThreadSystem &singleton();
Handle start(std::function<bool()> loopIteration);
Handle start(std::function<bool()> loopIteration, std::string threadName);
void stop(Handle handle);
private:
@ -34,7 +35,7 @@ namespace cpputils {
//TODO Rename to _doOnBeforeFork and _doAfterFork or similar, because they also handle locking _mutex for fork().
void _stopAllThreadsForRestart();
void _restartAllThreads();
boost::thread _startThread(std::function<bool()> loopIteration);
boost::thread _startThread(std::function<bool()> loopIteration, const std::string& threadName);
std::list<RunningThread> _runningThreads; // std::list, because we give out iterators as handles
boost::mutex _mutex;

16
src/cpp-utils/thread/debugging.h Normal file
View File

@ -0,0 +1,16 @@
#pragma once
#ifndef MESSMER_CPPUTILS_DEBUGGING_H
#define MESSMER_CPPUTILS_DEBUGGING_H
#include <string>
#include <thread>
namespace cpputils {
void set_thread_name(const char* name);
std::string get_thread_name();
std::string get_thread_name(std::thread* thread);
}
#endif

55
src/cpp-utils/thread/debugging_nonwindows.cpp Normal file
View File

@ -0,0 +1,55 @@
#if !defined(_MSC_VER)
#include "debugging.h"
#include <stdexcept>
#include <thread>
#include <pthread.h>
#include <cpp-utils/assert/assert.h>
namespace cpputils {
namespace {
constexpr size_t MAX_NAME_LEN = 16; // this length includes the terminating null character at the end
}
void set_thread_name(const char* name) {
std::string name_(name);
if (name_.size() > MAX_NAME_LEN - 1) {
name_.resize(MAX_NAME_LEN - 1);
}
#if defined(__APPLE__)
int result = pthread_setname_np(name_.c_str());
#else
int result = pthread_setname_np(pthread_self(), name_.c_str());
#endif
if (0 != result) {
throw std::runtime_error("Error setting thread name with pthread_setname_np. Code: " + std::to_string(result));
}
}
namespace {
std::string get_thread_name(pthread_t thread) {
char name[MAX_NAME_LEN];
int result = pthread_getname_np(thread, name, MAX_NAME_LEN);
if (0 != result) {
throw std::runtime_error("Error getting thread name with pthread_getname_np. Code: " + std::to_string(result));
}
// pthread_getname_np returns a null terminated string with maximum 16 bytes.
// but just to be safe against a buggy implementation, let's set the last byte to zero.
name[MAX_NAME_LEN - 1] = '\0';
return name;
}
}
std::string get_thread_name() {
return get_thread_name(pthread_self());
}
std::string get_thread_name(std::thread* thread) {
ASSERT(thread->joinable(), "Thread not running");
return get_thread_name(thread->native_handle());
}
}
#endif

111
src/cpp-utils/thread/debugging_windows.cpp Normal file
View File

@ -0,0 +1,111 @@
#if defined(_MSC_VER)
#include <Windows.h>
#include "debugging.h"
#include <codecvt>
#include <cpp-utils/assert/assert.h>
using std::string;
using std::wstring;
using std::wstring_convert;
namespace cpputils {
namespace {
struct NameData final {
wchar_t *name = nullptr;
~NameData() {
if (nullptr != LocalFree(name)) {
throw std::runtime_error("Error releasing thread description memory. Error code: " + std::to_string(GetLastError()));
}
}
};
struct ModuleHandle final {
HMODULE module;
ModuleHandle(const char* dll) {
bool success = GetModuleHandleExA(0, dll, &module);
if (!success) {
throw std::runtime_error(string() + "Error loading dll: " + dll + ". Error code: " + std::to_string(GetLastError()));
}
}
~ModuleHandle() {
bool success = FreeLibrary(module);
if (!success) {
throw std::runtime_error("Error unloading dll. Error code: " + std::to_string(GetLastError()));
}
}
};
template<class Fn>
class APIFunction final {
private:
ModuleHandle module_;
Fn func_;
public:
APIFunction(const char* dll, const char* function)
: module_(dll), func_(reinterpret_cast<Fn>(GetProcAddress(module_.module, function))) {
}
bool valid() const {
return func_ != nullptr;
}
Fn func() const {
return func_;
}
};
std::string get_thread_name(HANDLE thread) {
// The GetThreadDescription API was brought in version 1607 of Windows 10.
typedef HRESULT(WINAPI* GetThreadDescriptionFn)(HANDLE hThread, PWSTR* ppszThreadDescription);
static APIFunction<GetThreadDescriptionFn> get_thread_description_func("Kernel32.dll", "GetThreadDescription");
if (get_thread_description_func.valid()) {
NameData name_data;
HRESULT status = get_thread_description_func.func()(thread, &name_data.name);
if (FAILED(status)) {
throw std::runtime_error("Error getting thread description. Error code: " + std::to_string(status));
}
return wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().to_bytes(name_data.name);
}
else {
// GetThreadDescription API is not available.
return "";
}
}
}
void set_thread_name(const char* name) {
// The GetThreadDescription API was brought in version 1607 of Windows 10.
typedef HRESULT(WINAPI* SetThreadDescriptionFn)(HANDLE hThread, PCWSTR lpThreadDescription);
static APIFunction<SetThreadDescriptionFn> set_thread_description_func("Kernel32.dll", "SetThreadDescription");
if (set_thread_description_func.valid()) {
wstring wname = wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(name);
HRESULT status = set_thread_description_func.func()(GetCurrentThread(), wname.c_str());
if (FAILED(status)) {
throw std::runtime_error("Error setting thread description. Error code: " + std::to_string(status));
}
}
else {
// intentionally empty. SetThreadDescription API is not available.
}
}
std::string get_thread_name() {
return get_thread_name(GetCurrentThread());
}
std::string get_thread_name(std::thread* thread) {
ASSERT(thread->joinable(), "Thread not running");
return get_thread_name(static_cast<HANDLE>(thread->native_handle()));
}
}
#endif

12
src/cryfs-cli/CallAfterTimeout.h
View File

@ -1,15 +1,15 @@
#pragma once
#ifndef MESSMER_CRYFS_SRC_CLI_CALLAFTERTIMEOUT_H
#define MESSMER_CRYFS_SRC_CLI_CALLAFTERTIMEOUT_H
#ifndef MESSMER_CRYFSCLI_CALLAFTERTIMEOUT_H
#define MESSMER_CRYFSCLI_CALLAFTERTIMEOUT_H
#include <functional>
#include <mutex>
#include <cpp-utils/thread/LoopThread.h>
namespace cryfs {
namespace cryfs_cli {
class CallAfterTimeout final {
public:
CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback);
CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback, const std::string& timeoutName);
void resetTimer();
private:
bool _checkTimeoutThreadIteration();
@ -25,8 +25,8 @@ namespace cryfs {
DISALLOW_COPY_AND_ASSIGN(CallAfterTimeout);
};
inline CallAfterTimeout::CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback)
:_callback(std::move(callback)), _timeout(timeout), _start(), _checkTimeoutThread(std::bind(&CallAfterTimeout::_checkTimeoutThreadIteration, this)) {
inline CallAfterTimeout::CallAfterTimeout(boost::chrono::milliseconds timeout, std::function<void()> callback, const std::string& timeoutName)
:_callback(std::move(callback)), _timeout(timeout), _start(), _checkTimeoutThread(std::bind(&CallAfterTimeout::_checkTimeoutThreadIteration, this), "timeout_" + timeoutName) {
resetTimer();
_checkTimeoutThread.start();
}

28
src/cryfs-cli/Cli.cpp
View File

@ -26,10 +26,12 @@
#include <cryfs/localstate/BasedirMetadata.h>
#include "Environment.h"
#include <cryfs/CryfsException.h>
#include <cpp-utils/thread/debugging.h>
//TODO Many functions accessing the ProgramOptions object. Factor out into class that stores it as a member.
//TODO Factor out class handling askPassword
using namespace cryfs_cli;
using namespace cryfs;
namespace bf = boost::filesystem;
using namespace cpputils::logging;
@ -66,7 +68,7 @@ using gitversion::VersionCompare;
//TODO Replace ASSERTs with other error handling when it is not a programming error but an environment influence (e.g. a block is missing)
//TODO Can we improve performance by setting compiler parameter -maes for scrypt?
namespace cryfs {
namespace cryfs_cli {
Cli::Cli(RandomGenerator &keyGenerator, const SCryptSettings &scryptSettings, shared_ptr<Console> console):
_keyGenerator(keyGenerator), _scryptSettings(scryptSettings), _console(), _noninteractive(false), _idleUnmounter(none), _device(none) {
@ -262,12 +264,8 @@ namespace cryfs {
_initLogfile(options);
#ifdef __APPLE__
std::cout << "\nMounting filesystem. To unmount, call:\n$ umount " << options.mountDir() << "\n" << std::endl;
#else
std::cout << "\nMounting filesystem. To unmount, call:\n$ fusermount -u " << options.mountDir() << "\n"
std::cout << "\nMounting filesystem. To unmount, call:\n$ cryfs-unmount " << options.mountDir() << "\n"
<< std::endl;
#endif
fuse->run(options.mountDir(), options.fuseOptions());
if (stoppedBecauseOfIntegrityViolation) {
@ -300,7 +298,7 @@ namespace cryfs {
return none;
}
uint64_t millis = std::llround(60000 * (*minutes));
return make_unique_ref<CallAfterTimeout>(milliseconds(millis), callback);
return make_unique_ref<CallAfterTimeout>(milliseconds(millis), callback, "idlecallback");
}
void Cli::_initLogfile(const ProgramOptions &options) {
@ -316,10 +314,17 @@ namespace cryfs {
}
}
void Cli::_sanityChecks(const ProgramOptions &options) {
_checkDirAccessible(options.baseDir(), "base directory", ErrorCode::InaccessibleBaseDir);
_checkDirAccessible(options.mountDir(), "mount directory", ErrorCode::InaccessibleMountDir);
_checkMountdirDoesntContainBasedir(options);
void Cli::_sanityChecks(const ProgramOptions &options) {
_checkDirAccessible(bf::absolute(options.baseDir()), "base directory", ErrorCode::InaccessibleBaseDir);
if (!options.mountDirIsDriveLetter()) {
_checkDirAccessible(options.mountDir(), "mount directory", ErrorCode::InaccessibleMountDir);
_checkMountdirDoesntContainBasedir(options);
} else {
if (bf::exists(options.mountDir())) {
throw CryfsException("Drive " + options.mountDir().string() + " already exists.", ErrorCode::InaccessibleMountDir);
}
}
}
void Cli::_checkDirAccessible(const bf::path &dir, const std::string &name, ErrorCode errorCode) {
@ -392,6 +397,7 @@ namespace cryfs {
int Cli::main(int argc, const char *argv[], unique_ref<HttpClient> httpClient, std::function<void()> onMounted) {
cpputils::showBacktraceOnCrash();
cpputils::set_thread_name("cryfs");
try {
_showVersion(std::move(httpClient));

22
src/cryfs-cli/Cli.h
View File

@ -1,6 +1,6 @@
#pragma once
#ifndef MESSMER_CRYFS_CLI_H
#define MESSMER_CRYFS_CLI_H
#ifndef MESSMER_CRYFSCLI_CLI_H
#define MESSMER_CRYFSCLI_CLI_H
#include "program_options/ProgramOptions.h"
#include <cryfs/config/CryConfigFile.h>
@ -14,7 +14,7 @@
#include <cryfs/config/CryConfigLoader.h>
#include <cryfs/ErrorCodes.h>
namespace cryfs {
namespace cryfs_cli {
class Cli final {
public:
Cli(cpputils::RandomGenerator &keyGenerator, const cpputils::SCryptSettings& scryptSettings, std::shared_ptr<cpputils::Console> console);
@ -23,9 +23,9 @@ namespace cryfs {
private:
void _checkForUpdates(cpputils::unique_ref<cpputils::HttpClient> httpClient);
void _runFilesystem(const program_options::ProgramOptions &options, std::function<void()> onMounted);
CryConfigLoader::ConfigLoadResult _loadOrCreateConfig(const program_options::ProgramOptions &options, const LocalStateDir& localStateDir);
void _checkConfigIntegrity(const boost::filesystem::path& basedir, const LocalStateDir& localStateDir, const CryConfigFile& config, bool allowReplacedFilesystem);
boost::optional<CryConfigLoader::ConfigLoadResult> _loadOrCreateConfigFile(boost::filesystem::path configFilePath, LocalStateDir localStateDir, const boost::optional<std::string> &cipher, const boost::optional<uint32_t> &blocksizeBytes, bool allowFilesystemUpgrade, const boost::optional<bool> &missingBlockIsIntegrityViolation, bool allowReplacedFilesystem);
cryfs::CryConfigLoader::ConfigLoadResult _loadOrCreateConfig(const program_options::ProgramOptions &options, const cryfs::LocalStateDir& localStateDir);
void _checkConfigIntegrity(const boost::filesystem::path& basedir, const cryfs::LocalStateDir& localStateDir, const cryfs::CryConfigFile& config, bool allowReplacedFilesystem);
boost::optional<cryfs::CryConfigLoader::ConfigLoadResult> _loadOrCreateConfigFile(boost::filesystem::path configFilePath, cryfs::LocalStateDir localStateDir, const boost::optional<std::string> &cipher, const boost::optional<uint32_t> &blocksizeBytes, bool allowFilesystemUpgrade, const boost::optional<bool> &missingBlockIsIntegrityViolation, bool allowReplacedFilesystem);
boost::filesystem::path _determineConfigFile(const program_options::ProgramOptions &options);
static std::function<std::string()> _askPasswordForExistingFilesystem(std::shared_ptr<cpputils::Console> console);
static std::function<std::string()> _askPasswordForNewFilesystem(std::shared_ptr<cpputils::Console> console);
@ -37,11 +37,11 @@ namespace cryfs {
void _sanityChecks(const program_options::ProgramOptions &options);
void _checkMountdirDoesntContainBasedir(const program_options::ProgramOptions &options);
bool _pathContains(const boost::filesystem::path &parent, const boost::filesystem::path &child);
void _checkDirAccessible(const boost::filesystem::path &dir, const std::string &name, ErrorCode errorCode);
std::shared_ptr<cpputils::TempFile> _checkDirWriteable(const boost::filesystem::path &dir, const std::string &name, ErrorCode errorCode);
void _checkDirReadable(const boost::filesystem::path &dir, std::shared_ptr<cpputils::TempFile> tempfile, const std::string &name, ErrorCode errorCode);
void _checkDirAccessible(const boost::filesystem::path &dir, const std::string &name, cryfs::ErrorCode errorCode);
std::shared_ptr<cpputils::TempFile> _checkDirWriteable(const boost::filesystem::path &dir, const std::string &name, cryfs::ErrorCode errorCode);
void _checkDirReadable(const boost::filesystem::path &dir, std::shared_ptr<cpputils::TempFile> tempfile, const std::string &name, cryfs::ErrorCode errorCode);
boost::optional<cpputils::unique_ref<CallAfterTimeout>> _createIdleCallback(boost::optional<double> minutes, std::function<void()> callback);
void _sanityCheckFilesystem(CryDevice *device);
void _sanityCheckFilesystem(cryfs::CryDevice *device);
cpputils::RandomGenerator &_keyGenerator;
@ -49,7 +49,7 @@ namespace cryfs {
std::shared_ptr<cpputils::Console> _console;
bool _noninteractive;
boost::optional<cpputils::unique_ref<CallAfterTimeout>> _idleUnmounter;
boost::optional<cpputils::unique_ref<CryDevice>> _device;
boost::optional<cpputils::unique_ref<cryfs::CryDevice>> _device;
DISALLOW_COPY_AND_ASSIGN(Cli);
};

2
src/cryfs-cli/Environment.cpp
View File

@ -6,7 +6,7 @@
using std::string;
namespace bf = boost::filesystem;
namespace cryfs {
namespace cryfs_cli {
const string Environment::FRONTEND_KEY = "CRYFS_FRONTEND";
const string Environment::FRONTEND_NONINTERACTIVE = "noninteractive";
const string Environment::NOUPDATECHECK_KEY = "CRYFS_NO_UPDATE_CHECK";

6
src/cryfs-cli/Environment.h
View File

@ -1,11 +1,11 @@
#pragma once
#ifndef MESSMER_CRYFS_CLI_ENVIRONMENT_H
#define MESSMER_CRYFS_CLI_ENVIRONMENT_H
#ifndef MESSMER_CRYFSCLI_ENVIRONMENT_H
#define MESSMER_CRYFSCLI_ENVIRONMENT_H
#include <string>
#include <boost/filesystem/path.hpp>
namespace cryfs {
namespace cryfs_cli {
class Environment {
public:

2
src/cryfs-cli/VersionChecker.cpp
View File

@ -13,7 +13,7 @@ using boost::property_tree::ptree;
using boost::property_tree::json_parser_error;
using namespace cpputils::logging;
namespace cryfs {
namespace cryfs_cli {
VersionChecker::VersionChecker(HttpClient* httpClient)
: _versionInfo(_getVersionInfo(httpClient)) {}

6
src/cryfs-cli/VersionChecker.h
View File

@ -1,5 +1,5 @@
#ifndef MESSMER_CRYFS_SRC_CLI_VERSIONCHECKER_H
#define MESSMER_CRYFS_SRC_CLI_VERSIONCHECKER_H
#ifndef MESSMER_CRYFSCLI_VERSIONCHECKER_H
#define MESSMER_CRYFSCLI_VERSIONCHECKER_H
#include <cpp-utils/macros.h>
#include <string>
@ -8,7 +8,7 @@
#include <cpp-utils/network/HttpClient.h>
#include <cpp-utils/pointer/unique_ref.h>
namespace cryfs {
namespace cryfs_cli {
class VersionChecker final {
public:
//TODO Write a cpputils::shared_ref and use it

6
src/cryfs-cli/main.cpp
View File

@ -10,7 +10,7 @@
#include <cpp-utils/network/CurlHttpClient.h>
#endif
using namespace cryfs;
using namespace cryfs_cli;
using cpputils::Random;
using cpputils::SCrypt;
using cpputils::IOStreamConsole;
@ -35,13 +35,13 @@ int main(int argc, const char *argv[]) {
#endif
return Cli(keyGenerator, SCrypt::DefaultSettings, make_shared<IOStreamConsole>())
.main(argc, argv, std::move(httpClient), []{});
} catch (const CryfsException &e) {
} catch (const cryfs::CryfsException &e) {
if (e.what() != string()) {
std::cerr << "Error: " << e.what() << std::endl;
}
return exitCode(e.errorCode());
} catch (const std::exception &e) {
cerr << "Error: " << e.what();
return exitCode(ErrorCode::UnspecifiedError);
return exitCode(cryfs::ErrorCode::UnspecifiedError);
}
}

6
src/cryfs-cli/program_options/Parser.cpp
View File

@ -8,7 +8,7 @@
namespace po = boost::program_options;
namespace bf = boost::filesystem;
using namespace cryfs::program_options;
using namespace cryfs_cli::program_options;
using cryfs::CryConfigConsole;
using cryfs::CryfsException;
using cryfs::ErrorCode;
@ -49,8 +49,8 @@ ProgramOptions Parser::parse(const vector<string> &supportedCiphers) const {
if (!vm.count("mount-dir")) {
_showHelpAndExit("Please specify a mount directory.", ErrorCode::InvalidArguments);
}
bf::path baseDir = bf::absolute(vm["base-dir"].as<string>());
bf::path mountDir = bf::absolute(vm["mount-dir"].as<string>());
bf::path baseDir = vm["base-dir"].as<string>();
bf::path mountDir = vm["mount-dir"].as<string>();
optional<bf::path> configfile = none;
if (vm.count("config")) {
configfile = bf::absolute(vm["config"].as<string>());

8
src/cryfs-cli/program_options/Parser.h
View File

@ -1,12 +1,12 @@
#pragma once
#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_PARSER_H
#define MESSMER_CRYFS_PROGRAMOPTIONS_PARSER_H
#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_PARSER_H
#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_PARSER_H
#include "ProgramOptions.h"
#include <boost/program_options.hpp>
#include <cryfs/ErrorCodes.h>
namespace cryfs {
namespace cryfs_cli {
namespace program_options {
class Parser final {
public:
@ -21,7 +21,7 @@ namespace cryfs {
static void _addPositionalOptionForBaseDir(boost::program_options::options_description *desc,
boost::program_options::positional_options_description *positional);
static void _showHelp();
[[noreturn]] static void _showHelpAndExit(const std::string& message, ErrorCode errorCode);
[[noreturn]] static void _showHelpAndExit(const std::string& message, cryfs::ErrorCode errorCode);
[[noreturn]] static void _showCiphersAndExit(const std::vector<std::string> &supportedCiphers);
[[noreturn]] static void _showVersionAndExit();
static boost::program_options::variables_map _parseOptionsOrShowHelp(const std::vector<std::string> &options, const std::vector<std::string> &supportedCiphers);

19
src/cryfs-cli/program_options/ProgramOptions.cpp
View File

@ -1,8 +1,9 @@
#include "ProgramOptions.h"
#include <cstring>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/path.h>
using namespace cryfs::program_options;
using namespace cryfs_cli::program_options;
using std::string;
using std::vector;
using boost::optional;
@ -15,9 +16,15 @@ ProgramOptions::ProgramOptions(bf::path baseDir, bf::path mountDir, optional<bf:
bool allowIntegrityViolations,
boost::optional<bool> missingBlockIsIntegrityViolation,
vector<string> fuseOptions)
:_baseDir(std::move(baseDir)), _mountDir(std::move(mountDir)), _configFile(std::move(configFile)), _foreground(foreground), _allowFilesystemUpgrade(allowFilesystemUpgrade), _allowReplacedFilesystem(allowReplacedFilesystem), _allowIntegrityViolations(allowIntegrityViolations),
_cipher(std::move(cipher)), _blocksizeBytes(std::move(blocksizeBytes)), _unmountAfterIdleMinutes(std::move(unmountAfterIdleMinutes)),
_missingBlockIsIntegrityViolation(std::move(missingBlockIsIntegrityViolation)), _logFile(std::move(logFile)), _fuseOptions(std::move(fuseOptions)) {
: _configFile(std::move(configFile)), _baseDir(bf::absolute(std::move(baseDir))), _mountDir(std::move(mountDir)),
_mountDirIsDriveLetter(cpputils::path_is_just_drive_letter(_mountDir)),
_foreground(foreground),
_allowFilesystemUpgrade(allowFilesystemUpgrade), _allowReplacedFilesystem(allowReplacedFilesystem), _allowIntegrityViolations(allowIntegrityViolations),
_cipher(std::move(cipher)), _blocksizeBytes(std::move(blocksizeBytes)), _unmountAfterIdleMinutes(std::move(unmountAfterIdleMinutes)),
_missingBlockIsIntegrityViolation(std::move(missingBlockIsIntegrityViolation)), _logFile(std::move(logFile)), _fuseOptions(std::move(fuseOptions)) {
if (!_mountDirIsDriveLetter) {
_mountDir = bf::absolute(std::move(_mountDir));
}
}
const bf::path &ProgramOptions::baseDir() const {
@ -28,6 +35,10 @@ const bf::path &ProgramOptions::mountDir() const {
return _mountDir;
}
bool ProgramOptions::mountDirIsDriveLetter() const {
return _mountDirIsDriveLetter;
}
const optional<bf::path> &ProgramOptions::configFile() const {
return _configFile;
}

14
src/cryfs-cli/program_options/ProgramOptions.h
View File

@ -1,6 +1,6 @@
#pragma once
#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#define MESSMER_CRYFS_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#include <vector>
#include <string>
@ -8,7 +8,7 @@
#include <cpp-utils/macros.h>
#include <boost/filesystem.hpp>
namespace cryfs {
namespace cryfs_cli {
namespace program_options {
class ProgramOptions final {
public:
@ -25,6 +25,7 @@ namespace cryfs {
const boost::filesystem::path &baseDir() const;
const boost::filesystem::path &mountDir() const;
bool mountDirIsDriveLetter() const;
const boost::optional<boost::filesystem::path> &configFile() const;
bool foreground() const;
bool allowFilesystemUpgrade() const;
@ -38,9 +39,10 @@ namespace cryfs {
const std::vector<std::string> &fuseOptions() const;
private:
boost::filesystem::path _baseDir;
boost::filesystem::path _mountDir;
boost::optional<boost::filesystem::path> _configFile;
boost::optional<boost::filesystem::path> _configFile;
boost::filesystem::path _baseDir; // this is always absolute
boost::filesystem::path _mountDir; // this is absolute iff !_mountDirIsDriveLetter
bool _mountDirIsDriveLetter;
bool _foreground;
bool _allowFilesystemUpgrade;
bool _allowReplacedFilesystem;

2
src/cryfs-cli/program_options/utils.cpp
View File

@ -7,7 +7,7 @@ using std::make_pair;
using std::vector;
using std::string;
namespace cryfs {
namespace cryfs_cli {
namespace program_options {
pair<vector<string>, vector<string>> splitAtDoubleDash(const vector<string> &options) {
auto doubleDashIterator = std::find(options.begin(), options.end(), string("--"));

6
src/cryfs-cli/program_options/utils.h
View File

@ -1,12 +1,12 @@
#pragma once
#ifndef MESSMER_CRYFS_PROGRAMOPTIONS_UTILS_H
#define MESSMER_CRYFS_PROGRAMOPTIONS_UTILS_H
#ifndef MESSMER_CRYFSCLI_PROGRAMOPTIONS_UTILS_H
#define MESSMER_CRYFSCLI_PROGRAMOPTIONS_UTILS_H
#include <utility>
#include <vector>
#include <string>
namespace cryfs {
namespace cryfs_cli {
namespace program_options {
/**
* Splits an array of program options into two arrays of program options, split at a double dash '--' option.

24
src/cryfs-unmount/CMakeLists.txt Normal file
View File

@ -0,0 +1,24 @@
project (cryfs-unmount)
INCLUDE(GNUInstallDirs)
set(SOURCES
program_options/ProgramOptions.cpp
program_options/Parser.cpp
Cli.cpp
)
add_library(${PROJECT_NAME} ${SOURCES})
target_link_libraries(${PROJECT_NAME} PUBLIC cpp-utils cryfs fspp-fuse)
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})
add_executable(${PROJECT_NAME}_bin main_unmount.cpp)
set_target_properties(${PROJECT_NAME}_bin PROPERTIES OUTPUT_NAME cryfs-unmount)
target_link_libraries(${PROJECT_NAME}_bin PUBLIC ${PROJECT_NAME})
target_enable_style_warnings(${PROJECT_NAME}_bin)
target_activate_cpp14(${PROJECT_NAME}_bin)
install(TARGETS ${PROJECT_NAME}_bin
CONFIGURATIONS Debug Release RelWithDebInfo
DESTINATION ${CMAKE_INSTALL_BINDIR}
)

36
src/cryfs-unmount/Cli.cpp Normal file
View File

@ -0,0 +1,36 @@
#include "Cli.h"
#include <fspp/fuse/Fuse.h>
#include <cryfs-unmount/program_options/Parser.h>
#include <gitversion/gitversion.h>
#include <cryfs/CryfsException.h>
#include <iostream>
using fspp::fuse::Fuse;
using cryfs_unmount::program_options::Parser;
using cryfs_unmount::program_options::ProgramOptions;
namespace cryfs_unmount {
namespace {
void _showVersion() {
std::cout << "CryFS Version " << gitversion::VersionString() << std::endl;
}
}
void Cli::main(int argc, const char* argv[]) {
_showVersion();
ProgramOptions options = Parser(argc, argv).parse();
if (!boost::filesystem::exists(options.mountDir())) {
throw cryfs::CryfsException("Given mountdir doesn't exist", cryfs::ErrorCode::InaccessibleMountDir);
}
// TODO This doesn't seem to work with relative paths
std::cout << "Unmounting CryFS filesystem at " << options.mountDir() << "." << std::endl;
Fuse::unmount(options.mountDir());
// TODO Wait until it is actually unmounted and then show a better success message?
std::cout << "Filesystem is unmounting now." << std::endl;
}
}

14
src/cryfs-unmount/Cli.h Normal file
View File

@ -0,0 +1,14 @@
#pragma once
#ifndef MESSMER_CRYFSUNMOUNT_CLI_H
#define MESSMER_CRYFSUNMOUNT_CLI_H
namespace cryfs_unmount {
class Cli final {
public:
void main(int argc, const char* argv[]);
};
}
#endif

38
src/cryfs-unmount/main_unmount.cpp Normal file
View File

@ -0,0 +1,38 @@
#if defined(_MSC_VER)
#include <Windows.h>
#include <VersionHelpers.h>
#endif
#include <iostream>
#include <cryfs/CryfsException.h>
#include <cpp-utils/assert/backtrace.h>
#include "Cli.h"
using std::cerr;
using cryfs::ErrorCode;
int main(int argc, const char *argv[]) {
#if defined(_MSC_VER)
if (!IsWindows7SP1OrGreater()) {
std::cerr << "CryFS is currently only supported on Windows 7 SP1 (or later)." << std::endl;
exit(1);
}
#endif
cpputils::showBacktraceOnCrash();
try {
cryfs_unmount::Cli().main(argc, argv);
}
catch (const cryfs::CryfsException &e) {
if (e.what() != std::string()) {
std::cerr << "Error " << static_cast<int>(e.errorCode()) << ": " << e.what() << std::endl;
}
return exitCode(e.errorCode());
}
catch (const std::runtime_error &e) {
std::cerr << "Error: " << e.what() << std::endl;
return exitCode(ErrorCode::UnspecifiedError);
}
return exitCode(ErrorCode::Success);
}

128
src/cryfs-unmount/program_options/Parser.cpp Normal file
View File

@ -0,0 +1,128 @@
#include "Parser.h"
#include <iostream>
#include <boost/optional.hpp>
#include <cryfs/config/CryConfigConsole.h>
#include <cryfs/CryfsException.h>
#include <cryfs-cli/Environment.h>
namespace po = boost::program_options;
namespace bf = boost::filesystem;
using namespace cryfs_unmount::program_options;
using cryfs::CryConfigConsole;
using cryfs::CryfsException;
using cryfs::ErrorCode;
using std::vector;
using std::cerr;
using std::endl;
using std::string;
using namespace cpputils::logging;
Parser::Parser(int argc, const char *argv[])
:_options(_argsToVector(argc, argv)) {
}
vector<string> Parser::_argsToVector(int argc, const char *argv[]) {
vector<string> result;
for (int i = 0; i < argc; ++i) {
result.push_back(argv[i]);
}
return result;
}
ProgramOptions Parser::parse() const {
po::variables_map vm = _parseOptionsOrShowHelp(_options);
if (!vm.count("mount-dir")) {
_showHelpAndExit("Please specify a mount directory.", ErrorCode::InvalidArguments);
}
bf::path mountDir = vm["mount-dir"].as<string>();
return ProgramOptions(std::move(mountDir));
}
po::variables_map Parser::_parseOptionsOrShowHelp(const vector<string> &options) {
try {
return _parseOptions(options);
}
catch (const CryfsException& e) {
// If CryfsException is thrown, we already know what's wrong.
// Show usage information and pass through the exception, don't catch it.
if (e.errorCode() != ErrorCode::Success) {
_showHelp();
}
throw;
}
catch (const std::exception &e) {
std::cerr << e.what() << std::endl;
_showHelpAndExit("Invalid arguments", ErrorCode::InvalidArguments);
}
}
po::variables_map Parser::_parseOptions(const vector<string> &options) {
po::options_description desc;
po::positional_options_description positional_desc;
_addAllowedOptions(&desc);
_addPositionalOptionForBaseDir(&desc, &positional_desc);
po::variables_map vm;
vector<const char*> _options = _to_const_char_vector(options);
po::store(po::command_line_parser(_options.size(), _options.data())
.options(desc).positional(positional_desc).run(), vm);
if (vm.count("help")) {
_showHelpAndExit("", ErrorCode::Success);
}
if (vm.count("version")) {
_showVersionAndExit();
}
po::notify(vm);
return vm;
}
vector<const char*> Parser::_to_const_char_vector(const vector<string> &options) {
vector<const char*> result;
result.reserve(options.size());
for (const string &option : options) {
result.push_back(option.c_str());
}
return result;
}
void Parser::_addAllowedOptions(po::options_description *desc) {
po::options_description options("Allowed options");
string cipher_description = "Cipher to use for encryption. See possible values by calling cryfs with --show-ciphers. Default: ";
cipher_description += CryConfigConsole::DEFAULT_CIPHER;
string blocksize_description = "The block size used when storing ciphertext blocks (in bytes). Default: ";
blocksize_description += std::to_string(CryConfigConsole::DEFAULT_BLOCKSIZE_BYTES);
options.add_options()
("help,h", "show help message")
("version", "Show CryFS version number")
;
desc->add(options);
}
void Parser::_addPositionalOptionForBaseDir(po::options_description *desc, po::positional_options_description *positional) {
positional->add("mount-dir", 1);
po::options_description hidden("Hidden options");
hidden.add_options()
("mount-dir", po::value<string>(), "Mount directory")
;
desc->add(hidden);
}
void Parser::_showHelp() {
cerr << "Usage: cryfs-unmount [mountPoint]\n";
po::options_description desc;
_addAllowedOptions(&desc);
cerr << desc << endl;
}
[[noreturn]] void Parser::_showHelpAndExit(const std::string& message, ErrorCode errorCode) {
_showHelp();
throw CryfsException(message, errorCode);
}
[[noreturn]] void Parser::_showVersionAndExit() {
// no need to show version because it was already shown in the CryFS header before parsing program options
throw CryfsException("", ErrorCode::Success);
}

37
src/cryfs-unmount/program_options/Parser.h Normal file
View File

@ -0,0 +1,37 @@
#pragma once
#ifndef MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PARSER_H
#define MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PARSER_H
#include "ProgramOptions.h"
#include <boost/program_options.hpp>
#include <cryfs/ErrorCodes.h>
namespace cryfs_unmount {
namespace program_options {
class Parser final {
public:
Parser(int argc, const char *argv[]);
ProgramOptions parse() const;
private:
static std::vector<std::string> _argsToVector(int argc, const char *argv[]);
static std::vector<const char*> _to_const_char_vector(const std::vector<std::string> &options);
static void _addAllowedOptions(boost::program_options::options_description *desc);
static void _addPositionalOptionForBaseDir(boost::program_options::options_description *desc,
boost::program_options::positional_options_description *positional);
static void _showHelp();
[[noreturn]] static void _showHelpAndExit(const std::string& message, cryfs::ErrorCode errorCode);
[[noreturn]] static void _showCiphersAndExit(const std::vector<std::string> &supportedCiphers);
[[noreturn]] static void _showVersionAndExit();
static boost::program_options::variables_map _parseOptionsOrShowHelp(const std::vector<std::string> &options);
static boost::program_options::variables_map _parseOptions(const std::vector<std::string> &options);
std::vector<std::string> _options;
DISALLOW_COPY_AND_ASSIGN(Parser);
};
}
}
#endif

25
src/cryfs-unmount/program_options/ProgramOptions.cpp Normal file
View File

@ -0,0 +1,25 @@
#include "ProgramOptions.h"
#include <cstring>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/path.h>
using namespace cryfs_unmount::program_options;
using std::string;
namespace bf = boost::filesystem;
ProgramOptions::ProgramOptions(bf::path mountDir)
:_mountDir(std::move(mountDir)),
_mountDirIsDriveLetter(cpputils::path_is_just_drive_letter(_mountDir))
{
if (!_mountDirIsDriveLetter) {
_mountDir = bf::absolute(std::move(_mountDir));
}
}
const bf::path &ProgramOptions::mountDir() const {
return _mountDir;
}
bool ProgramOptions::mountDirIsDriveLetter() const {
return _mountDirIsDriveLetter;
}

30
src/cryfs-unmount/program_options/ProgramOptions.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#ifndef MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#define MESSMER_CRYFSUNMOUNT_PROGRAMOPTIONS_PROGRAMOPTIONS_H
#include <vector>
#include <string>
#include <boost/optional.hpp>
#include <cpp-utils/macros.h>
#include <boost/filesystem.hpp>
namespace cryfs_unmount {
namespace program_options {
class ProgramOptions final {
public:
ProgramOptions(boost::filesystem::path mountDir);
ProgramOptions(ProgramOptions &&rhs) = default;
const boost::filesystem::path &mountDir() const;
bool mountDirIsDriveLetter() const;
private:
boost::filesystem::path _mountDir;
bool _mountDirIsDriveLetter;
DISALLOW_COPY_AND_ASSIGN(ProgramOptions);
};
}
}
#endif

2
src/cryfs/config/CryConfigLoader.cpp
View File

@ -71,7 +71,7 @@ void CryConfigLoader::_checkVersion(const CryConfig &config, bool allowFilesyste
}
}
if (!allowFilesystemUpgrade && gitversion::VersionCompare::isOlderThan(config.Version(), CryConfig::FilesystemFormatVersion)) {
if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It can be migrated, but afterwards couldn't be opened anymore with older versions. Do you want to migrate it?", false)) {
if (!_console->askYesNo("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). You're running a CryFS version using storage format " + CryConfig::FilesystemFormatVersion + ". It can be migrated, but afterwards couldn't be opened anymore with older versions. Please make a backup of your data before attempting a migration. Do you want to migrate it now?", false)) {
throw CryfsException("This filesystem is for CryFS " + config.Version() + " (or a later version with the same storage format). It has to be migrated.", ErrorCode::TooOldFilesystemFormat);
}
}

2
src/cryfs/filesystem/cachingfsblobstore/CachingFsBlobStore.h
View File

@ -50,7 +50,7 @@ namespace cryfs {
inline CachingFsBlobStore::CachingFsBlobStore(cpputils::unique_ref<fsblobstore::FsBlobStore> baseBlobStore)
: _baseBlobStore(std::move(baseBlobStore)), _cache() {
: _baseBlobStore(std::move(baseBlobStore)), _cache("fsblobstore") {
}
inline CachingFsBlobStore::~CachingFsBlobStore() {

77
src/fspp/fuse/Fuse.cpp
View File

@ -7,9 +7,16 @@
#include <iostream>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/logging/logging.h>
#include <cpp-utils/process/subprocess.h>
#include <cpp-utils/thread/debugging.h>
#include <csignal>
#include "InvalidFilesystem.h"
#if defined(_MSC_VER)
#include <codecvt>
#include <dokan/dokan.h>
#endif
using std::vector;
using std::string;
@ -17,7 +24,9 @@ namespace bf = boost::filesystem;
using namespace cpputils::logging;
using std::make_shared;
using std::shared_ptr;
using std::string;
using namespace fspp::fuse;
using cpputils::set_thread_name;
namespace {
bool is_valid_fspp_path(const bf::path& path) {
@ -26,6 +35,18 @@ bool is_valid_fspp_path(const bf::path& path) {
&& !path.has_root_name() // on Windows, it shouldn't have a device specifier (i.e. no "C:")
&& (path.string() == path.generic_string()); // must use portable '/' as directory separator
}
class ThreadNameForDebugging final {
public:
ThreadNameForDebugging(const string& threadName) {
std::string name = "fspp_" + threadName;
set_thread_name(name.c_str());
}
~ThreadNameForDebugging() {
set_thread_name("fspp_idle");
}
};
}
#define FUSE_OBJ (static_cast<Fuse *>(fuse_get_context()->private_data))
@ -307,18 +328,29 @@ bool Fuse::running() const {
}
void Fuse::stop() {
unmount(_mountdir, false);
}
void Fuse::unmount(const bf::path& mountdir, bool force) {
//TODO Find better way to unmount (i.e. don't use external fusermount). Unmounting by kill(getpid(), SIGINT) worked, but left the mount directory transport endpoint as not connected.
#ifdef __APPLE__
int ret = system(("umount " + _mountdir.string()).c_str());
#if defined(__APPLE__)
int returncode = cpputils::Subprocess::call(std::string("umount ") + mountdir.string()).exitcode;
#elif defined(_MSC_VER)
std::wstring mountdir_ = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(mountdir.string());
BOOL success = DokanRemoveMountPoint(mountdir_.c_str());
int returncode = success ? 0 : -1;
#else
int ret = system(("fusermount -z -u " + _mountdir.string()).c_str()); // "-z" takes care that if the filesystem can't be unmounted right now because something is opened, it will be unmounted as soon as it can be.
std::string command = force ? "fusermount -u" : "fusermount -z -u"; // "-z" takes care that if the filesystem can't be unmounted right now because something is opened, it will be unmounted as soon as it can be.
int returncode = cpputils::Subprocess::call(
command + " " + mountdir.string()).exitcode;
#endif
if (ret != 0) {
LOG(ERR, "Could not unmount filesystem");
if (returncode != 0) {
throw std::runtime_error("Could not unmount filesystem");
}
}
int Fuse::getattr(const bf::path &path, fspp::fuse::STAT *stbuf) {
ThreadNameForDebugging _threadName("getattr");
#ifdef FSPP_LOG
LOG(DEBUG, "getattr({}, _, _)", path);
#endif
@ -341,6 +373,7 @@ int Fuse::getattr(const bf::path &path, fspp::fuse::STAT *stbuf) {
}
int Fuse::fgetattr(const bf::path &path, fspp::fuse::STAT *stbuf, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("fgetattr");
#ifdef FSPP_LOG
LOG(DEBUG, "fgetattr({}, _, _)\n", path);
#endif
@ -373,6 +406,7 @@ int Fuse::fgetattr(const bf::path &path, fspp::fuse::STAT *stbuf, fuse_file_info
}
int Fuse::readlink(const bf::path &path, char *buf, size_t size) {
ThreadNameForDebugging _threadName("readlink");
#ifdef FSPP_LOG
LOG(DEBUG, "readlink({}, _, {})", path, size);
#endif
@ -398,11 +432,13 @@ int Fuse::mknod(const bf::path &path, ::mode_t mode, dev_t rdev) {
UNUSED(rdev);
UNUSED(mode);
UNUSED(path);
ThreadNameForDebugging _threadName("mknod");
LOG(WARN, "Called non-implemented mknod({}, {}, _)", path, mode);
return ENOSYS;
}
int Fuse::mkdir(const bf::path &path, ::mode_t mode) {
ThreadNameForDebugging _threadName("mkdir");
#ifdef FSPP_LOG
LOG(DEBUG, "mkdir({}, {})", path, mode);
#endif
@ -426,6 +462,7 @@ int Fuse::mkdir(const bf::path &path, ::mode_t mode) {
}
int Fuse::unlink(const bf::path &path) {
ThreadNameForDebugging _threadName("unlink");
#ifdef FSPP_LOG
LOG(DEBUG, "unlink({})", path);
#endif
@ -448,6 +485,7 @@ int Fuse::unlink(const bf::path &path) {
}
int Fuse::rmdir(const bf::path &path) {
ThreadNameForDebugging _threadName("rmdir");
#ifdef FSPP_LOG
LOG(DEBUG, "rmdir({})", path);
#endif
@ -469,14 +507,15 @@ int Fuse::rmdir(const bf::path &path) {
}
}
int Fuse::symlink(const bf::path &from, const bf::path &to) {
int Fuse::symlink(const bf::path &to, const bf::path &from) {
ThreadNameForDebugging _threadName("symlink");
#ifdef FSPP_LOG
LOG(DEBUG, "symlink({}, {})", from, to);
LOG(DEBUG, "symlink({}, {})", to, from);
#endif
try {
ASSERT(is_valid_fspp_path(from), "has to be an absolute path");
auto context = fuse_get_context();
_fs->createSymlink(from, to, context->uid, context->gid);
_fs->createSymlink(to, from, context->uid, context->gid);
return 0;
} catch(const cpputils::AssertFailed &e) {
LOG(ERR, "AssertFailed in Fuse::symlink: {}", e.what());
@ -493,6 +532,7 @@ int Fuse::symlink(const bf::path &from, const bf::path &to) {
}
int Fuse::rename(const bf::path &from, const bf::path &to) {
ThreadNameForDebugging _threadName("rename");
#ifdef FSPP_LOG
LOG(DEBUG, "rename({}, {})", from, to);
#endif
@ -517,6 +557,7 @@ int Fuse::rename(const bf::path &from, const bf::path &to) {
//TODO
int Fuse::link(const bf::path &from, const bf::path &to) {
ThreadNameForDebugging _threadName("link");
LOG(WARN, "NOT IMPLEMENTED: link({}, {})", from, to);
//auto real_from = _impl->RootDir() / from;
//auto real_to = _impl->RootDir() / to;
@ -526,6 +567,7 @@ int Fuse::link(const bf::path &from, const bf::path &to) {
}
int Fuse::chmod(const bf::path &path, ::mode_t mode) {
ThreadNameForDebugging _threadName("chmod");
#ifdef FSPP_LOG
LOG(DEBUG, "chmod({}, {})", path, mode);
#endif
@ -548,6 +590,7 @@ int Fuse::chmod(const bf::path &path, ::mode_t mode) {
}
int Fuse::chown(const bf::path &path, ::uid_t uid, ::gid_t gid) {
ThreadNameForDebugging _threadName("chown");
#ifdef FSPP_LOG
LOG(DEBUG, "chown({}, {}, {})", path, uid, gid);
#endif
@ -570,6 +613,7 @@ int Fuse::chown(const bf::path &path, ::uid_t uid, ::gid_t gid) {
}
int Fuse::truncate(const bf::path &path, int64_t size) {
ThreadNameForDebugging _threadName("truncate");
#ifdef FSPP_LOG
LOG(DEBUG, "truncate({}, {})", path, size);
#endif
@ -592,6 +636,7 @@ int Fuse::truncate(const bf::path &path, int64_t size) {
}
int Fuse::ftruncate(const bf::path &path, int64_t size, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("ftruncate");
#ifdef FSPP_LOG
LOG(DEBUG, "ftruncate({}, {})", path, size);
#endif
@ -614,6 +659,7 @@ int Fuse::ftruncate(const bf::path &path, int64_t size, fuse_file_info *fileinfo
}
int Fuse::utimens(const bf::path &path, const timespec times[2]) {
ThreadNameForDebugging _threadName("utimens");
#ifdef FSPP_LOG
LOG(DEBUG, "utimens({}, _)", path);
#endif
@ -636,6 +682,7 @@ int Fuse::utimens(const bf::path &path, const timespec times[2]) {
}
int Fuse::open(const bf::path &path, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("open");
#ifdef FSPP_LOG
LOG(DEBUG, "open({}, _)", path);
#endif
@ -658,6 +705,7 @@ int Fuse::open(const bf::path &path, fuse_file_info *fileinfo) {
}
int Fuse::release(const bf::path &path, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("release");
#ifdef FSPP_LOG
LOG(DEBUG, "release({}, _)", path);
#endif
@ -680,6 +728,7 @@ int Fuse::release(const bf::path &path, fuse_file_info *fileinfo) {
}
int Fuse::read(const bf::path &path, char *buf, size_t size, int64_t offset, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("read");
#ifdef FSPP_LOG
LOG(DEBUG, "read({}, _, {}, {}, _)", path, size, offset);
#endif
@ -701,6 +750,7 @@ int Fuse::read(const bf::path &path, char *buf, size_t size, int64_t offset, fus
}
int Fuse::write(const bf::path &path, const char *buf, size_t size, int64_t offset, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("write");
#ifdef FSPP_LOG
LOG(DEBUG, "write({}, _, {}, {}, _)", path, size, offsset);
#endif
@ -723,6 +773,7 @@ int Fuse::write(const bf::path &path, const char *buf, size_t size, int64_t offs
}
int Fuse::statfs(const bf::path &path, struct ::statvfs *fsstat) {
ThreadNameForDebugging _threadName("statfs");
#ifdef FSPP_LOG
LOG(DEBUG, "statfs({}, _)", path);
#endif
@ -746,6 +797,7 @@ int Fuse::statfs(const bf::path &path, struct ::statvfs *fsstat) {
}
int Fuse::flush(const bf::path &path, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("flush");
#ifdef FSPP_LOG
LOG(WARN, "flush({}, _)", path);
#endif
@ -768,6 +820,7 @@ int Fuse::flush(const bf::path &path, fuse_file_info *fileinfo) {
}
int Fuse::fsync(const bf::path &path, int datasync, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("fsync");
#ifdef FSPP_LOG
LOG(DEBUG, "fsync({}, {}, _)", path, datasync);
#endif
@ -796,12 +849,14 @@ int Fuse::fsync(const bf::path &path, int datasync, fuse_file_info *fileinfo) {
int Fuse::opendir(const bf::path &path, fuse_file_info *fileinfo) {
UNUSED(path);
UNUSED(fileinfo);
ThreadNameForDebugging _threadName("opendir");
//LOG(DEBUG, "opendir({}, _)", path);
//We don't need opendir, because readdir works directly on the path
return 0;
}
int Fuse::readdir(const bf::path &path, void *buf, fuse_fill_dir_t filler, int64_t offset, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("readdir");
#ifdef FSPP_LOG
LOG(DEBUG, "readdir({}, _, _, {}, _)", path, offest);
#endif
@ -847,6 +902,7 @@ int Fuse::readdir(const bf::path &path, void *buf, fuse_fill_dir_t filler, int64
int Fuse::releasedir(const bf::path &path, fuse_file_info *fileinfo) {
UNUSED(path);
UNUSED(fileinfo);
ThreadNameForDebugging _threadName("releasedir");
//LOG(DEBUG, "releasedir({}, _)", path);
//We don't need releasedir, because readdir works directly on the path
return 0;
@ -857,12 +913,14 @@ int Fuse::fsyncdir(const bf::path &path, int datasync, fuse_file_info *fileinfo)
UNUSED(fileinfo);
UNUSED(datasync);
UNUSED(path);
ThreadNameForDebugging _threadName("fsyncdir");
//LOG(WARN, "Called non-implemented fsyncdir({}, {}, _)", path, datasync);
return 0;
}
void Fuse::init(fuse_conn_info *conn) {
UNUSED(conn);
ThreadNameForDebugging _threadName("init");
_fs = _init(this);
LOG(INFO, "Filesystem started.");
@ -876,6 +934,7 @@ void Fuse::init(fuse_conn_info *conn) {
}
void Fuse::destroy() {
ThreadNameForDebugging _threadName("destroy");
_fs = make_shared<InvalidFilesystem>();
LOG(INFO, "Filesystem stopped.");
_running = false;
@ -883,6 +942,7 @@ void Fuse::destroy() {
}
int Fuse::access(const bf::path &path, int mask) {
ThreadNameForDebugging _threadName("access");
#ifdef FSPP_LOG
LOG(DEBUG, "access({}, {})", path, mask);
#endif
@ -905,6 +965,7 @@ int Fuse::access(const bf::path &path, int mask) {
}
int Fuse::create(const bf::path &path, ::mode_t mode, fuse_file_info *fileinfo) {
ThreadNameForDebugging _threadName("create");
#ifdef FSPP_LOG
LOG(DEBUG, "create({}, {}, _)", path, mode);
#endif

2
src/fspp/fuse/Fuse.h
View File

@ -28,6 +28,8 @@ public:
bool running() const;
void stop();
static void unmount(const boost::filesystem::path &mountdir, bool force = false);
int getattr(const boost::filesystem::path &path, fspp::fuse::STAT *stbuf);
int fgetattr(const boost::filesystem::path &path, fspp::fuse::STAT *stbuf, fuse_file_info *fileinfo);
int readlink(const boost::filesystem::path &path, char *buf, size_t size);

2
test/blobstore/CMakeLists.txt
View File

@ -19,7 +19,7 @@ set(SOURCES
implementations/onblocks/datatreestore/DataTreeTest_NumStoredBytes.cpp
implementations/onblocks/datatreestore/DataTreeTest_ResizeNumBytes.cpp
implementations/onblocks/datatreestore/DataTreeStoreTest.cpp
implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp
implementations/onblocks/datatreestore/LeafTraverserTest.cpp
implementations/onblocks/BlobSizeTest.cpp
implementations/onblocks/BlobReadWriteTest.cpp
implementations/onblocks/BigBlobsTest.cpp

86
test/blobstore/implementations/onblocks/BlobReadWriteTest.cpp
View File

@ -63,6 +63,92 @@ TEST_F(BlobReadWriteTest, WritingCloseTo16ByteLimitDoesntDestroySize) {
EXPECT_EQ(32780u, blob->size());
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenTryReadInFirstLeaf_thenFails) {
Data data(5);
size_t read = blob->tryRead(data.data(), 3, 5);
EXPECT_EQ(0, read);
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenTryReadInLaterLeaf_thenFails) {
Data data(5);
size_t read = blob->tryRead(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5);
EXPECT_EQ(0, read);
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadInFirstLeaf_thenFails) {
Data data(5);
EXPECT_ANY_THROW(
blob->read(data.data(), 3, 5)
);
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadInLaterLeaf_thenFails) {
Data data(5);
EXPECT_ANY_THROW(
blob->read(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5)
);
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenReadAll_thenReturnsZeroSizedData) {
Data data = blob->readAll();
EXPECT_EQ(0, data.size());
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenWrite_thenGrows) {
Data data(5);
blob->write(data.data(), 4, 5);
EXPECT_EQ(9, blob->size());
}
TEST_F(BlobReadWriteTest, givenEmptyBlob_whenWriteZeroBytes_thenDoesntGrow) {
Data data(5);
blob->write(data.data(), 4, 0);
EXPECT_EQ(0, blob->size());;
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenTryReadInFirstLeaf_thenFails) {
Data data(5);
size_t read = blob->tryRead(data.data(), 3, 5);
EXPECT_EQ(0, read);
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenTryReadInLaterLeaf_thenFails) {
Data data(5);
size_t read = blob->tryRead(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5);
EXPECT_EQ(0, read);
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadInFirstLeaf_thenFails) {
Data data(5);
EXPECT_ANY_THROW(
blob->read(data.data(), 3, 5)
);
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadInLaterLeaf_thenFails) {
Data data(5);
EXPECT_ANY_THROW(
blob->read(data.data(), 2*LAYOUT.maxBytesPerLeaf(), 5)
);
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenReadAll_thenReturnsZeroSizedData) {
Data data = blob->readAll();
EXPECT_EQ(0, data.size());
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenWrite_thenGrows) {
Data data(5);
blob->write(data.data(), 4, 5);
EXPECT_EQ(9, blob->size());
}
TEST_F(BlobReadWriteTest, givenBlobResizedToZero_whenWriteZeroBytes_thenDoesntGrow) {
Data data(5);
blob->write(data.data(), 4, 0);
EXPECT_EQ(0, blob->size());
}
struct DataRange {
uint64_t blobsize;
uint64_t offset;

18
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_NumStoredBytes.cpp
View File

@ -13,7 +13,7 @@ public:
TEST_F(DataTreeTest_NumStoredBytes, CreatedTreeIsEmpty) {
auto tree = treeStore.createNewTree();
EXPECT_EQ(0u, tree->numStoredBytes());
EXPECT_EQ(0u, tree->numBytes());
}
class DataTreeTest_NumStoredBytes_P: public DataTreeTest_NumStoredBytes, public WithParamInterface<uint32_t> {};
@ -24,47 +24,47 @@ INSTANTIATE_TEST_CASE_P(FullLastLeaf, DataTreeTest_NumStoredBytes_P, Values(stat
TEST_P(DataTreeTest_NumStoredBytes_P, SingleLeaf) {
BlockId blockId = CreateLeafWithSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(GetParam(), tree->numStoredBytes());
EXPECT_EQ(GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, TwoLeafTree) {
BlockId blockId = CreateTwoLeafWithSecondLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, FullTwolevelTree) {
BlockId blockId = CreateFullTwoLevelWithLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithOneChild) {
BlockId blockId = CreateThreeLevelWithOneChildAndLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithTwoChildren) {
BlockId blockId = CreateThreeLevelWithTwoChildrenAndLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, ThreeLevelTreeWithThreeChildren) {
BlockId blockId = CreateThreeLevelWithThreeChildrenAndLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(2*nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numStoredBytes());
EXPECT_EQ(2*nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxBytesPerLeaf() + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, FullThreeLevelTree) {
BlockId blockId = CreateFullThreeLevelWithLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + nodeStore->layout().maxBytesPerLeaf()*(nodeStore->layout().maxChildrenPerInnerNode()-1) + GetParam(), tree->numBytes());
}
TEST_P(DataTreeTest_NumStoredBytes_P, FourLevelMinDataTree) {
BlockId blockId = CreateFourLevelMinDataWithLastLeafSize(GetParam())->blockId();
auto tree = treeStore.load(blockId).value();
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + GetParam(), tree->numStoredBytes());
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf()*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + GetParam(), tree->numBytes());
}

160
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_Performance.cpp
View File

@ -3,14 +3,24 @@
#include <gmock/gmock.h>
using blobstore::onblocks::datatreestore::DataTree;
using blobstore::onblocks::datatreestore::LeafHandle;
using blockstore::BlockId;
using cpputils::Data;
class DataTreeTest_Performance: public DataTreeTest {
public:
void Traverse(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
tree->traverseLeaves(beginIndex, endIndex, [] (uint32_t /*index*/, bool /*isRightBorderNode*/, LeafHandle /*leaf*/) {}, [this] (uint32_t /*index*/) -> Data {return Data(maxChildrenPerInnerNode).FillWithZeroes();});
void TraverseByWriting(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
uint64_t offset = beginIndex * maxBytesPerLeaf;
uint64_t count = endIndex * maxBytesPerLeaf - offset;
Data data(count);
data.FillWithZeroes();
tree->writeBytes(data.data(), offset, count);
}
void TraverseByReading(DataTree *tree, uint64_t beginIndex, uint64_t endIndex) {
uint64_t offset = beginIndex * maxBytesPerLeaf;
uint64_t count = endIndex * maxBytesPerLeaf - offset;
Data data(count);
tree->readBytes(data.data(), offset, count);
}
uint64_t maxChildrenPerInnerNode = nodeStore->layout().maxChildrenPerInnerNode();
@ -71,84 +81,168 @@ TEST_F(DataTreeTest_Performance, DeletingDoesntLoadLeaves_Threelevel_DeleteByKey
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All) {
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All_ByWriting) {
auto blockId = CreateFullTwoLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 0, maxChildrenPerInnerNode);
TraverseByWriting(tree.get(), 0, maxChildrenPerInnerNode);
EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Has to load the rightmost leaf once to adapt its size, rest of the leaves aren't loaded but just overwritten
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some) {
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_All_ByReading) {
auto blockId = CreateFullTwoLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 3, 5);
TraverseByReading(tree.get(), 0, maxChildrenPerInnerNode);
EXPECT_EQ(0u, blockStore->loadedBlocks().size()); // Doesn't actually load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Has to read the rightmost leaf an additional time in the beginning to determine size.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All) {
auto blockId = CreateFullThreeLevel()->blockId();
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some_ByWriting) {
auto blockId = CreateFullTwoLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
TraverseByWriting(tree.get(), 3, 5);
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner nodes. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(0u, blockStore->loadedBlocks().size());
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Twolevel_Some_ByReading) {
auto blockId = CreateFullTwoLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByReading(tree.get(), 3, 5);
EXPECT_EQ(3u, blockStore->loadedBlocks().size()); // reads 2 leaves and the rightmost leaf to determine size
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner) {
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All_ByWriting) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 3, 5);
TraverseByWriting(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(maxChildrenPerInnerNode + 1, blockStore->loadedBlocks().size()); // Loads inner nodes and has to load the rightmost leaf once to adapt its size, rest of the leaves aren't loaded but just overwritten.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(maxChildrenPerInnerNode*maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_All_ByReading) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByReading(tree.get(), 0, maxChildrenPerInnerNode * maxChildrenPerInnerNode);
EXPECT_EQ(maxChildrenPerInnerNode*maxChildrenPerInnerNode + maxChildrenPerInnerNode + 2, blockStore->loadedBlocks().size()); // Loads inner nodes and leaves. Has to load the rightmost inner node and leaf an additional time at the beginning to compute size
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner) {
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner_ByWriting) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 3, 3 + maxChildrenPerInnerNode);
TraverseByWriting(tree.get(), 3, 5);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, they're just overwritten.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InOneInner_ByReading) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByReading(tree.get(), 3, 5);
EXPECT_EQ(5u, blockStore->loadedBlocks().size()); // reads 2 leaves and the inner node, also has to read the rightmost inner node and leaf additionally at the beginning to determine size
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner_ByWriting) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByWriting(tree.get(), 3, 3 + maxChildrenPerInnerNode);
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads both inner node
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner) {
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_InTwoInner_ByReading) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
TraverseByReading(tree.get(), 3, 3 + maxChildrenPerInnerNode);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, but returns the keys of the leaves to the callback
EXPECT_EQ(4u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads both inner nodes and the requested leaves. Also has to load rightmost inner node and leaf additionally in the beginning to determine size.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner_ByWriting) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByWriting(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads inner node. Doesn't load the leaves, they're just overwritten.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(maxChildrenPerInnerNode, blockStore->distinctWrittenBlocks().size());
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
TEST_F(DataTreeTest_Performance, TraverseLeaves_Threelevel_WholeInner_ByReading) {
auto blockId = CreateFullThreeLevel()->blockId();
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
TraverseByReading(tree.get(), maxChildrenPerInnerNode, 2*maxChildrenPerInnerNode);
EXPECT_EQ(3u + maxChildrenPerInnerNode, blockStore->loadedBlocks().size()); // Loads inner node and all requested leaves. Also has to load rightmost inner node and leaf additionally in the beginning to determine size.
EXPECT_EQ(0u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(0u, blockStore->distinctWrittenBlocks().size());
@ -160,12 +254,12 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingInside) {
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 1, 4);
TraverseByWriting(tree.get(), 1, 4);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old child (for growing it)
EXPECT_EQ(2u, blockStore->createdBlocks());
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to inner node
EXPECT_EQ(2u, blockStore->distinctWrittenBlocks().size()); // write the data and add children to inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
@ -174,7 +268,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingOutside_TwoL
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 4, 5);
TraverseByWriting(tree.get(), 4, 5);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(3u, blockStore->createdBlocks());
@ -188,7 +282,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingOutside_Thre
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 2*maxChildrenPerInnerNode+1, 2*maxChildrenPerInnerNode+2);
TraverseByWriting(tree.get(), 2*maxChildrenPerInnerNode+1, 2*maxChildrenPerInnerNode+2);
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads last old leaf (and its inner node) for growing it
EXPECT_EQ(3u, blockStore->createdBlocks()); // inner node and two leaves
@ -202,12 +296,12 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTree_StartingAtBeginOfChi
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode);
TraverseByWriting(tree.get(), maxChildrenPerInnerNode, 3*maxChildrenPerInnerNode);
EXPECT_EQ(2u, blockStore->loadedBlocks().size()); // Loads inner node and one leaf to check whether we have to grow it. Doesn't load the leaves, but returns the keys of the leaves to the callback.
EXPECT_EQ(1u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // Creates an inner node and its leaves
EXPECT_EQ(0u, blockStore->removedBlocks().size());
EXPECT_EQ(1u, blockStore->distinctWrittenBlocks().size()); // add children to existing inner node
EXPECT_EQ(maxChildrenPerInnerNode + 1u, blockStore->distinctWrittenBlocks().size()); // write data and add children to existing inner node
EXPECT_EQ(0u, blockStore->resizedBlocks().size());
}
@ -216,7 +310,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
TraverseByWriting(tree.get(), 4, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@ -230,7 +324,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInOldDe
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), 4, maxChildrenPerInnerNode+2);
TraverseByWriting(tree.get(), 4, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@ -244,7 +338,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
TraverseByWriting(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves
@ -258,7 +352,7 @@ TEST_F(DataTreeTest_Performance, TraverseLeaves_GrowingTreeDepth_StartingInNewDe
auto tree = treeStore.load(blockId).value();
blockStore->resetCounters();
Traverse(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
TraverseByWriting(tree.get(), maxChildrenPerInnerNode, maxChildrenPerInnerNode+2);
EXPECT_EQ(1u, blockStore->loadedBlocks().size()); // Loads last old leaf for growing it
EXPECT_EQ(2u + maxChildrenPerInnerNode, blockStore->createdBlocks()); // 2x new inner node + leaves

35
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeByTraversing.cpp
View File

@ -19,7 +19,6 @@ using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blobstore::onblocks::datatreestore::DataTree;
using blobstore::onblocks::datatreestore::LeafHandle;
using blobstore::onblocks::utils::ceilDivision;
using blockstore::BlockId;
using cpputils::Data;
@ -109,9 +108,13 @@ public:
GrowTree(tree.get().get());
}
void GrowTree(DataTree *tree, std::function<void (int32_t)> traverse = [] (uint32_t){}) {
void GrowTree(DataTree *tree) {
uint64_t maxBytesPerLeaf = tree->maxBytesPerLeaf();
tree->traverseLeaves(traversalBeginIndex, newNumberOfLeaves, [&traverse] (uint32_t index, bool, LeafHandle){traverse(index);}, [maxBytesPerLeaf, &traverse] (uint32_t index) -> Data { traverse(index); return Data(maxBytesPerLeaf).FillWithZeroes();});
uint64_t offset = traversalBeginIndex * maxBytesPerLeaf;
uint64_t count = newNumberOfLeaves * maxBytesPerLeaf - offset;
Data data(count);
data.FillWithZeroes();
tree->writeBytes(data.data(), offset, count);
tree->flush();
}
@ -163,7 +166,6 @@ INSTANTIATE_TEST_CASE_P(DataTreeTest_ResizeByTraversing_P, DataTreeTest_ResizeBy
),
//Decide the traversal begin index
Values(
[] (uint32_t /*oldNumberOfLeaves*/, uint32_t newNumberOfLeaves) {return newNumberOfLeaves;}, // Don't traverse any leaves, just resize (begin==end)
[] (uint32_t /*oldNumberOfLeaves*/, uint32_t newNumberOfLeaves) {return newNumberOfLeaves-1;}, // Traverse last leaf (begin==end-1)
[] (uint32_t oldNumberOfLeaves, uint32_t newNumberOfLeaves) {return (oldNumberOfLeaves+newNumberOfLeaves)/2;}, // Start traversal in middle of new leaves
[] (uint32_t oldNumberOfLeaves, uint32_t /*newNumberOfLeaves*/) {return oldNumberOfLeaves-1;}, // Start traversal with last old leaf
@ -189,9 +191,9 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, NumLeavesIsCorrect_FromCache) {
TEST_P(DataTreeTest_ResizeByTraversing_P, NumLeavesIsCorrect) {
GrowTree(tree.get());
// tree->_forceComputeNumLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
// tree->forceComputeNumLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
// This is what the StructureIsValid test case is for.
EXPECT_EQ(newNumberOfLeaves, tree->_forceComputeNumLeaves());
EXPECT_EQ(newNumberOfLeaves, tree->forceComputeNumLeaves());
}
TEST_P(DataTreeTest_ResizeByTraversing_P, DepthFlagsAreCorrect) {
@ -208,7 +210,8 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, KeyDoesntChange) {
}
TEST_P(DataTreeTest_ResizeByTraversing_P, DataStaysIntact) {
uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numStoredBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
TwoLevelDataFixture data(nodeStore, TwoLevelDataFixture::SizePolicy::Unchanged);
BlockId blockId = tree->blockId();
cpputils::destruct(std::move(tree));
@ -216,15 +219,13 @@ TEST_P(DataTreeTest_ResizeByTraversing_P, DataStaysIntact) {
GrowTree(blockId);
data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), oldNumberOfLeaves, oldLastLeafSize);
}
TEST_P(DataTreeTest_ResizeByTraversing_P, AllLeavesAreTraversed) {
std::vector<uint32_t> traversedLeaves;
GrowTree(tree.get(), [&traversedLeaves] (uint32_t index) {traversedLeaves.push_back(index);});
EXPECT_EQ(newNumberOfLeaves-traversalBeginIndex, traversedLeaves.size());
for (uint32_t i = traversalBeginIndex; i < newNumberOfLeaves; ++i) {
EXPECT_NE(traversedLeaves.end(), std::find(traversedLeaves.begin(), traversedLeaves.end(), i));
if (traversalBeginIndex < oldNumberOfLeaves) {
// Traversal wrote over part of the pre-existing data, we can only check the data before it.
if (traversalBeginIndex != 0) {
data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), traversalBeginIndex - 1);
}
} else {
// Here, traversal was entirely outside the preexisting data, we can check all preexisting data.
data.EXPECT_DATA_CORRECT(nodeStore->load(blockId).get().get(), oldNumberOfLeaves, oldLastLeafSize);
}
}

8
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_ResizeNumBytes.cpp
View File

@ -171,9 +171,9 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, StructureIsValid) {
TEST_P(DataTreeTest_ResizeNumBytes_P, NumBytesIsCorrect) {
tree->resizeNumBytes(newSize);
tree->flush();
// tree->numStoredBytes() only goes down the right border nodes and expects the tree to be a left max data tree.
// tree->numBytes() only goes down the right border nodes and expects the tree to be a left max data tree.
// This is what the StructureIsValid test case is for.
EXPECT_EQ(newSize, tree->numStoredBytes());
EXPECT_EQ(newSize, tree->numBytes());
}
TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect) {
@ -181,7 +181,7 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect) {
tree->flush();
// tree->numLeaves() only goes down the right border nodes and expects the tree to be a left max data tree.
// This is what the StructureIsValid test case is for.
EXPECT_EQ(newNumberOfLeaves, tree->_forceComputeNumLeaves());
EXPECT_EQ(newNumberOfLeaves, tree->forceComputeNumLeaves());
}
TEST_P(DataTreeTest_ResizeNumBytes_P, NumLeavesIsCorrect_FromCache) {
@ -208,7 +208,7 @@ TEST_P(DataTreeTest_ResizeNumBytes_P, KeyDoesntChange) {
}
TEST_P(DataTreeTest_ResizeNumBytes_P, DataStaysIntact) {
uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numStoredBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
uint32_t oldNumberOfLeaves = std::max(UINT64_C(1), ceilDivision(tree->numBytes(), static_cast<uint64_t>(nodeStore->layout().maxBytesPerLeaf())));
TwoLevelDataFixture data(nodeStore, TwoLevelDataFixture::SizePolicy::Unchanged);
BlockId blockId = tree->blockId();
cpputils::destruct(std::move(tree));

170
test/blobstore/implementations/onblocks/datatreestore/DataTreeTest_TraverseLeaves.cpp → test/blobstore/implementations/onblocks/datatreestore/LeafTraverserTest.cpp
View File

@ -1,4 +1,5 @@
#include "testutils/DataTreeTest.h"
#include <blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h>
#include <gmock/gmock.h>
using ::testing::_;
@ -9,6 +10,7 @@ using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datatreestore::LeafHandle;
using blobstore::onblocks::datatreestore::LeafTraverser;
using blockstore::BlockId;
using cpputils::unique_ref;
@ -26,9 +28,9 @@ MATCHER_P(KeyEq, expected, "node blockId equals") {
return arg->blockId() == expected;
}
class DataTreeTest_TraverseLeaves: public DataTreeTest {
class LeafTraverserTest: public DataTreeTest {
public:
DataTreeTest_TraverseLeaves() :traversor() {}
LeafTraverserTest() :traversor() {}
unique_ref<DataInnerNode> CreateThreeLevel() {
return CreateInner({
@ -70,166 +72,172 @@ public:
EXPECT_CALL(traversor, calledCreateLeaf(_)).Times(0);
}
void TraverseLeaves(DataNode *root, uint32_t beginIndex, uint32_t endIndex) {
void TraverseLeaves(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool expectReadOnly) {
root->flush();
auto tree = treeStore.load(root->blockId()).value();
tree->traverseLeaves(beginIndex, endIndex, [this] (uint32_t nodeIndex, bool isRightBorderNode,LeafHandle leaf) {
auto* old_root = root.get();
LeafTraverser(nodeStore, expectReadOnly).traverseAndUpdateRoot(&root, beginIndex, endIndex, [this] (uint32_t nodeIndex, bool isRightBorderNode,LeafHandle leaf) {
traversor.calledExistingLeaf(leaf.node(), isRightBorderNode, nodeIndex);
}, [this] (uint32_t nodeIndex) -> Data {
return traversor.calledCreateLeaf(nodeIndex)->copy();
});
}, [] (auto) {});
if (expectReadOnly) {
EXPECT_EQ(old_root, root.get());
} else {
EXPECT_NE(old_root, root.get());
}
}
TraversorMock traversor;
};
TEST_F(DataTreeTest_TraverseLeaves, TraverseSingleLeafTree) {
auto root = CreateLeaf();
TEST_F(LeafTraverserTest, TraverseSingleLeafTree) {
unique_ref<DataNode> root = CreateLeaf();
EXPECT_TRAVERSE_LEAF(root->blockId(), true, 0);
TraverseLeaves(root.get(), 0, 1);
TraverseLeaves(std::move(root), 0, 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInSingleLeafTree1) {
auto root = CreateLeaf();
TEST_F(LeafTraverserTest, TraverseNothingInSingleLeafTree1) {
unique_ref<DataNode> root = CreateLeaf();
EXPECT_DONT_TRAVERSE_ANY_LEAVES();
TraverseLeaves(root.get(), 0, 0);
TraverseLeaves(std::move(root), 0, 0, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInSingleLeafTree2) {
auto root = CreateLeaf();
TEST_F(LeafTraverserTest, TraverseNothingInSingleLeafTree2) {
unique_ref<DataNode> root = CreateLeaf();
EXPECT_DONT_TRAVERSE_ANY_LEAVES();
TraverseLeaves(root.get(), 1, 1);
TraverseLeaves(std::move(root), 1, 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseFirstLeafOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
EXPECT_TRAVERSE_LEAF(root->readChild(0).blockId(), false, 0);
TraverseLeaves(root.get(), 0, 1);
TraverseLeaves(std::move(root), 0, 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseMiddleLeafOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
EXPECT_TRAVERSE_LEAF(root->readChild(5).blockId(), false, 5);
TraverseLeaves(root.get(), 5, 6);
TraverseLeaves(std::move(root), 5, 6, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseLastLeafOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
EXPECT_TRAVERSE_LEAF(root->readChild(nodeStore->layout().maxChildrenPerInnerNode()-1).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode()-1);
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode()-1, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode()-1, nodeStore->layout().maxChildrenPerInnerNode(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInFullTwolevelTree1) {
TEST_F(LeafTraverserTest, TraverseNothingInFullTwolevelTree1) {
auto root = CreateFullTwoLevel();
EXPECT_DONT_TRAVERSE_ANY_LEAVES();
TraverseLeaves(root.get(), 0, 0);
TraverseLeaves(std::move(root), 0, 0, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseNothingInFullTwolevelTree2) {
TEST_F(LeafTraverserTest, TraverseNothingInFullTwolevelTree2) {
auto root = CreateFullTwoLevel();
EXPECT_DONT_TRAVERSE_ANY_LEAVES();
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfThreeLevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseFirstLeafOfThreeLevelMinDataTree) {
auto root = CreateThreeLevelMinData();
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(0).blockId(), false, 0);
TraverseLeaves(root.get(), 0, 1);
TraverseLeaves(std::move(root), 0, 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfThreeLevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseMiddleLeafOfThreeLevelMinDataTree) {
auto root = CreateThreeLevelMinData();
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(5).blockId(), false, 5);
TraverseLeaves(root.get(), 5, 6);
TraverseLeaves(std::move(root), 5, 6, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfThreeLevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseLastLeafOfThreeLevelMinDataTree) {
auto root = CreateThreeLevelMinData();
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode()+1);
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode(), nodeStore->layout().maxChildrenPerInnerNode()+1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseAllLeavesOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
EXPECT_TRAVERSE_ALL_CHILDREN_OF(*root, true, 0);
TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseAllLeavesOfThreelevelMinDataTree) {
auto root = CreateThreeLevelMinData();
EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(root->readChild(0).blockId()), false, 0);
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode()+1);
TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode()+1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstChildOfThreelevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseFirstChildOfThreelevelMinDataTree) {
auto root = CreateThreeLevelMinData();
EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(root->readChild(0).blockId()), false, 0);
TraverseLeaves(root.get(), 0, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(std::move(root), 0, nodeStore->layout().maxChildrenPerInnerNode(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseFirstPartOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
for (unsigned int i = 0; i < 5; ++i) {
EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), false, i);
}
TraverseLeaves(root.get(), 0, 5);
TraverseLeaves(std::move(root), 0, 5, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseInnerPartOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseInnerPartOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
for (unsigned int i = 5; i < 10; ++i) {
EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), false, i);
}
TraverseLeaves(root.get(), 5, 10);
TraverseLeaves(std::move(root), 5, 10, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfFullTwolevelTree) {
TEST_F(LeafTraverserTest, TraverseLastPartOfFullTwolevelTree) {
auto root = CreateFullTwoLevel();
for (unsigned int i = 5; i < nodeStore->layout().maxChildrenPerInnerNode(); ++i) {
EXPECT_TRAVERSE_LEAF(root->readChild(i).blockId(), i==nodeStore->layout().maxChildrenPerInnerNode()-1, i);
}
TraverseLeaves(root.get(), 5, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(std::move(root), 5, nodeStore->layout().maxChildrenPerInnerNode(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseFirstPartOfThreelevelMinDataTree) {
auto root = CreateThreeLevelMinData();
auto node = LoadInnerNode(root->readChild(0).blockId());
for (unsigned int i = 0; i < 5; ++i) {
EXPECT_TRAVERSE_LEAF(node->readChild(i).blockId(), false, i);
}
TraverseLeaves(root.get(), 0, 5);
TraverseLeaves(std::move(root), 0, 5, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseInnerPartOfThreelevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseInnerPartOfThreelevelMinDataTree) {
auto root = CreateThreeLevelMinData();
auto node = LoadInnerNode(root->readChild(0).blockId());
for (unsigned int i = 5; i < 10; ++i) {
EXPECT_TRAVERSE_LEAF(node->readChild(i).blockId(), false, i);
}
TraverseLeaves(root.get(), 5, 10);
TraverseLeaves(std::move(root), 5, 10, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelMinDataTree) {
TEST_F(LeafTraverserTest, TraverseLastPartOfThreelevelMinDataTree) {
auto root = CreateThreeLevelMinData();
auto node = LoadInnerNode(root->readChild(0).blockId());
for (unsigned int i = 5; i < nodeStore->layout().maxChildrenPerInnerNode(); ++i) {
@ -237,33 +245,33 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelMinDataTree) {
}
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(1).blockId())->readChild(0).blockId(), true, nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(root.get(), 5, nodeStore->layout().maxChildrenPerInnerNode()+1);
TraverseLeaves(std::move(root), 5, nodeStore->layout().maxChildrenPerInnerNode()+1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstLeafOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseFirstLeafOfThreelevelTree) {
auto root = CreateThreeLevel();
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(0).blockId())->readChild(0).blockId(), false, 0);
TraverseLeaves(root.get(), 0, 1);
TraverseLeaves(std::move(root), 0, 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastLeafOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseLastLeafOfThreelevelTree) {
auto root = CreateThreeLevel();
uint32_t numLeaves = nodeStore->layout().maxChildrenPerInnerNode() * 5 + 3;
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readLastChild().blockId())->readLastChild().blockId(), true, numLeaves-1);
TraverseLeaves(root.get(), numLeaves-1, numLeaves);
TraverseLeaves(std::move(root), numLeaves-1, numLeaves, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddleLeafOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseMiddleLeafOfThreelevelTree) {
auto root = CreateThreeLevel();
uint32_t wantedLeafIndex = nodeStore->layout().maxChildrenPerInnerNode() * 2 + 5;
EXPECT_TRAVERSE_LEAF(LoadInnerNode(root->readChild(2).blockId())->readChild(5).blockId(), false, wantedLeafIndex);
TraverseLeaves(root.get(), wantedLeafIndex, wantedLeafIndex+1);
TraverseLeaves(std::move(root), wantedLeafIndex, wantedLeafIndex+1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseFirstPartOfThreelevelTree) {
auto root = CreateThreeLevel();
//Traverse all leaves in the first two children of the root
for(unsigned int i = 0; i < 2; ++i) {
@ -275,10 +283,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseFirstPartOfThreelevelTree) {
EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 2 * nodeStore->layout().maxChildrenPerInnerNode() + i);
}
TraverseLeaves(root.get(), 0, 2 * nodeStore->layout().maxChildrenPerInnerNode() + 5);
TraverseLeaves(std::move(root), 0, 2 * nodeStore->layout().maxChildrenPerInnerNode() + 5, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_OnlyFullChildren) {
TEST_F(LeafTraverserTest, TraverseMiddlePartOfThreelevelTree_OnlyFullChildren) {
auto root = CreateThreeLevel();
//Traverse some of the leaves in the second child of the root
auto child = LoadInnerNode(root->readChild(1).blockId());
@ -295,10 +303,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_OnlyFullC
EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 4 * nodeStore->layout().maxChildrenPerInnerNode() + i);
}
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 4 * nodeStore->layout().maxChildrenPerInnerNode() + 5);
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 4 * nodeStore->layout().maxChildrenPerInnerNode() + 5, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_AlsoLastNonfullChild) {
TEST_F(LeafTraverserTest, TraverseMiddlePartOfThreelevelTree_AlsoLastNonfullChild) {
auto root = CreateThreeLevel();
//Traverse some of the leaves in the second child of the root
auto child = LoadInnerNode(root->readChild(1).blockId());
@ -315,10 +323,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfThreelevelTree_AlsoLastN
EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), false, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
}
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + 2);
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + 2, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseLastPartOfThreelevelTree) {
auto root = CreateThreeLevel();
//Traverse some of the leaves in the second child of the root
auto child = LoadInnerNode(root->readChild(1).blockId());
@ -335,10 +343,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseLastPartOfThreelevelTree) {
EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), i == child->numChildren()-1, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
}
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren());
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode() + 5, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelTree) {
TEST_F(LeafTraverserTest, TraverseAllLeavesOfThreelevelTree) {
auto root = CreateThreeLevel();
//Traverse all leaves in the third, fourth and fifth child of the root
for(unsigned int i = 0; i < 5; ++i) {
@ -350,10 +358,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfThreelevelTree) {
EXPECT_TRAVERSE_LEAF(child->readChild(i).blockId(), i==child->numChildren()-1, 5 * nodeStore->layout().maxChildrenPerInnerNode() + i);
}
TraverseLeaves(root.get(), 0, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren());
TraverseLeaves(std::move(root), 0, 5 * nodeStore->layout().maxChildrenPerInnerNode() + child->numChildren(), true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFourLevelTree) {
TEST_F(LeafTraverserTest, TraverseAllLeavesOfFourLevelTree) {
auto root = CreateFourLevel();
//Traverse all leaves of the full threelevel tree in the first child
auto firstChild = LoadInnerNode(root->readChild(0).blockId());
@ -370,10 +378,10 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseAllLeavesOfFourLevelTree) {
EXPECT_TRAVERSE_ALL_CHILDREN_OF(*LoadInnerNode(thirdChild->readChild(0).blockId()), false, 2 * nodeStore->layout().maxChildrenPerInnerNode() * nodeStore->layout().maxChildrenPerInnerNode());
EXPECT_TRAVERSE_LEAF(LoadInnerNode(thirdChild->readChild(1).blockId())->readChild(0).blockId(), true, 2 * nodeStore->layout().maxChildrenPerInnerNode() * nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode());
TraverseLeaves(root.get(), 0, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() + 1);
TraverseLeaves(std::move(root), 0, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() + 1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfFourLevelTree) {
TEST_F(LeafTraverserTest, TraverseMiddlePartOfFourLevelTree) {
auto root = CreateFourLevel();
//Traverse some leaves of the full threelevel tree in the first child
auto firstChild = LoadInnerNode(root->readChild(0).blockId());
@ -396,14 +404,15 @@ TEST_F(DataTreeTest_TraverseLeaves, TraverseMiddlePartOfFourLevelTree) {
EXPECT_TRAVERSE_LEAF(firstChildOfThirdChild->readChild(i).blockId(), false, 2 * nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode()+i);
}
TraverseLeaves(root.get(), nodeStore->layout().maxChildrenPerInnerNode()+5, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() -1);
TraverseLeaves(std::move(root), nodeStore->layout().maxChildrenPerInnerNode()+5, 2*nodeStore->layout().maxChildrenPerInnerNode()*nodeStore->layout().maxChildrenPerInnerNode() + nodeStore->layout().maxChildrenPerInnerNode() -1, true);
}
TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) {
auto root = CreateLeaf();
TEST_F(LeafTraverserTest, LastLeafIsAlreadyResizedInCallback) {
unique_ref<DataNode> root = CreateLeaf();
root->flush();
auto* old_root = root.get();
auto tree = treeStore.load(root->blockId()).value();
tree->traverseLeaves(0, 2, [this] (uint32_t leafIndex, bool /*isRightBorderNode*/, LeafHandle leaf) {
LeafTraverser(nodeStore, false).traverseAndUpdateRoot(&root, 0, 2, [this] (uint32_t leafIndex, bool /*isRightBorderNode*/, LeafHandle leaf) {
if (leafIndex == 0) {
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
} else {
@ -411,28 +420,31 @@ TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback) {
}
}, [] (uint32_t /*nodeIndex*/) -> Data {
return Data(1);
});
}, [] (auto) {});
EXPECT_NE(old_root, root.get()); // expect that we grew the tree
}
TEST_F(DataTreeTest_TraverseLeaves, LastLeafIsAlreadyResizedInCallback_TwoLevel) {
auto root = CreateFullTwoLevelWithLastLeafSize(5);
TEST_F(LeafTraverserTest, LastLeafIsAlreadyResizedInCallback_TwoLevel) {
unique_ref<DataNode> root = CreateFullTwoLevelWithLastLeafSize(5);
root->flush();
auto* old_root = root.get();
auto tree = treeStore.load(root->blockId()).value();
tree->traverseLeaves(0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, bool /*isRightBorderNode*/, LeafHandle leaf) {
LeafTraverser(nodeStore, false).traverseAndUpdateRoot(&root, 0, nodeStore->layout().maxChildrenPerInnerNode()+1, [this] (uint32_t /*leafIndex*/, bool /*isRightBorderNode*/, LeafHandle leaf) {
EXPECT_EQ(nodeStore->layout().maxBytesPerLeaf(), leaf.node()->numBytes());
}, [] (uint32_t /*nodeIndex*/) -> Data {
return Data(1);
});
}, [] (auto) {});
EXPECT_NE(old_root, root.get()); // expect that we grew the tree
}
TEST_F(DataTreeTest_TraverseLeaves, ResizeFromOneLeafToMultipleLeaves) {
TEST_F(LeafTraverserTest, ResizeFromOneLeafToMultipleLeaves) {
auto root = CreateLeaf();
EXPECT_TRAVERSE_LEAF(root->blockId(), false, 0);
//EXPECT_CALL(traversor, calledExistingLeaf(_, false, 0)).Times(1);
for (uint32_t i = 1; i < 10; ++i) {
EXPECT_CREATE_LEAF(i);
}
TraverseLeaves(root.get(), 0, 10);
TraverseLeaves(std::move(root), 0, 10, false);
}
//TODO Refactor the test cases that are too long
////TODO Refactor the test cases that are too long

2
test/blockstore/implementations/caching/cache/CacheTest_MoveConstructor.cpp vendored
View File

@ -13,7 +13,7 @@ using ::testing::Test;
//Test that Cache uses a move constructor for Value if possible
class CacheTest_MoveConstructor: public Test {
public:
CacheTest_MoveConstructor(): cache(make_unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>>()) {
CacheTest_MoveConstructor(): cache(make_unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>>("test")) {
CopyableMovableValueType::numCopyConstructorCalled = 0;
}
unique_ref<Cache<MinimalKeyType, CopyableMovableValueType, 100>> cache;

2
test/blockstore/implementations/caching/cache/CacheTest_RaceCondition.cpp vendored
View File

@ -36,7 +36,7 @@ private:
class CacheTest_RaceCondition: public ::testing::Test {
public:
CacheTest_RaceCondition(): cache(), destructorStarted(), destructorFinished(false) {}
CacheTest_RaceCondition(): cache("test"), destructorStarted(), destructorFinished(false) {}
static constexpr unsigned int MAX_ENTRIES = 100;

6
test/blockstore/implementations/caching/cache/PeriodicTaskTest.cpp vendored
View File

@ -37,7 +37,7 @@ class PeriodicTaskTest: public Test {
};
TEST_F(PeriodicTaskTest, DoesntDeadlockInDestructorWhenDestructedImmediately) {
PeriodicTask task([](){}, 1);
PeriodicTask task([](){}, 1, "test");
}
TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) {
@ -45,7 +45,7 @@ TEST_F(PeriodicTaskTest, CallsCallbackAtLeast10Times) {
PeriodicTask task([&counter](){
counter.decrease();
}, 0.001);
}, 0.001, "test");
counter.waitForZero();
}
@ -55,7 +55,7 @@ TEST_F(PeriodicTaskTest, DoesntCallCallbackAfterDestruction) {
{
PeriodicTask task([&callCount](){
callCount += 1;
}, 0.001);
}, 0.001, "test");
}
int callCountDirectlyAfterDestruction = callCount;
boost::this_thread::sleep_for(boost::chrono::seconds(1));

2
test/blockstore/implementations/caching/cache/testutils/CacheTest.h vendored
View File

@ -13,7 +13,7 @@
// Furthermore, the class checks that there are no memory leaks left after destructing the QueueMap (by counting leftover instances of Keys/Values).
class CacheTest: public ::testing::Test {
public:
CacheTest(): _cache() {}
CacheTest(): _cache("test") {}
void push(int key, int value);
boost::optional<int> pop(int key);

2
test/cpp-utils/CMakeLists.txt
View File

@ -49,10 +49,12 @@ set(SOURCES
assert/assert_debug_test.cpp
system/GetTotalMemoryTest.cpp
system/TimeTest.cpp
system/PathTest.cpp
system/FiletimeTest.cpp
system/MemoryTest.cpp
system/HomedirTest.cpp
system/EnvTest.cpp
thread/debugging_test.cpp
value_type/ValueTypeTest.cpp
either_test.cpp
)

32
test/cpp-utils/system/PathTest.cpp Normal file
View File

@ -0,0 +1,32 @@
#include <gtest/gtest.h>
#include <cpp-utils/system/path.h>
using cpputils::path_is_just_drive_letter;
#if defined(_MSC_VER)
TEST(PathTest, pathIsJustDriveLetter) {
EXPECT_FALSE(path_is_just_drive_letter("C"));
EXPECT_TRUE(path_is_just_drive_letter("C:"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\"));
EXPECT_FALSE(path_is_just_drive_letter("C:/"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\test"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\test\\"));
EXPECT_FALSE(path_is_just_drive_letter("/"));
EXPECT_FALSE(path_is_just_drive_letter(""));
}
#else
TEST(PathTest, onNonWindowsWeDontHaveDriveLetterPaths) {
EXPECT_FALSE(path_is_just_drive_letter("C"));
EXPECT_FALSE(path_is_just_drive_letter("C:"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\"));
EXPECT_FALSE(path_is_just_drive_letter("C:/"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\test"));
EXPECT_FALSE(path_is_just_drive_letter("C:\\test\\"));
EXPECT_FALSE(path_is_just_drive_letter("/"));
EXPECT_FALSE(path_is_just_drive_letter(""));
}
#endif

62
test/cpp-utils/thread/debugging_test.cpp Normal file
View File

@ -0,0 +1,62 @@
#include <cpp-utils/thread/debugging.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/lock/ConditionBarrier.h>
#include <gtest/gtest.h>
using namespace cpputils;
using std::string;
TEST(ThreadDebuggingTest_ThreadName, givenMainThread_whenSettingAndGetting_thenDoesntCrash) {
set_thread_name("my_thread_name");
get_thread_name();
}
TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenSettingAndGetting_thenDoesntCrash) {
ConditionBarrier nameIsChecked;
bool child_didnt_crash = false;
std::thread child([&] {
set_thread_name("my_thread_name");
get_thread_name();
child_didnt_crash = true;
nameIsChecked.wait();
});
get_thread_name(&child);
nameIsChecked.release(); // getting the name of a not-running thread would cause errors, so let's make sure we only exit after getting the name
child.join();
EXPECT_TRUE(child_didnt_crash);
}
TEST(ThreadDebuggingTest_ThreadName, givenMainThread_whenGettingFromInside_thenIsCorrect) {
set_thread_name("my_thread_name");
string name = get_thread_name();
EXPECT_EQ("my_thread_name", name);
}
TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenGettingFromInside_thenIsCorrect) {
std::thread child([] {
set_thread_name("my_thread_name");
string name = get_thread_name();
EXPECT_EQ("my_thread_name", name);
});
child.join();
}
TEST(ThreadDebuggingTest_ThreadName, givenChildThread_whenGettingFromOutside_thenIsCorrect) {
ConditionBarrier nameIsSet;
ConditionBarrier nameIsChecked;
std::thread child([&] {
set_thread_name("my_thread_name");
nameIsSet.release();
nameIsChecked.wait();
});
nameIsSet.wait();
string name = get_thread_name(&child);
EXPECT_EQ("my_thread_name", name);
nameIsChecked.release();
child.join();
}

4
test/cryfs-cli/CMakeLists.txt
View File

@ -12,12 +12,12 @@ set(SOURCES
EnvironmentTest.cpp
VersionCheckerTest.cpp
CliTest_IntegrityCheck.cpp
CryfsUnmountTest.cpp
)
add_executable(${PROJECT_NAME} ${SOURCES})
target_link_libraries(${PROJECT_NAME} googletest cryfs-cli)
target_link_libraries(${PROJECT_NAME} googletest cryfs-cli cryfs-unmount fspp-fuse)
add_test(${PROJECT_NAME} ${PROJECT_NAME})
target_enable_style_warnings(${PROJECT_NAME})
target_activate_cpp14(${PROJECT_NAME})

4
test/cryfs-cli/CallAfterTimeoutTest.cpp
View File

@ -8,14 +8,14 @@ using cpputils::make_unique_ref;
using boost::chrono::milliseconds;
using boost::chrono::minutes;
using boost::this_thread::sleep_for;
using namespace cryfs;
using namespace cryfs_cli;
class CallAfterTimeoutTest : public ::testing::Test {
public:
CallAfterTimeoutTest(): called(false) {}
unique_ref<CallAfterTimeout> callAfterTimeout(milliseconds timeout) {
return make_unique_ref<CallAfterTimeout>(timeout, [this] {called = true;});
return make_unique_ref<CallAfterTimeout>(timeout, [this] {called = true;}, "test");
}
std::atomic<bool> called;

25
test/cryfs-cli/CryfsUnmountTest.cpp Normal file
View File

@ -0,0 +1,25 @@
#include "testutils/CliTest.h"
#include <cryfs-cli/Cli.h>
#include <cryfs-unmount/Cli.h>
using CliTest_Unmount = CliTest;
namespace bf = boost::filesystem;
namespace {
void unmount(const bf::path& mountdir) {
std::vector<const char*> _args = {"cryfs-unmount", mountdir.string().c_str()};
cryfs_unmount::Cli().main(2, _args.data());
}
TEST_F(CliTest_Unmount, givenMountedFilesystem_whenUnmounting_thenSucceeds) {
// we're passing in boost::none as mountdir so EXPECT_RUN_SUCCESS doesn't unmount itself.
// if the unmount we're calling here in the onMounted callback wouldn't work, EXPECT_RUN_SUCCESS
// would never return and this would be a deadlock.
EXPECT_RUN_SUCCESS({basedir.string().c_str(), mountdir.string().c_str(), "-f"}, boost::none, [this] () {
unmount(mountdir);
});
}
// TODO Test calling with invalid args, valid '--version' or '--help' args, with a non-mounted mountdir and a nonexisting mountdir.
}

2
test/cryfs-cli/EnvironmentTest.cpp
View File

@ -4,7 +4,7 @@
#include <boost/filesystem.hpp>
#include <cpp-utils/system/env.h>
using namespace cryfs;
using namespace cryfs_cli;
using std::string;
using boost::optional;
using boost::none;

2
test/cryfs-cli/VersionCheckerTest.cpp
View File

@ -8,7 +8,7 @@ using cpputils::FakeHttpClient;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::none;
using namespace cryfs;
using namespace cryfs_cli;
class VersionCheckerTest: public ::testing::Test {
public:

2
test/cryfs-cli/program_options/ParserTest.cpp
View File

@ -7,7 +7,7 @@
#include <cpp-utils/testutils/CaptureStderrRAII.h>
using namespace cryfs;
using namespace cryfs::program_options;
using namespace cryfs_cli::program_options;
using std::vector;
using std::string;
using boost::none;

2
test/cryfs-cli/program_options/ProgramOptionsTest.cpp
View File

@ -2,7 +2,7 @@
#include <cryfs-cli/program_options/ProgramOptions.h>
#include <cpp-utils/pointer/unique_ref_boost_optional_gtest_workaround.h>
using namespace cryfs::program_options;
using namespace cryfs_cli::program_options;
using boost::none;
using boost::optional;
using std::ostream;

2
test/cryfs-cli/program_options/UtilsTest.cpp
View File

@ -1,7 +1,7 @@
#include "testutils/ProgramOptionsTestBase.h"
#include <cryfs-cli/program_options/utils.h>
using namespace cryfs::program_options;
using namespace cryfs_cli::program_options;
using std::pair;
using std::vector;
using std::string;

18
test/cryfs-cli/testutils/CliTest.h
View File

@ -19,6 +19,7 @@
#include <cpp-utils/lock/ConditionBarrier.h>
#include "../../cryfs/testutils/MockConsole.h"
#include "../../cryfs/testutils/TestWithFakeHomeDirectory.h"
#include <fspp/fuse/Fuse.h>
#include <cryfs/ErrorCodes.h>
#include <cpp-utils/testutils/CaptureStderrRAII.h>
#include <regex>
@ -45,7 +46,7 @@ public:
int run(const std::vector<std::string>& args, std::function<void()> onMounted) {
std::vector<const char*> _args;
_args.reserve(args.size() + 1);
_args.emplace_back("cryfs");
_args.emplace_back("cryfs");
for (const std::string& arg : args) {
_args.emplace_back(arg.c_str());
}
@ -53,7 +54,7 @@ public:
ON_CALL(*console, askPassword(testing::StrEq("Password: "))).WillByDefault(testing::Return("pass"));
ON_CALL(*console, askPassword(testing::StrEq("Confirm Password: "))).WillByDefault(testing::Return("pass"));
// Run Cryfs
return cryfs::Cli(keyGenerator, cpputils::SCrypt::TestSettings, console).main(_args.size(), _args.data(), _httpClient(), std::move(onMounted));
return cryfs_cli::Cli(keyGenerator, cpputils::SCrypt::TestSettings, console).main(_args.size(), _args.data(), _httpClient(), std::move(onMounted));
}
void EXPECT_EXIT_WITH_HELP_MESSAGE(const std::vector<std::string>& args, const std::string &message, cryfs::ErrorCode errorCode) {
@ -90,18 +91,7 @@ public:
};
static void _unmount(const boost::filesystem::path &mountDir) {
int returncode = -1;
#if defined(__APPLE__)
returncode = cpputils::Subprocess::call(std::string("umount ") + mountDir.string().c_str() + " 2>/dev/null").exitcode;
#elif defined(_MSC_VER)
std::wstring mountDir_ = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().from_bytes(mountDir.string());
BOOL success = DokanRemoveMountPoint(mountDir_.c_str());
returncode = success ? 0 : -1;
#else
returncode = cpputils::Subprocess::call(
std::string("fusermount -u ") + mountDir.string().c_str() + " 2>/dev/null").exitcode;
#endif
EXPECT_EQ(0, returncode);
fspp::fuse::Fuse::unmount(mountDir, true);
}
FilesystemOutput run_filesystem(const std::vector<std::string>& args, boost::optional<boost::filesystem::path> mountDirForUnmounting, std::function<void()> onMounted) {

6
test/cryfs/config/CryConfigLoaderTest.cpp
View File

@ -310,7 +310,7 @@ TEST_F(CryConfigLoaderTest, AsksWhenLoadingNewerFilesystem_AnswerNo) {
}
TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), false)).Times(1).WillOnce(Return(true));
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(true));
string version = olderVersion();
CreateWithVersion(version, version);
@ -318,14 +318,14 @@ TEST_F(CryConfigLoaderTest, AsksWhenMigratingOlderFilesystem) {
}
TEST_F(CryConfigLoaderTest, DoesNotAskForMigrationWhenCorrectVersion) {
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), _)).Times(0);
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), _)).Times(0);
CreateWithVersion(gitversion::VersionString(), CryConfig::FilesystemFormatVersion);
EXPECT_NE(boost::none, Load());
}
TEST_F(CryConfigLoaderTest, DontMigrateWhenAnsweredNo) {
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it?"), false)).Times(1).WillOnce(Return(false));
EXPECT_CALL(*console, askYesNo(HasSubstr("Do you want to migrate it now?"), false)).Times(1).WillOnce(Return(false));
string version = olderVersion();
CreateWithVersion(version, version);

6
vendor/README vendored
View File

@ -2,6 +2,6 @@ This directory contains external projects, taken from the following locations:
googletest: https://github.com/google/googletest/tree/4e4df226fc197c0dda6e37f5c8c3845ca1e73a49
- changed: added NOLINT comment as workaround for clang-tidy warning https://github.com/google/googletest/issues/853
spdlog: https://github.com/gabime/spdlog/tree/v0.16.3/include/spdlog
cryptopp: https://github.com/weidai11/cryptopp/tree/CRYPTOPP_7_0_0
- changed: added CMakeLists.txt and cryptopp-config.cmake from https://github.com/noloader/cryptopp-cmake/tree/7f3b3540740b365bed673936852d70795c0104f3
- changed: removed cmake warning from CMakeLists.txt
cryptopp: https://github.com/weidai11/cryptopp/tree/CRYPTOPP_8_0_0
- changed: added CMakeLists.txt and cryptopp-config.cmake from https://github.com/noloader/cryptopp-cmake/tree/CRYPTOPP_8_0_0
- changed: commented out line including winapifamily.h in CMakeLists.txt

1
vendor/cryptopp/CMakeLists.txt vendored
View File

@ -88,6 +88,7 @@ set(BUILD_TESTING OFF CACHE BOOL "")
set(BUILD_DOCUMENTATION OFF CACHE BOOL "")
set(BUILD_SHARED OFF CACHE BOOL "")
set(BUILD_STATIC ON CACHE BOOL "")
set(cryptocpp_DISPLAY_CMAKE_SUPPORT_WARNING OFF CACHE BOOL "")
add_subdirectory(vendor_cryptopp EXCLUDE_FROM_ALL)
target_link_libraries(cryptopp PRIVATE cryptopp-static)

1466
vendor/cryptopp/vendor_cryptopp/CMakeLists.txt vendored
View File

File diff suppressed because it is too large Load Diff

15
vendor/cryptopp/vendor_cryptopp/Doxyfile vendored
View File

@ -41,7 +41,7 @@ PROJECT_NAME = Crypto++
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = 7.0
PROJECT_NUMBER = 8.0
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
@ -1225,8 +1225,7 @@ DOCSET_PUBLISHER_NAME = Crypto++
# compressed HTML files.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_HTMLHELP = YES
GENERATE_HTMLHELP = NO
# The CHM_FILE tag can be used to specify the file name of the resulting .chm
# file. You can add a path in front of the file if the result should not be
@ -1494,7 +1493,10 @@ MATHJAX_CODEFILE =
# The default value is: YES.
# This tag requires that the tag GENERATE_HTML is set to YES.
SEARCHENGINE = NO
# JW: Do _not_ enable server-side search. It consumes too many resources
# and results in OOM Kills on MySQL, which corrupts the database.
# https://github.com/weidai11/cryptopp/issues/720
SEARCHENGINE = NO
# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
# implemented using a web server instead of a web client using Javascript. There
@ -1506,7 +1508,10 @@ SEARCHENGINE = NO
# The default value is: NO.
# This tag requires that the tag SEARCHENGINE is set to YES.
SERVER_BASED_SEARCH = NO
# JW: Do _not_ enable server-side search. It consumes too many resources
# and results in OOM Kills on MySQL, which corrupts the database.
# https://github.com/weidai11/cryptopp/issues/720
SERVER_BASED_SEARCH = NO
# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
# script for searching. Instead the search results are written to an XML file

158
vendor/cryptopp/vendor_cryptopp/Filelist.txt vendored
View File

@ -1,10 +1,12 @@
3way.cpp
3way.h
adhoc.cpp.proto
adv-simd.h
adv_simd.h
adler32.cpp
adler32.h
aes.h
aes_armv4.h
aes_armv4.S
algebra.cpp
algebra.h
algparam.cpp
@ -13,7 +15,7 @@ arc4.cpp
arc4.h
ariatab.cpp
aria.cpp
aria-simd.cpp
aria_simd.cpp
aria.h
argnames.h
asn.cpp
@ -30,9 +32,11 @@ bench.h
bds10.zip
bench1.cpp
bench2.cpp
bench3.cpp
bfinit.cpp
blake2.cpp
blake2-simd.cpp
blake2s_simd.cpp
blake2b_simd.cpp
blake2.h
blowfish.cpp
blowfish.h
@ -48,7 +52,12 @@ cbcmac.h
ccm.cpp
ccm.h
chacha.cpp
chacha_avx.cpp
chacha_simd.cpp
chacha.h
cham.cpp
cham_simd.cpp
cham.h
channels.cpp
channels.h
cmac.cpp
@ -57,7 +66,7 @@ config.h
cpu.cpp
cpu.h
crc.cpp
crc-simd.cpp
crc_simd.cpp
crc.h
cryptdll.vcxproj
cryptdll.vcxproj.filters
@ -73,6 +82,8 @@ cryptlib.h
cryptlib.vcxproj
cryptlib.vcxproj.filters
cryptopp.rc
darn.cpp
darn.h
datatest.cpp
default.cpp
default.h
@ -90,6 +101,13 @@ dlltest.vcxproj
dlltest.vcxproj.filters
dmac.h
drbg.h
donna.h
donna_32.h
donna_64.h
donna_sse.h
donna_32.cpp
donna_64.cpp
donna_sse.cpp
dsa.cpp
dsa.h
eax.cpp
@ -121,7 +139,7 @@ fips140.h
fipsalgt.cpp
fipstest.cpp
fltrimpl.h
gcm-simd.cpp
gcm_simd.cpp
gcm.cpp
gcm.h
gf256.cpp
@ -137,8 +155,14 @@ gost.h
gzip.cpp
gzip.h
hashfwd.h
hc128.cpp
hc128.h
hc256.cpp
hc256.h
hex.cpp
hex.h
hight.h
hight.cpp
hkdf.h
hmac.cpp
hmac.h
@ -159,7 +183,12 @@ kalyna.cpp
kalyna.h
keccak.cpp
keccak.h
keccakc.cpp
keccakc.h
lubyrack.h
lea.cpp
lea_simd.cpp
lea.h
luc.cpp
luc.h
mars.cpp
@ -186,9 +215,7 @@ mqv.h
naclite.h
nbtheory.cpp
nbtheory.h
neon-simd.cpp
network.cpp
network.h
neon_simd.cpp
nr.h
oaep.cpp
oaep.h
@ -208,8 +235,11 @@ poly1305.cpp
poly1305.h
polynomi.cpp
polynomi.h
ppc-simd.h
ppc-simd.cpp
ppc_power7.cpp
ppc_power8.cpp
ppc_power9.cpp
ppc_simd.cpp
ppc_simd.h
pssr.cpp
pssr.h
pubkey.cpp
@ -221,6 +251,8 @@ rabin.cpp
rabin.h
randpool.cpp
randpool.h
rabbit.cpp
rabbit.h
rc2.cpp
rc2.h
rc5.cpp
@ -237,9 +269,10 @@ rdtables.cpp
regtest1.cpp
regtest2.cpp
regtest3.cpp
regtest4.cpp
resource.h
rijndael.cpp
rijndael-simd.cpp
rijndael_simd.cpp
rijndael.h
ripemd.cpp
ripemd.h
@ -265,12 +298,12 @@ serpent.cpp
serpent.h
serpentp.h
sha.cpp
sha-simd.cpp
sha_simd.cpp
sha.h
sha3.cpp
sha3.h
shacal2.cpp
shacal2-simd.cpp
shacal2_simd.cpp
shacal2.h
shark.cpp
shark.h
@ -278,27 +311,31 @@ sharkbox.cpp
simple.cpp
simple.h
siphash.h
simeck.cpp
simeck_simd.cpp
simeck.h
simon.cpp
simon-simd.cpp
simon64_simd.cpp
simon128_simd.cpp
simon.h
skipjack.cpp
skipjack.h
sm3.cpp
sm3.h
sm4.cpp
sm4_simd.cpp
sm4.h
smartptr.h
socketft.cpp
socketft.h
sosemanuk.cpp
sosemanuk.h
speck.cpp
speck-simd.cpp
speck64_simd.cpp
speck128_simd.cpp
speck.h
square.cpp
square.h
squaretb.cpp
sse-simd.cpp
sse_simd.cpp
stdcpp.h
strciphr.cpp
strciphr.h
@ -312,8 +349,6 @@ threefish.h
tiger.cpp
tiger.h
tigertab.cpp
trdlocal.cpp
trdlocal.h
trunhash.h
ttmac.cpp
ttmac.h
@ -326,21 +361,25 @@ validat1.cpp
validat2.cpp
validat3.cpp
validat4.cpp
validat5.cpp
validat6.cpp
validat7.cpp
validat8.cpp
validat9.cpp
validat10.cpp
validate.h
vmac.cpp
vmac.h
vs2005.zip
wait.cpp
wait.h
wake.cpp
wake.h
whrlpool.cpp
whrlpool.h
winpipes.cpp
winpipes.h
words.h
x64dll.asm
x64masm.asm
xed25519.h
xed25519.cpp
xtr.cpp
xtr.h
xtrcrypt.cpp
@ -366,6 +405,8 @@ TestData/aria.dat
TestData/camellia.dat
TestData/cast128v.dat
TestData/cast256v.dat
TestData/defdmac1.bin
TestData/defdmac2.bin
TestData/descert.dat
TestData/dh1024.dat
TestData/dh2048.dat
@ -374,6 +415,10 @@ TestData/dlie2048.dat
TestData/dsa1024.dat
TestData/dsa1024b.dat
TestData/dsa512.dat
TestData/ed25519.dat
TestData/ed25519_ver.dat
TestData/ed25519v0.dat
TestData/ed25519v1.dat
TestData/elgc1024.dat
TestData/esig1023.dat
TestData/esig1536.dat
@ -409,10 +454,10 @@ TestData/rc6val.dat
TestData/rijndael.dat
TestData/rsa1024.dat
TestData/rsa2048.dat
TestData/rsa2048a.dat
TestData/rsa400pb.dat
TestData/rsa400pv.dat
TestData/rsa512a.dat
TestData/rsa2048a.dat
TestData/rw1024.dat
TestData/rw2048.dat
TestData/saferval.dat
@ -423,57 +468,68 @@ TestData/skipjack.dat
TestData/squareva.dat
TestData/twofishv.dat
TestData/usage.dat
TestData/x25519.dat
TestData/x25519v0.dat
TestData/x25519v1.dat
TestData/xtrdh171.dat
TestData/xtrdh342.dat
TestVectors/Readme.txt
TestVectors/aes.txt
TestVectors/all.txt
TestVectors/aria.txt
TestVectors/blake2.txt
TestVectors/blake2b.txt
TestVectors/blake2s.txt
TestVectors/aria.txt
TestVectors/camellia.txt
TestVectors/ccm.txt
TestVectors/chacha.txt
TestVectors/chacha_tls.txt
TestVectors/cham.txt
TestVectors/cmac.txt
TestVectors/dlies.txt
TestVectors/dsa.txt
TestVectors/dsa_1363.txt
TestVectors/dsa_rfc6979.txt
TestVectors/eax.txt
TestVectors/esign.txt
TestVectors/gcm.txt
TestVectors/hc128.txt
TestVectors/hc256.txt
TestVectors/hight.txt
TestVectors/hkdf.txt
TestVectors/hmac.txt
TestVectors/kalyna.txt
TestVectors/keccak.txt
TestVectors/lea.txt
TestVectors/mars.txt
TestVectors/nr.txt
TestVectors/ocb.txt
TestVectors/panama.txt
TestVectors/rabbit.txt
TestVectors/Readme.txt
TestVectors/rsa_oaep.txt
TestVectors/rsa_pkcs1_1_5.txt
TestVectors/rsa_pss.txt
TestVectors/dsa_rfc6979.txt
TestVectors/rw.txt
TestVectors/salsa.txt
TestVectors/seal.txt
TestVectors/seed.txt
TestVectors/sha.txt
TestVectors/sha2.txt
TestVectors/sha3.txt
TestVectors/sha1_fips_180.txt
TestVectors/sha1_160_fips_180.txt
TestVectors/sha2_fips_180.txt
TestVectors/sha1_fips_180.txt
TestVectors/sha2.txt
TestVectors/sha2_224_fips_180.txt
TestVectors/sha2_256_fips_180.txt
TestVectors/sha2_384_fips_180.txt
TestVectors/sha2_512_fips_180.txt
TestVectors/sha3_fips_202.txt
TestVectors/sha2_fips_180.txt
TestVectors/sha3.txt
TestVectors/sha3_224_fips_202.txt
TestVectors/sha3_256_fips_202.txt
TestVectors/sha3_384_fips_202.txt
TestVectors/sha3_512_fips_202.txt
TestVectors/sha3_fips_202.txt
TestVectors/shacal2.txt
TestVectors/simeck.txt
TestVectors/simon.txt
TestVectors/siphash.txt
TestVectors/sm3.txt
@ -486,3 +542,39 @@ TestVectors/ttmac.txt
TestVectors/vmac.txt
TestVectors/wake.txt
TestVectors/whrlpool.txt
TestPrograms/test_32bit.cxx
TestPrograms/test_64bit.cxx
TestPrograms/test_arm_acle.cxx
TestPrograms/test_arm_aes.cxx
TestPrograms/test_arm_asimd.cxx
TestPrograms/test_arm_crc.cxx
TestPrograms/test_arm_neon.cxx
TestPrograms/test_arm_pmull.cxx
TestPrograms/test_arm_sha.cxx
TestPrograms/test_arm_sha3.cxx
TestPrograms/test_arm_sha512.cxx
TestPrograms/test_arm_sm3.cxx
TestPrograms/test_arm_sm4.cxx
TestPrograms/test_cxx.cxx
TestPrograms/test_mixed_asm.cxx
TestPrograms/test_newlib.cxx
TestPrograms/test_ppc_aes.cxx
TestPrograms/test_ppc_altivec.cxx
TestPrograms/test_ppc_power7.cxx
TestPrograms/test_ppc_power8.cxx
TestPrograms/test_ppc_power9.cxx
TestPrograms/test_ppc_sha.cxx
TestPrograms/test_ppc_vmull.cxx
TestPrograms/test_pthreads.cxx
TestPrograms/test_x86_aes.cxx
TestPrograms/test_x86_avx.cxx
TestPrograms/test_x86_avx2.cxx
TestPrograms/test_x86_avx512.cxx
TestPrograms/test_x86_clmul.cxx
TestPrograms/test_x86_cpuid.cxx
TestPrograms/test_x86_sha.cxx
TestPrograms/test_x86_sse2.cxx
TestPrograms/test_x86_sse3.cxx
TestPrograms/test_x86_sse41.cxx
TestPrograms/test_x86_sse42.cxx
TestPrograms/test_x86_ssse3.cxx

1277
vendor/cryptopp/vendor_cryptopp/GNUmakefile vendored
View File

File diff suppressed because it is too large Load Diff

703
vendor/cryptopp/vendor_cryptopp/GNUmakefile-cross vendored
View File

@ -3,6 +3,14 @@
SHELL = /bin/sh
# If needed
TMPDIR ?= /tmp
# Used for ARMv7 and NEON.
FP_ABI ?= hard
# Used for feature tests
TOUT ?= a.out
TOUT := $(strip $(TOUT))
# Default CXXFLAGS if none were provided
CXXFLAGS ?= -DNDEBUG -g2 -O3 -fPIC -pipe
@ -22,35 +30,25 @@ INSTALL = install
INSTALL_PROGRAM = $(INSTALL)
INSTALL_DATA = $(INSTALL) -m 644
# Attempt to determine host machine, fallback to "this" machine.
# The host machine is the one the package runs on. Most people
# call this the "target", but not Autotools.
HOSTX := $(shell $(CXX) $(CXXFLAGS) -dumpmachine 2>/dev/null | cut -f 1 -d '-')
ifeq ($(HOSTX),)
HOSTX := $(shell uname -m 2>/dev/null)
endif
IS_i686 := $(shell echo "$HOSTX" | $(EGREP) -v 64 | $(EGREP) -i -c 'i.86')
IS_x86_64 := $(shell echo "$HOSTX" | $(EGREP) -i -c 'x86_64|amd64')
IS_ARM := $(shell echo "$HOSTX" | $(EGREP) -i -c 'arm')
IS_ARMv8 := $(shell echo "$HOSTX" | $(EGREP) -i -c 'aarch32|aarch64')
CLANG_COMPILER := $(shell $(CXX) --version 2>&1 | $(EGREP) -i -c "clang")
IS_IOS ?= 0
IS_ANDROID ?= 0
IS_ARM_EMBEDDED ?= 0
IS_NEON ?= 0
# Fixup ARM
ifeq ($(IS_ARMv8),1)
IS_ARM := 0
endif
# Can be used by Android and Embeeded cross-compiles. Disable by default because
# Android and embedded users typically don't run this configuration.
HAS_SOLIB_VERSION ?= 0
# Formely adhoc.cpp was created from adhoc.cpp.proto when needed.
# This is now needed because ISA tests are performed using adhoc.cpp.
ifeq ($(wildcard adhoc.cpp),)
$(shell cp adhoc.cpp.proto adhoc.cpp)
endif
###########################################################
##### General Variables #####
###########################################################
# Default prefix for make install
ifeq ($(PREFIX),)
PREFIX = /usr/local
@ -72,29 +70,37 @@ endif
# We honor ARFLAGS, but the "v" option used by default causes a noisy make
ifeq ($(ARFLAGS),rv)
ARFLAGS = r
ARFLAGS = r
endif
# Sadly, we can't actually use GCC_PRAGMA_AWARE because of GCC bug 53431.
# Its a shame because GCC has so much to offer by the way of analysis.
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53431
ifneq ($(CLANG_COMPILER),0)
CXXFLAGS += -Wall
CXXFLAGS += -Wall
endif
###########################################################
##### iOS #####
###########################################################
# iOS cross-compile configuration.
# See http://www.cryptopp.com/wiki/iOS_(Command_Line).
ifeq ($(IS_IOS),1)
CXX = clang++
CXXFLAGS += $(IOS_FLAGS) -arch $(IOS_ARCH)
CXXFLAGS += -isysroot $(IOS_SYSROOT) -stdlib=libc++
CXXFLAGS += -isysroot "$(IOS_SYSROOT)" -stdlib=libc++
AR = libtool
ARFLAGS = -static -o
RANLIB = ranlib
endif
###########################################################
##### Android #####
###########################################################
# Android cross-compile configuration.
# See http://www.cryptopp.com/wiki/Android_(Command_Line).
ifeq ($(IS_ANDROID),1)
@ -116,6 +122,10 @@ ifeq ($(IS_ANDROID),1)
AOSP_CPU_OBJ = cpu-features.o
endif
###########################################################
##### Embedded #####
###########################################################
# ARM embedded cross-compile configuration.
# See http://www.cryptopp.com/wiki/ARM_Embedded_(Command_Line)
# and http://www.cryptopp.com/wiki/ARM_Embedded_(Bare Metal).
@ -124,6 +134,364 @@ ifeq ($(IS_ARM_EMBEDDED),1)
CXXFLAGS += $(ARM_EMBEDDED_FLAGS) --sysroot=$(ARM_EMBEDDED_SYSROOT)
endif
###########################################################
##### Compiler and Platform #####
###########################################################
# Wait until CXXFLAGS have been set by setenv scripts.
GCC_COMPILER := $(shell $(CXX) --version 2>/dev/null | $(EGREP) -v -E 'llvm|clang' | $(EGREP) -i -c -E '(gcc|g\+\+)')
CLANG_COMPILER := $(shell $(CXX) --version 2>/dev/null | $(EGREP) -i -c -E 'llvm|clang')
HOSTX := $(shell $(CXX) $(CXXFLAGS) -dumpmachine 2>/dev/null | cut -f 1 -d '-')
ifeq ($(HOSTX),)
HOSTX := $(shell uname -m 2>/dev/null)
endif
# This dance is because Clang reports the host architecture instead
# of the target architecture. Running Clang on an x86_64 machine with
# -arch arm64 yields x86_64 instead of aarch64 or arm64.
ifeq ($(CLANG_COMPILER),1)
IS_X86 := $(shell echo $(CXXFLAGS) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'i.86')
IS_X64 := $(shell echo $(CXXFLAGS) | $(EGREP) -i -c -E 'x86_64|amd64')
IS_ARM32 := $(shell echo $(CXXFLAGS) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'arm|armhf|arm7l|eabihf')
IS_ARMV8 := $(shell echo $(CXXFLAGS) | $(EGREP) -i -c -E 'aarch32|aarch64|arm64|armv8')
else
IS_X86 := $(shell echo $(HOSTX) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'i.86')
IS_X64 := $(shell echo $(HOSTX) | $(EGREP) -i -c -E 'x86_64|amd64')
IS_ARM32 := $(shell echo $(HOSTX) | $(EGREP) -v 64 | $(EGREP) -i -c -E 'arm|armhf|arm7l|eabihf')
IS_ARMV8 := $(shell echo $(HOSTX) | $(EGREP) -i -c -E 'aarch32|aarch64|arm64|armv8')
endif
$(info Here's what we found... IS_X86: $(IS_X86), IS_X64: $(IS_X64), IS_ARM32: $(IS_ARM32), IS_ARMV8: $(IS_ARMV8))
###########################################################
##### Test Program #####
###########################################################
# Hack to skip CPU feature tests for some recipes
DETECT_FEATURES ?= 1
ifeq ($(findstring -DCRYPTOPP_DISABLE_ASM,$(CXXFLAGS)),-DCRYPTOPP_DISABLE_ASM)
DETECT_FEATURES := 0
else ifeq ($(findstring clean,$(MAKECMDGOALS)),clean)
DETECT_FEATURES := 0
else ifeq ($(findstring distclean,$(MAKECMDGOALS)),distclean)
DETECT_FEATURES := 0
else ifeq ($(findstring distclean,$(MAKECMDGOALS)),trim)
DETECT_FEATURES := 0
endif
# Strip out -Wall, -Wextra and friends for feature testing
ifeq ($(DETECT_FEATURES),1)
TCXXFLAGS := $(filter-out -Wall -Wextra -Werror% -Wunused -Wconversion -Wp%, $(CXXFLAGS))
ifneq ($(strip $(TCXXFLAGS)),)
$(info Using testing flags: $(TCXXFLAGS))
endif
endif
# For the previous messages
$(info )
###########################################################
##### X86/X32/X64 Options #####
###########################################################
ifneq ($(IS_X86)$(IS_X64),00)
ifeq ($(DETECT_FEATURES),1)
SSE2_FLAG = -msse2
SSE3_FLAG = -msse3
SSSE3_FLAG = -mssse3
SSE41_FLAG = -msse4.1
SSE42_FLAG = -msse4.2
CLMUL_FLAG = -mpclmul
AESNI_FLAG = -maes
AVX_FLAG = -mavx
AVX2_FLAG = -mavx2
SHANI_FLAG = -msha
TPROG = TestPrograms/test_x86_sse2.cxx
TOPT = $(SSE2_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
CHACHA_FLAG = $(SSE2_FLAG)
else
SSE2_FLAG =
endif
TPROG = TestPrograms/test_x86_ssse3.cxx
TOPT = $(SSSE3_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
ARIA_FLAG = $(SSSE3_FLAG)
CHAM_FLAG = $(SSSE3_FLAG)
LEA_FLAG = $(SSSE3_FLAG)
SIMECK_FLAG = $(SSSE3_FLAG)
SIMON64_FLAG = $(SSSE3_FLAG)
SIMON128_FLAG = $(SSSE3_FLAG)
SPECK64_FLAG = $(SSSE3_FLAG)
SPECK128_FLAG = $(SSSE3_FLAG)
else
SSSE3_FLAG =
endif
TPROG = TestPrograms/test_x86_sse41.cxx
TOPT = $(SSE41_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
BLAKE2B_FLAG = $(SSE41_FLAG)
BLAKE2S_FLAG = $(SSE41_FLAG)
SIMON64_FLAG = $(SSE41_FLAG)
SPECK64_FLAG = $(SSE41_FLAG)
else
SSE41_FLAG =
endif
TPROG = TestPrograms/test_x86_sse42.cxx
TOPT = $(SSE42_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
CRC_FLAG = $(SSE42_FLAG)
else
SSE42_FLAG =
endif
TPROG = TestPrograms/test_x86_clmul.cxx
TOPT = $(CLMUL_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
GCM_FLAG = $(SSSE3_FLAG) $(CLMUL_FLAG)
else
CLMUL_FLAG =
endif
TPROG = TestPrograms/test_x86_aes.cxx
TOPT = $(AESNI_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
AES_FLAG = $(SSE41_FLAG) $(AESNI_FLAG)
SM4_FLAG = $(SSSE3_FLAG) $(AESNI_FLAG)
else
AESNI_FLAG =
endif
TPROG = TestPrograms/test_x86_avx.cxx
TOPT = $(AVX_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
# XXX_FLAG = $(AVX_FLAG)
else
AVX_FLAG =
endif
TPROG = TestPrograms/test_x86_avx2.cxx
TOPT = $(AVX2_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
CHACHA_AVX2_FLAG = $(AVX2_FLAG)
else
AVX2_FLAG =
endif
TPROG = TestPrograms/test_x86_sha.cxx
TOPT = $(SHANI_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
SHA_FLAG = $(SSE42_FLAG) $(SHANI_FLAG)
else
SHANI_FLAG =
endif
ifeq ($(SSE2_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_ASM
else ifeq ($(SSE3_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_SSE3
else ifeq ($(SSSE3_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_SSSE3
else ifeq ($(SSE41_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_SSE4
else ifeq ($(SSE42_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_SSE4
endif
ifneq ($(SSE42_FLAG),)
# Unusual GCC/Clang on Macports. It assembles AES, but not CLMUL.
# test_x86_clmul.s:15: no such instruction: 'pclmulqdq $0, %xmm1,%xmm0'
ifeq ($(CLMUL_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_CLMUL
endif
ifeq ($(AESNI_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_AESNI
endif
ifeq ($(AVX_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_AVX
else ifeq ($(AVX2_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_AVX2
else ifeq ($(SHANI_FLAG),)
CXXFLAGS += -DCRYPTOPP_DISABLE_SHANI
endif
endif
# DETECT_FEATURES
endif
# IS_X86, IS_X32 and IS_X64
endif
###########################################################
##### ARM A-32, Aach64 and NEON #####
###########################################################
ifneq ($(IS_ARM32)$(IS_ARMV8),00)
ifeq ($(DETECT_FEATURES),1)
ifeq ($(IS_ARM32),1)
ifneq ($(IS_IOS)$(IS_ANDROID),00)
NEON_FLAG =
else
NEON_FLAG = -march=armv7-a -mfloat-abi=$(FP_ABI) -mfpu=neon
endif
TPROG = TestPrograms/test_arm_neon.cxx
TOPT = $(NEON_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
ARIA_FLAG = $(NEON_FLAG)
AES_FLAG = $(NEON_FLAG)
CRC_FLAG = $(NEON_FLAG)
GCM_FLAG = $(NEON_FLAG)
BLAKE2B_FLAG = $(NEON_FLAG)
BLAKE2S_FLAG = $(NEON_FLAG)
CHACHA_FLAG = $(NEON_FLAG)
CHAM_FLAG = $(NEON_FLAG)
LEA_FLAG = $(NEON_FLAG)
SHA_FLAG = $(NEON_FLAG)
SIMECK_FLAG = $(NEON_FLAG)
SIMON64_FLAG = $(NEON_FLAG)
SIMON128_FLAG = $(NEON_FLAG)
SPECK64_FLAG = $(NEON_FLAG)
SPECK128_FLAG = $(NEON_FLAG)
SM4_FLAG = $(NEON_FLAG)
else
NEON_FLAG =
CXXFLAGS += -DCRYPTOPP_DISABLE_NEON
endif
# IS_NEON
endif
ifeq ($(IS_ARMV8),1)
ifeq ($(IS_IOS),1)
ASIMD_FLAG =
CRC_FLAG =
AES_FLAG =
PMUL_FLAG =
SHA_FLAG =
else
ASIMD_FLAG = -march=armv8-a
CRC_FLAG = -march=armv8-a+crc
AES_FLAG = -march=armv8-a+crypto
PMULL_FLAG = -march=armv8-a+crypto
SHA_FLAG = -march=armv8-a+crypto
endif
TPROG = TestPrograms/test_arm_acle.cxx
TOPT = $(ASIMD_FLAG)
HAVE_OPT = $(shell $(CXX) $(TCXXFLAGS) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
ACLE_FLAG += -DCRYPTOPP_ARM_ACLE_AVAILABLE=1
else
CXXFLAGS += -DCRYPTOPP_ARM_ACLE_AVAILABLE=0
endif
TPROG = TestPrograms/test_arm_asimd.cxx
TOPT = $(ASIMD_FLAG)
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
ARIA_FLAG = $(ASIMD_FLAG)
BLAKE2B_FLAG = $(ASIMD_FLAG)
BLAKE2S_FLAG = $(ASIMD_FLAG)
CHACHA_FLAG = $(ASIMD_FLAG)
CHAM_FLAG = $(ASIMD_FLAG)
LEA_FLAG = $(ASIMD_FLAG)
NEON_FLAG = $(ASIMD_FLAG)
SIMECK_FLAG = $(ASIMD_FLAG)
SIMON64_FLAG = $(ASIMD_FLAG)
SIMON128_FLAG = $(ASIMD_FLAG)
SPECK64_FLAG = $(ASIMD_FLAG)
SPECK128_FLAG = $(ASIMD_FLAG)
SM4_FLAG = $(ASIMD_FLAG)
else
ASIMD_FLAG =
CXXFLAGS += -DCRYPTOPP_DISABLE_ASM
endif
TPROG = TestPrograms/test_arm_crc.cxx
TOPT = $(CRC_FLAG)
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifneq ($(strip $(HAVE_OPT)),0)
CRC_FLAG =
CXXFLAGS += -DCRYPTOPP_ARM_CRC32_AVAILABLE=0
endif
TPROG = TestPrograms/test_arm_aes.cxx
TOPT = $(AES_FLAG)
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifneq ($(strip $(HAVE_OPT)),0)
AES_FLAG =
CXXFLAGS += -DCRYPTOPP_ARM_AES_AVAILABLE=0
endif
TPROG = TestPrograms/test_arm_pmull.cxx
TOPT = $(PMULL_FLAG)
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifneq ($(strip $(HAVE_OPT)),0)
PMULL_FLAG =
CXXFLAGS += -DCRYPTOPP_ARM_PMULL_AVAILABLE=0
endif
TPROG = TestPrograms/test_arm_sha.cxx
TOPT = $(SHA_FLAG)
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifneq ($(strip $(HAVE_OPT)),0)
SHA_FLAG =
CXXFLAGS += -DCRYPTOPP_ARM_SHA_AVAILABLE=0
endif
TPROG = TestPrograms/test_arm_sm3.cxx
TOPT = -march=armv8.4-a+crypto
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
SM3_FLAG = -march=armv8.4-a+crypto
SM4_FLAG = -march=armv8.4-a+crypto
endif
TPROG = TestPrograms/test_arm_sha3.cxx
TOPT = -march=armv8.4-a+crypto
HAVE_OPT = $(shell $(CXX) $(CXXFLAGS) $(ACLE_FLAG) $(ZOPT) $(TOPT) $(TPROG) -o $(TOUT) 2>&1 | tr ' ' '\n' | wc -l)
ifeq ($(strip $(HAVE_OPT)),0)
SHA3_FLAG = -march=armv8.4-a+crypto
SHA512_FLAG = -march=armv8.4-a+crypto
endif
# IS_ARMV8
endif
# DETECT_FEATURES
endif
# IS_ARM32, IS_ARMV8, IS_NEON
endif
###########################################################
##### Common #####
###########################################################
# No ASM for Travis testing
ifeq ($(findstring no-asm,$(MAKECMDGOALS)),no-asm)
ifeq ($(findstring -DCRYPTOPP_DISABLE_ASM,$(CXXFLAGS)),)
@ -178,7 +546,7 @@ endif # Valgrind
# Debug testing on GNU systems. Triggered by -DDEBUG.
# Newlib test due to http://sourceware.org/bugzilla/show_bug.cgi?id=20268
ifneq ($(filter -DDEBUG -DDEBUG=1,$(CXXFLAGS)),)
USING_GLIBCXX := $(shell $(CXX) -x c++ $(CXXFLAGS) -E adhoc.cpp.proto 2>&1 | $(EGREP) -i -c "__GLIBCXX__")
USING_GLIBCXX := $(shell $(CXX) $(CXXFLAGS) -E pch.cpp 2>&1 | $(EGREP) -i -c "__GLIBCXX__")
ifneq ($(USING_GLIBCXX),0)
ifeq ($(HAS_NEWLIB),0)
ifeq ($(findstring -D_GLIBCXX_DEBUG,$(CXXFLAGS)),)
@ -207,115 +575,34 @@ ifeq ($(findstring lean,$(MAKECMDGOALS)),lean)
endif # MAKECMDGOALS
endif # Dead code stripping
# Pickup ARMv7 and NEON. Do it after Android, iOS and Embedded flags have been set.
ifeq ($(IS_ARM),1)
IS_ARMv7 := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -dM -E - 2>/dev/null | $(EGREP) -i -c '__ARM_ARCH 7')
ifeq ($(IS_ARMv7),1)
IS_NEON := 1
else
IS_NEON := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -dM -E - 2>/dev/null | $(EGREP) -i -c -E '\<__ARM_NEON\>')
endif
endif
# ARMv7-a
ifeq ($(IS_ARMv7),1)
ifeq ($(findstring -march=armv7-a,$(CXXFLAGS)),)
NEON_FLAG = -march=armv7-a
GCM_FLAG = -march=armv7-a
ARIA_FLAG = -march=armv7-a
BLAKE2_FLAG = -march=armv7-a
endif
endif
# NEON
ifeq ($(IS_NEON),1)
ifeq ($(findstring -mfpu=neon,$(CXXFLAGS)),)
NEON_FLAG += -mfpu=neon
GCM_FLAG += -mfpu=neon
ARIA_FLAG += -mfpu=neon
BLAKE2_FLAG += -mfpu=neon
SIMON_FLAG += -mfpu=neon
SPECK_FLAG += -mfpu=neon
ifeq ($(IS_ANDROID),1)
ifeq ($(findstring -mfloat-abi=softfp,$(CXXFLAGS)),)
NEON_FLAG += -mfloat-abi=softfp
GCM_FLAG += -mfloat-abi=softfp
ARIA_FLAG += -mfloat-abi=softfp
BLAKE2_FLAG += -mfloat-abi=softfp
SIMON_FLAG += -mfloat-abi=softfp
SPECK_FLAG += -mfloat-abi=softfp
endif
endif
endif
endif
# ARMv8-a
ifneq ($(IS_ARMv8),0)
IS_NEON := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_NEON)
ifeq ($(IS_NEON),1)
ARIA_FLAG = -march=armv8-a
BLAKE2_FLAG = -march=armv8-a
NEON_FLAG = -march=armv8-a
SIMON_FLAG = -march=armv8-a
SPECK_FLAG = -march=armv8-a
endif
HAVE_CRC := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a+crc -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_FEATURE_CRC32)
ifeq ($(HAVE_CRC),1)
CRC_FLAG = -march=armv8-a+crc
endif
HAVE_CRYPTO := $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -march=armv8-a+crypto -dM -E - 2>/dev/null | $(EGREP) -i -c __ARM_FEATURE_CRYPTO)
ifeq ($(HAVE_CRYPTO),1)
AES_FLAG = -march=armv8-a+crypto
GCM_FLAG = -march=armv8-a+crypto
SHA_FLAG = -march=armv8-a+crypto
endif
endif
# i686 and x86_64
ifneq ($(IS_i686)$(IS_x86_64),00)
HAVE_SSSE3 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -mssse3 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSSE3__)
ifeq ($(HAVE_SSSE3),1)
ARIA_FLAG = -mssse3
SSSE3_FLAG = -mssse3
SIMON_FLAG = -mssse3
SPECK_FLAG = -mssse3
endif
HAVE_SSE4 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.1 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSE4_1__)
ifeq ($(HAVE_SSE4),1)
SIMON_FLAG = -msse4.1
SPECK_FLAG = -msse4.1
endif
HAVE_SSE4 = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.2 -dM -E - 2>/dev/null | $(EGREP) -i -c __SSE4_2__)
ifeq ($(HAVE_SSE4),1)
BLAKE2_FLAG = -msse4.2
CRC_FLAG = -msse4.2
endif
HAVE_CLMUL = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -mssse3 -mpclmul -dM -E - 2>/dev/null | $(EGREP) -i -c __PCLMUL__ )
ifeq ($(HAVE_CLMUL),1)
GCM_FLAG = -mssse3 -mpclmul
endif
HAVE_AES = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.1 -maes -dM -E - 2>/dev/null | $(EGREP) -i -c __AES__)
ifeq ($(HAVE_AES),1)
AES_FLAG = -msse4.1 -maes
endif
HAVE_SHA = $(shell echo | $(CXX) -x c++ $(CXXFLAGS) -msse4.2 -msha -dM -E - 2>/dev/null | $(EGREP) -i -c __SHA__)
ifeq ($(HAVE_SHA),1)
SHA_FLAG = -msse4.2 -msha
endif
endif
###########################################################
##### Source and object files #####
###########################################################
# List cryptlib.cpp first, then cpu.cpp, then integer.cpp to tame C++ static initialization problems.
SRCS := cryptlib.cpp cpu.cpp integer.cpp $(filter-out cryptlib.cpp cpu.cpp integer.cpp pch.cpp simple.cpp winpipes.cpp cryptlib_bds.cpp,$(sort $(wildcard *.cpp)))
SRCS := cryptlib.cpp cpu.cpp integer.cpp $(filter-out cryptlib.cpp cpu.cpp integer.cpp pch.cpp simple.cpp cryptlib_bds.cpp,$(sort $(wildcard *.cpp)))
# For Makefile.am; resource.h is Windows
INCL := $(filter-out resource.h,$(sort $(wildcard *.h)))
# Cryptogams AES for ARMv4 and above. We couple to ARMv7.
# Avoid iOS. It cannot consume the assembly.
ifeq ($(IS_ARM32),1)
ifneq ($(IS_IOS),1)
CRYPTOGAMS_AES_FLAG = -march=armv7-a
CRYPTOGAMS_AES_FLAG += -Wa,--noexecstack
SRCS += aes_armv4.S
endif
endif
# List cryptlib.cpp first, then cpu.cpp, then integer.cpp to tame C++ static initialization problems.
OBJS := $(SRCS:.cpp=.o)
OBJS := $(OBJS:.S=.o)
# List test.cpp first to tame C++ static initialization problems.
TESTSRCS := adhoc.cpp test.cpp bench1.cpp bench2.cpp validat0.cpp validat1.cpp validat2.cpp validat3.cpp validat4.cpp datatest.cpp regtest1.cpp regtest2.cpp regtest3.cpp fipsalgt.cpp dlltest.cpp
TESTSRCS := adhoc.cpp test.cpp bench1.cpp bench2.cpp bench3.cpp datatest.cpp dlltest.cpp fipsalgt.cpp validat0.cpp validat1.cpp validat2.cpp validat3.cpp validat4.cpp validat5.cpp validat6.cpp validat7.cpp validat8.cpp validat9.cpp validat10.cpp regtest1.cpp regtest2.cpp regtest3.cpp regtest4.cpp
TESTINCL := bench.h factory.h validate.h
# Test objects
TESTOBJS := $(TESTSRCS:.cpp=.o)
LIBOBJS := $(filter-out $(TESTOBJS),$(OBJS))
@ -337,11 +624,15 @@ SOLIB_COMPAT_SUFFIX=.$(LIB_MAJOR)
SOLIB_FLAGS=-Wl,-soname,libcryptopp.so$(SOLIB_COMPAT_SUFFIX)
endif # HAS_SOLIB_VERSION
###########################################################
##### Targets and Recipes #####
###########################################################
# Default builds program with static library only
.PHONY: default
default: cryptest.exe
.PHONY: all
.PHONY: all static dynamic
all: static dynamic cryptest.exe
ifneq ($(IS_IOS),0)
@ -363,10 +654,10 @@ lean: static dynamic cryptest.exe
.PHONY: clean
clean:
-$(RM) adhoc.cpp.o adhoc.cpp.proto.o $(LIBOBJS) $(AOSP_CPU_OBJ) $(TESTOBJS) $(DLLOBJS) $(LIBIMPORTOBJS) $(TESTIMPORTOBJS) $(DLLTESTOBJS)
-$(RM) adhoc.cpp.o adhoc.cpp.proto.o $(LIBOBJS) $(AOSP_CPU_OBJ) rdrand-*.o $(TESTOBJS) $(DLLOBJS) $(LIBIMPORTOBJS) $(TESTIMPORTOBJS) $(DLLTESTOBJS)
@-$(RM) libcryptopp.a libcryptopp.dylib cryptopp.dll libcryptopp.dll.a libcryptopp.import.a
@-$(RM) libcryptopp.so libcryptopp.so$(SOLIB_COMPAT_SUFFIX) libcryptopp.so$(SOLIB_VERSION_SUFFIX)
@-$(RM) cryptest.exe dlltest.exe cryptest.import.exe cryptest.info ct rdrand-???.o
@-$(RM) cryptest.exe dlltest.exe cryptest.import.exe cryptest.info ct et
@-$(RM) *.la *.lo *.gcov *.gcno *.gcda *.stackdump core core-*
@-$(RM) /tmp/adhoc.exe
@-$(RM) -r /tmp/cryptopp_test/
@ -374,31 +665,46 @@ clean:
@-$(RM) -r *.dylib.dSYM/
@-$(RM) -r cov-int/
.PHONY: distclean
distclean: clean
-$(RM) adhoc.cpp adhoc.cpp.copied GNUmakefile.deps benchmarks.html cryptest.txt cryptest-*.txt
@-$(RM) cryptopp.tgz *.o *.bc *.ii *~
@-$(RM) -r $(SRCS:.cpp=.obj) cryptlib.lib cryptest.exe *.suo *.sdf *.pdb Win32/ x64/ ipch/
.PHONY: autotools-clean
autotools-clean:
@-$(RM) -f configure.ac configure configure.in Makefile.am Makefile.in Makefile
@-$(RM) -f config.guess config.status config.sub depcomp install-sh compile
@-$(RM) -f stamp-h1 ar-lib *.m4 local.* lt*.sh missing libtool* libcryptopp.pc*
@-$(RM) -f config.guess config.status config.sub config.h.in compile depcomp
@-$(RM) -f install-sh stamp-h1 ar-lib *.lo *.la *.m4 local.* lt*.sh missing
@-$(RM) -f cryptest cryptestcwd libtool* libcryptopp.la libcryptopp.pc*
@-$(RM) -rf m4/ auto*.cache/ .deps/ .libs/
@-$(RM) -r TestCoverage/
@-$(RM) cryptopp$(LIB_VER)\.*
@-$(RM) CryptoPPRef.zip
.PHONY: cmake-clean
cmake-clean:
@-$(RM) -f cryptopp-config.cmake CMakeLists.txt
@-$(RM) -rf cmake_build/
.PHONY: distclean
distclean: clean autotools-clean cmake-clean
-$(RM) adhoc.cpp adhoc.cpp.copied GNUmakefile.deps benchmarks.html cryptest.txt
@-$(RM) cryptest-*.txt cryptopp.tgz libcryptopp.pc *.o *.bc *.ii *~
@-$(RM) -r cryptlib.lib cryptest.exe *.suo *.sdf *.pdb Win32/ x64/ ipch/
@-$(RM) -r $(LIBOBJS:.o=.obj) $(TESTOBJS:.o=.obj)
@-$(RM) -r $(LIBOBJS:.o=.lst) $(TESTOBJS:.o=.lst)
@-$(RM) -r TestCoverage/ ref*/
@-$(RM) cryptopp$(LIB_VER)\.* CryptoPPRef.zip
# Install cryptest.exe, libcryptopp.a and libcryptopp.so.
# The library install was broken-out into its own recipe at GH #653.
.PHONY: install
install:
@-$(MKDIR) $(DESTDIR)$(INCLUDEDIR)/cryptopp
$(INSTALL_DATA) *.h $(DESTDIR)$(INCLUDEDIR)/cryptopp
ifneq ($(wildcard cryptest.exe),)
install: cryptest.exe install-lib
@-$(MKDIR) $(DESTDIR)$(BINDIR)
$(INSTALL_PROGRAM) cryptest.exe $(DESTDIR)$(BINDIR)
@-$(MKDIR) $(DESTDIR)$(DATADIR)/cryptopp/TestData
@-$(MKDIR) $(DESTDIR)$(DATADIR)/cryptopp/TestVectors
$(INSTALL_DATA) TestData/*.dat $(DESTDIR)$(DATADIR)/cryptopp/TestData
$(INSTALL_DATA) TestVectors/*.txt $(DESTDIR)$(DATADIR)/cryptopp/TestVectors
endif
# A recipe to install only the library, and not cryptest.exe. Also
# see https://github.com/weidai11/cryptopp/issues/653.
.PHONY: install-lib
install-lib:
@-$(MKDIR) $(DESTDIR)$(INCLUDEDIR)/cryptopp
$(INSTALL_DATA) *.h $(DESTDIR)$(INCLUDEDIR)/cryptopp
ifneq ($(wildcard libcryptopp.a),)
@-$(MKDIR) $(DESTDIR)$(LIBDIR)
$(INSTALL_DATA) libcryptopp.a $(DESTDIR)$(LIBDIR)
@ -435,15 +741,15 @@ ifeq ($(HAS_SOLIB_VERSION),1)
libcryptopp.so: libcryptopp.so$(SOLIB_VERSION_SUFFIX)
endif
libcryptopp.so$(SOLIB_VERSION_SUFFIX): $(LIBOBJS)
$(CXX) -shared $(SOLIB_FLAGS) -o $@ $(strip $(CXXFLAGS)) -Wl,--exclude-libs,ALL $(LIBOBJS) $(LDFLAGS) $(LDLIBS)
libcryptopp.so$(SOLIB_VERSION_SUFFIX): $(LIBOBJS) $(AOSP_CPU_OBJ)
$(CXX) -shared $(SOLIB_FLAGS) -o $@ $(strip $(CXXFLAGS)) -Wl,--exclude-libs,ALL $(LIBOBJS) $(AOSP_CPU_OBJ) $(LDFLAGS) $(LDLIBS)
ifeq ($(HAS_SOLIB_VERSION),1)
-$(LN) libcryptopp.so$(SOLIB_VERSION_SUFFIX) libcryptopp.so
-$(LN) libcryptopp.so$(SOLIB_VERSION_SUFFIX) libcryptopp.so$(SOLIB_COMPAT_SUFFIX)
endif
libcryptopp.dylib: $(LIBOBJS)
$(CXX) -dynamiclib -o $@ $(strip $(CXXFLAGS)) -install_name "$@" -current_version "$(LIB_MAJOR).$(LIB_MINOR).$(LIB_PATCH)" -compatibility_version "$(LIB_MAJOR).$(LIB_MINOR)" -headerpad_max_install_names $(LDFLAGS) $(LIBOBJS)
libcryptopp.dylib: $(LIBOBJS) $(AOSP_CPU_OBJ)
$(CXX) -dynamiclib -o $@ $(strip $(CXXFLAGS)) -install_name "$@" -current_version "$(LIB_MAJOR).$(LIB_MINOR).$(LIB_PATCH)" -compatibility_version "$(LIB_MAJOR).$(LIB_MINOR)" -headerpad_max_install_names $(LDFLAGS) $(LIBOBJS) $(AOSP_CPU_OBJ)
cryptest.exe: libcryptopp.a $(TESTOBJS)
$(CXX) -o $@ $(strip $(CXXFLAGS)) $(TESTOBJS) ./libcryptopp.a $(LDFLAGS) $(LDLIBS)
@ -451,13 +757,17 @@ cryptest.exe: libcryptopp.a $(TESTOBJS)
# Used to generate list of source files for Autotools, CMakeList and Android.mk
.PHONY: sources
sources:
$(info Library sources: $(filter-out $(TESTSRCS),$(SRCS)))
$(info ***** Library sources *****)
$(info $(filter-out $(TESTSRCS),$(SRCS)))
$(info )
$(info Library headers: $(filter-out $(TESTINCL),$(INCL)))
$(info ***** Library headers *****)
$(info $(filter-out $(TESTINCL),$(INCL)))
$(info )
$(info Test sources: $(TESTSRCS))
$(info ***** Test sources *****)
$(info $(TESTSRCS))
$(info )
$(info Test headers: $(TESTINCL))
$(info ***** Test headers *****)
$(info $(TESTINCL))
adhoc.cpp: adhoc.cpp.proto
ifeq ($(wildcard adhoc.cpp),)
@ -471,55 +781,108 @@ ifeq ($(wildcard GNUmakefile.deps),GNUmakefile.deps)
-include GNUmakefile.deps
endif # Dependencies
# Cryptogams ARM asm implementation.
aes_armv4.o : aes_armv4.S
$(CXX) $(strip $(CXXFLAGS) -fpermissive $(CRYPTOGAMS_AES_FLAG) -c) $<
cpu-features.o: cpu-features.h cpu-features.c
$(CXX) $(strip $(CXXFLAGS) -fpermissive -c) cpu-features.c
# SSE4.2 or NEON available
aria-simd.o : aria-simd.cpp
# SSSE3 or NEON available
aria_simd.o : aria_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(ARIA_FLAG) -c) $<
# SSE4.2 or ARMv8a available
blake2-simd.o : blake2-simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(BLAKE2_FLAG) -c) $<
# SSE, NEON or POWER7 available
blake2s_simd.o : blake2s_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(BLAKE2S_FLAG) -c) $<
# SSE2 on i586
cpu.o : cpu.cpp
$(CXX) $(strip $(CXXFLAGS) $(CPU_FLAG) -c) $<
# SSE, NEON or POWER8 available
blake2b_simd.o : blake2b_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(BLAKE2B_FLAG) -c) $<
# SSE2 or NEON available
chacha_simd.o : chacha_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(CHACHA_FLAG) -c) $<
# AVX2 available
chacha_avx.o : chacha_avx.cpp
$(CXX) $(strip $(CXXFLAGS) $(CHACHA_AVX2_FLAG) -c) $<
# SSSE3 available
cham_simd.o : cham_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(CHAM_FLAG) -c) $<
# Power9 available
darn.o : darn.cpp
$(CXX) $(strip $(CXXFLAGS) $(DARN_FLAG) -c) $<
# SSE2 on i686
donna_sse.o : donna_sse.cpp
$(CXX) $(strip $(CXXFLAGS) $(SSE2_FLAG) -c) $<
# SSE2 on i686
sse_simd.o : sse_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SSE2_FLAG) -c) $<
# SSE4.2 or ARMv8a available
crc-simd.o : crc-simd.cpp
crc_simd.o : crc_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(CRC_FLAG) -c) $<
# PCLMUL or ARMv7a/ARMv8a available
gcm-simd.o : gcm-simd.cpp
gcm_simd.o : gcm_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(GCM_FLAG) -c) $<
# SSSE3 available
lea_simd.o : lea_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(LEA_FLAG) -c) $<
# NEON available
neon-simd.o : neon-simd.cpp
neon_simd.o : neon_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(NEON_FLAG) -c) $<
# AESNI or ARMv7a/ARMv8a available
rijndael-simd.o : rijndael-simd.cpp
rijndael_simd.o : rijndael_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(AES_FLAG) -c) $<
# SSE4.2/SHA-NI or ARMv8a available
sha-simd.o : sha-simd.cpp
sha_simd.o : sha_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SHA_FLAG) -c) $<
# SSE4.2/SHA-NI or ARMv8a available
shacal2-simd.o : shacal2-simd.cpp
shacal2_simd.o : shacal2_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SHA_FLAG) -c) $<
# SSSE3 or NEON available
simon-simd.o : simon-simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SIMON_FLAG) -c) $<
simeck_simd.o : simeck_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SIMECK_FLAG) -c) $<
# SSSE3 or NEON available
speck-simd.o : speck-simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SPECK_FLAG) -c) $<
# SSE4.1, NEON or POWER7 available
simon64_simd.o : simon64_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SIMON64_FLAG) -c) $<
# SSSE3, NEON or POWER8 available
simon128_simd.o : simon128_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SIMON128_FLAG) -c) $<
# SSE4.1, NEON or POWER7 available
speck64_simd.o : speck64_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SPECK64_FLAG) -c) $<
# SSSE3, NEON or POWER8 available
speck128_simd.o : speck128_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SPECK128_FLAG) -c) $<
# ARMv8.4 available
sm3_simd.o : sm3_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SM3_FLAG) -c) $<
# AESNI available
sm4_simd.o : sm4_simd.cpp
$(CXX) $(strip $(CXXFLAGS) $(SM4_FLAG) -c) $<
%.o : %.cpp
$(CXX) $(strip $(CXXFLAGS) -c) $<
GNUmakefile.deps:
$(CXX) $(strip $(CXXFLAGS) -MM) *.cpp > GNUmakefile.deps
.PHONY: dep deps depend
dep deps depend GNUmakefile.deps:
$(CXX) $(strip $(CXXFLAGS) -DCRYPTOPP_DISABLE_ASM) -MM *.cpp > GNUmakefile.deps

22
vendor/cryptopp/vendor_cryptopp/History.txt vendored
View File

@ -4,7 +4,8 @@ The History file contains the items that comprise the release notes. The
items in the list below used to be in Readme.txt. Readme.txt now contans the
last several releases.
1.0 - First public release. Withdrawn at the request of RSA DSI.
1.0 - First public release
- Withdrawn at the request of RSA DSI over patent claims
- included Blowfish, BBS, DES, DH, Diamond, DSA, ElGamal, IDEA,
MD5, RC4, RC5, RSA, SHA, WAKE, secret sharing, DEFLATE compression
- had a serious bug in the RSA key generation code.
@ -432,3 +433,22 @@ last several releases.
- remove s_nullNameValuePairs from unnamed namespace
- ported to MSVC 2017, Xcode 9.3, Sun Studio 12.5, GCC 8.0.1,
MacPorts GCC 7.0, Clang 4.0, Intel C++ 17.00, IBM XL C/C++ 13.1
8.0.0 - December 28, 2018
- major release, recompile of programs required
- expanded community input and support
* 54 unique contributors as of this release
- add x25519 key exchange and ed25519 signature scheme
- add limited Asymmetric Key Package support from RFC 5958
- add Power9 DARN random number generator support
- add CHAM, HC-128, HC-256, Hight, LEA, Rabbit, Simeck
- fix FixedSizeAllocatorWithCleanup may be unaligned on some platforms
- cutover to GNU Make-based cpu feature tests
- rename files with dashes to underscores
- fix LegacyDecryptor and LegacyDecryptorWithMAC use wrong MAC
- fix incorrect AES/CBC decryption on Windows
- avoid Singleton<T> when possible, avoid std::call_once completely
- fix SPARC alignment problems due to GetAlignmentOf<T>() on word64
- add ARM AES asm implementation from Cryptogams
- remove CRYPTOPP_ALLOW_UNALIGNED_DATA_ACCESS support

101
vendor/cryptopp/vendor_cryptopp/Readme.txt vendored
View File

@ -1,5 +1,5 @@
Crypto++: free C++ Class Library of Cryptographic Schemes
Version 7.0 - APR/08/2018
Version 8.0 - DEC/28/2018
Crypto++ Library is a free C++ class library of cryptographic schemes.
Currently the library contains the following algorithms:
@ -39,11 +39,12 @@ Currently the library contains the following algorithms:
padding schemes for public-key PKCS#1 v2.0, OAEP, PSS, PSSR, IEEE P1363
systems EMSA2 and EMSA5
Diffie-Hellman (DH), Unified Diffie-Hellman
key agreement schemes (DH2), Menezes-Qu-Vanstone (MQV), Hashed MQV (HMQV),
Diffie-Hellman (DH), Unified Diffie-Hellman (DH2),
key agreement schemes Menezes-Qu-Vanstone (MQV), Hashed MQV (HMQV),
Fully Hashed MQV (FHMQV), LUCDIF, XTR-DH
elliptic curve cryptography ECDSA, Determinsitic ECDSA, ECNR, ECIES, ECDH, ECMQV
elliptic curve cryptography ECDSA, Determinsitic ECDSA, ed25519, ECNR, ECIES,
ECDH, ECMQV, x25519
insecure or obsolescent MD2, MD4, MD5, Panama Hash, DES, ARC4, SEAL
algorithms retained for backwards 3.0, WAKE-OFB, DESX (DES-XEX3), RC2,
@ -53,7 +54,7 @@ algorithms retained for backwards 3.0, WAKE-OFB, DESX (DES-XEX3), RC2,
Other features include:
* pseudo random number generators (PRNG): ANSI X9.17 appendix C, RandomPool,
VIA Padlock, RDRAND, RDSEED, NIST Hash and HMAC DRBGs
DARN, VIA Padlock, RDRAND, RDSEED, NIST Hash and HMAC DRBGs
* password based key derivation functions: PBKDF1 and PBKDF2 from PKCS #5,
PBKDF from PKCS #12 appendix B, HKDF from RFC 5869, Scrypt from RFC 7914
* Shamir's secret sharing scheme and Rabin's information dispersal algorithm
@ -91,13 +92,13 @@ The following compilers are supported for this release. Please visit
http://www.cryptopp.com the most up to date build instructions and porting notes.
* Visual Studio 2003 - 2017
* GCC 3.3 - 8.0
* GCC 3.3 - 9.0
* Apple Clang 4.3 - 9.3
* LLVM Clang 2.9 - 4.0
* C++Builder 2010
* LLVM Clang 2.9 - 7.0
* C++Builder 2015
* Intel C++ Compiler 9 - 16.0
* Sun Studio 12u1 - 12.5
* IBM XL C/C++ 10.0 - 13.1
* Sun Studio 12u1 - 12.6
* IBM XL C/C++ 10.0 - 13.3
*** Important Usage Notes ***
@ -282,6 +283,25 @@ documentation is one of the highest returns on investment.
The items in this section comprise the most recent history. Please see History.txt
for the record back to Crypto++ 1.0.
8.0.0 - December 28, 2018
- major release, recompile of programs required
- expanded community input and support
* 54 unique contributors as of this release
- add x25519 key exchange and ed25519 signature scheme
- add limited Asymmetric Key Package support from RFC 5958
- add Power9 DARN random number generator support
- add CHAM, HC-128, HC-256, Hight, LEA, Rabbit, Simeck
- fix FixedSizeAllocatorWithCleanup may be unaligned on some platforms
- cutover to GNU Make-based cpu feature tests
- rename files with dashes to underscores
- fix LegacyDecryptor and LegacyDecryptorWithMAC use wrong MAC
- fix incorrect AES/CBC decryption on Windows
- avoid Singleton<T> when possible, avoid std::call_once completely
- fix SPARC alignment problems due to GetAlignmentOf<T>() on word64
- add ARM AES asm implementation from Cryptogams
- remove CRYPTOPP_ALLOW_UNALIGNED_DATA_ACCESS support
7.0.0 - April 8, 2018
- major release, recompile of programs required
- expanded community input and support
@ -340,67 +360,6 @@ for the record back to Crypto++ 1.0.
- ported to MSVC 2017, Xcode 8.1, Sun Studio 12.5, GCC 7.3,
MacPorts GCC 7.0, Clang 4.0, Intel C++ 17.00, IBM XL C/C++ 13.1
5.6.5 - October 11, 2016
- maintenance release, recompile of programs recommended
- expanded community input and support
* 25 unique contributors as of this release
- fixed CVE-2016-7420 (Issue 277, document NDEBUG for production/release)
- fixed CVE-2016-7544 (Issue 302, avoid _malloca and _freea)
- shipped library in recommended state
* backwards compatibility achieved with <config.compat>
- Visual Studio project file cleanup
* improved X86 and X64 MSBuild support
* added ARM-based MSBuild awareness
- improved Testing and QA
* expanded platforms and compilers
* expanded Coverity into OS X and Windows platforms
* added Windows test scripts using Strawberry Perl
- ported to MSVC 2015 SP3, Xcode 7.3, Sun Studio 12.5, GCC 7.0,
MacPorts GCC 7.0, Clang 3.8, Intel C++ 17.00
5.6.4 - September 11, 2016
- maintenance release, honored API/ABI/Versioning requirements
- expanded community input and support
* 22 unique contributors for this release
- fixed CVE-2016-3995
- changed SHA3 to FIPS 202 (F1600, XOF d=0x06)
- added Keccak (F1600, XOF d=0x01)
- added ChaCha (ChaCha8/12/20)
- added HMQV and FHMQV
* Hashed and Fully Hashed MQV
- added BLAKE2 (BLAKE2s and BLAKE2b)
* C++, SSE2, SSE4, ARM NEON and ARMv8 ASIMD
- added CRC32-C
* C/C++, Amd64 CRC, and ARMv8 CRC
- improved Rabin-William signatures
* Tweaked roots <em>e</em> and <em>f</em>
- improved C++11 support
* atomics, threads and fences
* alginof, alignas
* constexpr
* noexcept
- improved GCM mode
* ARM NEON and ARMv8 ASIMD
* ARMv8 carry-less multiply
- improved Windows 8 and 10 support
* Windows Phone, Universal Windows Platform, Windows Store
- improved MIPS, ARMv7 and ARMv8 support
* added scripts setenv-{android|embedded|ios}.sh for GNUmakefile-cross
* aggressive use of -march=<arch> and -mfpu=<fpu> in cryptest.sh
- improved build systems
* Visual Studio 2010 default
* added CMake support (lacks FindCryptopp.cmake)
* archived VC++ 5/0/6.0 project files (vc60.zip)
* archived VS2005 project files (vs2005.zip)
* archived Borland project files (bds10.zip)
- improved Testing and QA
* expanded platforms and compilers
* added code generation tests based on CPU features
* added C++03, C++11, C++14, C++17 testing
* added -O3, -O5, -Ofast and -Os testing
- ported to MSVC 2015 SP3, Xcode 9.0, Sun Studio 12.5, GCC 7.0,
MacPorts GCC 7.0, Clang 3.8, Intel C++ 17.00
June 2015 - Changing of the guard. Wei Dai turned the library over to the
community. The first community release was Crypto++ 5.6.3. Wei is
no longer involved with the daily operations of the project. Wei

1
vendor/cryptopp/vendor_cryptopp/TestData/defdmac1.bin vendored Normal file
View File

@ -0,0 +1 @@
nCBţ pIbź_¤_ĺ|źâ+"w˙¶¶†Ž˝í#Aýíż‡„nŢmÖéŠČ‡ű%±––kXřťąRňłüw`Ză‘^s.+ÚÚµ<C39A>ř„Š`ŠÝBf]zó•}¦R`}\ŘdiśbfttÉ^ř˛

BIN
vendor/cryptopp/vendor_cryptopp/TestData/defdmac2.bin vendored Normal file
View File

Binary file not shown.

1
vendor/cryptopp/vendor_cryptopp/TestData/ed25519.dat vendored Normal file
View File

@ -0,0 +1 @@
302E020100300506032B65700422042098C59D3F097FB23D44BA90791281B453258D691A55AF5CE4F1EE712FDF91AE6D

1
vendor/cryptopp/vendor_cryptopp/TestData/ed25519v0.dat vendored Normal file
View File

@ -0,0 +1 @@
302E020100300506032B65700422042030BF776A497D7F1E0E26AC4FB03F5BE7E187DDFEFB914CD292A6FEDB7F70CE6B

1
vendor/cryptopp/vendor_cryptopp/TestData/ed25519v1.dat vendored Normal file
View File

@ -0,0 +1 @@
3053020101300506032B6570042204206861FD53C7643DABDCDF4C3969CE44A156BAC261242A5AAEC140EDE510071C6CA12303210029CF90E6C1CF1ADC7105720303B2EE303412D2B682C6FEEF3D8736A286B2E27F

1
vendor/cryptopp/vendor_cryptopp/TestData/x25519.dat vendored Normal file
View File

@ -0,0 +1 @@
302E020100300506032B656E0422042030D407BB0CC97D0EC493BDB00A4A8EFA06A50D2388F5BA62947030E7D9873F49

1
vendor/cryptopp/vendor_cryptopp/TestData/x25519v0.dat vendored Normal file
View File

@ -0,0 +1 @@
302E020100300506032B656E042204202818E54DE6B88EEF3E99E25042CB98E69373B222E4C1E8B3FB10AC9B26C1007B

1
vendor/cryptopp/vendor_cryptopp/TestData/x25519v1.dat vendored Normal file
View File

@ -0,0 +1 @@
3053020101300506032B656E04220420A00FADD6D29BE764B851F64F7620E80B700DF65914BED31E486362281BB5D061A123032100E9AD4CC54DAA36F312D98B253854F0076E2BC26FCE5802B3AC79A5B59B3D2C4F

7
vendor/cryptopp/vendor_cryptopp/TestPrograms/test_32bit.cxx vendored Normal file
View File

@ -0,0 +1,7 @@
#include <cstddef>
int main(int argc, char* argv[])
{
enum {N = (sizeof(std::size_t) == 4 ? 4 : -1)};
int x[N];
return 0;
}

7
vendor/cryptopp/vendor_cryptopp/TestPrograms/test_64bit.cxx vendored Normal file
View File

@ -0,0 +1,7 @@
#include <cstddef>
int main(int argc, char* argv[])
{
enum {N = (sizeof(std::size_t) == 8 ? 8 : -1)};
int x[N];
return 0;
}

5
vendor/cryptopp/vendor_cryptopp/TestPrograms/test_arm_acle.cxx vendored Normal file
View File

@ -0,0 +1,5 @@
#include <arm_acle.h>
int main(int argc, char* argv[])
{
return 0;
}

19
vendor/cryptopp/vendor_cryptopp/TestPrograms/test_arm_aes.cxx vendored Normal file
View File

@ -0,0 +1,19 @@
#include <arm_neon.h>
#include <stdint.h>
// test_acle.h determines if this is available. Then,
// -DCRYPTOPP_ARM_ACLE_AVAILABLE=0 is added to CXXFLAGS
// if the ACLE header is not available.
#if (CRYPTOPP_ARM_ACLE_AVAILABLE)
# include <arm_acle.h>
#endif
int main(int argc, char* argv[])
{
uint8x16_t x={0};
x=vaeseq_u8(x,x);
x=vaesmcq_u8(x);
x=vaesdq_u8(x,x);
x=vaesimcq_u8(x);
return 0;
}

Some files were not shown because too many files have changed in this diff Show More