hardcoresushi
/
libcryfs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Merge from next

This commit is contained in:
Sebastian Messmer 2018-09-09 14:48:06 -07:00
parent fb792ec353 6ada5166e9
commit 1a7625ea9b
1535 changed files with 223529 additions and 55421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
.circleci/config.yml
View File

@ -7,7 +7,7 @@ references:
run:
name: Initialize Cache
command: |
echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}" > /tmp/_build_env_vars
echo "${APT_COMPILER_PACKAGE}_${BUILD_TOOLSET}_${CXX}_${CC}_${BUILD_TYPE}_${CXXFLAGS}" > /tmp/_build_env_vars
echo Build env vars used for cache keys:
cat /tmp/_build_env_vars
container_setup_pre: &container_setup_pre
@ -49,7 +49,7 @@ references:
sudo chmod o-w /etc/apt/sources.list.d/clang.list
DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq
DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
DEBIAN_FRONTEND=noninteractive sudo apt-get install -y git ccache $APT_COMPILER_PACKAGE cmake3 make libcurl4-openssl-dev libssl-dev libfuse-dev python
# Use /dev/urandom when /dev/random is accessed to use less entropy
sudo cp -a /dev/urandom /dev/random
@ -150,26 +150,34 @@ references:
ccache --max-size=512M
ccache --show-stats
# Disable OpenMP if it is clang, because Ubuntu 14.04 doesn't have the libomp-dev package needed to support OpenMP for clang.
if [[ ${APT_COMPILER_PACKAGE} == clang* ]]; then
OPENMP_PARAMS="-DDISABLE_OPENMP=ON"
else
OPENMP_PARAMS=""
fi
# Build
mkdir cmake
cd cmake
cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ${OPENMP_PARAMS}
make -j$NUMCORES
ccache --show-stats
test: &test
run:
name: Test
no_output_timeout: 120m
command: |
cd cmake
./test/gitversion/gitversion-test
./test/cpp-utils/cpp-utils-test
./test/fspp/fspp-test
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cpp-utils/cpp-utils-test ; fi
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] && [ ! "$DISABLE_BROKEN_ASAN_TESTS" = true ] ; then ./test/fspp/fspp-test ; fi
./test/parallelaccessstore/parallelaccessstore-test
./test/blockstore/blockstore-test
./test/blobstore/blobstore-test
./test/cryfs/cryfs-test
./test/cryfs-cli/cryfs-cli-test
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs/cryfs-test ; fi
if [ ! "$DISABLE_BROKEN_TSAN_TESTS" = true ] ; then ./test/cryfs-cli/cryfs-cli-test ; fi
job_definition: &job_definition
<<: *container_config
steps:
@ -191,82 +199,156 @@ references:
only: /.*/
jobs:
gcc_4_8:
<<: *job_definition
environment:
CC: gcc-4.8
CXX: g++-4.8
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-4.8"
gcc_5:
gcc_5_debug:
<<: *job_definition
environment:
CC: gcc-5
CXX: g++-5
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-5"
gcc_6:
CXXFLAGS: ""
BUILD_TYPE: "Debug"
gcc_5_release:
<<: *job_definition
environment:
CC: gcc-5
CXX: g++-5
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-5"
CXXFLAGS: ""
BUILD_TYPE: "Release"
gcc_6_debug:
<<: *job_definition
environment:
CC: gcc-6
CXX: g++-6
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-6"
gcc_7:
CXXFLAGS: ""
BUILD_TYPE: "Debug"
gcc_6_release:
<<: *job_definition
environment:
CC: gcc-6
CXX: g++-6
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-6"
CXXFLAGS: ""
BUILD_TYPE: "Release"
gcc_7_debug:
<<: *job_definition
environment:
CC: gcc-7
CXX: g++-7
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-7"
clang_3_7:
CXXFLAGS: ""
BUILD_TYPE: "Debug"
gcc_7_release:
<<: *job_definition
environment:
CC: clang-3.7
CXX: clang++-3.7
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-3.7
clang_3_8:
<<: *job_definition
environment:
CC: clang-3.8
CXX: clang++-3.8
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-3.8
clang_4_0:
CC: gcc-7
CXX: g++-7
BUILD_TOOLSET: gcc
APT_COMPILER_PACKAGE: "g++-7"
CXXFLAGS: ""
BUILD_TYPE: "Release"
clang_4_0_debug:
<<: *job_definition
environment:
CC: clang-4.0
CXX: clang++-4.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-4.0
clang_5_0:
CXXFLAGS: ""
BUILD_TYPE: "Debug"
clang_4_0_release:
<<: *job_definition
environment:
CC: clang-4.0
CXX: clang++-4.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-4.0
CXXFLAGS: ""
BUILD_TYPE: "Release"
clang_5_0_debug:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: ""
BUILD_TYPE: "Debug"
clang_5_0_release:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: ""
BUILD_TYPE: "Release"
no_compatibility:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: "-DCRYFS_NO_COMPATIBILITY"
BUILD_TYPE: "Debug"
address_sanitizer:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: "-O2 -fsanitize=address -fno-omit-frame-pointer -fno-common -fsanitize-address-use-after-scope"
BUILD_TYPE: "Debug"
# Note: Leak detection is disabled because libfuse itself is leaky...
ASAN_OPTIONS: "detect_leaks=0 check_initialization_order=1 detect_stack_use_after_return=1 detect_invalid_pointer_pairs=1 atexit=1"
DISABLE_BROKEN_ASAN_TESTS: true
thread_sanitizer:
<<: *job_definition
environment:
CC: clang-5.0
CXX: clang++-5.0
BUILD_TOOLSET: clang
APT_COMPILER_PACKAGE: clang-5.0
CXXFLAGS: "-O2 -fsanitize=thread -fno-omit-frame-pointer"
BUILD_TYPE: "Debug"
DISABLE_BROKEN_TSAN_TESTS: true
workflows:
version: 2
build_and_test:
jobs:
- gcc_4_8:
- gcc_5_debug:
<<: *enable_for_tags
- gcc_5:
- gcc_5_release:
<<: *enable_for_tags
- gcc_6:
- gcc_6_debug:
<<: *enable_for_tags
- gcc_7:
- gcc_6_release:
<<: *enable_for_tags
- clang_3_7:
- gcc_7_debug:
<<: *enable_for_tags
- clang_3_8:
- gcc_7_release:
<<: *enable_for_tags
- clang_4_0:
- clang_4_0_debug:
<<: *enable_for_tags
- clang_5_0:
- clang_4_0_release:
<<: *enable_for_tags
- clang_5_0_debug:
<<: *enable_for_tags
- clang_5_0_release:
<<: *enable_for_tags
- no_compatibility:
<<: *enable_for_tags
- address_sanitizer:
<<: *enable_for_tags
- thread_sanitizer:
<<: *enable_for_tags

50
.clang-tidy Normal file
View File

@ -0,0 +1,50 @@
---
# TODO Enable (some of) the explicitly disabled checks. Possibly needs helper types from gsl library or similar to enable full cppcoreguidelines.
# TODO Enable more checks (google-*, hicpp-*, llvm-*, modernize-*, mpi-*, performance-*, readability-*)
# TODO Maybe just enable * and disable a list instead?
Checks: |
clang-diagnostic-*,
clang-analyzer-*,
bugprone-*,
cert-*,
cppcoreguidelines-*,
misc-*,
boost-use-to-string,
-cert-env33-c,
-cert-err58-cpp,
-cppcoreguidelines-owning-memory,
-cppcoreguidelines-no-malloc,
-cppcoreguidelines-pro-type-const-cast,
-cppcoreguidelines-pro-bounds-pointer-arithmetic,
-cppcoreguidelines-pro-type-reinterpret-cast,
-cppcoreguidelines-special-member-functions,
-cppcoreguidelines-pro-type-cstyle-cast,
-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
-cppcoreguidelines-pro-type-vararg,
-misc-macro-parentheses,
-misc-unused-raii
WarningsAsErrors: ''
HeaderFilterRegex: '/src/|/test/'
CheckOptions:
- key: google-readability-braces-around-statements.ShortStatementLines
value: '1'
- key: google-readability-function-size.StatementThreshold
value: '800'
- key: google-readability-namespace-comments.ShortNamespaceLines
value: '10'
- key: google-readability-namespace-comments.SpacesBeforeComments
value: '2'
- key: modernize-loop-convert.MaxCopySize
value: '16'
- key: modernize-loop-convert.MinConfidence
value: reasonable
- key: modernize-loop-convert.NamingStyle
value: CamelCase
- key: modernize-pass-by-value.IncludeStyle
value: llvm
- key: modernize-replace-auto-ptr.IncludeStyle
value: llvm
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
...

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ umltest.inner.sh
umltest.status
/build
/cmake
/cmake-build-*
/.idea
*~

34
.travis.yml
View File

@ -5,7 +5,7 @@ compiler:
- gcc
- clang
os:
- linux
#- linux
- osx
addons:
apt:
@ -18,34 +18,6 @@ addons:
- libcrypto++-dev
- libfuse-dev
install:
# Use new clang
- if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
# Detect number of CPU cores
- export NUMCORES=`grep -c ^processor /proc/cpuinfo` && if [ ! -n "$NUMCORES" ]; then export NUMCORES=`sysctl -n hw.ncpu`; fi
- echo Using $NUMCORES cores
# Install dependencies
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./travis.install_boost.sh; fi
- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew cask install osxfuse && brew install cryptopp; fi
# Install run_with_fuse.sh
- mkdir cmake
- cd cmake
- wget https://raw.githubusercontent.com/smessmer/travis-utils/master/run_with_fuse.sh
- chmod +x run_with_fuse.sh
- cmake --version
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then sudo cp -a /dev/urandom /dev/random; fi
- .travisci/install.sh
script:
- cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
- make -j$NUMCORES
- ./test/gitversion/gitversion-test
- ./test/cpp-utils/cpp-utils-test
# TODO Also run on osx once fixed
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./run_with_fuse.sh ./test/fspp/fspp-test || exit 1; fi
- ./test/parallelaccessstore/parallelaccessstore-test
- ./test/blockstore/blockstore-test
- ./test/blobstore/blobstore-test
# TODO Also run on osx once fixed
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs/cryfs-test || exit 1; fi
- if [ "${TRAVIS_OS_NAME}" == "linux" ]; then ./test/cryfs-cli/cryfs-cli-test || exit 1; fi
after_script:
- rm run_with_fuse.sh
- .travisci/build_and_test.sh

33
.travisci/build_and_test.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash
set -e
# Detect number of CPU cores
export NUMCORES=`grep -c ^processor /proc/cpuinfo`
if [ ! -n "$NUMCORES" ]; then
export NUMCORES=`sysctl -n hw.ncpu`
fi
echo Using $NUMCORES cores
# Setup target directory
mkdir cmake
cd cmake
cmake --version
# Build
cmake .. -DBUILD_TESTING=on -DCMAKE_BUILD_TYPE=Debug
make -j$NUMCORES
# Test
./test/gitversion/gitversion-test
./test/cpp-utils/cpp-utils-test
./test/parallelaccessstore/parallelaccessstore-test
./test/blockstore/blockstore-test
./test/blobstore/blobstore-test
./test/cryfs/cryfs-test
# TODO Also run on osx once fixed
if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
./test/fspp/fspp-test
./test/cryfs-cli/cryfs-cli-test
fi

31
.travisci/install.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
set -e
# Use new clang on linux
if [ "${TRAVIS_OS_NAME}" == "linux" ] && [ "$CXX" = "clang++" ]; then
export CXX="clang++-3.7" CC="clang-3.7"
fi
# If using gcc on mac, actually use it ("gcc" just links to clang, but "gcc-4.8" is gcc, https://github.com/travis-ci/travis-ci/issues/2423)
if [ "${TRAVIS_OS_NAME}" == "osx" ] && ["${CXX}" = "g++" ]; then
export CXX="g++-4.8" CC="gcc-4.8"
fi
# Install dependencies
if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
./.travisci/install_boost.sh
fi
if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
brew cask install osxfuse
brew install libomp
fi
# By default, travis only fetches the newest 50 commits. We need more in case we're further from the last version tag, so the build doesn't fail because it can't generate the version number.
git fetch --unshallow
# Use /dev/urandom when /dev/random is accessed, because travis doesn't have enough entropy
if [ "${TRAVIS_OS_NAME}" == "linux" ]; then
sudo cp -a /dev/urandom /dev/random
fi

0
travis.install_boost.sh → .travisci/install_boost.sh
View File

29
CMakeLists.txt
View File

@ -1,26 +1,33 @@
cmake_minimum_required(VERSION 2.8 FATAL_ERROR)
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
cmake_policy(SET CMP0054 NEW)
# note: for clang-tidy, we need cmake 3.6, or (if the return code should be handled correctly, e.g. on CI), we need 3.8.
# TODO Perf test:
# - try if setting CRYPTOPP_NATIVE_ARCH=ON and adding -march=native to the compile commands for cryfs source files makes a difference
# -> if yes, offer a cmake option to enable both of these
project(cryfs)
include(utils.cmake)
require_gcc_version(4.8)
require_clang_version(3.7)
require_gcc_version(5.0)
require_clang_version(4.0)
# Default value is not to build test cases
if(NOT BUILD_TESTING)
set(BUILD_TESTING OFF CACHE BOOL "BUILD_TESTING")
endif(NOT BUILD_TESTING)
option(BUILD_TESTING "build test cases" OFF)
option(CRYFS_UPDATE_CHECKS "let cryfs check for updates and security vulnerabilities" ON)
option(DISABLE_OPENMP "allow building without OpenMP libraries. This will cause performance degradations." OFF)
# Default vaule is to build in release mode
# Default value is to build in release mode
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE Release CACHE INTERNAL "CMAKE_BUILD_TYPE")
endif(NOT CMAKE_BUILD_TYPE)
# Default value is to do update checks
if(NOT CRYFS_UPDATE_CHECKS)
set(CRYFS_UPDATE_CHECKS ON CACHE BOOL "CRYFS_UPDATE_CHECKS")
endif(NOT CRYFS_UPDATE_CHECKS)
# The MSVC version on AppVeyor CI needs this
if(MSVC)
add_definitions(/bigobj)
endif()
add_subdirectory(vendor)
add_subdirectory(src)

20
ChangeLog.txt
View File

@ -1,11 +1,29 @@
Version 0.10.0 (unreleased)
---------------
New Features & Improvements:
* Integrity checks ensure you notice when someone modifies your file system.
* File system nodes (files, directories, symlinks) store a parent pointer to the directory that contains them. This information can be used in later versions to resolve some synchronization conflicts.
* Allow mounting using system mount tool and /etc/fstab (e.g. mount -t fuse.cryfs basedir mountdir)
* Performance improvements
* Use relatime instead of strictatime (further performance improvement)
* Pass fuse options directly to cryfs (i.e. 'cryfs basedir mountdir -o allow_other' instead of 'cryfs basedir mountdir -- -o allow_other')
* CryFS tells the operating system to not swap the encryption key to the disk (note: this is best-effort and cannot be guaranteed. Hibernation, for example, will still write the encryption key to the disk)
* New block size options: 4KB and 16KB
* New default block size: 16KB
* Increased scrypt hardness to (N=1048576, r=4, p=8) to make it harder to crack the key while allowing cryfs to take advantage of multicore machines.
Fixed bugs:
* `du` shows correct file system size on Mac OS X.
Version 0.9.9
--------------
Improvements:
* Add --allow-filesystem-upgrade option which will upgrade old file systems without asking the user. This will be especially helpful for GUI tools.
* Add --version option that shows the CryFS version and exits.
* When CryFS fails to load a file system, the process stops with a helpful error code, which can be used by GUI tools to show detailed messages.
* Only migrate a file system if the underlying storage format changed
Version 0.9.8
--------------
Compatibility:

23
README.md
View File

@ -1,4 +1,5 @@
# CryFS [![Build Status](https://travis-ci.org/cryfs/cryfs.svg?branch=master)](https://travis-ci.org/cryfs/cryfs)
# CryFS [![Build Status](https://travis-ci.org/cryfs/cryfs.svg?branch=master)](https://travis-ci.org/cryfs/cryfs) [![CircleCI](https://circleci.com/gh/cryfs/cryfs/tree/master.svg?style=svg)](https://circleci.com/gh/cryfs/cryfs/tree/master)
CryFS encrypts your files, so you can safely store them anywhere. It works well together with cloud services like Dropbox, iCloud, OneDrive and others.
See [https://www.cryfs.org](https://www.cryfs.org).
@ -22,8 +23,8 @@ Building from source
Requirements
------------
- Git (for getting the source code)
- GCC version >= 4.8 or Clang >= 3.7
- CMake version >= 2.8
- GCC version >= 5.0 or Clang >= 4.0
- CMake version >= 3.0
- libcurl4 (including development headers)
- Boost libraries version >= 1.56 (including development headers)
- filesystem
@ -31,21 +32,21 @@ Requirements
- chrono
- program_options
- thread
- Crypto++ version >= 5.6.3 (including development headers)
- SSL development libraries (including development headers, e.g. libssl-dev)
- libFUSE version >= 2.8.6 (including development headers), on Mac OS X instead install osxfuse from https://osxfuse.github.io/
- Python >= 2.7
- OpenMP
You can use the following commands to install these requirements
# Ubuntu
$ sudo apt-get install git g++ cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libcrypto++-dev libssl-dev libfuse-dev python
$ sudo apt-get install git g++ cmake make libcurl4-openssl-dev libboost-filesystem-dev libboost-system-dev libboost-chrono-dev libboost-program-options-dev libboost-thread-dev libssl-dev libfuse-dev python
# Fedora
sudo dnf install git gcc-c++ cmake make libcurl-devel boost-devel boost-static cryptopp-devel openssl-devel fuse-devel python
sudo dnf install git gcc-c++ cmake make libcurl-devel boost-devel boost-static openssl-devel fuse-devel python
# Macintosh
brew install cmake boost cryptopp openssl
brew install cmake boost openssl libomp
Build & Install
---------------
@ -97,13 +98,7 @@ On most systems, CMake should find the libraries automatically. However, that do
cmake .. -DCMAKE_CXX_FLAGS="-I/path/to/fuse/or/osxfuse/headers"
4. **CryptoPP library not found**
Pass in the library path with
cmake .. -DCRYPTOPP_LIB_PATH=/path/to/cryptopp
5. **Openssl headers not found**
4. **Openssl headers not found**
Pass in the include path with

1
TODO-0.10.txt Normal file
View File

@ -0,0 +1 @@
Change homebrew recipe to "brew install libomp" but not "brew install cryptopp"

27
doc/CMakeLists.txt
View File

@ -1,15 +1,20 @@
project (doc)
INCLUDE(GNUInstallDirs)
find_program(GZIP gzip)
IF (WIN32)
MESSAGE(STATUS "This is Windows. Will not install man page")
ELSE (WIN32)
INCLUDE(GNUInstallDirs)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
COMMAND ${GZIP} -c ${CMAKE_CURRENT_SOURCE_DIR}/man/cryfs.1 > ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
)
add_custom_target(man ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz)
find_program(GZIP gzip)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
DESTINATION ${CMAKE_INSTALL_MANDIR}/man1
CONFIGURATIONS Release
)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
COMMAND ${GZIP} -c ${CMAKE_CURRENT_SOURCE_DIR}/man/cryfs.1 > ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
)
add_custom_target(man ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/cryfs.1.gz
DESTINATION ${CMAKE_INSTALL_MANDIR}/man1
CONFIGURATIONS Release
)
ENDIF(WIN32)

27
doc/man/cryfs.1
View File

@ -169,6 +169,24 @@ Allow upgrading the file system if it was created with an old CryFS version. Aft
.
.
.TP
\fB\-\-allow-integrity-violations\fI
.
By default, CryFS checks for integrity violations, i.e. will notice if an adversary modified or rolled back the file system. Using this flag, you can disable the integrity checks. This can for example be helpful for loading an old snapshot of your file system without CryFS thinking an adversary rolled it back.
.
.
.TP
\fB\-\-allow-replaced-filesystem\fI
.
By default, CryFS remembers file systems it has seen in this base directory and checks that it didn't get replaced by an attacker with an entirely different file system since the last time it was loaded. However, if you do want to replace the file system with an entirely new one, you can pass in this option to disable the check.
.
.
.TP
\fB\-\-missing-block-is-integrity-violation\fR=true
.
When CryFS encounters a missing ciphertext block, it cannot cannot (yet) know if it was deleted by an unauthorized adversary or by a second authorized client. This is one of the restrictions of the integrity checks currently in place. You can enable this flag to treat missing ciphertext blocks as integrity violations, but then your file system will not be usable by multiple clients anymore. By default, this flag is disabled.
.
.
.TP
\fB\-\-logfile\fR \fIfile\fR
.
Write status information to \fIfile\fR. If no logfile is given, CryFS will
@ -205,6 +223,15 @@ By default, CryFS connects to the internet to check for known security
vulnerabilities and new versions. This option disables this.
.
.
.TP
\fBCRYFS_LOCAL_STATE_DIR\fR=[path]
.
Sets the directory cryfs uses to store local state. This local state
is used to recognize known file systems and run integrity checks
(i.e. check that they haven't been modified by an attacker.
Default value: ${HOME}/.cryfs
.
.
.
.SH SEE ALSO
.

20
run-clang-tidy.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
# Note: Call this from a cmake build directory (e.g. cmake/) for out-of-source builds
# Examples:
# mkdir cmake && cd cmake && ../run-clang-tidy.sh
# mkdir cmake && cd cmake && ../run-clang-tidy.sh -fix
# mkdir cmake && cd cmake && ../run-clang-tidy.sh -export-fixes fixes.yaml
set -e
NUMCORES=`nproc`
# Run cmake in current working directory, but on source that is in the same directory as this script file
cmake -DBUILD_TESTING=on -DCMAKE_EXPORT_COMPILE_COMMANDS=ON "${0%/*}"
# Build scrypt first. Our Makefiles call ino theirs, and this is needed to generate some header files. Clang-tidy will otherwise complain they're missing.
make -j${NUMCORES} scrypt
run-clang-tidy.py -j${NUMCORES} -quiet -header-filter "$(realpath ${0%/*})/(src|test)/.*" $@

1
src/CMakeLists.txt
View File

@ -8,3 +8,4 @@ add_subdirectory(blockstore)
add_subdirectory(blobstore)
add_subdirectory(cryfs)
add_subdirectory(cryfs-cli)
add_subdirectory(stats)

2
src/blobstore/CMakeLists.txt
View File

@ -11,6 +11,8 @@ set(SOURCES
implementations/onblocks/datanodestore/DataInnerNode.cpp
implementations/onblocks/datanodestore/DataNodeStore.cpp
implementations/onblocks/datatreestore/impl/algorithms.cpp
implementations/onblocks/datatreestore/impl/LeafTraverser.cpp
implementations/onblocks/datatreestore/LeafHandle.cpp
implementations/onblocks/datatreestore/DataTree.cpp
implementations/onblocks/datatreestore/DataTreeStore.cpp
implementations/onblocks/BlobOnBlocks.cpp

115
src/blobstore/implementations/onblocks/BlobOnBlocks.cpp
View File

@ -2,16 +2,19 @@
#include "BlobOnBlocks.h"
#include "datanodestore/DataLeafNode.h"
#include "datanodestore/DataNodeStore.h"
#include "utils/Math.h"
#include <cmath>
#include <cpp-utils/assert/assert.h>
#include "datatreestore/LeafHandle.h"
using std::function;
using std::unique_lock;
using std::mutex;
using cpputils::unique_ref;
using cpputils::Data;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using blockstore::Key;
using blockstore::BlockId;
using blobstore::onblocks::datatreestore::LeafHandle;
namespace blobstore {
namespace onblocks {
@ -19,11 +22,11 @@ namespace onblocks {
using parallelaccessdatatreestore::DataTreeRef;
BlobOnBlocks::BlobOnBlocks(unique_ref<DataTreeRef> datatree)
: _datatree(std::move(datatree)), _sizeCache(boost::none) {
: _datatree(std::move(datatree)), _sizeCache(boost::none), _mutex() {
}
BlobOnBlocks::~BlobOnBlocks() {
}
} // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
uint64_t BlobOnBlocks::size() const {
if (_sizeCache == boost::none) {
@ -37,22 +40,50 @@ void BlobOnBlocks::resize(uint64_t numBytes) {
_sizeCache = numBytes;
}
void BlobOnBlocks::traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t, DataLeafNode *leaf, uint32_t, uint32_t)> func) const {
void BlobOnBlocks::_traverseLeaves(uint64_t beginByte, uint64_t sizeBytes, function<void (uint64_t leafOffset, LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, function<Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const {
unique_lock<mutex> lock(_mutex); // TODO Multiple traverse calls in parallel?
uint64_t endByte = beginByte + sizeBytes;
uint32_t firstLeaf = beginByte / _datatree->maxBytesPerLeaf();
uint32_t endLeaf = utils::ceilDivision(endByte, _datatree->maxBytesPerLeaf());
bool writingOutside = size() < endByte; // TODO Calling size() is slow because it has to traverse the tree
_datatree->traverseLeaves(firstLeaf, endLeaf, [&func, beginByte, endByte, endLeaf, writingOutside](DataLeafNode *leaf, uint32_t leafIndex) {
uint64_t indexOfFirstLeafByte = leafIndex * leaf->maxStoreableBytes();
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(leaf->maxStoreableBytes(), endByte - indexOfFirstLeafByte);
if (leafIndex == endLeaf-1 && writingOutside) {
// If we are traversing an area that didn't exist before, then the last leaf was just created with a wrong size. We have to fix it.
leaf->resize(dataEnd);
}
func(indexOfFirstLeafByte, leaf, dataBegin, dataEnd-dataBegin);
});
if (writingOutside) {
uint64_t maxBytesPerLeaf = _datatree->maxBytesPerLeaf();
uint32_t firstLeaf = beginByte / maxBytesPerLeaf;
uint32_t endLeaf = utils::ceilDivision(endByte, maxBytesPerLeaf);
bool blobIsGrowingFromThisTraversal = false;
auto _onExistingLeaf = [&onExistingLeaf, beginByte, endByte, endLeaf, maxBytesPerLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex, bool isRightBorderLeaf, LeafHandle leafHandle) {
uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
// If we are traversing exactly until the last leaf, then the last leaf wasn't resized by the traversal and might have a wrong size. We have to fix it.
if (isRightBorderLeaf) {
ASSERT(leafIndex == endLeaf-1, "If we traversed further right, this wouldn't be the right border leaf.");
auto leaf = leafHandle.node();
if (leaf->numBytes() < dataEnd) {
leaf->resize(dataEnd);
blobIsGrowingFromThisTraversal = true;
}
}
onExistingLeaf(indexOfFirstLeafByte, std::move(leafHandle), dataBegin, dataEnd-dataBegin);
};
auto _onCreateLeaf = [&onCreateLeaf, maxBytesPerLeaf, beginByte, firstLeaf, endByte, endLeaf, &blobIsGrowingFromThisTraversal] (uint32_t leafIndex) -> Data {
blobIsGrowingFromThisTraversal = true;
uint64_t indexOfFirstLeafByte = leafIndex * maxBytesPerLeaf;
ASSERT(endByte > indexOfFirstLeafByte, "Traversal went too far right");
uint32_t dataBegin = utils::maxZeroSubtraction(beginByte, indexOfFirstLeafByte);
uint32_t dataEnd = std::min(maxBytesPerLeaf, endByte - indexOfFirstLeafByte);
ASSERT(leafIndex == firstLeaf || dataBegin == 0, "Only the leftmost leaf can have a gap on the left.");
ASSERT(leafIndex == endLeaf-1 || dataEnd == maxBytesPerLeaf, "Only the rightmost leaf can have a gap on the right");
Data data = onCreateLeaf(indexOfFirstLeafByte + dataBegin, dataEnd-dataBegin);
ASSERT(data.size() == dataEnd-dataBegin, "Returned leaf data with wrong size");
// If this leaf is created but only partly in the traversed region (i.e. dataBegin > leafBegin), we have to fill the data before the traversed region with zeroes.
if (dataBegin != 0) {
Data actualData(dataBegin + data.size());
std::memset(actualData.data(), 0, dataBegin);
std::memcpy(actualData.dataOffset(dataBegin), data.data(), data.size());
data = std::move(actualData);
}
return data;
};
_datatree->traverseLeaves(firstLeaf, endLeaf, _onExistingLeaf, _onCreateLeaf);
if (blobIsGrowingFromThisTraversal) {
ASSERT(_datatree->numStoredBytes() == endByte, "Writing didn't grow by the correct number of bytes");
_sizeCache = endByte;
}
@ -67,38 +98,60 @@ Data BlobOnBlocks::readAll() const {
}
void BlobOnBlocks::read(void *target, uint64_t offset, uint64_t count) const {
ASSERT(offset <= size() && offset + count <= size(), "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t _size = size();
ASSERT(offset <= _size && offset + count <= _size, "BlobOnBlocks::read() read outside blob. Use BlobOnBlocks::tryRead() if this should be allowed.");
uint64_t read = tryRead(target, offset, count);
ASSERT(read == count, "BlobOnBlocks::read() couldn't read all requested bytes. Use BlobOnBlocks::tryRead() if this should be allowed.");
}
uint64_t BlobOnBlocks::tryRead(void *target, uint64_t offset, uint64_t count) const {
//TODO Quite inefficient to call size() here, because that has to traverse the tree
uint64_t realCount = std::max(UINT64_C(0), std::min(count, size()-offset));
uint64_t realCount = std::max(INT64_C(0), std::min(static_cast<int64_t>(count), static_cast<int64_t>(size())-static_cast<int64_t>(offset)));
_read(target, offset, realCount);
return realCount;
}
void BlobOnBlocks::_read(void *target, uint64_t offset, uint64_t count) const {
traverseLeaves(offset, count, [target, offset] (uint64_t indexOfFirstLeafByte, const DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
auto onExistingLeaf = [target, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Writing to target out of bounds");
//TODO Simplify formula, make it easier to understand
leaf->read((uint8_t*)target + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
leaf.node()->read(static_cast<uint8_t*>(target) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
};
auto onCreateLeaf = [] (uint64_t /*beginByte*/, uint32_t /*count*/) -> Data {
ASSERT(false, "Reading shouldn't create new leaves.");
};
_traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
}
void BlobOnBlocks::write(const void *source, uint64_t offset, uint64_t count) {
traverseLeaves(offset, count, [source, offset] (uint64_t indexOfFirstLeafByte, DataLeafNode *leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
//TODO Simplify formula, make it easier to understand
leaf->write((uint8_t*)source + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset, leafDataSize);
});
auto onExistingLeaf = [source, offset, count] (uint64_t indexOfFirstLeafByte, LeafHandle leaf, uint32_t leafDataOffset, uint32_t leafDataSize) {
ASSERT(indexOfFirstLeafByte+leafDataOffset>=offset && indexOfFirstLeafByte-offset+leafDataOffset <= count && indexOfFirstLeafByte-offset+leafDataOffset+leafDataSize <= count, "Reading from source out of bounds");
if (leafDataOffset == 0 && leafDataSize == leaf.nodeStore()->layout().maxBytesPerLeaf()) {
Data leafData(leafDataSize);
std::memcpy(leafData.data(), static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset, leafDataSize);
leaf.nodeStore()->overwriteLeaf(leaf.blockId(), std::move(leafData));
} else {
//TODO Simplify formula, make it easier to understand
leaf.node()->write(static_cast<const uint8_t*>(source) + indexOfFirstLeafByte - offset + leafDataOffset, leafDataOffset,
leafDataSize);
}
};
auto onCreateLeaf = [source, offset, count] (uint64_t beginByte, uint32_t numBytes) -> Data {
ASSERT(beginByte >= offset && beginByte-offset <= count && beginByte-offset+numBytes <= count, "Reading from source out of bounds");
Data result(numBytes);
//TODO Simplify formula, make it easier to understand
std::memcpy(result.data(), static_cast<const uint8_t*>(source) + beginByte - offset, numBytes);
return result;
};
_traverseLeaves(offset, count, onExistingLeaf, onCreateLeaf);
}
void BlobOnBlocks::flush() {
_datatree->flush();
}
const Key &BlobOnBlocks::key() const {
return _datatree->key();
const BlockId &BlobOnBlocks::blockId() const {
return _datatree->blockId();
}
unique_ref<DataTreeRef> BlobOnBlocks::releaseTree() {

6
src/blobstore/implementations/onblocks/BlobOnBlocks.h
View File

@ -3,6 +3,7 @@
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_BLOBONBLOCKS_H_
#include "../../interface/Blob.h"
#include "datatreestore/LeafHandle.h"
#include <memory>
#include <boost/optional.hpp>
@ -21,7 +22,7 @@ public:
BlobOnBlocks(cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> datatree);
~BlobOnBlocks();
const blockstore::Key &key() const override;
const blockstore::BlockId &blockId() const override;
uint64_t size() const override;
void resize(uint64_t numBytes) override;
@ -38,10 +39,11 @@ public:
private:
void _read(void *target, uint64_t offset, uint64_t count) const;
void traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t, datanodestore::DataLeafNode *, uint32_t, uint32_t)>) const;
void _traverseLeaves(uint64_t offsetBytes, uint64_t sizeBytes, std::function<void (uint64_t leafOffset, datatreestore::LeafHandle leaf, uint32_t begin, uint32_t count)> onExistingLeaf, std::function<cpputils::Data (uint64_t beginByte, uint32_t count)> onCreateLeaf) const;
cpputils::unique_ref<parallelaccessdatatreestore::DataTreeRef> _datatree;
mutable boost::optional<uint64_t> _sizeCache;
mutable std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(BlobOnBlocks);
};

10
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.cpp
View File

@ -15,7 +15,7 @@ using cpputils::make_unique_ref;
using blockstore::BlockStore;
using blockstore::parallelaccess::ParallelAccessBlockStore;
using blockstore::Key;
using blockstore::BlockId;
using cpputils::dynamic_pointer_move;
using boost::optional;
using boost::none;
@ -38,8 +38,8 @@ unique_ref<Blob> BlobStoreOnBlocks::create() {
return make_unique_ref<BlobOnBlocks>(_dataTreeStore->createNewTree());
}
optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const Key &key) {
auto tree = _dataTreeStore->load(key);
optional<unique_ref<Blob>> BlobStoreOnBlocks::load(const BlockId &blockId) {
auto tree = _dataTreeStore->load(blockId);
if (tree == none) {
return none;
}
@ -52,6 +52,10 @@ void BlobStoreOnBlocks::remove(unique_ref<Blob> blob) {
_dataTreeStore->remove((*_blob)->releaseTree());
}
void BlobStoreOnBlocks::remove(const BlockId &blockId) {
_dataTreeStore->remove(blockId);
}
uint64_t BlobStoreOnBlocks::virtualBlocksizeBytes() const {
return _dataTreeStore->virtualBlocksizeBytes();
}

3
src/blobstore/implementations/onblocks/BlobStoreOnBlocks.h
View File

@ -20,9 +20,10 @@ public:
~BlobStoreOnBlocks();
cpputils::unique_ref<Blob> create() override;
boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) override;
boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::BlockId &blockId) override;
void remove(cpputils::unique_ref<Blob> blob) override;
void remove(const blockstore::BlockId &blockId) override;
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
//virtual means "space we can use" as opposed to "space it takes on the disk" (i.e. virtual is without headers, checksums, ...)

82
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.cpp
View File

@ -3,10 +3,12 @@
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using blockstore::BlockStore;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using blockstore::Key;
using blockstore::BlockId;
using std::vector;
namespace blobstore {
namespace onblocks {
@ -23,62 +25,62 @@ DataInnerNode::DataInnerNode(DataNodeView view)
DataInnerNode::~DataInnerNode() {
}
unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNode &first_child) {
DataNodeView node(std::move(block));
node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
node.setDepth(first_child.depth() + 1);
node.setSize(1);
auto result = make_unique_ref<DataInnerNode>(std::move(node));
result->ChildrenBegin()->setKey(first_child.key());
return result;
unique_ref<DataInnerNode> DataInnerNode::InitializeNewNode(unique_ref<Block> block, const DataNodeLayout &layout, uint8_t depth, const vector<BlockId> &children) {
ASSERT(children.size() >= 1, "An inner node must have at least one child");
Data data = _serializeChildren(children);
return make_unique_ref<DataInnerNode>(DataNodeView::initialize(std::move(block), layout, DataNode::FORMAT_VERSION_HEADER, depth, children.size(), std::move(data)));
}
unique_ref<DataInnerNode> DataInnerNode::CreateNewNode(BlockStore *blockStore, const DataNodeLayout &layout, uint8_t depth, const vector<BlockId> &children) {
ASSERT(children.size() >= 1, "An inner node must have at least one child");
Data data = _serializeChildren(children);
return make_unique_ref<DataInnerNode>(DataNodeView::create(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, depth, children.size(), std::move(data)));
}
Data DataInnerNode::_serializeChildren(const vector<BlockId> &children) {
Data data(sizeof(ChildEntry) * children.size());
uint32_t i = 0;
for (const BlockId &child : children) {
child.ToBinary(data.dataOffset(i * BlockId::BINARY_LENGTH));
++i;
}
return data;
}
uint32_t DataInnerNode::numChildren() const {
return node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenBegin());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenBegin() const {
return node().DataBegin<ChildEntry>();
}
DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->ChildrenEnd());
}
const DataInnerNode::ChildEntry *DataInnerNode::ChildrenEnd() const {
return ChildrenBegin() + node().Size();
}
DataInnerNode::ChildEntry *DataInnerNode::LastChild() {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->LastChild());
}
const DataInnerNode::ChildEntry *DataInnerNode::LastChild() const {
return getChild(numChildren()-1);
}
DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) {
return const_cast<ChildEntry*>(const_cast<const DataInnerNode*>(this)->getChild(index));
}
const DataInnerNode::ChildEntry *DataInnerNode::getChild(unsigned int index) const {
DataInnerNode::ChildEntry DataInnerNode::readChild(unsigned int index) const {
ASSERT(index < numChildren(), "Accessing child out of range");
return ChildrenBegin()+index;
return ChildEntry(BlockId::FromBinary(static_cast<const uint8_t*>(node().data()) + index * sizeof(ChildEntry)));
}
void DataInnerNode::_writeChild(unsigned int index, const ChildEntry& child) {
ASSERT(index < numChildren(), "Accessing child out of range");
node().write(child.blockId().data().data(), index * sizeof(ChildEntry), sizeof(ChildEntry));
}
DataInnerNode::ChildEntry DataInnerNode::readLastChild() const {
return readChild(numChildren() - 1);
}
void DataInnerNode::_writeLastChild(const ChildEntry& child) {
_writeChild(numChildren() - 1, child);
}
void DataInnerNode::addChild(const DataNode &child) {
ASSERT(numChildren() < maxStoreableChildren(), "Adding more children than we can store");
ASSERT(child.depth() == depth()-1, "The child that should be added has wrong depth");
node().setSize(node().Size()+1);
LastChild()->setKey(child.key());
_writeLastChild(ChildEntry(child.blockId()));
}
void DataInnerNode::removeLastChild() {
ASSERT(node().Size() > 1, "There is no child to remove");
_writeLastChild(ChildEntry(BlockId::Null()));
node().setSize(node().Size()-1);
}

24
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode.h
View File

@ -11,33 +11,29 @@ namespace datanodestore {
class DataInnerNode final: public DataNode {
public:
static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNode &first_child_key);
static cpputils::unique_ref<DataInnerNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block, const DataNodeLayout &layout, uint8_t depth, const std::vector<blockstore::BlockId> &children);
static cpputils::unique_ref<DataInnerNode> CreateNewNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint8_t depth, const std::vector<blockstore::BlockId> &children);
using ChildEntry = DataInnerNode_ChildEntry;
DataInnerNode(DataNodeView block);
~DataInnerNode();
using ChildEntry = DataInnerNode_ChildEntry;
uint32_t maxStoreableChildren() const;
ChildEntry *getChild(unsigned int index);
const ChildEntry *getChild(unsigned int index) const;
ChildEntry readChild(unsigned int index) const;
ChildEntry readLastChild() const;
uint32_t numChildren() const;
void addChild(const DataNode &child_key);
void addChild(const DataNode &child_blockId);
void removeLastChild();
ChildEntry *LastChild();
const ChildEntry *LastChild() const;
private:
ChildEntry *ChildrenBegin();
ChildEntry *ChildrenEnd();
const ChildEntry *ChildrenBegin() const;
const ChildEntry *ChildrenEnd() const;
void _writeChild(unsigned int index, const ChildEntry& child);
void _writeLastChild(const ChildEntry& child);
static cpputils::Data _serializeChildren(const std::vector<blockstore::BlockId> &children);
DISALLOW_COPY_AND_ASSIGN(DataInnerNode);
};

22
src/blobstore/implementations/onblocks/datanodestore/DataInnerNode_ChildEntry.h
View File

@ -10,17 +10,19 @@ namespace datanodestore{
struct DataInnerNode_ChildEntry final {
public:
blockstore::Key key() const {
return blockstore::Key::FromBinary(_keydata);
}
private:
void setKey(const blockstore::Key &key) {
key.ToBinary(_keydata);
}
friend class DataInnerNode;
uint8_t _keydata[blockstore::Key::BINARY_LENGTH];
DataInnerNode_ChildEntry(const blockstore::BlockId &blockId): _blockId(blockId) {}
DISALLOW_COPY_AND_ASSIGN(DataInnerNode_ChildEntry);
const blockstore::BlockId& blockId() const {
return _blockId;
}
DataInnerNode_ChildEntry(const DataInnerNode_ChildEntry&) = delete;
DataInnerNode_ChildEntry& operator=(const DataInnerNode_ChildEntry&) = delete;
DataInnerNode_ChildEntry(DataInnerNode_ChildEntry&&) = default;
DataInnerNode_ChildEntry& operator=(DataInnerNode_ChildEntry&&) = default;
private:
blockstore::BlockId _blockId;
};
}

23
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.cpp
View File

@ -2,9 +2,9 @@
#include "DataInnerNode.h"
#include <cpp-utils/assert/assert.h>
using blockstore::Block;
using cpputils::Data;
using blockstore::Key;
using blockstore::BlockId;
using blockstore::BlockStore;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
@ -24,18 +24,21 @@ DataLeafNode::DataLeafNode(DataNodeView view)
DataLeafNode::~DataLeafNode() {
}
unique_ref<DataLeafNode> DataLeafNode::InitializeNewNode(unique_ref<Block> block) {
DataNodeView node(std::move(block));
node.setFormatVersion(DataNode::FORMAT_VERSION_HEADER);
node.setDepth(0);
node.setSize(0);
//fillDataWithZeroes(); not needed, because a newly created block will be zeroed out. DataLeafNodeTest.SpaceIsZeroFilledWhenGrowing ensures this.
return make_unique_ref<DataLeafNode>(std::move(node));
unique_ref<DataLeafNode> DataLeafNode::CreateNewNode(BlockStore *blockStore, const DataNodeLayout &layout, Data data) {
ASSERT(data.size() <= layout.maxBytesPerLeaf(), "Data passed in is too large for one leaf.");
uint32_t size = data.size();
return make_unique_ref<DataLeafNode>(DataNodeView::create(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, std::move(data)));
}
unique_ref<DataLeafNode> DataLeafNode::OverwriteNode(BlockStore *blockStore, const DataNodeLayout &layout, const BlockId &blockId, Data data) {
ASSERT(data.size() == layout.maxBytesPerLeaf(), "Data passed in is too large for one leaf.");
uint32_t size = data.size();
return make_unique_ref<DataLeafNode>(DataNodeView::overwrite(blockStore, layout, DataNode::FORMAT_VERSION_HEADER, 0, size, blockId, std::move(data)));
}
void DataLeafNode::read(void *target, uint64_t offset, uint64_t size) const {
ASSERT(offset <= node().Size() && offset + size <= node().Size(), "Read out of valid area"); // Also check offset, because the addition could lead to overflows
std::memcpy(target, (uint8_t*)node().data() + offset, size);
std::memcpy(target, static_cast<const uint8_t*>(node().data()) + offset, size);
}
void DataLeafNode::write(const void *source, uint64_t offset, uint64_t size) {

3
src/blobstore/implementations/onblocks/datanodestore/DataLeafNode.h
View File

@ -11,7 +11,8 @@ class DataInnerNode;
class DataLeafNode final: public DataNode {
public:
static cpputils::unique_ref<DataLeafNode> InitializeNewNode(cpputils::unique_ref<blockstore::Block> block);
static cpputils::unique_ref<DataLeafNode> CreateNewNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, cpputils::Data data);
static cpputils::unique_ref<DataLeafNode> OverwriteNode(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, const blockstore::BlockId &blockId, cpputils::Data data);
DataLeafNode(DataNodeView block);
~DataLeafNode();

13
src/blobstore/implementations/onblocks/datanodestore/DataNode.cpp
View File

@ -4,10 +4,8 @@
#include "DataNodeStore.h"
#include <blockstore/utils/BlockStoreUtils.h>
using blockstore::Block;
using blockstore::Key;
using blockstore::BlockId;
using std::runtime_error;
using cpputils::unique_ref;
namespace blobstore {
@ -31,20 +29,19 @@ const DataNodeView &DataNode::node() const {
return _node;
}
const Key &DataNode::key() const {
return _node.key();
const BlockId &DataNode::blockId() const {
return _node.blockId();
}
uint8_t DataNode::depth() const {
return _node.Depth();
}
unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNode &first_child) {
Key key = node->key();
unique_ref<DataInnerNode> DataNode::convertToNewInnerNode(unique_ref<DataNode> node, const DataNodeLayout &layout, const DataNode &first_child) {
auto block = node->_node.releaseBlock();
blockstore::utils::fillWithZeroes(block.get());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
return DataInnerNode::InitializeNewNode(std::move(block), layout, first_child.depth()+1, {first_child.blockId()});
}
void DataNode::flush() const {

4
src/blobstore/implementations/onblocks/datanodestore/DataNode.h
View File

@ -15,11 +15,11 @@ class DataNode {
public:
virtual ~DataNode();
const blockstore::Key &key() const;
const blockstore::BlockId &blockId() const;
uint8_t depth() const;
static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNode &first_child);
static cpputils::unique_ref<DataInnerNode> convertToNewInnerNode(cpputils::unique_ref<DataNode> node, const DataNodeLayout &layout, const DataNode &first_child);
void flush() const;

96
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.cpp
View File

@ -8,13 +8,15 @@
using blockstore::BlockStore;
using blockstore::Block;
using blockstore::Key;
using blockstore::BlockId;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::dynamic_pointer_move;
using std::runtime_error;
using boost::optional;
using boost::none;
using std::vector;
namespace blobstore {
namespace onblocks {
@ -28,7 +30,6 @@ DataNodeStore::~DataNodeStore() {
}
unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
ASSERT(block->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
DataNodeView node(std::move(block));
if (node.Depth() == 0) {
@ -40,24 +41,25 @@ unique_ref<DataNode> DataNodeStore::load(unique_ref<Block> block) {
}
}
unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(const DataNode &first_child) {
ASSERT(first_child.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataInnerNode::InitializeNewNode(std::move(block), first_child);
unique_ref<DataInnerNode> DataNodeStore::createNewInnerNode(uint8_t depth, const vector<BlockId> &children) {
ASSERT(children.size() >= 1, "Inner node must have at least one child");
return DataInnerNode::CreateNewNode(_blockstore.get(), _layout, depth, children);
}
unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode() {
//TODO Initialize block and then create it in the blockstore - this is more efficient than creating it and then writing to it
auto block = _blockstore->create(Data(_layout.blocksizeBytes()).FillWithZeroes());
return DataLeafNode::InitializeNewNode(std::move(block));
unique_ref<DataLeafNode> DataNodeStore::createNewLeafNode(Data data) {
return DataLeafNode::CreateNewNode(_blockstore.get(), _layout, std::move(data));
}
optional<unique_ref<DataNode>> DataNodeStore::load(const Key &key) {
auto block = _blockstore->load(key);
unique_ref<DataLeafNode> DataNodeStore::overwriteLeaf(const BlockId &blockId, Data data) {
return DataLeafNode::OverwriteNode(_blockstore.get(), _layout, blockId, std::move(data));
}
optional<unique_ref<DataNode>> DataNodeStore::load(const BlockId &blockId) {
auto block = _blockstore->load(blockId);
if (block == none) {
return none;
} else {
ASSERT((*block)->size() == _layout.blocksizeBytes(), "Loading block of wrong size");
return load(std::move(*block));
}
}
@ -71,21 +73,52 @@ unique_ref<DataNode> DataNodeStore::createNewNodeAsCopyFrom(const DataNode &sour
unique_ref<DataNode> DataNodeStore::overwriteNodeWith(unique_ref<DataNode> target, const DataNode &source) {
ASSERT(target->node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Target node has wrong layout. Is it from the same DataNodeStore?");
ASSERT(source.node().layout().blocksizeBytes() == _layout.blocksizeBytes(), "Source node has wrong layout. Is it from the same DataNodeStore?");
Key key = target->key();
{
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
}
auto loaded = load(key);
ASSERT(loaded != none, "Couldn't load the target node after overwriting it");
return std::move(*loaded);
auto targetBlock = target->node().releaseBlock();
cpputils::destruct(std::move(target)); // Call destructor
blockstore::utils::copyTo(targetBlock.get(), source.node().block());
return DataNodeStore::load(std::move(targetBlock));
}
void DataNodeStore::remove(unique_ref<DataNode> node) {
auto block = node->node().releaseBlock();
cpputils::destruct(std::move(node)); // Call destructor
_blockstore->remove(std::move(block));
BlockId blockId = node->blockId();
cpputils::destruct(std::move(node));
remove(blockId);
}
void DataNodeStore::remove(const BlockId &blockId) {
_blockstore->remove(blockId);
}
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
auto leaf = dynamic_pointer_move<DataLeafNode>(node);
if (leaf != none) {
remove(std::move(*leaf));
return;
}
auto inner = dynamic_pointer_move<DataInnerNode>(node);
ASSERT(inner != none, "Is neither a leaf nor an inner node");
for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
removeSubtree((*inner)->depth()-1, (*inner)->readChild(i).blockId());
}
remove(std::move(*inner));
}
void DataNodeStore::removeSubtree(uint8_t depth, const BlockId &blockId) {
if (depth == 0) {
remove(blockId);
} else {
auto node = load(blockId);
ASSERT(node != none, "Node for removeSubtree not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*node);
ASSERT(inner != none, "Is not an inner node, but depth was not zero");
ASSERT((*inner)->depth() == depth, "Wrong depth given");
for (uint32_t i = 0; i < (*inner)->numChildren(); ++i) {
removeSubtree(depth-1, (*inner)->readChild(i).blockId());
}
remove(std::move(*inner));
}
}
uint64_t DataNodeStore::numNodes() const {
@ -100,19 +133,6 @@ uint64_t DataNodeStore::virtualBlocksizeBytes() const {
return _layout.blocksizeBytes();
}
void DataNodeStore::removeSubtree(unique_ref<DataNode> node) {
//TODO Make this faster by not loading the leaves but just deleting them. Can be recognized, because of the depth of their parents.
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(node.get());
if (inner != nullptr) {
for (uint32_t i = 0; i < inner->numChildren(); ++i) {
auto child = load(inner->getChild(i)->key());
ASSERT(child != none, "Couldn't load child node");
removeSubtree(std::move(*child));
}
}
remove(std::move(node));
}
DataNodeLayout DataNodeStore::layout() const {
return _layout;
}

15
src/blobstore/implementations/onblocks/datanodestore/DataNodeStore.h
View File

@ -5,7 +5,7 @@
#include <memory>
#include <cpp-utils/macros.h>
#include "DataNodeView.h"
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
namespace blockstore{
class Block;
@ -28,17 +28,21 @@ public:
DataNodeLayout layout() const;
boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::Key &key);
boost::optional<cpputils::unique_ref<DataNode>> load(const blockstore::BlockId &blockId);
static cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
cpputils::unique_ref<DataLeafNode> createNewLeafNode();
cpputils::unique_ref<DataInnerNode> createNewInnerNode(const DataNode &first_child);
cpputils::unique_ref<DataLeafNode> createNewLeafNode(cpputils::Data data);
cpputils::unique_ref<DataInnerNode> createNewInnerNode(uint8_t depth, const std::vector<blockstore::BlockId> &children);
cpputils::unique_ref<DataNode> createNewNodeAsCopyFrom(const DataNode &source);
cpputils::unique_ref<DataNode> overwriteNodeWith(cpputils::unique_ref<DataNode> target, const DataNode &source);
void remove(cpputils::unique_ref<DataNode> node);
cpputils::unique_ref<DataLeafNode> overwriteLeaf(const blockstore::BlockId &blockId, cpputils::Data data);
void remove(cpputils::unique_ref<DataNode> node);
void remove(const blockstore::BlockId &blockId);
void removeSubtree(uint8_t depth, const blockstore::BlockId &blockId);
void removeSubtree(cpputils::unique_ref<DataNode> node);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
@ -48,7 +52,6 @@ public:
//TODO Test overwriteNodeWith(), createNodeAsCopyFrom(), removeSubtree()
private:
cpputils::unique_ref<DataNode> load(cpputils::unique_ref<blockstore::Block> block);
cpputils::unique_ref<blockstore::BlockStore> _blockstore;
const DataNodeLayout _layout;

57
src/blobstore/implementations/onblocks/datanodestore/DataNodeView.h
View File

@ -65,10 +65,33 @@ public:
}
~DataNodeView() {}
static DataNodeView create(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
ASSERT(data.size() <= layout.datasizeBytes(), "Data is too large for node");
cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
ASSERT(serialized.size() == layout.blocksizeBytes(), "Wrong block size");
auto block = blockStore->create(serialized);
return DataNodeView(std::move(block));
}
static DataNodeView initialize(cpputils::unique_ref<blockstore::Block> block, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
ASSERT(data.size() <= DataNodeLayout(block->size()).datasizeBytes(), "Data is too large for node");
cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
ASSERT(serialized.size() == block->size(), "Block has wrong size");
block->write(serialized.data(), 0, serialized.size());
return DataNodeView(std::move(block));
}
static DataNodeView overwrite(blockstore::BlockStore *blockStore, const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, const blockstore::BlockId &blockId, cpputils::Data data) {
ASSERT(data.size() <= layout.datasizeBytes(), "Data is too large for node");
cpputils::Data serialized = _serialize(layout, formatVersion, depth, size, std::move(data));
auto block = blockStore->overwrite(blockId, std::move(serialized));
return DataNodeView(std::move(block));
}
DataNodeView(DataNodeView &&rhs) = default;
uint16_t FormatVersion() const {
return *((uint8_t*)_block->data()+DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES);
return cpputils::deserializeWithOffset<uint16_t>(_block->data(), DataNodeLayout::FORMAT_VERSION_OFFSET_BYTES);
}
void setFormatVersion(uint16_t value) {
@ -76,7 +99,7 @@ public:
}
uint8_t Depth() const {
return *((uint8_t*)_block->data()+DataNodeLayout::DEPTH_OFFSET_BYTES);
return cpputils::deserializeWithOffset<uint8_t>(_block->data(), DataNodeLayout::DEPTH_OFFSET_BYTES);
}
void setDepth(uint8_t value) {
@ -84,7 +107,7 @@ public:
}
uint32_t Size() const {
return *(uint32_t*)((uint8_t*)_block->data()+DataNodeLayout::SIZE_OFFSET_BYTES);
return cpputils::deserializeWithOffset<uint32_t>(_block->data(), DataNodeLayout::SIZE_OFFSET_BYTES);
}
void setSize(uint32_t value) {
@ -92,24 +115,13 @@ public:
}
const void *data() const {
return (uint8_t*)_block->data() + DataNodeLayout::HEADERSIZE_BYTES;
return static_cast<const uint8_t*>(_block->data()) + DataNodeLayout::HEADERSIZE_BYTES;
}
void write(const void *source, uint64_t offset, uint64_t size) {
_block->write(source, offset + DataNodeLayout::HEADERSIZE_BYTES, size);
}
template<typename Entry>
const Entry *DataBegin() const {
return GetOffset<DataNodeLayout::HEADERSIZE_BYTES, Entry>();
}
template<typename Entry>
const Entry *DataEnd() const {
const unsigned int NUM_ENTRIES = layout().datasizeBytes() / sizeof(Entry);
return DataBegin<Entry>() + NUM_ENTRIES;
}
DataNodeLayout layout() const {
return DataNodeLayout(_block->size());
}
@ -122,8 +134,8 @@ public:
return *_block;
}
const blockstore::Key &key() const {
return _block->key();
const blockstore::BlockId &blockId() const {
return _block->blockId();
}
void flush() const {
@ -131,9 +143,14 @@ public:
}
private:
template<int offset, class Type>
const Type *GetOffset() const {
return (Type*)(((const int8_t*)_block->data())+offset);
static cpputils::Data _serialize(const DataNodeLayout &layout, uint16_t formatVersion, uint8_t depth, uint32_t size, cpputils::Data data) {
cpputils::Data result(layout.blocksizeBytes());
cpputils::serialize<uint16_t>(result.dataOffset(layout.FORMAT_VERSION_OFFSET_BYTES), formatVersion);
cpputils::serialize<uint8_t>(result.dataOffset(layout.DEPTH_OFFSET_BYTES), depth);
cpputils::serialize<uint32_t>(result.dataOffset(layout.SIZE_OFFSET_BYTES), size);
std::memcpy(result.dataOffset(layout.HEADERSIZE_BYTES), data.data(), data.size());
std::memset(result.dataOffset(layout.HEADERSIZE_BYTES+data.size()), 0, layout.datasizeBytes()-data.size());
return result;
}
cpputils::unique_ref<blockstore::Block> _block;

298
src/blobstore/implementations/onblocks/datatreestore/DataTree.cpp
View File

@ -11,118 +11,37 @@
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include <cmath>
#include <cpp-utils/assert/assert.h>
#include "impl/LeafTraverser.h"
using blockstore::Key;
using blockstore::BlockId;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeLayout;
using std::dynamic_pointer_cast;
using std::function;
using boost::shared_mutex;
using boost::shared_lock;
using boost::unique_lock;
using boost::none;
using std::vector;
using cpputils::dynamic_pointer_move;
using cpputils::optional_ownership_ptr;
using cpputils::WithOwnership;
using cpputils::WithoutOwnership;
using cpputils::unique_ref;
using cpputils::Data;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
DataTree::DataTree(DataNodeStore *nodeStore, unique_ref<DataNode> rootNode)
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)) {
: _mutex(), _nodeStore(nodeStore), _rootNode(std::move(rootNode)), _blockId(_rootNode->blockId()), _numLeavesCache(none) {
}
DataTree::~DataTree() {
}
void DataTree::removeLastDataLeaf() {
auto deletePosOrNull = algorithms::GetLowestRightBorderNodeWithMoreThanOneChildOrNull(_nodeStore, _rootNode.get());
ASSERT(deletePosOrNull.get() != nullptr, "Tree has only one leaf, can't shrink it.");
deleteLastChildSubtree(deletePosOrNull.get());
ifRootHasOnlyOneChildReplaceRootWithItsChild();
}
void DataTree::ifRootHasOnlyOneChildReplaceRootWithItsChild() {
DataInnerNode *rootNode = dynamic_cast<DataInnerNode*>(_rootNode.get());
ASSERT(rootNode != nullptr, "RootNode is not an inner node");
if (rootNode->numChildren() == 1) {
auto child = _nodeStore->load(rootNode->getChild(0)->key());
ASSERT(child != none, "Couldn't load first child of root node");
_rootNode = _nodeStore->overwriteNodeWith(std::move(_rootNode), **child);
_nodeStore->remove(std::move(*child));
}
}
void DataTree::deleteLastChildSubtree(DataInnerNode *node) {
auto lastChild = _nodeStore->load(node->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
_nodeStore->removeSubtree(std::move(*lastChild));
node->removeLastChild();
}
unique_ref<DataLeafNode> DataTree::addDataLeaf() {
auto insertPosOrNull = algorithms::GetLowestInnerRightBorderNodeWithLessThanKChildrenOrNull(_nodeStore, _rootNode.get());
if (insertPosOrNull) {
return addDataLeafAt(insertPosOrNull.get());
} else {
return addDataLeafToFullTree();
}
}
unique_ref<DataLeafNode> DataTree::addDataLeafAt(DataInnerNode *insertPos) {
auto new_leaf = _nodeStore->createNewLeafNode();
auto chain = createChainOfInnerNodes(insertPos->depth()-1, new_leaf.get());
insertPos->addChild(*chain);
return new_leaf;
}
optional_ownership_ptr<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, DataNode *child) {
//TODO This function is implemented twice, once with optional_ownership_ptr, once with unique_ref. Redundancy!
optional_ownership_ptr<DataNode> chain = cpputils::WithoutOwnership<DataNode>(child);
for(unsigned int i=0; i<num; ++i) {
auto newnode = _nodeStore->createNewInnerNode(*chain);
chain = cpputils::WithOwnership<DataNode>(std::move(newnode));
}
return chain;
}
unique_ref<DataNode> DataTree::createChainOfInnerNodes(unsigned int num, unique_ref<DataNode> child) {
unique_ref<DataNode> chain = std::move(child);
for(unsigned int i=0; i<num; ++i) {
chain = _nodeStore->createNewInnerNode(*chain);
}
return chain;
}
DataInnerNode* DataTree::increaseTreeDepth(unsigned int levels) {
ASSERT(levels >= 1, "Parameter out of bounds: tried to increase tree depth by zero.");
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*_rootNode);
auto chain = createChainOfInnerNodes(levels-1, copyOfOldRoot.get());
auto newRootNode = DataNode::convertToNewInnerNode(std::move(_rootNode), *chain);
DataInnerNode *result = newRootNode.get();
_rootNode = std::move(newRootNode);
return result;
}
unique_ref<DataLeafNode> DataTree::addDataLeafToFullTree() {
DataInnerNode *rootNode = increaseTreeDepth(1);
auto newLeaf = addDataLeafAt(rootNode);
return newLeaf;
}
const Key &DataTree::key() const {
return _rootNode->key();
const BlockId &DataTree::blockId() const {
return _blockId;
}
void DataTree::flush() const {
@ -133,124 +52,67 @@ void DataTree::flush() const {
}
unique_ref<DataNode> DataTree::releaseRootNode() {
unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
return std::move(_rootNode);
}
//TODO Test numLeaves(), for example also two configurations with same number of bytes but different number of leaves (last leaf has 0 bytes)
uint32_t DataTree::numLeaves() const {
shared_lock<shared_mutex> lock(_mutex);
return _numLeaves(*_rootNode);
shared_lock<shared_mutex> lock(_mutex);
return _numLeaves();
}
uint32_t DataTree::_numLeaves(const DataNode &node) const {
uint32_t DataTree::_numLeaves() const {
if (_numLeavesCache == none) {
_numLeavesCache = _computeNumLeaves(*_rootNode);
}
return *_numLeavesCache;
}
uint32_t DataTree::_forceComputeNumLeaves() const {
unique_lock<shared_mutex> lock(_mutex); // Lock ensures that the root node is currently set (traversing unsets it temporarily)
_numLeavesCache = _computeNumLeaves(*_rootNode);
return *_numLeavesCache;
}
uint32_t DataTree::_computeNumLeaves(const DataNode &node) const {
const DataLeafNode *leaf = dynamic_cast<const DataLeafNode*>(&node);
if (leaf != nullptr) {
return 1;
}
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(node);
uint64_t numLeavesInLeftChildren = (uint64_t)(inner.numChildren()-1) * leavesPerFullChild(inner);
auto lastChild = _nodeStore->load(inner.LastChild()->key());
uint64_t numLeavesInLeftChildren = static_cast<uint64_t>(inner.numChildren()-1) * leavesPerFullChild(inner);
auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numLeavesInRightChild = _numLeaves(**lastChild);
uint64_t numLeavesInRightChild = _computeNumLeaves(**lastChild);
return numLeavesInLeftChildren + numLeavesInRightChild;
}
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
//TODO Can we traverse in parallel?
unique_lock<shared_mutex> lock(_mutex); //TODO Only lock when resizing. Otherwise parallel read/write to a blob is not possible!
void DataTree::traverseLeaves(uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf) {
//TODO Can we allow multiple runs of traverseLeaves() in parallel? Also in parallel with resizeNumBytes()?
std::unique_lock<shared_mutex> lock(_mutex);
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == endIndex) {
// In this case the utils::ceilLog(_, endIndex) below would fail
return;
}
uint8_t neededTreeDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)endIndex);
uint32_t numLeaves = this->_numLeaves(*_rootNode); // TODO Querying the size causes a tree traversal down to the leaves. Possible without querying the size?
if (_rootNode->depth() < neededTreeDepth) {
//TODO Test cases that actually increase it here by 0 level / 1 level / more than 1 level
increaseTreeDepth(neededTreeDepth - _rootNode->depth());
}
auto onBacktrackFromSubtree = [] (DataInnerNode* /*node*/) {};
if (numLeaves <= beginIndex) {
//TODO Test cases with numLeaves < / >= beginIndex
// There is a gap between the current size and the begin of the traversal
return _traverseLeaves(_rootNode.get(), 0, numLeaves-1, endIndex, [beginIndex, numLeaves, &func, this](DataLeafNode* node, uint32_t index) {
if (index >= beginIndex) {
func(node, index);
} else if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
});
} else if (numLeaves < endIndex) {
// We are starting traversal in the valid region, but traverse until after it (we grow new leaves)
return _traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, [numLeaves, &func, this] (DataLeafNode *node, uint32_t index) {
if (index == numLeaves - 1) {
// It is the old last leaf - resize it to maximum
node->resize(_nodeStore->layout().maxBytesPerLeaf());
}
func(node, index);
});
} else {
//We are traversing entirely inside the valid region
_traverseLeaves(_rootNode.get(), 0, beginIndex, endIndex, func);
_traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
if (_numLeavesCache != none && *_numLeavesCache < endIndex) {
_numLeavesCache = endIndex;
}
}
void DataTree::_traverseLeaves(DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, function<void (DataLeafNode*, uint32_t)> func) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
ASSERT(beginIndex <= 1 && endIndex <= 1, "If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
if (beginIndex == 0 && endIndex == 1) {
func(leaf, leafOffset);
}
return;
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
uint32_t leavesPerChild = leavesPerFullChild(*inner);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
vector<unique_ref<DataNode>> children = getOrCreateChildren(inner, beginChild, endChild);
for (uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto child = std::move(children[childIndex-beginChild]);
_traverseLeaves(child.get(), leafOffset + childOffset, localBeginIndex, localEndIndex, func);
}
}
vector<unique_ref<DataNode>> DataTree::getOrCreateChildren(DataInnerNode *node, uint32_t begin, uint32_t end) {
vector<unique_ref<DataNode>> children;
children.reserve(end-begin);
for (uint32_t childIndex = begin; childIndex < std::min(node->numChildren(), end); ++childIndex) {
auto child = _nodeStore->load(node->getChild(childIndex)->key());
ASSERT(child != none, "Couldn't load child node");
children.emplace_back(std::move(*child));
}
for (uint32_t childIndex = node->numChildren(); childIndex < end; ++childIndex) {
//TODO This creates each child with one chain to one leaf only, and then on the next lower level it
// has to create the children for the child. Would be faster to directly create full trees if necessary.
children.emplace_back(addChildTo(node));
}
ASSERT(children.size() == end-begin, "Number of children in the result is wrong");
return children;
}
unique_ref<DataNode> DataTree::addChildTo(DataInnerNode *node) {
auto new_leaf = _nodeStore->createNewLeafNode();
new_leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
auto chain = createChainOfInnerNodes(node->depth()-1, std::move(new_leaf));
node->addChild(*chain);
return chain;
void DataTree::_traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
function<Data (uint32_t index)> onCreateLeaf,
function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
_rootNode = LeafTraverser(_nodeStore).traverseAndReturnRoot(std::move(_rootNode), beginIndex, endIndex, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
uint32_t DataTree::leavesPerFullChild(const DataInnerNode &root) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), (uint64_t)root.depth()-1);
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(root.depth())-1);
}
uint64_t DataTree::numStoredBytes() const {
@ -270,7 +132,7 @@ uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
const DataInnerNode &inner = dynamic_cast<const DataInnerNode&>(root);
uint64_t numBytesInLeftChildren = (inner.numChildren()-1) * leavesPerFullChild(inner) * _nodeStore->layout().maxBytesPerLeaf();
auto lastChild = _nodeStore->load(inner.LastChild()->key());
auto lastChild = _nodeStore->load(inner.readLastChild().blockId());
ASSERT(lastChild != none, "Couldn't load last child");
uint64_t numBytesInRightChild = _numStoredBytes(**lastChild);
@ -278,57 +140,49 @@ uint64_t DataTree::_numStoredBytes(const DataNode &root) const {
}
void DataTree::resizeNumBytes(uint64_t newNumBytes) {
//TODO Can we resize in parallel? Especially creating new blocks (i.e. encrypting them) is expensive and should be done in parallel.
boost::upgrade_lock<shared_mutex> lock(_mutex);
{
boost::upgrade_to_unique_lock<shared_mutex> exclusiveLock(lock);
//TODO Faster implementation possible (no addDataLeaf()/removeLastDataLeaf() in a loop, but directly resizing)
LastLeaf(_rootNode.get())->resize(_nodeStore->layout().maxBytesPerLeaf());
uint64_t currentNumBytes = _numStoredBytes();
ASSERT(currentNumBytes % _nodeStore->layout().maxBytesPerLeaf() == 0, "The last leaf is not a max data leaf, although we just resized it to be one.");
uint32_t currentNumLeaves = currentNumBytes / _nodeStore->layout().maxBytesPerLeaf();
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
std::unique_lock<shared_mutex> lock(_mutex); // TODO Multiple ones in parallel? Also in parallel with traverseLeaves()?
for(uint32_t i = currentNumLeaves; i < newNumLeaves; ++i) {
addDataLeaf()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
for(uint32_t i = currentNumLeaves; i > newNumLeaves; --i) {
removeLastDataLeaf();
}
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1)*_nodeStore->layout().maxBytesPerLeaf();
LastLeaf(_rootNode.get())->resize(newLastLeafSize);
}
ASSERT(newNumBytes == _numStoredBytes(), "We resized to the wrong number of bytes ("+std::to_string(numStoredBytes())+" instead of "+std::to_string(newNumBytes)+")");
}
uint32_t newNumLeaves = std::max(UINT64_C(1), utils::ceilDivision(newNumBytes, _nodeStore->layout().maxBytesPerLeaf()));
uint32_t newLastLeafSize = newNumBytes - (newNumLeaves-1) * _nodeStore->layout().maxBytesPerLeaf();
uint32_t maxChildrenPerInnerNode = _nodeStore->layout().maxChildrenPerInnerNode();
auto onExistingLeaf = [newLastLeafSize] (uint32_t /*index*/, bool /*isRightBorderLeaf*/, LeafHandle leafHandle) {
auto leaf = leafHandle.node();
// This is only called, if the new last leaf was already existing
if (leaf->numBytes() != newLastLeafSize) {
leaf->resize(newLastLeafSize);
}
};
auto onCreateLeaf = [newLastLeafSize] (uint32_t /*index*/) -> Data {
// This is only called, if the new last leaf was not existing yet
return Data(newLastLeafSize).FillWithZeroes();
};
auto onBacktrackFromSubtree = [this, newNumLeaves, maxChildrenPerInnerNode] (DataInnerNode* node) {
// This is only called for the right border nodes of the new tree.
// When growing size, the following is a no-op. When shrinking, we're deleting the children that aren't needed anymore.
uint32_t maxLeavesPerChild = utils::intPow(static_cast<uint64_t>(maxChildrenPerInnerNode), (static_cast<uint64_t>(node->depth())-1));
uint32_t neededNodesOnChildLevel = utils::ceilDivision(newNumLeaves, maxLeavesPerChild);
uint32_t neededSiblings = utils::ceilDivision(neededNodesOnChildLevel, maxChildrenPerInnerNode);
uint32_t neededChildrenForRightBorderNode = neededNodesOnChildLevel - (neededSiblings-1) * maxChildrenPerInnerNode;
ASSERT(neededChildrenForRightBorderNode <= node->numChildren(), "Node has too few children");
// All children to the right of the new right-border-node are removed including their subtree.
while(node->numChildren() > neededChildrenForRightBorderNode) {
_nodeStore->removeSubtree(node->depth()-1, node->readLastChild().blockId());
node->removeLastChild();
}
};
optional_ownership_ptr<DataLeafNode> DataTree::LastLeaf(DataNode *root) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root);
if (leaf != nullptr) {
return WithoutOwnership(leaf);
}
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root);
auto lastChild = _nodeStore->load(inner->LastChild()->key());
ASSERT(lastChild != none, "Couldn't load last child");
return WithOwnership(LastLeaf(std::move(*lastChild)));
}
unique_ref<DataLeafNode> DataTree::LastLeaf(unique_ref<DataNode> root) {
auto leaf = dynamic_pointer_move<DataLeafNode>(root);
if (leaf != none) {
return std::move(*leaf);
}
auto inner = dynamic_pointer_move<DataInnerNode>(root);
ASSERT(inner != none, "Root node is neither a leaf nor an inner node");
auto child = _nodeStore->load((*inner)->LastChild()->key());
ASSERT(child != none, "Couldn't load last child");
return LastLeaf(std::move(*child));
_traverseLeaves(newNumLeaves - 1, newNumLeaves, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
_numLeavesCache = newNumLeaves;
}
uint64_t DataTree::maxBytesPerLeaf() const {
return _nodeStore->layout().maxBytesPerLeaf();
}
uint8_t DataTree::depth() const {
return _rootNode->depth();
}
}
}
}

38
src/blobstore/implementations/onblocks/datatreestore/DataTree.h
View File

@ -8,7 +8,8 @@
#include "../datanodestore/DataNodeView.h"
//TODO Replace with C++14 once std::shared_mutex is supported
#include <boost/thread/shared_mutex.hpp>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include "LeafHandle.h"
namespace blobstore {
namespace onblocks {
@ -26,48 +27,43 @@ public:
DataTree(datanodestore::DataNodeStore *nodeStore, cpputils::unique_ref<datanodestore::DataNode> rootNode);
~DataTree();
const blockstore::Key &key() const;
const blockstore::BlockId &blockId() const;
//Returning uint64_t, because calculations handling this probably need to be done in 64bit to support >4GB blobs.
uint64_t maxBytesPerLeaf() const;
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf);
void resizeNumBytes(uint64_t newNumBytes);
uint32_t numLeaves() const;
uint64_t numStoredBytes() const;
uint8_t depth() const;
// only used by test cases
uint32_t _forceComputeNumLeaves() const;
void flush() const;
private:
mutable boost::shared_mutex _mutex;
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _rootNode;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeaf();
void removeLastDataLeaf();
blockstore::BlockId _blockId; // BlockId is stored in a member variable, since _rootNode is nullptr while traversing, but we still want to be able to return the blockId.
mutable boost::optional<uint32_t> _numLeavesCache;
cpputils::unique_ref<datanodestore::DataNode> releaseRootNode();
friend class DataTreeStore;
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafAt(datanodestore::DataInnerNode *insertPos);
cpputils::optional_ownership_ptr<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, datanodestore::DataNode *child);
cpputils::unique_ref<datanodestore::DataNode> createChainOfInnerNodes(unsigned int num, cpputils::unique_ref<datanodestore::DataNode> child);
cpputils::unique_ref<datanodestore::DataLeafNode> addDataLeafToFullTree();
void deleteLastChildSubtree(datanodestore::DataInnerNode *node);
void ifRootHasOnlyOneChildReplaceRootWithItsChild();
//TODO Use underscore for private methods
void _traverseLeaves(datanodestore::DataNode *root, uint32_t leafOffset, uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func);
void _traverseLeaves(uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
uint32_t leavesPerFullChild(const datanodestore::DataInnerNode &root) const;
uint64_t _numStoredBytes() const;
uint64_t _numStoredBytes(const datanodestore::DataNode &root) const;
uint32_t _numLeaves(const datanodestore::DataNode &node) const;
cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> LastLeaf(datanodestore::DataNode *root);
cpputils::unique_ref<datanodestore::DataLeafNode> LastLeaf(cpputils::unique_ref<datanodestore::DataNode> root);
datanodestore::DataInnerNode* increaseTreeDepth(unsigned int levels);
std::vector<cpputils::unique_ref<datanodestore::DataNode>> getOrCreateChildren(datanodestore::DataInnerNode *node, uint32_t begin, uint32_t end);
cpputils::unique_ref<datanodestore::DataNode> addChildTo(datanodestore::DataInnerNode *node);
uint32_t _numLeaves() const;
uint32_t _computeNumLeaves(const datanodestore::DataNode &node) const;
DISALLOW_COPY_AND_ASSIGN(DataTree);
};

19
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.cpp
View File

@ -1,15 +1,14 @@
#include "DataTreeStore.h"
#include "../datanodestore/DataNodeStore.h"
#include "../datanodestore/DataLeafNode.h"
#include "DataTree.h"
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::Data;
using boost::optional;
using boost::none;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
namespace blobstore {
namespace onblocks {
@ -22,8 +21,8 @@ DataTreeStore::DataTreeStore(unique_ref<DataNodeStore> nodeStore)
DataTreeStore::~DataTreeStore() {
}
optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
auto node = _nodeStore->load(key);
optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::BlockId &blockId) {
auto node = _nodeStore->load(blockId);
if (node == none) {
return none;
}
@ -31,14 +30,18 @@ optional<unique_ref<DataTree>> DataTreeStore::load(const blockstore::Key &key) {
}
unique_ref<DataTree> DataTreeStore::createNewTree() {
auto newleaf = _nodeStore->createNewLeafNode();
auto newleaf = _nodeStore->createNewLeafNode(Data(0));
return make_unique_ref<DataTree>(_nodeStore.get(), std::move(newleaf));
}
void DataTreeStore::remove(unique_ref<DataTree> tree) {
auto root = tree->releaseRootNode();
cpputils::destruct(std::move(tree)); // Destruct tree
_nodeStore->removeSubtree(std::move(root));
_nodeStore->removeSubtree(tree->releaseRootNode());
}
void DataTreeStore::remove(const blockstore::BlockId &blockId) {
auto tree = load(blockId);
ASSERT(tree != none, "Tree to remove not found");
remove(std::move(*tree));
}
}

5
src/blobstore/implementations/onblocks/datatreestore/DataTreeStore.h
View File

@ -5,7 +5,7 @@
#include <memory>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include <boost/optional.hpp>
#include "../datanodestore/DataNodeStore.h"
@ -19,11 +19,12 @@ public:
DataTreeStore(cpputils::unique_ref<datanodestore::DataNodeStore> nodeStore);
~DataTreeStore();
boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::Key &key);
boost::optional<cpputils::unique_ref<DataTree>> load(const blockstore::BlockId &blockId);
cpputils::unique_ref<DataTree> createNewTree();
void remove(cpputils::unique_ref<DataTree> tree);
void remove(const blockstore::BlockId &blockId);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;

40
src/blobstore/implementations/onblocks/datatreestore/LeafHandle.cpp Normal file
View File

@ -0,0 +1,40 @@
#include "LeafHandle.h"
#include "../datanodestore/DataLeafNode.h"
#include "../datanodestore/DataNodeStore.h"
using cpputils::WithOwnership;
using cpputils::WithoutOwnership;
using boost::none;
using cpputils::dynamic_pointer_move;
using blobstore::onblocks::datanodestore::DataLeafNode;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blockstore::BlockId;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
LeafHandle::LeafHandle(DataNodeStore *nodeStore, const BlockId &blockId)
: _nodeStore(nodeStore), _blockId(blockId), _leaf(cpputils::null<DataLeafNode>()) {
}
LeafHandle::LeafHandle(DataNodeStore *nodeStore, DataLeafNode *node)
: _nodeStore(nodeStore), _blockId(node->blockId()),
_leaf(WithoutOwnership<DataLeafNode>(node)) {
}
DataLeafNode *LeafHandle::node() {
if (_leaf.get() == nullptr) {
auto loaded = _nodeStore->load(_blockId);
ASSERT(loaded != none, "Leaf not found");
auto leaf = dynamic_pointer_move<DataLeafNode>(*loaded);
ASSERT(leaf != none, "Loaded leaf is not leaf node");
_leaf = WithOwnership(std::move(*leaf));
}
return _leaf.get();
}
}
}
}

46
src/blobstore/implementations/onblocks/datatreestore/LeafHandle.h Normal file
View File

@ -0,0 +1,46 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_LEAFHANDLE_H_
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/optional_ownership_ptr.h>
#include <blockstore/utils/BlockId.h>
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataLeafNode;
}
namespace datatreestore {
class LeafHandle final {
public:
LeafHandle(datanodestore::DataNodeStore *nodeStore, const blockstore::BlockId &blockId);
LeafHandle(datanodestore::DataNodeStore *nodeStore, datanodestore::DataLeafNode *node);
LeafHandle(LeafHandle &&rhs) = default;
const blockstore::BlockId &blockId() {
return _blockId;
}
datanodestore::DataLeafNode *node();
datanodestore::DataNodeStore *nodeStore() {
return _nodeStore;
}
private:
datanodestore::DataNodeStore *_nodeStore;
blockstore::BlockId _blockId;
cpputils::optional_ownership_ptr<datanodestore::DataLeafNode> _leaf;
DISALLOW_COPY_AND_ASSIGN(LeafHandle);
};
}
}
}
#endif

250
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.cpp Normal file
View File

@ -0,0 +1,250 @@
#include "LeafTraverser.h"
#include <cpp-utils/assert/assert.h>
#include "../../datanodestore/DataLeafNode.h"
#include "../../datanodestore/DataInnerNode.h"
#include "../../datanodestore/DataNodeStore.h"
#include "../../utils/Math.h"
using std::function;
using std::vector;
using boost::none;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::dynamic_pointer_move;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataLeafNode;
namespace blobstore {
namespace onblocks {
namespace datatreestore {
LeafTraverser::LeafTraverser(DataNodeStore *nodeStore)
: _nodeStore(nodeStore) {
}
unique_ref<DataNode> LeafTraverser::traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
return _traverseAndReturnRoot(std::move(root), beginIndex, endIndex, true, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
unique_ref<DataNode> LeafTraverser::_traverseAndReturnRoot(unique_ref<DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
//TODO Test cases with numLeaves < / >= beginIndex, ideally test all configurations:
// beginIndex<endIndex<numLeaves, beginIndex=endIndex<numLeaves, beginIndex<endIndex=numLeaves, beginIndex=endIndex=numLeaves
// beginIndex<numLeaves<endIndex, beginIndex=numLeaves<endIndex,
// numLeaves<beginIndex<endIndex, numLeaves<beginIndex=endIndex
uint32_t maxLeavesForDepth = _maxLeavesForTreeDepth(root->depth());
bool increaseTreeDepth = endIndex > maxLeavesForDepth;
if (root->depth() == 0) {
DataLeafNode *leaf = dynamic_cast<DataLeafNode*>(root.get());
ASSERT(leaf != nullptr, "Depth 0 has to be leaf node");
if (increaseTreeDepth && leaf->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
leaf->resize(_nodeStore->layout().maxBytesPerLeaf());
}
if (beginIndex == 0 && endIndex >= 1) {
bool isRightBorderLeaf = (endIndex == 1);
onExistingLeaf(0, isRightBorderLeaf, LeafHandle(_nodeStore, leaf));
}
} else {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
ASSERT(inner != nullptr, "Depth != 0 has to be leaf node");
_traverseExistingSubtree(inner, std::min(beginIndex, maxLeavesForDepth),
std::min(endIndex, maxLeavesForDepth), 0, isLeftBorderOfTraversal, !increaseTreeDepth,
increaseTreeDepth, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
// If the traversal goes too far right for a tree this depth, increase tree depth by one and continue traversal.
// This is recursive, i.e. will be repeated if the tree is still not deep enough.
// We don't increase to the full needed tree depth in one step, because we want the traversal to go as far as possible
// and only then increase the depth - this causes the tree to be in consistent shape (balanced) for longer.
if (increaseTreeDepth) {
// TODO Test cases that increase tree depth by 0, 1, 2, ... levels
auto newRoot = _increaseTreeDepth(std::move(root));
return _traverseAndReturnRoot(std::move(newRoot), std::max(beginIndex, maxLeavesForDepth), endIndex, false, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
} else {
// Once we're done growing the tree and done with the traversal, we might have to decrease tree depth,
// because the callbacks could have deleted nodes (this happens for example when shrinking the tree using a traversal).
return _whileRootHasOnlyOneChildReplaceRootWithItsChild(std::move(root));
}
}
unique_ref<DataInnerNode> LeafTraverser::_increaseTreeDepth(unique_ref<DataNode> root) {
auto copyOfOldRoot = _nodeStore->createNewNodeAsCopyFrom(*root);
return DataNode::convertToNewInnerNode(std::move(root), _nodeStore->layout(), *copyOfOldRoot);
}
void LeafTraverser::_traverseExistingSubtree(const blockstore::BlockId &blockId, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
if (depth == 0) {
ASSERT(beginIndex <= 1 && endIndex <= 1,
"If root node is a leaf, the (sub)tree has only one leaf - access indices must be 0 or 1.");
LeafHandle leafHandle(_nodeStore, blockId);
if (growLastLeaf) {
if (leafHandle.node()->numBytes() != _nodeStore->layout().maxBytesPerLeaf()) {
leafHandle.node()->resize(_nodeStore->layout().maxBytesPerLeaf());
}
}
if (beginIndex == 0 && endIndex == 1) {
onExistingLeaf(leafOffset, isRightBorderNode, std::move(leafHandle));
}
} else {
auto node = _nodeStore->load(blockId);
if (node == none) {
throw std::runtime_error("Couldn't find child node " + blockId.ToString());
}
auto inner = dynamic_pointer_move<DataInnerNode>(*node);
ASSERT(inner != none, "Has to be either leaf or inner node");
ASSERT((*inner)->depth() == depth, "Wrong depth given");
_traverseExistingSubtree(inner->get(), beginIndex, endIndex, leafOffset, isLeftBorderOfTraversal,
isRightBorderNode, growLastLeaf, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
}
void LeafTraverser::_traverseExistingSubtree(DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf, function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
//TODO Call callbacks for different leaves in parallel.
uint32_t leavesPerChild = _maxLeavesForTreeDepth(root->depth()-1);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
ASSERT(endChild <= _nodeStore->layout().maxChildrenPerInnerNode(), "Traversal region would need increasing the tree depth. This should have happened before calling this function.");
uint32_t numChildren = root->numChildren();
ASSERT(!growLastLeaf || endChild >= numChildren, "Can only grow last leaf if it exists");
bool shouldGrowLastExistingLeaf = growLastLeaf || endChild > numChildren;
// If we traverse outside of the valid region (i.e. usually would only traverse to new leaves and not to the last leaf),
// we still have to descend to the last old child to fill it with leaves and grow the last old leaf.
if (isLeftBorderOfTraversal && beginChild >= numChildren) {
ASSERT(numChildren > 0, "Node doesn't have children.");
auto childBlockId = root->readLastChild().blockId();
uint32_t childOffset = (numChildren-1) * leavesPerChild;
_traverseExistingSubtree(childBlockId, root->depth()-1, leavesPerChild, leavesPerChild, childOffset, true, false, true,
[] (uint32_t /*index*/, bool /*isRightBorderNode*/, LeafHandle /*leaf*/) {ASSERT(false, "We don't actually traverse any leaves.");},
[] (uint32_t /*index*/) -> Data {ASSERT(false, "We don't actually traverse any leaves.");},
[] (DataInnerNode* /*node*/) {ASSERT(false, "We don't actually traverse any leaves.");});
}
// Traverse existing children
for (uint32_t childIndex = beginChild; childIndex < std::min(endChild, numChildren); ++childIndex) {
auto childBlockId = root->readChild(childIndex).blockId();
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
bool isFirstChild = (childIndex == beginChild);
bool isLastExistingChild = (childIndex == numChildren - 1);
bool isLastChild = isLastExistingChild && (numChildren == endChild);
ASSERT(localEndIndex <= leavesPerChild, "We don't want the child to add a tree level because it doesn't have enough space for the traversal.");
_traverseExistingSubtree(childBlockId, root->depth()-1, localBeginIndex, localEndIndex, leafOffset + childOffset, isLeftBorderOfTraversal && isFirstChild,
isRightBorderNode && isLastChild, shouldGrowLastExistingLeaf && isLastExistingChild, onExistingLeaf, onCreateLeaf, onBacktrackFromSubtree);
}
// Traverse new children (including gap children, i.e. children that are created but not traversed because they're to the right of the current size, but to the left of the traversal region)
for (uint32_t childIndex = numChildren; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = std::min(leavesPerChild, utils::maxZeroSubtraction(beginIndex, childOffset));
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto leafCreator = (childIndex >= beginChild) ? onCreateLeaf : _createMaxSizeLeaf();
auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, root->depth() - 1, leafCreator, onBacktrackFromSubtree);
root->addChild(*child);
}
// This is only a backtrack, if we actually visited a leaf here.
if (endIndex > beginIndex) {
onBacktrackFromSubtree(root);
}
}
unique_ref<DataNode> LeafTraverser::_createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth, function<Data (uint32_t index)> onCreateLeaf, function<void (DataInnerNode *node)> onBacktrackFromSubtree) {
ASSERT(beginIndex <= endIndex, "Invalid parameters");
if (0 == depth) {
ASSERT(beginIndex <= 1 && endIndex == 1, "With depth 0, we can only traverse one or zero leaves (i.e. traverse one leaf or traverse a gap leaf).");
auto leafCreator = (beginIndex
== 0) ? onCreateLeaf : _createMaxSizeLeaf();
return _nodeStore->createNewLeafNode(leafCreator(leafOffset));
}
uint8_t minNeededDepth = utils::ceilLog(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(endIndex));
ASSERT(depth >= minNeededDepth, "Given tree depth doesn't fit given number of leaves to create.");
uint32_t leavesPerChild = _maxLeavesForTreeDepth(depth-1);
uint32_t beginChild = beginIndex/leavesPerChild;
uint32_t endChild = utils::ceilDivision(endIndex, leavesPerChild);
vector<blockstore::BlockId> children;
children.reserve(endChild);
// TODO Remove redundancy of following two for loops by using min/max for calculating the parameters of the recursive call.
// Create gap children (i.e. children before the traversal but after the current size)
for (uint32_t childIndex = 0; childIndex < beginChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
auto child = _createNewSubtree(leavesPerChild, leavesPerChild, leafOffset + childOffset, depth - 1,
[] (uint32_t /*index*/)->Data {ASSERT(false, "We're only creating gap leaves here, not traversing any.");},
[] (DataInnerNode* /*node*/) {});
ASSERT(child->depth() == depth-1, "Created child node has wrong depth");
children.push_back(child->blockId());
}
// Create new children that are traversed
for(uint32_t childIndex = beginChild; childIndex < endChild; ++childIndex) {
uint32_t childOffset = childIndex * leavesPerChild;
uint32_t localBeginIndex = utils::maxZeroSubtraction(beginIndex, childOffset);
uint32_t localEndIndex = std::min(leavesPerChild, endIndex - childOffset);
auto child = _createNewSubtree(localBeginIndex, localEndIndex, leafOffset + childOffset, depth - 1, onCreateLeaf, onBacktrackFromSubtree);
ASSERT(child->depth() == depth-1, "Created child node has wrong depth");
children.push_back(child->blockId());
}
ASSERT(children.size() > 0, "No children created");
auto newNode = _nodeStore->createNewInnerNode(depth, children);
// This is only a backtrack, if we actually created a leaf here.
if (endIndex > beginIndex) {
onBacktrackFromSubtree(newNode.get());
}
return std::move(newNode);
}
uint32_t LeafTraverser::_maxLeavesForTreeDepth(uint8_t depth) const {
return utils::intPow(_nodeStore->layout().maxChildrenPerInnerNode(), static_cast<uint64_t>(depth));
}
function<Data (uint32_t index)> LeafTraverser::_createMaxSizeLeaf() const {
uint64_t maxBytesPerLeaf = _nodeStore->layout().maxBytesPerLeaf();
return [maxBytesPerLeaf] (uint32_t /*index*/) -> Data {
return Data(maxBytesPerLeaf).FillWithZeroes();
};
}
unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildReplaceRootWithItsChild(unique_ref<DataNode> root) {
DataInnerNode *inner = dynamic_cast<DataInnerNode*>(root.get());
if (inner != nullptr && inner->numChildren() == 1) {
auto newRoot = _whileRootHasOnlyOneChildRemoveRootReturnChild(inner->readChild(0).blockId());
auto result = _nodeStore->overwriteNodeWith(std::move(root), *newRoot);
_nodeStore->remove(std::move(newRoot));
return result;
} else {
return root;
}
}
unique_ref<DataNode> LeafTraverser::_whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId) {
auto current = _nodeStore->load(blockId);
ASSERT(current != none, "Node not found");
auto inner = dynamic_pointer_move<DataInnerNode>(*current);
if (inner == none) {
return std::move(*current);
} else if ((*inner)->numChildren() == 1) {
auto result = _whileRootHasOnlyOneChildRemoveRootReturnChild((*inner)->readChild(0).blockId());
_nodeStore->remove(std::move(*inner));
return result;
} else {
return std::move(*inner);
}
}
}
}
}

69
src/blobstore/implementations/onblocks/datatreestore/impl/LeafTraverser.h Normal file
View File

@ -0,0 +1,69 @@
#pragma once
#ifndef MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_LEAFTRAVERSER_H_
#define MESSMER_BLOBSTORE_IMPLEMENTATIONS_ONBLOCKS_IMPL_LEAFTRAVERSER_H_
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/data/Data.h>
#include <blockstore/utils/BlockId.h>
#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
namespace blobstore {
namespace onblocks {
namespace datanodestore {
class DataNodeStore;
class DataNode;
class DataLeafNode;
class DataInnerNode;
}
namespace datatreestore {
/**
* LeafTraverser can create leaves if they don't exist yet (i.e. endIndex > numLeaves), but
* it cannot increase the tree depth. That is, the tree has to be deep enough to allow
* creating the number of leaves.
*/
class LeafTraverser final {
public:
LeafTraverser(datanodestore::DataNodeStore *nodeStore);
cpputils::unique_ref<datanodestore::DataNode> traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
private:
datanodestore::DataNodeStore *_nodeStore;
cpputils::unique_ref<datanodestore::DataNode> _traverseAndReturnRoot(
cpputils::unique_ref<datanodestore::DataNode> root, uint32_t beginIndex, uint32_t endIndex, bool isLeftBorderOfTraversal,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
void _traverseExistingSubtree(datanodestore::DataInnerNode *root, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
void _traverseExistingSubtree(const blockstore::BlockId &blockId, uint8_t depth, uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, bool isLeftBorderOfTraversal, bool isRightBorderNode, bool growLastLeaf,
std::function<void (uint32_t index, bool isRightBorderLeaf, LeafHandle leaf)> onExistingLeaf,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
cpputils::unique_ref<datanodestore::DataInnerNode> _increaseTreeDepth(cpputils::unique_ref<datanodestore::DataNode> root);
cpputils::unique_ref<datanodestore::DataNode> _createNewSubtree(uint32_t beginIndex, uint32_t endIndex, uint32_t leafOffset, uint8_t depth,
std::function<cpputils::Data (uint32_t index)> onCreateLeaf,
std::function<void (datanodestore::DataInnerNode *node)> onBacktrackFromSubtree);
uint32_t _maxLeavesForTreeDepth(uint8_t depth) const;
std::function<cpputils::Data (uint32_t index)> _createMaxSizeLeaf() const;
cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildReplaceRootWithItsChild(cpputils::unique_ref<datanodestore::DataNode> root);
cpputils::unique_ref<datanodestore::DataNode> _whileRootHasOnlyOneChildRemoveRootReturnChild(const blockstore::BlockId &blockId);
DISALLOW_COPY_AND_ASSIGN(LeafTraverser);
};
}
}
}
#endif

16
src/blobstore/implementations/onblocks/datatreestore/impl/algorithms.cpp
View File

@ -1,6 +1,6 @@
#include "algorithms.h"
#include <cpp-utils/pointer/cast.h>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include "../../datanodestore/DataInnerNode.h"
#include "../../datanodestore/DataNodeStore.h"
@ -13,7 +13,7 @@ using cpputils::unique_ref;
using blobstore::onblocks::datanodestore::DataInnerNode;
using blobstore::onblocks::datanodestore::DataNode;
using blobstore::onblocks::datanodestore::DataNodeStore;
using blockstore::Key;
using blockstore::BlockId;
using boost::optional;
using boost::none;
@ -23,8 +23,8 @@ namespace datatreestore {
namespace algorithms {
optional<unique_ref<DataInnerNode>> getLastChildAsInnerNode(DataNodeStore *nodeStore, const DataInnerNode &node) {
Key key = node.LastChild()->key();
auto lastChild = nodeStore->load(key);
BlockId blockId = node.readLastChild().blockId();
auto lastChild = nodeStore->load(blockId);
ASSERT(lastChild != none, "Couldn't load last child");
return dynamic_pointer_move<DataInnerNode>(*lastChild);
}
@ -40,10 +40,12 @@ optional_ownership_ptr<DataInnerNode> GetLowestInnerRightBorderNodeWithCondition
if (condition(*currentNode)) {
result = std::move(currentNode);
}
ASSERT(lastChild != none || static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
if (lastChild != none) {
currentNode = cpputils::WithOwnership(std::move(*lastChild));
if (lastChild == none) {
// lastChild is a leaf
ASSERT(static_cast<int>(i) == rootNode->depth()-1, "Couldn't get last child as inner node but we're not deep enough yet for the last child to be a leaf");
break;
}
currentNode = cpputils::WithOwnership(std::move(*lastChild));
}
return result;

11
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/DataTreeRef.h
View File

@ -4,25 +4,26 @@
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTree.h"
#include "blobstore/implementations/onblocks/datatreestore/LeafHandle.h"
namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key>::ResourceRefBase {
class DataTreeRef final: public parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::BlockId>::ResourceRefBase {
public:
DataTreeRef(datatreestore::DataTree *baseTree): _baseTree(baseTree) {}
const blockstore::Key &key() const {
return _baseTree->key();
const blockstore::BlockId &blockId() const {
return _baseTree->blockId();
}
uint64_t maxBytesPerLeaf() const {
return _baseTree->maxBytesPerLeaf();
}
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (datanodestore::DataLeafNode*, uint32_t)> func) {
return _baseTree->traverseLeaves(beginIndex, endIndex, func);
void traverseLeaves(uint32_t beginIndex, uint32_t endIndex, std::function<void (uint32_t index, bool isRightBorderLeaf, datatreestore::LeafHandle leaf)> onExistingLeaf, std::function<cpputils::Data (uint32_t index)> onCreateLeaf) {
return _baseTree->traverseLeaves(beginIndex, endIndex, onExistingLeaf, onCreateLeaf);
}
uint32_t numLeaves() const {

20
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.cpp
View File

@ -9,12 +9,11 @@ using cpputils::make_unique_ref;
using boost::optional;
using blobstore::onblocks::datatreestore::DataTreeStore;
using blockstore::Key;
using blockstore::BlockId;
namespace blobstore {
namespace onblocks {
using datatreestore::DataTreeStore;
using datatreestore::DataTree;
namespace parallelaccessdatatreestore {
//TODO Here and for other stores (DataTreeStore, ...): Make small functions inline
@ -26,23 +25,26 @@ ParallelAccessDataTreeStore::ParallelAccessDataTreeStore(unique_ref<DataTreeStor
ParallelAccessDataTreeStore::~ParallelAccessDataTreeStore() {
}
optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::Key &key) {
return _parallelAccessStore.load(key);
optional<unique_ref<DataTreeRef>> ParallelAccessDataTreeStore::load(const blockstore::BlockId &blockId) {
return _parallelAccessStore.load(blockId);
}
unique_ref<DataTreeRef> ParallelAccessDataTreeStore::createNewTree() {
auto dataTree = _dataTreeStore->createNewTree();
Key key = dataTree->key();
return _parallelAccessStore.add(key, std::move(dataTree));
BlockId blockId = dataTree->blockId();
return _parallelAccessStore.add(blockId, std::move(dataTree)); // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
}
void ParallelAccessDataTreeStore::remove(unique_ref<DataTreeRef> tree) {
Key key = tree->key();
return _parallelAccessStore.remove(key, std::move(tree));
BlockId blockId = tree->blockId();
return _parallelAccessStore.remove(blockId, std::move(tree));
}
void ParallelAccessDataTreeStore::remove(const BlockId &blockId) {
return _parallelAccessStore.remove(blockId);
}
}
}
}

7
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStore.h
View File

@ -4,7 +4,7 @@
#include <memory>
#include <cpp-utils/macros.h>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include <parallelaccessstore/ParallelAccessStore.h>
#include "../datatreestore/DataTreeStore.h"
@ -20,11 +20,12 @@ public:
ParallelAccessDataTreeStore(cpputils::unique_ref<datatreestore::DataTreeStore> dataTreeStore);
~ParallelAccessDataTreeStore();
boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::Key &key);
boost::optional<cpputils::unique_ref<DataTreeRef>> load(const blockstore::BlockId &blockId);
cpputils::unique_ref<DataTreeRef> createNewTree();
void remove(cpputils::unique_ref<DataTreeRef> tree);
void remove(const blockstore::BlockId &blockId);
//TODO Test blocksizeBytes/numBlocks/estimateSpaceForNumBlocksLeft
uint64_t virtualBlocksizeBytes() const;
@ -33,7 +34,7 @@ public:
private:
cpputils::unique_ref<datatreestore::DataTreeStore> _dataTreeStore;
parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::Key> _parallelAccessStore;
parallelaccessstore::ParallelAccessStore<datatreestore::DataTree, DataTreeRef, blockstore::BlockId> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessDataTreeStore);
};

12
src/blobstore/implementations/onblocks/parallelaccessdatatreestore/ParallelAccessDataTreeStoreAdapter.h
View File

@ -11,20 +11,24 @@ namespace blobstore {
namespace onblocks {
namespace parallelaccessdatatreestore {
class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::Key> {
class ParallelAccessDataTreeStoreAdapter final: public parallelaccessstore::ParallelAccessBaseStore<datatreestore::DataTree, blockstore::BlockId> {
public:
ParallelAccessDataTreeStoreAdapter(datatreestore::DataTreeStore *baseDataTreeStore)
:_baseDataTreeStore(std::move(baseDataTreeStore)) {
:_baseDataTreeStore(baseDataTreeStore) {
}
boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::Key &key) override {
return _baseDataTreeStore->load(key);
boost::optional<cpputils::unique_ref<datatreestore::DataTree>> loadFromBaseStore(const blockstore::BlockId &blockId) override {
return _baseDataTreeStore->load(blockId);
}
void removeFromBaseStore(cpputils::unique_ref<datatreestore::DataTree> dataTree) override {
return _baseDataTreeStore->remove(std::move(dataTree));
}
void removeFromBaseStore(const blockstore::BlockId &blockId) override {
return _baseDataTreeStore->remove(blockId);
}
private:
datatreestore::DataTreeStore *_baseDataTreeStore;

2
src/blobstore/implementations/onblocks/utils/Math.h
View File

@ -33,7 +33,7 @@ inline INT_TYPE maxZeroSubtraction(INT_TYPE minuend, INT_TYPE subtrahend) {
template<typename INT_TYPE>
inline INT_TYPE ceilLog(INT_TYPE base, INT_TYPE value) {
return std::ceil((long double)std::log(value)/(long double)std::log(base));
return std::ceil(static_cast<long double>(std::log(value))/static_cast<long double>(std::log(base)));
}
}

6
src/blobstore/interface/Blob.h
View File

@ -4,7 +4,7 @@
#include <cstring>
#include <cstdint>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include <cpp-utils/data/Data.h>
namespace blobstore {
@ -13,8 +13,8 @@ class Blob {
public:
virtual ~Blob() {}
//TODO Use own Key class for blobstore
virtual const blockstore::Key &key() const = 0;
//TODO Use own Id class for blobstore
virtual const blockstore::BlockId &blockId() const = 0;
virtual uint64_t size() const = 0;
virtual void resize(uint64_t numBytes) = 0;

5
src/blobstore/interface/BlobStore.h
View File

@ -6,7 +6,7 @@
#include <string>
#include <memory>
#include <blockstore/utils/Key.h>
#include <blockstore/utils/BlockId.h>
#include <cpp-utils/pointer/unique_ref.h>
namespace blobstore {
@ -17,8 +17,9 @@ public:
virtual ~BlobStore() {}
virtual cpputils::unique_ref<Blob> create() = 0;
virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::Key &key) = 0;
virtual boost::optional<cpputils::unique_ref<Blob>> load(const blockstore::BlockId &blockId) = 0;
virtual void remove(cpputils::unique_ref<Blob> blob) = 0;
virtual void remove(const blockstore::BlockId &blockId) = 0;
virtual uint64_t numBlocks() const = 0;
virtual uint64_t estimateSpaceForNumBlocksLeft() const = 0;

25
src/blockstore/CMakeLists.txt
View File

@ -1,14 +1,13 @@
project (blockstore)
set(SOURCES
utils/Key.cpp
utils/BlockId.cpp
utils/IdWrapper.cpp
utils/BlockStoreUtils.cpp
utils/FileDoesntExistException.cpp
interface/helpers/BlockStoreWithRandomKeys.cpp
implementations/testfake/FakeBlockStore.cpp
implementations/testfake/FakeBlock.cpp
implementations/inmemory/InMemoryBlock.cpp
implementations/inmemory/InMemoryBlockStore.cpp
implementations/inmemory/InMemoryBlockStore2.cpp
implementations/parallelaccess/ParallelAccessBlockStore.cpp
implementations/parallelaccess/BlockRef.cpp
implementations/parallelaccess/ParallelAccessBlockStoreAdapter.cpp
@ -16,17 +15,21 @@ set(SOURCES
implementations/compressing/CompressedBlock.cpp
implementations/compressing/compressors/RunLengthEncoding.cpp
implementations/compressing/compressors/Gzip.cpp
implementations/encrypted/EncryptedBlockStore.cpp
implementations/encrypted/EncryptedBlock.cpp
implementations/ondisk/OnDiskBlockStore.cpp
implementations/ondisk/OnDiskBlock.cpp
implementations/caching/CachingBlockStore.cpp
implementations/encrypted/EncryptedBlockStore2.cpp
implementations/ondisk/OnDiskBlockStore2.cpp
implementations/caching/CachingBlockStore2.cpp
implementations/caching/cache/PeriodicTask.cpp
implementations/caching/cache/CacheEntry.cpp
implementations/caching/cache/Cache.cpp
implementations/caching/cache/QueueMap.cpp
implementations/caching/CachedBlock.cpp
implementations/caching/NewBlock.cpp
implementations/low2highlevel/LowToHighLevelBlock.cpp
implementations/low2highlevel/LowToHighLevelBlockStore.cpp
implementations/integrity/IntegrityBlockStore2.cpp
implementations/integrity/KnownBlockVersions.cpp
implementations/integrity/ClientIdAndBlockId.cpp
implementations/integrity/IntegrityViolationError.cpp
implementations/mock/MockBlockStore.cpp
implementations/mock/MockBlock.cpp
)
add_library(${PROJECT_NAME} STATIC ${SOURCES})

46
src/blockstore/implementations/caching/CachedBlock.cpp
View File

@ -1,46 +0,0 @@
#include "CachedBlock.h"
#include "CachingBlockStore.h"
using cpputils::unique_ref;
namespace blockstore {
namespace caching {
CachedBlock::CachedBlock(unique_ref<Block> baseBlock, CachingBlockStore *blockStore)
:Block(baseBlock->key()),
_blockStore(blockStore),
_baseBlock(std::move(baseBlock)) {
}
CachedBlock::~CachedBlock() {
if (_baseBlock.get() != nullptr) {
_blockStore->release(std::move(_baseBlock));
}
}
const void *CachedBlock::data() const {
return _baseBlock->data();
}
void CachedBlock::write(const void *source, uint64_t offset, uint64_t size) {
return _baseBlock->write(source, offset, size);
}
void CachedBlock::flush() {
return _baseBlock->flush();
}
size_t CachedBlock::size() const {
return _baseBlock->size();
}
void CachedBlock::resize(size_t newSize) {
return _baseBlock->resize(newSize);
}
unique_ref<Block> CachedBlock::releaseBlock() {
return std::move(_baseBlock);
}
}
}

39
src/blockstore/implementations/caching/CachedBlock.h
View File

@ -1,39 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHEDBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace caching {
class CachingBlockStore;
class CachedBlock final: public Block {
public:
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
CachedBlock(cpputils::unique_ref<Block> baseBlock, CachingBlockStore *blockStore);
~CachedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
CachingBlockStore *_blockStore;
cpputils::unique_ref<Block> _baseBlock;
DISALLOW_COPY_AND_ASSIGN(CachedBlock);
};
}
}
#endif

100
src/blockstore/implementations/caching/CachingBlockStore.cpp
View File

@ -1,100 +0,0 @@
#include "CachedBlock.h"
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include "../../interface/Block.h"
#include <algorithm>
#include <cpp-utils/pointer/cast.h>
#include <cpp-utils/assert/assert.h>
using cpputils::dynamic_pointer_move;
using cpputils::Data;
using boost::optional;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::none;
namespace blockstore {
namespace caching {
CachingBlockStore::CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore)
:_baseBlockStore(std::move(baseBlockStore)), _cache(), _numNewBlocks(0) {
}
Key CachingBlockStore::createKey() {
return _baseBlockStore->createKey();
}
optional<unique_ref<Block>> CachingBlockStore::tryCreate(const Key &key, Data data) {
ASSERT(_cache.pop(key) == none, "Key already exists in cache");
//TODO Shouldn't we return boost::none if the key already exists?
//TODO Key can also already exist but not be in the cache right now.
++_numNewBlocks;
return unique_ref<Block>(make_unique_ref<CachedBlock>(make_unique_ref<NewBlock>(key, std::move(data), this), this));
}
optional<unique_ref<Block>> CachingBlockStore::load(const Key &key) {
optional<unique_ref<Block>> optBlock = _cache.pop(key);
//TODO an optional<> class with .getOrElse() would make this code simpler. boost::optional<>::value_or_eval didn't seem to work with unique_ptr members.
if (optBlock != none) {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*optBlock), this));
} else {
auto block = _baseBlockStore->load(key);
if (block == none) {
return none;
} else {
return optional<unique_ref<Block>>(make_unique_ref<CachedBlock>(std::move(*block), this));
}
}
}
void CachingBlockStore::remove(cpputils::unique_ref<Block> block) {
auto cached_block = dynamic_pointer_move<CachedBlock>(block);
ASSERT(cached_block != none, "Passed block is not a CachedBlock");
auto baseBlock = (*cached_block)->releaseBlock();
auto baseNewBlock = dynamic_pointer_move<NewBlock>(baseBlock);
if (baseNewBlock != none) {
if(!(*baseNewBlock)->alreadyExistsInBaseStore()) {
--_numNewBlocks;
}
(*baseNewBlock)->remove();
} else {
_baseBlockStore->remove(std::move(baseBlock));
}
}
uint64_t CachingBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks() + _numNewBlocks;
}
uint64_t CachingBlockStore::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
void CachingBlockStore::release(unique_ref<Block> block) {
Key key = block->key();
_cache.push(key, std::move(block));
}
optional<unique_ref<Block>> CachingBlockStore::tryCreateInBaseStore(const Key &key, Data data) {
auto block = _baseBlockStore->tryCreate(key, std::move(data));
if (block != none) {
--_numNewBlocks;
}
return block;
}
void CachingBlockStore::removeFromBaseStore(cpputils::unique_ref<Block> block) {
_baseBlockStore->remove(std::move(block));
}
void CachingBlockStore::flush() {
_cache.flush();
}
uint64_t CachingBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
}
}

42
src/blockstore/implementations/caching/CachingBlockStore.h
View File

@ -1,42 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE_H_
#include "cache/Cache.h"
#include "../../interface/BlockStore.h"
namespace blockstore {
namespace caching {
//TODO Check that this blockstore allows parallel destructing of blocks (otherwise we won't encrypt blocks in parallel)
class CachingBlockStore final: public BlockStore {
public:
explicit CachingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void release(cpputils::unique_ref<Block> block);
boost::optional<cpputils::unique_ref<Block>> tryCreateInBaseStore(const Key &key, cpputils::Data data);
void removeFromBaseStore(cpputils::unique_ref<Block> block);
void flush();
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
Cache<Key, cpputils::unique_ref<Block>, 1000> _cache;
uint32_t _numNewBlocks;
DISALLOW_COPY_AND_ASSIGN(CachingBlockStore);
};
}
}
#endif

155
src/blockstore/implementations/caching/CachingBlockStore2.cpp Normal file
View File

@ -0,0 +1,155 @@
#include "CachingBlockStore2.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/get_total_memory.h>
using std::string;
using std::mutex;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
using std::unique_lock;
using std::mutex;
namespace blockstore {
namespace caching {
CachingBlockStore2::CachedBlock::CachedBlock(const CachingBlockStore2* blockStore, const BlockId &blockId, cpputils::Data data, bool isDirty)
: _blockStore(blockStore), _blockId(blockId), _data(std::move(data)), _dirty(isDirty) {
}
CachingBlockStore2::CachedBlock::~CachedBlock() {
if (_dirty) {
_blockStore->_baseBlockStore->store(_blockId, _data);
}
// remove it from the list of blocks not in the base store, if it's on it
unique_lock<mutex> lock(_blockStore->_cachedBlocksNotInBaseStoreMutex);
_blockStore->_cachedBlocksNotInBaseStore.erase(_blockId);
}
const Data& CachingBlockStore2::CachedBlock::read() const {
return _data;
}
void CachingBlockStore2::CachedBlock::markNotDirty() && {
_dirty = false; // Prevent writing it back into the base store
}
void CachingBlockStore2::CachedBlock::write(Data data) {
_data = std::move(data);
_dirty = true;
}
CachingBlockStore2::CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)), _cachedBlocksNotInBaseStoreMutex(), _cachedBlocksNotInBaseStore(), _cache() {
}
bool CachingBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
//TODO Check if block exists in base store? Performance hit? It's very unlikely it exists.
auto popped = _cache.pop(blockId);
if (popped != boost::none) {
// entry already exists in cache
_cache.push(blockId, std::move(*popped)); // push the just popped element back to the cache
return false;
} else {
_cache.push(blockId, make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, data.copy(), true));
unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
_cachedBlocksNotInBaseStore.insert(blockId);
return true;
}
}
bool CachingBlockStore2::remove(const BlockId &blockId) {
// TODO Don't write-through but cache remove operations
auto popped = _cache.pop(blockId);
if (popped != boost::none) {
// Remove from base store if it exists in the base store
{
unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
if (_cachedBlocksNotInBaseStore.count(blockId) == 0) {
const bool existedInBaseStore = _baseBlockStore->remove(blockId);
if (!existedInBaseStore) {
throw std::runtime_error("Tried to remove block. Block existed in cache and stated it exists in base store, but wasn't found there.");
}
}
}
// Don't write back the cached block when it is destructed
std::move(**popped).markNotDirty();
return true;
} else {
return _baseBlockStore->remove(blockId);
}
}
optional<unique_ref<CachingBlockStore2::CachedBlock>> CachingBlockStore2::_loadFromCacheOrBaseStore(const BlockId &blockId) const {
auto popped = _cache.pop(blockId);
if (popped != boost::none) {
return std::move(*popped);
} else {
auto loaded = _baseBlockStore->load(blockId);
if (loaded == boost::none) {
return boost::none;
}
return make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, std::move(*loaded), false);
}
}
optional<Data> CachingBlockStore2::load(const BlockId &blockId) const {
auto loaded = _loadFromCacheOrBaseStore(blockId);
if (loaded == boost::none) {
// TODO Cache non-existence?
return boost::none;
}
optional<Data> result = (*loaded)->read().copy();
_cache.push(blockId, std::move(*loaded));
return result;
}
void CachingBlockStore2::store(const BlockId &blockId, const Data &data) {
auto popped = _cache.pop(blockId);
if (popped != boost::none) {
(*popped)->write(data.copy());
} else {
popped = make_unique_ref<CachingBlockStore2::CachedBlock>(this, blockId, data.copy(), false);
// TODO Instead of storing it to the base store, we could just keep it dirty in the cache
// and (if it doesn't exist in base store yet) add it to _cachedBlocksNotInBaseStore
_baseBlockStore->store(blockId, data);
}
_cache.push(blockId, std::move(*popped));
}
uint64_t CachingBlockStore2::numBlocks() const {
uint64_t numInCacheButNotInBaseStore = 0;
{
unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
numInCacheButNotInBaseStore = _cachedBlocksNotInBaseStore.size();
}
return _baseBlockStore->numBlocks() + numInCacheButNotInBaseStore;
}
uint64_t CachingBlockStore2::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
uint64_t CachingBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
void CachingBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
{
unique_lock<mutex> lock(_cachedBlocksNotInBaseStoreMutex);
for (const BlockId &blockId : _cachedBlocksNotInBaseStore) {
callback(blockId);
}
}
_baseBlockStore->forEachBlock(std::move(callback));
}
void CachingBlockStore2::flush() {
_cache.flush();
}
}
}

63
src/blockstore/implementations/caching/CachingBlockStore2.h Normal file
View File

@ -0,0 +1,63 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE2_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_CACHINGBLOCKSTORE2_H_
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
#include "../caching/cache/Cache.h"
#include <unordered_set>
namespace blockstore {
namespace caching {
class CachingBlockStore2 final: public BlockStore2 {
public:
CachingBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore);
bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
bool remove(const BlockId &blockId) override;
boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
void store(const BlockId &blockId, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
void flush();
private:
// TODO Is a cache implementation with onEvict callback instead of destructor simpler?
class CachedBlock final {
public:
CachedBlock(const CachingBlockStore2* blockStore, const BlockId &blockId, cpputils::Data data, bool isDirty);
~CachedBlock();
const cpputils::Data& read() const;
void write(cpputils::Data data);
void markNotDirty() &&; // only on rvalue because the destructor should be called after calling markNotDirty(). It shouldn't be put back into the cache.
private:
const CachingBlockStore2* _blockStore;
BlockId _blockId;
cpputils::Data _data;
bool _dirty;
DISALLOW_COPY_AND_ASSIGN(CachedBlock);
};
boost::optional<cpputils::unique_ref<CachedBlock>> _loadFromCacheOrBaseStore(const BlockId &blockId) const;
cpputils::unique_ref<BlockStore2> _baseBlockStore;
friend class CachedBlock;
// TODO Store CachedBlock directly, without unique_ref
mutable std::mutex _cachedBlocksNotInBaseStoreMutex;
mutable std::unordered_set<BlockId> _cachedBlocksNotInBaseStore;
mutable Cache<BlockId, cpputils::unique_ref<CachedBlock>, 1000> _cache;
DISALLOW_COPY_AND_ASSIGN(CachingBlockStore2);
};
}
}
#endif

75
src/blockstore/implementations/caching/NewBlock.cpp
View File

@ -1,75 +0,0 @@
#include "NewBlock.h"
#include "CachingBlockStore.h"
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
using cpputils::Data;
using boost::none;
namespace blockstore {
namespace caching {
NewBlock::NewBlock(const Key &key, Data data, CachingBlockStore *blockStore)
:Block(key),
_blockStore(blockStore),
_data(std::move(data)),
_baseBlock(none),
_dataChanged(true) {
}
NewBlock::~NewBlock() {
writeToBaseBlockIfChanged();
}
const void *NewBlock::data() const {
return _data.data();
}
void NewBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area");
std::memcpy((uint8_t*)_data.data()+offset, source, size);
_dataChanged = true;
}
void NewBlock::writeToBaseBlockIfChanged() {
if (_dataChanged) {
if (_baseBlock == none) {
//TODO _data.copy() necessary?
auto newBase = _blockStore->tryCreateInBaseStore(key(), _data.copy());
ASSERT(newBase != boost::none, "Couldn't create base block"); //TODO What if tryCreate fails due to a duplicate key? We should ensure we don't use duplicate keys.
_baseBlock = std::move(*newBase);
} else {
(*_baseBlock)->write(_data.data(), 0, _data.size());
}
_dataChanged = false;
}
}
void NewBlock::remove() {
if (_baseBlock != none) {
_blockStore->removeFromBaseStore(std::move(*_baseBlock));
}
_dataChanged = false;
}
void NewBlock::flush() {
writeToBaseBlockIfChanged();
ASSERT(_baseBlock != none, "At this point, the base block should already have been created but wasn't");
(*_baseBlock)->flush();
}
size_t NewBlock::size() const {
return _data.size();
}
void NewBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
bool NewBlock::alreadyExistsInBaseStore() const {
return _baseBlock != none;
}
}
}

52
src/blockstore/implementations/caching/NewBlock.h
View File

@ -1,52 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_CACHING_NEWBLOCK_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/data/Data.h>
#include <cpp-utils/macros.h>
#include <memory>
namespace blockstore {
namespace caching {
class CachingBlockStore;
//TODO Does it make sense to write a general DataBackedBlock that just stores a Data object and maps the block operations to it?
// Can we reuse that object somewhere else?
// Maybe a second abstract class for BlockRefBackedBlock?
// This is a block that was created in CachingBlockStore, but doesn't exist in the base block store yet.
// It only exists in the cache and it is created in the base block store when destructed.
class NewBlock final: public Block {
public:
NewBlock(const Key &key, cpputils::Data data, CachingBlockStore *blockStore);
~NewBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
void remove();
bool alreadyExistsInBaseStore() const;
private:
CachingBlockStore *_blockStore;
cpputils::Data _data;
boost::optional<cpputils::unique_ref<Block>> _baseBlock;
bool _dataChanged;
void writeToBaseBlockIfChanged();
DISALLOW_COPY_AND_ASSIGN(NewBlock);
};
}
}
#endif

2
src/blockstore/implementations/caching/cache/Cache.h vendored
View File

@ -132,7 +132,7 @@ void Cache<Key, Value, MAX_ENTRIES>::_deleteOldEntriesParallel() {
template<class Key, class Value, uint32_t MAX_ENTRIES>
void Cache<Key, Value, MAX_ENTRIES>::_deleteMatchingEntriesAtBeginningParallel(std::function<bool (const CacheEntry<Key, Value> &)> matches) {
// Twice the number of cores, so we use full CPU even if half the threads are doing I/O
unsigned int numThreads = 2 * std::max(1u, std::thread::hardware_concurrency());
unsigned int numThreads = 2 * (std::max)(1u, std::thread::hardware_concurrency());
std::vector<std::future<void>> waitHandles;
for (unsigned int i = 0; i < numThreads; ++i) {
waitHandles.push_back(std::async(std::launch::async, [this, matches] {

4
src/blockstore/implementations/caching/cache/CacheEntry.h vendored
View File

@ -16,10 +16,10 @@ public:
explicit CacheEntry(Value value): _lastAccess(currentTime()), _value(std::move(value)) {
}
CacheEntry(CacheEntry &&) = default;
CacheEntry(CacheEntry&& rhs) noexcept: _lastAccess(std::move(rhs._lastAccess)), _value(std::move(rhs._value)) {}
double ageSeconds() const {
return ((double)(currentTime() - _lastAccess).total_nanoseconds()) / ((double)1000000000);
return static_cast<double>((currentTime() - _lastAccess).total_nanoseconds()) / static_cast<double>(1000000000);
}
Value releaseValue() {

3
src/blockstore/implementations/caching/cache/PeriodicTask.cpp vendored
View File

@ -2,7 +2,6 @@
#include <cpp-utils/logging/logging.h>
using std::function;
using std::endl;
using namespace cpputils::logging;
namespace blockstore {
@ -10,7 +9,7 @@ namespace caching {
PeriodicTask::PeriodicTask(function<void ()> task, double intervalSec) :
_task(task),
_interval((uint64_t)(UINT64_C(1000000000) * intervalSec)),
_interval(static_cast<uint64_t>(UINT64_C(1000000000) * intervalSec)),
_thread(std::bind(&PeriodicTask::_loopIteration, this)) {
_thread.start();
}

23
src/blockstore/implementations/compressing/CompressedBlock.h
View File

@ -18,7 +18,8 @@ template<class Compressor> class CompressingBlockStore;
template<class Compressor>
class CompressedBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData);
static boost::optional<cpputils::unique_ref<CompressedBlock>> TryCreateNew(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Overwrite(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData);
static cpputils::unique_ref<CompressedBlock> Decompress(cpputils::unique_ref<Block> baseBlock);
CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData);
@ -46,9 +47,9 @@ private:
};
template<class Compressor>
boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data decompressedData) {
boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlock<Compressor>::TryCreateNew(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData) {
cpputils::Data compressed = Compressor::Compress(decompressedData);
auto baseBlock = baseBlockStore->tryCreate(key, std::move(compressed));
auto baseBlock = baseBlockStore->tryCreate(blockId, std::move(compressed));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
@ -57,15 +58,23 @@ boost::optional<cpputils::unique_ref<CompressedBlock<Compressor>>> CompressedBlo
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(*baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Overwrite(BlockStore *baseBlockStore, const BlockId &blockId, cpputils::Data decompressedData) {
cpputils::Data compressed = Compressor::Compress(decompressedData);
auto baseBlock = baseBlockStore->overwrite(blockId, std::move(compressed));
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressedData));
}
template<class Compressor>
cpputils::unique_ref<CompressedBlock<Compressor>> CompressedBlock<Compressor>::Decompress(cpputils::unique_ref<Block> baseBlock) {
cpputils::Data decompressed = Compressor::Decompress((CryptoPP::byte*)baseBlock->data(), baseBlock->size());
cpputils::Data decompressed = Compressor::Decompress(baseBlock->data(), baseBlock->size());
return cpputils::make_unique_ref<CompressedBlock<Compressor>>(std::move(baseBlock), std::move(decompressed));
}
template<class Compressor>
CompressedBlock<Compressor>::CompressedBlock(cpputils::unique_ref<Block> baseBlock, cpputils::Data decompressedData)
: Block(baseBlock->key()),
: Block(baseBlock->blockId()),
_baseBlock(std::move(baseBlock)),
_decompressedData(std::move(decompressedData)),
_dataChanged(false) {
@ -84,7 +93,7 @@ const void *CompressedBlock<Compressor>::data() const {
template<class Compressor>
void CompressedBlock<Compressor>::write(const void *source, uint64_t offset, uint64_t size) {
std::memcpy((uint8_t*)_decompressedData.dataOffset(offset), source, size);
std::memcpy(_decompressedData.dataOffset(offset), source, size);
_dataChanged = true;
}
@ -102,7 +111,7 @@ size_t CompressedBlock<Compressor>::size() const {
template<class Compressor>
void CompressedBlock<Compressor>::resize(size_t newSize) {
_decompressedData = cpputils::DataUtils::resize(std::move(_decompressedData), newSize);
_decompressedData = cpputils::DataUtils::resize(_decompressedData, newSize);
_dataChanged = true;
}

39
src/blockstore/implementations/compressing/CompressingBlockStore.h
View File

@ -14,13 +14,15 @@ public:
CompressingBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
~CompressingBlockStore();
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
BlockId createBlockId() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
cpputils::unique_ref<Block> overwrite(const blockstore::BlockId &blockId, cpputils::Data data) override;
void remove(const BlockId &blockId) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
@ -38,13 +40,13 @@ CompressingBlockStore<Compressor>::~CompressingBlockStore() {
}
template<class Compressor>
Key CompressingBlockStore<Compressor>::createKey() {
return _baseBlockStore->createKey();
BlockId CompressingBlockStore<Compressor>::createBlockId() {
return _baseBlockStore->createBlockId();
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const Key &key, cpputils::Data data) {
auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), key, std::move(data));
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::tryCreate(const BlockId &blockId, cpputils::Data data) {
auto result = CompressedBlock<Compressor>::TryCreateNew(_baseBlockStore.get(), blockId, std::move(data));
if (result == boost::none) {
return boost::none;
}
@ -52,8 +54,13 @@ boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const Key &key) {
auto loaded = _baseBlockStore->load(key);
cpputils::unique_ref<Block> CompressingBlockStore<Compressor>::overwrite(const blockstore::BlockId &blockId, cpputils::Data data) {
return CompressedBlock<Compressor>::Overwrite(_baseBlockStore.get(), blockId, std::move(data));
}
template<class Compressor>
boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::load(const BlockId &blockId) {
auto loaded = _baseBlockStore->load(blockId);
if (loaded == boost::none) {
return boost::none;
}
@ -61,11 +68,8 @@ boost::optional<cpputils::unique_ref<Block>> CompressingBlockStore<Compressor>::
}
template<class Compressor>
void CompressingBlockStore<Compressor>::remove(cpputils::unique_ref<Block> block) {
auto _block = cpputils::dynamic_pointer_move<CompressedBlock<Compressor>>(block);
ASSERT(_block != boost::none, "Wrong block type");
auto baseBlock = (*_block)->releaseBaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
void CompressingBlockStore<Compressor>::remove(const BlockId &blockId) {
return _baseBlockStore->remove(blockId);
}
template<class Compressor>
@ -78,6 +82,11 @@ uint64_t CompressingBlockStore<Compressor>::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
template<class Compressor>
void CompressingBlockStore<Compressor>::forEachBlock(std::function<void (const BlockId &)> callback) const {
return _baseBlockStore->forEachBlock(callback);
}
template<class Compressor>
uint64_t CompressingBlockStore<Compressor>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
//We probably have more since we're compressing, but we don't know exactly how much.

10
src/blockstore/implementations/compressing/compressors/Gzip.cpp
View File

@ -1,6 +1,6 @@
#include "cpp-utils/crypto/cryptopp_byte.h"
#include "Gzip.h"
#include <cryptopp/gzip.h>
#include <vendor_cryptopp/gzip.h>
using cpputils::Data;
@ -9,20 +9,20 @@ namespace blockstore {
Data Gzip::Compress(const Data &data) {
CryptoPP::Gzip zipper;
zipper.Put((CryptoPP::byte *) data.data(), data.size());
zipper.Put(static_cast<const CryptoPP::byte *>(data.data()), data.size());
zipper.MessageEnd();
Data compressed(zipper.MaxRetrievable());
zipper.Get((CryptoPP::byte *) compressed.data(), compressed.size());
zipper.Get(static_cast<CryptoPP::byte *>(compressed.data()), compressed.size());
return compressed;
}
Data Gzip::Decompress(const void *data, size_t size) {
//TODO Change interface to taking cpputils::Data objects (needs changing blockstore so we can read their "class Data", because this is called from CompressedBlock::Decompress()).
CryptoPP::Gunzip zipper;
zipper.Put((CryptoPP::byte *) data, size);
zipper.Put(static_cast<const CryptoPP::byte *>(data), size);
zipper.MessageEnd();
Data decompressed(zipper.MaxRetrievable());
zipper.Get((CryptoPP::byte *) decompressed.data(), decompressed.size());
zipper.Get(static_cast<CryptoPP::byte *>(decompressed.data()), decompressed.size());
return decompressed;
}

44
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.cpp
View File

@ -18,8 +18,8 @@ namespace blockstore {
Data RunLengthEncoding::Compress(const Data &data) {
ostringstream compressed;
uint8_t *current = (uint8_t*)data.data();
uint8_t *end = (uint8_t*)data.data()+data.size();
const uint8_t *current = static_cast<const uint8_t*>(data.data());
const uint8_t *end = static_cast<const uint8_t*>(data.data())+data.size();
while (current < end) {
_encodeArbitraryWords(&current, end, &compressed);
ASSERT(current <= end, "Overflow");
@ -32,25 +32,25 @@ namespace blockstore {
return _extractData(&compressed);
}
void RunLengthEncoding::_encodeArbitraryWords(uint8_t **current, uint8_t* end, ostringstream *output) {
void RunLengthEncoding::_encodeArbitraryWords(const uint8_t **current, const uint8_t* end, ostringstream *output) {
uint16_t size = _arbitraryRunLength(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, size);
output->write(reinterpret_cast<const char*>(&size), sizeof(uint16_t));
output->write(reinterpret_cast<const char*>(*current), size);
*current += size;
}
uint16_t RunLengthEncoding::_arbitraryRunLength(uint8_t *start, uint8_t* end) {
uint16_t RunLengthEncoding::_arbitraryRunLength(const uint8_t *start, const uint8_t* end) {
// Each stopping of an arbitrary bytes run costs us 5 byte, because we have to store the length
// for the identical bytes run (2 byte), the identical byte itself (1 byte) and the length for the next arbitrary bytes run (2 byte).
// So to get an advantage from stopping an arbitrary bytes run, at least 6 bytes have to be identical.
// realEnd avoids an overflow of the 16bit counter
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
const uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max());
// Count the number of identical bytes and return if it finds a run of more than 6 identical bytes.
uint8_t lastByte = *start + 1; // Something different from the first byte
uint8_t numIdenticalBytes = 1;
for(uint8_t *current = start; current != realEnd; ++current) {
for(const uint8_t *current = start; current != realEnd; ++current) {
if (*current == lastByte) {
++numIdenticalBytes;
if (numIdenticalBytes == 6) {
@ -65,16 +65,16 @@ namespace blockstore {
return realEnd-start;
}
void RunLengthEncoding::_encodeIdenticalWords(uint8_t **current, uint8_t* end, ostringstream *output) {
void RunLengthEncoding::_encodeIdenticalWords(const uint8_t **current, const uint8_t* end, ostringstream *output) {
uint16_t size = _countIdenticalBytes(*current, end);
output->write((const char*)&size, sizeof(uint16_t));
output->write((const char*)*current, 1);
output->write(reinterpret_cast<const char*>(&size), sizeof(uint16_t));
output->write(reinterpret_cast<const char*>(*current), 1);
*current += size;
}
uint16_t RunLengthEncoding::_countIdenticalBytes(uint8_t *start, uint8_t *end) {
uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
for (uint8_t *current = start+1; current != realEnd; ++current) {
uint16_t RunLengthEncoding::_countIdenticalBytes(const uint8_t *start, const uint8_t *end) {
const uint8_t *realEnd = std::min(end, start + std::numeric_limits<uint16_t>::max()); // This prevents overflow of the 16bit counter
for (const uint8_t *current = start+1; current != realEnd; ++current) {
if (*current != *start) {
return current-start;
}
@ -92,7 +92,7 @@ namespace blockstore {
Data RunLengthEncoding::Decompress(const void *data, size_t size) {
istringstream stream;
_parseData((uint8_t*)data, size, &stream);
_parseData(static_cast<const uint8_t*>(data), size, &stream);
ostringstream decompressed;
while(_hasData(&stream)) {
_decodeArbitraryWords(&stream, &decompressed);
@ -110,29 +110,29 @@ namespace blockstore {
}
void RunLengthEncoding::_parseData(const uint8_t *data, size_t size, istringstream *result) {
result->str(string((const char*)data, size));
result->str(string(reinterpret_cast<const char*>(data), size));
}
void RunLengthEncoding::_decodeArbitraryWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
stream->read(reinterpret_cast<char*>(&size), sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
stream->read((char*)run.data(), size);
stream->read(static_cast<char*>(run.data()), size);
ASSERT(stream->good(), "Premature end of stream");
decompressed->write((const char*)run.data(), run.size());
decompressed->write(static_cast<const char*>(run.data()), run.size());
}
void RunLengthEncoding::_decodeIdenticalWords(istringstream *stream, ostringstream *decompressed) {
uint16_t size;
stream->read((char*)&size, sizeof(uint16_t));
stream->read(reinterpret_cast<char*>(&size), sizeof(uint16_t));
ASSERT(stream->good(), "Premature end of stream");
uint8_t value;
stream->read((char*)&value, 1);
stream->read(reinterpret_cast<char*>(&value), 1);
ASSERT(stream->good(), "Premature end of stream");
Data run(size);
std::memset(run.data(), value, run.size());
decompressed->write((const char*)run.data(), run.size());
decompressed->write(static_cast<const char*>(run.data()), run.size());
}
}

8
src/blockstore/implementations/compressing/compressors/RunLengthEncoding.h
View File

@ -13,10 +13,10 @@ namespace blockstore {
static cpputils::Data Decompress(const void *data, size_t size);
private:
static void _encodeArbitraryWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _arbitraryRunLength(uint8_t *start, uint8_t* end);
static void _encodeIdenticalWords(uint8_t **current, uint8_t* end, std::ostringstream *output);
static uint16_t _countIdenticalBytes(uint8_t *start, uint8_t *end);
static void _encodeArbitraryWords(const uint8_t **current, const uint8_t* end, std::ostringstream *output);
static uint16_t _arbitraryRunLength(const uint8_t *start, const uint8_t* end);
static void _encodeIdenticalWords(const uint8_t **current, const uint8_t* end, std::ostringstream *output);
static uint16_t _countIdenticalBytes(const uint8_t *start, const uint8_t *end);
static bool _hasData(std::istringstream *stream);
static cpputils::Data _extractData(std::ostringstream *stream);
static void _parseData(const uint8_t *data, size_t size, std::istringstream *result);

1
src/blockstore/implementations/encrypted/EncryptedBlock.cpp
View File

@ -1 +0,0 @@
#include "EncryptedBlock.h"

219
src/blockstore/implementations/encrypted/EncryptedBlock.h
View File

@ -1,219 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCK_H_
#include "cpp-utils/crypto/cryptopp_byte.h"
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
namespace blockstore {
namespace encrypted {
template<class Cipher> class EncryptedBlockStore;
//TODO Test EncryptedBlock
//TODO Fix mutexes & locks (basically true for all blockstores)
template<class Cipher>
class EncryptedBlock final: public Block {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey);
static boost::optional<cpputils::unique_ref<EncryptedBlock>> TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key);
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
//TODO Storing key twice (in parent class and in object pointed to). Once would be enough.
EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &key, cpputils::Data plaintextWithHeader);
~EncryptedBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
cpputils::unique_ref<Block> releaseBlock();
private:
cpputils::unique_ref<Block> _baseBlock; // TODO Do I need the ciphertext block in memory or is the key enough?
cpputils::Data _plaintextWithHeader;
typename Cipher::EncryptionKey _encKey;
bool _dataChanged;
static constexpr unsigned int HEADER_LENGTH = Key::BINARY_LENGTH;
void _encryptToBaseBlock();
static cpputils::Data _prependKeyHeaderToData(const Key &key, cpputils::Data data);
static bool _keyHeaderIsCorrect(const Key &key, const cpputils::Data &data);
static cpputils::Data _prependFormatHeader(const cpputils::Data &data);
static void _checkFormatHeader(const void *data);
// This header is prepended to blocks to allow future versions to have compatibility.
static constexpr uint16_t FORMAT_VERSION_HEADER = 0;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlock);
};
template<class Cipher>
constexpr unsigned int EncryptedBlock<Cipher>::HEADER_LENGTH;
template<class Cipher>
constexpr uint16_t EncryptedBlock<Cipher>::FORMAT_VERSION_HEADER;
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryCreateNew(BlockStore *baseBlockStore, const Key &key, cpputils::Data data, const typename Cipher::EncryptionKey &encKey) {
//TODO Is it possible to avoid copying the whole plaintext data into plaintextWithHeader? Maybe an encrypt() object that has an .addData() function and concatenates all data for encryption? Maybe Crypto++ offers this functionality already.
cpputils::Data plaintextWithHeader = _prependKeyHeaderToData(key, std::move(data));
cpputils::Data encrypted = Cipher::encrypt((CryptoPP::byte*)plaintextWithHeader.data(), plaintextWithHeader.size(), encKey);
//TODO Avoid copying the whole encrypted block into a encryptedWithFormatHeader by creating a Data object with full size and then giving it as an encryption target to Cipher::encrypt()
cpputils::Data encryptedWithFormatHeader = _prependFormatHeader(std::move(encrypted));
auto baseBlock = baseBlockStore->tryCreate(key, std::move(encryptedWithFormatHeader));
if (baseBlock == boost::none) {
//TODO Test this code branch
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock>(std::move(*baseBlock), encKey, std::move(plaintextWithHeader));
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependFormatHeader(const cpputils::Data &data) {
cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
std::memcpy(dataWithHeader.dataOffset(0), &FORMAT_VERSION_HEADER, sizeof(FORMAT_VERSION_HEADER));
std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
return dataWithHeader;
}
template<class Cipher>
boost::optional<cpputils::unique_ref<EncryptedBlock<Cipher>>> EncryptedBlock<Cipher>::TryDecrypt(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey) {
_checkFormatHeader(baseBlock->data());
boost::optional<cpputils::Data> plaintextWithHeader = Cipher::decrypt((CryptoPP::byte*)baseBlock->data() + sizeof(FORMAT_VERSION_HEADER), baseBlock->size() - sizeof(FORMAT_VERSION_HEADER), encKey);
if(plaintextWithHeader == boost::none) {
//Decryption failed (e.g. an authenticated cipher detected modifications to the ciphertext)
cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed. Was the block modified by an attacker?", baseBlock->key().ToString());
return boost::none;
}
if(!_keyHeaderIsCorrect(baseBlock->key(), *plaintextWithHeader)) {
//The stored key in the block data is incorrect - an attacker might have exchanged the contents with the encrypted data from a different block
cpputils::logging::LOG(cpputils::logging::WARN, "Decrypting block {} failed due to invalid block key. Was the block modified by an attacker?", baseBlock->key().ToString());
return boost::none;
}
return cpputils::make_unique_ref<EncryptedBlock<Cipher>>(std::move(baseBlock), encKey, std::move(*plaintextWithHeader));
}
template<class Cipher>
void EncryptedBlock<Cipher>::_checkFormatHeader(const void *data) {
if (*reinterpret_cast<decltype(FORMAT_VERSION_HEADER)*>(data) != FORMAT_VERSION_HEADER) {
throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
}
}
template<class Cipher>
cpputils::Data EncryptedBlock<Cipher>::_prependKeyHeaderToData(const Key &key, cpputils::Data data) {
static_assert(HEADER_LENGTH >= Key::BINARY_LENGTH, "Key doesn't fit into the header");
cpputils::Data result(data.size() + HEADER_LENGTH);
std::memcpy(result.data(), key.data(), Key::BINARY_LENGTH);
std::memcpy((uint8_t*)result.data() + Key::BINARY_LENGTH, data.data(), data.size());
return result;
}
template<class Cipher>
bool EncryptedBlock<Cipher>::_keyHeaderIsCorrect(const Key &key, const cpputils::Data &data) {
return 0 == std::memcmp(key.data(), data.data(), Key::BINARY_LENGTH);
}
template<class Cipher>
EncryptedBlock<Cipher>::EncryptedBlock(cpputils::unique_ref<Block> baseBlock, const typename Cipher::EncryptionKey &encKey, cpputils::Data plaintextWithHeader)
:Block(baseBlock->key()),
_baseBlock(std::move(baseBlock)),
_plaintextWithHeader(std::move(plaintextWithHeader)),
_encKey(encKey),
_dataChanged(false),
_mutex() {
}
template<class Cipher>
EncryptedBlock<Cipher>::~EncryptedBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
}
template<class Cipher>
const void *EncryptedBlock<Cipher>::data() const {
return (uint8_t*)_plaintextWithHeader.data() + HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy((uint8_t*)_plaintextWithHeader.data()+HEADER_LENGTH+offset, source, count);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::flush() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return _baseBlock->flush();
}
template<class Cipher>
size_t EncryptedBlock<Cipher>::size() const {
return _plaintextWithHeader.size() - HEADER_LENGTH;
}
template<class Cipher>
void EncryptedBlock<Cipher>::resize(size_t newSize) {
_plaintextWithHeader = cpputils::DataUtils::resize(std::move(_plaintextWithHeader), newSize + HEADER_LENGTH);
_dataChanged = true;
}
template<class Cipher>
void EncryptedBlock<Cipher>::_encryptToBaseBlock() {
if (_dataChanged) {
cpputils::Data encrypted = Cipher::encrypt((CryptoPP::byte*)_plaintextWithHeader.data(), _plaintextWithHeader.size(), _encKey);
if (_baseBlock->size() != sizeof(FORMAT_VERSION_HEADER) + encrypted.size()) {
_baseBlock->resize(sizeof(FORMAT_VERSION_HEADER) + encrypted.size());
}
_baseBlock->write(&FORMAT_VERSION_HEADER, 0, sizeof(FORMAT_VERSION_HEADER));
_baseBlock->write(encrypted.data(), sizeof(FORMAT_VERSION_HEADER), encrypted.size());
_dataChanged = false;
}
}
template<class Cipher>
cpputils::unique_ref<Block> EncryptedBlock<Cipher>::releaseBlock() {
std::unique_lock<std::mutex> lock(_mutex);
_encryptToBaseBlock();
return std::move(_baseBlock);
}
template<class Cipher>
uint64_t EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
if (blockSize <= Cipher::ciphertextSize(HEADER_LENGTH) + sizeof(FORMAT_VERSION_HEADER)) {
return 0;
}
return Cipher::plaintextSize(blockSize - sizeof(FORMAT_VERSION_HEADER)) - HEADER_LENGTH;
}
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlockStore.cpp
View File

@ -1 +0,0 @@
#include "EncryptedBlockStore.h"

102
src/blockstore/implementations/encrypted/EncryptedBlockStore.h
View File

@ -1,102 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include "EncryptedBlock.h"
#include <iostream>
namespace blockstore {
namespace encrypted {
template<class Cipher>
class EncryptedBlockStore final: public BlockStore {
public:
EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
//TODO Are createKey() tests included in generic BlockStoreTest? If not, add it!
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
typename Cipher::EncryptionKey _encKey;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore);
};
template<class Cipher>
EncryptedBlockStore<Cipher>::EncryptedBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
}
template<class Cipher>
Key EncryptedBlockStore<Cipher>::createKey() {
return _baseBlockStore->createKey();
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::tryCreate(const Key &key, cpputils::Data data) {
//TODO Test that this returns boost::none when base blockstore returns nullptr (for all pass-through-blockstores)
//TODO Easier implementation? This is only so complicated because of the cast EncryptedBlock -> Block
auto result = EncryptedBlock<Cipher>::TryCreateNew(_baseBlockStore.get(), key, std::move(data), _encKey);
if (result == boost::none) {
return boost::none;
}
return cpputils::unique_ref<Block>(std::move(*result));
}
template<class Cipher>
boost::optional<cpputils::unique_ref<Block>> EncryptedBlockStore<Cipher>::load(const Key &key) {
auto block = _baseBlockStore->load(key);
if (block == boost::none) {
//TODO Test this path (for all pass-through-blockstores)
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(EncryptedBlock<Cipher>::TryDecrypt(std::move(*block), _encKey));
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::remove(cpputils::unique_ref<Block> block) {
auto encryptedBlock = cpputils::dynamic_pointer_move<EncryptedBlock<Cipher>>(block);
ASSERT(encryptedBlock != boost::none, "Block is not an EncryptedBlock");
auto baseBlock = (*encryptedBlock)->releaseBlock();
return _baseBlockStore->remove(std::move(baseBlock));
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
template<class Cipher>
void EncryptedBlockStore<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
template<class Cipher>
uint64_t EncryptedBlockStore<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return EncryptedBlock<Cipher>::blockSizeFromPhysicalBlockSize(_baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize));
}
}
}
#endif

1
src/blockstore/implementations/encrypted/EncryptedBlockStore2.cpp Normal file
View File

@ -0,0 +1 @@
#include "EncryptedBlockStore2.h"

199
src/blockstore/implementations/encrypted/EncryptedBlockStore2.h Normal file
View File

@ -0,0 +1,199 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ENCRYPTED_ENCRYPTEDBLOCKSTORE2_H_
#include "../../interface/BlockStore2.h"
#include "cpp-utils/crypto/cryptopp_byte.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/data/SerializationHelper.h>
namespace blockstore {
namespace encrypted {
//TODO Format version headers
template<class Cipher>
class EncryptedBlockStore2 final: public BlockStore2 {
public:
BOOST_CONCEPT_ASSERT((cpputils::CipherConcept<Cipher>));
EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey);
bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
bool remove(const BlockId &blockId) override;
boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
void store(const BlockId &blockId, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
//This function should only be used by test cases
void __setKey(const typename Cipher::EncryptionKey &encKey);
private:
// This header is prepended to blocks to allow future versions to have compatibility.
#ifndef CRYFS_NO_COMPATIBILITY
static constexpr uint16_t FORMAT_VERSION_HEADER_OLD = 0;
#endif
static constexpr uint16_t FORMAT_VERSION_HEADER = 1;
cpputils::Data _encrypt(const cpputils::Data &data) const;
boost::optional<cpputils::Data> _tryDecrypt(const BlockId &blockId, const cpputils::Data &data) const;
static cpputils::Data _prependFormatHeaderToData(const cpputils::Data &data);
#ifndef CRYFS_NO_COMPATIBILITY
static bool _blockIdHeaderIsCorrect(const BlockId &blockId, const cpputils::Data &data);
static cpputils::Data _migrateBlock(const cpputils::Data &data);
#endif
static void _checkFormatHeader(const cpputils::Data &data);
static uint16_t _readFormatHeader(const cpputils::Data &data);
cpputils::unique_ref<BlockStore2> _baseBlockStore;
typename Cipher::EncryptionKey _encKey;
DISALLOW_COPY_AND_ASSIGN(EncryptedBlockStore2);
};
#ifndef CRYFS_NO_COMPATIBILITY
template<class Cipher>
constexpr uint16_t EncryptedBlockStore2<Cipher>::FORMAT_VERSION_HEADER_OLD;
#endif
template<class Cipher>
constexpr uint16_t EncryptedBlockStore2<Cipher>::FORMAT_VERSION_HEADER;
template<class Cipher>
inline EncryptedBlockStore2<Cipher>::EncryptedBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const typename Cipher::EncryptionKey &encKey)
: _baseBlockStore(std::move(baseBlockStore)), _encKey(encKey) {
}
template<class Cipher>
inline bool EncryptedBlockStore2<Cipher>::tryCreate(const BlockId &blockId, const cpputils::Data &data) {
cpputils::Data encrypted = _encrypt(data);
return _baseBlockStore->tryCreate(blockId, encrypted);
}
template<class Cipher>
inline bool EncryptedBlockStore2<Cipher>::remove(const BlockId &blockId) {
return _baseBlockStore->remove(blockId);
}
template<class Cipher>
inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::load(const BlockId &blockId) const {
auto loaded = _baseBlockStore->load(blockId);
if (boost::none == loaded) {
return boost::optional<cpputils::Data>(boost::none);
}
return _tryDecrypt(blockId, *loaded);
}
template<class Cipher>
inline void EncryptedBlockStore2<Cipher>::store(const BlockId &blockId, const cpputils::Data &data) {
cpputils::Data encrypted = _encrypt(data);
return _baseBlockStore->store(blockId, encrypted);
}
template<class Cipher>
inline uint64_t EncryptedBlockStore2<Cipher>::numBlocks() const {
return _baseBlockStore->numBlocks();
}
template<class Cipher>
inline uint64_t EncryptedBlockStore2<Cipher>::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
template<class Cipher>
inline uint64_t EncryptedBlockStore2<Cipher>::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
if (baseBlockSize <= Cipher::ciphertextSize(0) + sizeof(FORMAT_VERSION_HEADER)) {
return 0;
}
return Cipher::plaintextSize(baseBlockSize - sizeof(FORMAT_VERSION_HEADER));
}
template<class Cipher>
inline void EncryptedBlockStore2<Cipher>::forEachBlock(std::function<void (const BlockId &)> callback) const {
return _baseBlockStore->forEachBlock(std::move(callback));
}
template<class Cipher>
inline cpputils::Data EncryptedBlockStore2<Cipher>::_encrypt(const cpputils::Data &data) const {
cpputils::Data encrypted = Cipher::encrypt(static_cast<const CryptoPP::byte*>(data.data()), data.size(), _encKey);
return _prependFormatHeaderToData(encrypted);
}
template<class Cipher>
inline boost::optional<cpputils::Data> EncryptedBlockStore2<Cipher>::_tryDecrypt(const BlockId &blockId, const cpputils::Data &data) const {
_checkFormatHeader(data);
boost::optional<cpputils::Data> decrypted = Cipher::decrypt(static_cast<const CryptoPP::byte*>(data.dataOffset(sizeof(FORMAT_VERSION_HEADER))), data.size() - sizeof(FORMAT_VERSION_HEADER), _encKey);
if (decrypted == boost::none) {
// TODO Log warning
return boost::none;
}
#ifndef CRYFS_NO_COMPATIBILITY
if (FORMAT_VERSION_HEADER_OLD == _readFormatHeader(data)) {
if (!_blockIdHeaderIsCorrect(blockId, *decrypted)) {
return boost::none;
}
*decrypted = _migrateBlock(*decrypted);
// no need to write migrated back to block store because
// this migration happens in line with a migration in IntegrityBlockStore2
// which then writes it back
}
#endif
return decrypted;
}
#ifndef CRYFS_NO_COMPATIBILITY
template<class Cipher>
inline cpputils::Data EncryptedBlockStore2<Cipher>::_migrateBlock(const cpputils::Data &data) {
return data.copyAndRemovePrefix(BlockId::BINARY_LENGTH);
}
template<class Cipher>
inline bool EncryptedBlockStore2<Cipher>::_blockIdHeaderIsCorrect(const BlockId &blockId, const cpputils::Data &data) {
return blockId == BlockId::FromBinary(data.data());
}
#endif
template<class Cipher>
inline cpputils::Data EncryptedBlockStore2<Cipher>::_prependFormatHeaderToData(const cpputils::Data &data) {
cpputils::Data dataWithHeader(sizeof(FORMAT_VERSION_HEADER) + data.size());
cpputils::serialize<uint16_t>(dataWithHeader.dataOffset(0), FORMAT_VERSION_HEADER);
std::memcpy(dataWithHeader.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.data(), data.size());
return dataWithHeader;
}
template<class Cipher>
inline void EncryptedBlockStore2<Cipher>::_checkFormatHeader(const cpputils::Data &data) {
const uint16_t formatVersionHeader = _readFormatHeader(data);
#ifndef CRYFS_NO_COMPATIBILITY
const bool formatVersionHeaderValid = formatVersionHeader == FORMAT_VERSION_HEADER || formatVersionHeader == FORMAT_VERSION_HEADER_OLD;
#else
const bool formatVersionHeaderValid = formatVersionHeader == FORMAT_VERSION_HEADER;
#endif
if (!formatVersionHeaderValid) {
throw std::runtime_error("The encrypted block has the wrong format. Was it created with a newer version of CryFS?");
}
}
template<class Cipher>
uint16_t EncryptedBlockStore2<Cipher>::_readFormatHeader(const cpputils::Data &data) {
return cpputils::deserialize<uint16_t>(data.data());
}
template<class Cipher>
void EncryptedBlockStore2<Cipher>::__setKey(const typename Cipher::EncryptionKey &encKey) {
_encKey = encKey;
}
}
}
#endif

50
src/blockstore/implementations/inmemory/InMemoryBlock.cpp
View File

@ -1,50 +0,0 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <cstring>
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
using std::make_shared;
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using cpputils::Data;
namespace blockstore {
namespace inmemory {
InMemoryBlock::InMemoryBlock(const Key &key, Data data)
: Block(key), _data(make_shared<Data>(std::move(data))) {
}
InMemoryBlock::InMemoryBlock(const InMemoryBlock &rhs)
: Block(rhs), _data(rhs._data) {
}
InMemoryBlock::~InMemoryBlock() {
}
const void *InMemoryBlock::data() const {
return _data->data();
}
void InMemoryBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data->size() && offset + size <= _data->size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy((uint8_t*)_data->data()+offset, source, size);
}
size_t InMemoryBlock::size() const {
return _data->size();
}
void InMemoryBlock::resize(size_t newSize) {
*_data = cpputils::DataUtils::resize(std::move(*_data), newSize);
}
void InMemoryBlock::flush() {
}
}
}

33
src/blockstore/implementations/inmemory/InMemoryBlock.h
View File

@ -1,33 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore;
class InMemoryBlock final: public Block {
public:
InMemoryBlock(const Key &key, cpputils::Data size);
InMemoryBlock(const InMemoryBlock &rhs);
~InMemoryBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
std::shared_ptr<cpputils::Data> _data;
};
}
}
#endif

65
src/blockstore/implementations/inmemory/InMemoryBlockStore.cpp
View File

@ -1,65 +0,0 @@
#include "InMemoryBlock.h"
#include "InMemoryBlockStore.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/get_total_memory.h>
using std::make_unique;
using std::string;
using std::mutex;
using std::lock_guard;
using std::piecewise_construct;
using std::make_tuple;
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using boost::optional;
using boost::none;
namespace blockstore {
namespace inmemory {
InMemoryBlockStore::InMemoryBlockStore()
: _blocks() {}
optional<unique_ref<Block>> InMemoryBlockStore::tryCreate(const Key &key, Data data) {
auto insert_result = _blocks.emplace(piecewise_construct, make_tuple(key), make_tuple(key, std::move(data)));
if (!insert_result.second) {
return none;
}
//Return a pointer to the stored InMemoryBlock
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(insert_result.first->second));
}
optional<unique_ref<Block>> InMemoryBlockStore::load(const Key &key) {
//Return a pointer to the stored InMemoryBlock
try {
return optional<unique_ref<Block>>(make_unique_ref<InMemoryBlock>(_blocks.at(key)));
} catch (const std::out_of_range &e) {
return none;
}
}
void InMemoryBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
int numRemoved = _blocks.erase(key);
ASSERT(1==numRemoved, "Didn't find block to remove");
}
uint64_t InMemoryBlockStore::numBlocks() const {
return _blocks.size();
}
uint64_t InMemoryBlockStore::estimateNumFreeBytes() const {
return cpputils::system::get_total_memory();
}
uint64_t InMemoryBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return blockSize;
}
}
}

35
src/blockstore/implementations/inmemory/InMemoryBlockStore.h
View File

@ -1,35 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE_H_
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
#include <mutex>
#include <unordered_map>
#include "InMemoryBlock.h"
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore final: public BlockStoreWithRandomKeys {
public:
InMemoryBlockStore();
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
private:
std::unordered_map<Key, InMemoryBlock> _blocks;
DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore);
};
}
}
#endif

96
src/blockstore/implementations/inmemory/InMemoryBlockStore2.cpp Normal file
View File

@ -0,0 +1,96 @@
#include "InMemoryBlockStore2.h"
#include <memory>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/system/get_total_memory.h>
using std::string;
using std::mutex;
using std::make_pair;
using std::vector;
using cpputils::Data;
using boost::optional;
using boost::none;
namespace blockstore {
namespace inmemory {
InMemoryBlockStore2::InMemoryBlockStore2()
: _blocks() {}
bool InMemoryBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
return _tryCreate(blockId, data);
}
bool InMemoryBlockStore2::_tryCreate(const BlockId &blockId, const Data &data) {
auto result = _blocks.insert(make_pair(blockId, data.copy()));
return result.second; // Return if insertion was successful (i.e. blockId didn't exist yet)
}
bool InMemoryBlockStore2::remove(const BlockId &blockId) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(blockId);
if (found == _blocks.end()) {
// BlockId not found
return false;
}
_blocks.erase(found);
return true;
}
optional<Data> InMemoryBlockStore2::load(const BlockId &blockId) const {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(blockId);
if (found == _blocks.end()) {
return boost::none;
}
return found->second.copy();
}
void InMemoryBlockStore2::store(const BlockId &blockId, const Data &data) {
std::unique_lock<std::mutex> lock(_mutex);
auto found = _blocks.find(blockId);
if (found == _blocks.end()) {
bool success = _tryCreate(blockId, data);
if (!success) {
throw std::runtime_error("Could neither save nor create the block in InMemoryBlockStore::store()");
}
} else {
// TODO Would have better performance: found->second.overwriteWith(data)
found->second = data.copy();
}
}
uint64_t InMemoryBlockStore2::numBlocks() const {
std::unique_lock<std::mutex> lock(_mutex);
return _blocks.size();
}
uint64_t InMemoryBlockStore2::estimateNumFreeBytes() const {
return cpputils::system::get_total_memory();
}
uint64_t InMemoryBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return blockSize;
}
vector<BlockId> InMemoryBlockStore2::_allBlockIds() const {
std::unique_lock<std::mutex> lock(_mutex);
vector<BlockId> result;
result.reserve(_blocks.size());
for (const auto &entry : _blocks) {
result.push_back(entry.first);
}
return result;
}
void InMemoryBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
auto blockIds = _allBlockIds();
for (const auto &blockId : blockIds) {
callback(blockId);
}
}
}
}

38
src/blockstore/implementations/inmemory/InMemoryBlockStore2.h Normal file
View File

@ -0,0 +1,38 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE2_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INMEMORY_INMEMORYBLOCKSTORE2_H_
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
#include <unordered_map>
namespace blockstore {
namespace inmemory {
class InMemoryBlockStore2 final: public BlockStore2 {
public:
InMemoryBlockStore2();
bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
bool remove(const BlockId &blockId) override;
boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
void store(const BlockId &blockId, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
std::vector<BlockId> _allBlockIds() const;
bool _tryCreate(const BlockId &blockId, const cpputils::Data &data);
std::unordered_map<BlockId, cpputils::Data> _blocks;
mutable std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(InMemoryBlockStore2);
};
}
}
#endif

1
src/blockstore/implementations/integrity/ClientIdAndBlockId.cpp Normal file
View File

@ -0,0 +1 @@
#include "ClientIdAndBlockId.h"

36
src/blockstore/implementations/integrity/ClientIdAndBlockId.h Normal file
View File

@ -0,0 +1,36 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_CLIENTIDANDBLOCKID_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_CLIENTIDANDBLOCKID_H_
#include <utility>
#include "blockstore/utils/BlockId.h"
namespace blockstore {
namespace integrity {
struct ClientIdAndBlockId final {
ClientIdAndBlockId(uint32_t clientId_, BlockId blockId_): clientId(clientId_), blockId(blockId_) {}
uint32_t clientId;
BlockId blockId;
};
}
}
// Allow using it in std::unordered_set / std::unordered_map
namespace std {
template<> struct hash<blockstore::integrity::ClientIdAndBlockId> {
size_t operator()(const blockstore::integrity::ClientIdAndBlockId &ref) const {
return std::hash<uint32_t>()(ref.clientId) ^ std::hash<blockstore::BlockId>()(ref.blockId);
}
};
template<> struct equal_to<blockstore::integrity::ClientIdAndBlockId> {
size_t operator()(const blockstore::integrity::ClientIdAndBlockId &lhs, const blockstore::integrity::ClientIdAndBlockId &rhs) const {
return lhs.clientId == rhs.clientId && lhs.blockId == rhs.blockId;
}
};
}
#endif

223
src/blockstore/implementations/integrity/IntegrityBlockStore2.cpp Normal file
View File

@ -0,0 +1,223 @@
#include <blockstore/interface/BlockStore2.h>
#include "IntegrityBlockStore2.h"
#include "KnownBlockVersions.h"
#include <cpp-utils/data/SerializationHelper.h>
using cpputils::Data;
using cpputils::unique_ref;
using cpputils::serialize;
using cpputils::deserialize;
using std::string;
using boost::optional;
using boost::none;
using namespace cpputils::logging;
namespace blockstore {
namespace integrity {
#ifndef CRYFS_NO_COMPATIBILITY
constexpr uint16_t IntegrityBlockStore2::FORMAT_VERSION_HEADER_OLD;
#endif
constexpr uint16_t IntegrityBlockStore2::FORMAT_VERSION_HEADER;
constexpr uint64_t IntegrityBlockStore2::VERSION_ZERO;
constexpr unsigned int IntegrityBlockStore2::ID_HEADER_OFFSET;
constexpr unsigned int IntegrityBlockStore2::CLIENTID_HEADER_OFFSET;
constexpr unsigned int IntegrityBlockStore2::VERSION_HEADER_OFFSET;
constexpr unsigned int IntegrityBlockStore2::HEADER_LENGTH;
Data IntegrityBlockStore2::_prependHeaderToData(const BlockId& blockId, uint32_t myClientId, uint64_t version, const Data &data) {
static_assert(HEADER_LENGTH == sizeof(FORMAT_VERSION_HEADER) + BlockId::BINARY_LENGTH + sizeof(myClientId) + sizeof(version), "Wrong header length");
Data result(data.size() + HEADER_LENGTH);
serialize<uint16_t>(result.dataOffset(0), FORMAT_VERSION_HEADER);
blockId.ToBinary(result.dataOffset(ID_HEADER_OFFSET));
serialize<uint32_t>(result.dataOffset(CLIENTID_HEADER_OFFSET), myClientId);
serialize<uint64_t>(result.dataOffset(VERSION_HEADER_OFFSET), version);
std::memcpy(result.dataOffset(HEADER_LENGTH), data.data(), data.size());
return result;
}
void IntegrityBlockStore2::_checkHeader(const BlockId &blockId, const Data &data) const {
_checkFormatHeader(data);
_checkIdHeader(blockId, data);
_checkVersionHeader(blockId, data);
}
void IntegrityBlockStore2::_checkFormatHeader(const Data &data) const {
if (FORMAT_VERSION_HEADER != _readFormatHeader(data)) {
throw std::runtime_error("The versioned block has the wrong format. Was it created with a newer version of CryFS?");
}
}
void IntegrityBlockStore2::_checkVersionHeader(const BlockId &blockId, const Data &data) const {
uint32_t clientId = _readClientId(data);
uint64_t version = _readVersion(data);
if(!_knownBlockVersions.checkAndUpdateVersion(clientId, blockId, version)) {
integrityViolationDetected("The block version number is too low. Did an attacker try to roll back the block or to re-introduce a deleted block?");
}
}
void IntegrityBlockStore2::_checkIdHeader(const BlockId &expectedBlockId, const Data &data) const {
BlockId actualBlockId = _readBlockId(data);
if (expectedBlockId != actualBlockId) {
integrityViolationDetected("The block id is wrong. Did an attacker try to rename some blocks?");
}
}
uint16_t IntegrityBlockStore2::_readFormatHeader(const Data &data) {
return deserialize<uint16_t>(data.data());
}
uint32_t IntegrityBlockStore2::_readClientId(const Data &data) {
return deserialize<uint32_t>(data.dataOffset(CLIENTID_HEADER_OFFSET));
}
BlockId IntegrityBlockStore2::_readBlockId(const Data &data) {
return BlockId::FromBinary(data.dataOffset(ID_HEADER_OFFSET));
}
uint64_t IntegrityBlockStore2::_readVersion(const Data &data) {
return deserialize<uint64_t>(data.dataOffset(VERSION_HEADER_OFFSET));
}
Data IntegrityBlockStore2::_removeHeader(const Data &data) {
return data.copyAndRemovePrefix(HEADER_LENGTH);
}
void IntegrityBlockStore2::_checkNoPastIntegrityViolations() const {
if (_integrityViolationDetected) {
throw std::runtime_error(string() +
"There was an integrity violation detected. Preventing any further access to the file system. " +
"If you want to reset the integrity data (i.e. accept changes made by a potential attacker), " +
"please unmount the file system and delete the following file before re-mounting it: " +
_knownBlockVersions.path().string());
}
}
void IntegrityBlockStore2::integrityViolationDetected(const string &reason) const {
if (_allowIntegrityViolations) {
LOG(WARN, "Integrity violation (but integrity checks are disabled): {}", reason);
return;
}
_integrityViolationDetected = true;
throw IntegrityViolationError(reason);
}
IntegrityBlockStore2::IntegrityBlockStore2(unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool allowIntegrityViolations, bool missingBlockIsIntegrityViolation)
: _baseBlockStore(std::move(baseBlockStore)), _knownBlockVersions(integrityFilePath, myClientId), _allowIntegrityViolations(allowIntegrityViolations), _missingBlockIsIntegrityViolation(missingBlockIsIntegrityViolation), _integrityViolationDetected(false) {
}
bool IntegrityBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
_checkNoPastIntegrityViolations();
uint64_t version = _knownBlockVersions.incrementVersion(blockId);
Data dataWithHeader = _prependHeaderToData(blockId, _knownBlockVersions.myClientId(), version, data);
return _baseBlockStore->tryCreate(blockId, dataWithHeader);
}
bool IntegrityBlockStore2::remove(const BlockId &blockId) {
_checkNoPastIntegrityViolations();
_knownBlockVersions.markBlockAsDeleted(blockId);
return _baseBlockStore->remove(blockId);
}
optional<Data> IntegrityBlockStore2::load(const BlockId &blockId) const {
_checkNoPastIntegrityViolations();
auto loaded = _baseBlockStore->load(blockId);
if (none == loaded) {
if (_missingBlockIsIntegrityViolation && _knownBlockVersions.blockShouldExist(blockId)) {
integrityViolationDetected("A block that should exist wasn't found. Did an attacker delete it?");
}
return optional<Data>(none);
}
#ifndef CRYFS_NO_COMPATIBILITY
if (FORMAT_VERSION_HEADER_OLD == _readFormatHeader(*loaded)) {
Data migrated = _migrateBlock(blockId, *loaded);
_checkHeader(blockId, migrated);
Data content = _removeHeader(migrated);
const_cast<IntegrityBlockStore2*>(this)->store(blockId, content);
return optional<Data>(_removeHeader(migrated));
}
#endif
_checkHeader(blockId, *loaded);
return optional<Data>(_removeHeader(*loaded));
}
#ifndef CRYFS_NO_COMPATIBILITY
Data IntegrityBlockStore2::_migrateBlock(const BlockId &blockId, const Data &data) {
Data migrated(data.size() + BlockId::BINARY_LENGTH);
serialize<uint16_t>(migrated.dataOffset(0), FORMAT_VERSION_HEADER);
blockId.ToBinary(migrated.dataOffset(ID_HEADER_OFFSET));
std::memcpy(migrated.dataOffset(ID_HEADER_OFFSET + BlockId::BINARY_LENGTH), data.dataOffset(sizeof(FORMAT_VERSION_HEADER)), data.size() - sizeof(FORMAT_VERSION_HEADER));
ASSERT(migrated.size() == sizeof(FORMAT_VERSION_HEADER) + BlockId::BINARY_LENGTH + (data.size() - sizeof(FORMAT_VERSION_HEADER)), "Wrong offset computation");
return migrated;
}
#endif
void IntegrityBlockStore2::store(const BlockId &blockId, const Data &data) {
_checkNoPastIntegrityViolations();
uint64_t version = _knownBlockVersions.incrementVersion(blockId);
Data dataWithHeader = _prependHeaderToData(blockId, _knownBlockVersions.myClientId(), version, data);
return _baseBlockStore->store(blockId, dataWithHeader);
}
uint64_t IntegrityBlockStore2::numBlocks() const {
return _baseBlockStore->numBlocks();
}
uint64_t IntegrityBlockStore2::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
uint64_t IntegrityBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
uint64_t baseBlockSize = _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
if (baseBlockSize <= HEADER_LENGTH) {
return 0;
}
return baseBlockSize - HEADER_LENGTH;
}
void IntegrityBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
if (!_missingBlockIsIntegrityViolation) {
return _baseBlockStore->forEachBlock(std::move(callback));
}
std::unordered_set<blockstore::BlockId> existingBlocks = _knownBlockVersions.existingBlocks();
_baseBlockStore->forEachBlock([&existingBlocks, callback] (const BlockId &blockId) {
callback(blockId);
auto found = existingBlocks.find(blockId);
if (found != existingBlocks.end()) {
existingBlocks.erase(found);
}
});
if (!existingBlocks.empty()) {
integrityViolationDetected("A block that should have existed wasn't found.");
}
}
#ifndef CRYFS_NO_COMPATIBILITY
void IntegrityBlockStore2::migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId) {
std::cout << "Migrating file system for integrity features. Please don't interrupt this process. This can take a while..." << std::flush;
KnownBlockVersions knownBlockVersions(integrityFilePath, myClientId);
baseBlockStore->forEachBlock([&baseBlockStore, &knownBlockVersions] (const BlockId &blockId) {
migrateBlockFromBlockstoreWithoutVersionNumbers(baseBlockStore, blockId, &knownBlockVersions);
});
std::cout << "done" << std::endl;
}
void IntegrityBlockStore2::migrateBlockFromBlockstoreWithoutVersionNumbers(blockstore::BlockStore2* baseBlockStore, const blockstore::BlockId& blockId, KnownBlockVersions *knownBlockVersions) {
uint64_t version = knownBlockVersions->incrementVersion(blockId);
auto data_ = baseBlockStore->load(blockId);
if (data_ == boost::none) {
LOG(WARN, "Block not found, but was returned from forEachBlock before");
return;
}
cpputils::Data data = std::move(*data_);
cpputils::Data dataWithHeader = _prependHeaderToData(blockId, knownBlockVersions->myClientId(), version, data);
baseBlockStore->store(blockId, dataWithHeader);
}
#endif
}
}

79
src/blockstore/implementations/integrity/IntegrityBlockStore2.h Normal file
View File

@ -0,0 +1,79 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYBLOCKSTORE2_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYBLOCKSTORE2_H_
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
#include "KnownBlockVersions.h"
#include "IntegrityViolationError.h"
namespace blockstore {
namespace integrity {
//TODO Format version headers
// This blockstore implements integrity measures.
// It depends on being used on top of an encrypted block store that protects integrity of the block contents (i.e. uses an authenticated cipher).
class IntegrityBlockStore2 final: public BlockStore2 {
public:
IntegrityBlockStore2(cpputils::unique_ref<BlockStore2> baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId, bool allowIntegrityViolations, bool missingBlockIsIntegrityViolation);
bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
bool remove(const BlockId &blockId) override;
boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
void store(const BlockId &blockId, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
// This format version is prepended to blocks to allow future versions to have compatibility.
#ifndef CRYFS_NO_COMPATIBILITY
static constexpr uint16_t FORMAT_VERSION_HEADER_OLD = 0;
#endif
static constexpr uint16_t FORMAT_VERSION_HEADER = 1;
public:
static constexpr uint64_t VERSION_ZERO = 0;
static constexpr unsigned int ID_HEADER_OFFSET = sizeof(FORMAT_VERSION_HEADER);
static constexpr unsigned int CLIENTID_HEADER_OFFSET = ID_HEADER_OFFSET + BlockId::BINARY_LENGTH;
static constexpr unsigned int VERSION_HEADER_OFFSET = CLIENTID_HEADER_OFFSET + sizeof(uint32_t);
static constexpr unsigned int HEADER_LENGTH = VERSION_HEADER_OFFSET + sizeof(VERSION_ZERO);
#ifndef CRYFS_NO_COMPATIBILITY
static void migrateFromBlockstoreWithoutVersionNumbers(BlockStore2 *baseBlockStore, const boost::filesystem::path &integrityFilePath, uint32_t myClientId);
static void migrateBlockFromBlockstoreWithoutVersionNumbers(BlockStore2* baseBlockStore, const blockstore::BlockId &blockId, KnownBlockVersions *knownBlockVersions);
#endif
private:
static cpputils::Data _prependHeaderToData(const BlockId &blockId, uint32_t myClientId, uint64_t version, const cpputils::Data &data);
void _checkHeader(const BlockId &blockId, const cpputils::Data &data) const;
void _checkFormatHeader(const cpputils::Data &data) const;
void _checkIdHeader(const BlockId &expectedBlockId, const cpputils::Data &data) const;
void _checkVersionHeader(const BlockId &blockId, const cpputils::Data &data) const;
static uint16_t _readFormatHeader(const cpputils::Data &data);
static uint32_t _readClientId(const cpputils::Data &data);
static BlockId _readBlockId(const cpputils::Data &data);
static uint64_t _readVersion(const cpputils::Data &data);
#ifndef CRYFS_NO_COMPATIBILITY
static cpputils::Data _migrateBlock(const BlockId &blockId, const cpputils::Data &data);
#endif
static cpputils::Data _removeHeader(const cpputils::Data &data);
void _checkNoPastIntegrityViolations() const;
void integrityViolationDetected(const std::string &reason) const;
cpputils::unique_ref<BlockStore2> _baseBlockStore;
mutable KnownBlockVersions _knownBlockVersions;
const bool _allowIntegrityViolations;
const bool _missingBlockIsIntegrityViolation;
mutable bool _integrityViolationDetected;
DISALLOW_COPY_AND_ASSIGN(IntegrityBlockStore2);
};
}
}
#endif

1
src/blockstore/implementations/integrity/IntegrityViolationError.cpp Normal file
View File

@ -0,0 +1 @@
#include "IntegrityViolationError.h"

27
src/blockstore/implementations/integrity/IntegrityViolationError.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYVIOLATIONERROR_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_INTEGRITYVIOLATIONERROR_H_
#include <cpp-utils/macros.h>
#include <string>
#include <stdexcept>
namespace blockstore {
namespace integrity {
class IntegrityViolationError final : public std::runtime_error {
private:
// Constructor is private to make sure that only IntegrityBlockStore can throw this exception.
// This is because IntegrityBlockStore wants to know about integrity violations and
// block all further file system access if it happens.
IntegrityViolationError(const std::string &reason)
: std::runtime_error("Integrity violation: " + reason) {
}
friend class IntegrityBlockStore2;
};
}
}
#endif

218
src/blockstore/implementations/integrity/KnownBlockVersions.cpp Normal file
View File

@ -0,0 +1,218 @@
#include <fstream>
#include <cpp-utils/random/Random.h>
#include <unordered_set>
#include "KnownBlockVersions.h"
namespace bf = boost::filesystem;
using std::pair;
using std::string;
using std::unique_lock;
using std::mutex;
using boost::optional;
using boost::none;
using cpputils::Data;
using cpputils::Serializer;
using cpputils::Deserializer;
namespace blockstore {
namespace integrity {
const string KnownBlockVersions::HEADER = "cryfs.integritydata.knownblockversions;0";
constexpr uint32_t KnownBlockVersions::CLIENT_ID_FOR_DELETED_BLOCK;
KnownBlockVersions::KnownBlockVersions(const bf::path &stateFilePath, uint32_t myClientId)
:_knownVersions(), _lastUpdateClientId(), _stateFilePath(stateFilePath), _myClientId(myClientId), _mutex(), _valid(true) {
unique_lock<mutex> lock(_mutex);
ASSERT(_myClientId != CLIENT_ID_FOR_DELETED_BLOCK, "This is not a valid client id");
_loadStateFile();
}
KnownBlockVersions::KnownBlockVersions(KnownBlockVersions &&rhs) // NOLINT (intentionally not noexcept)
: _knownVersions(), _lastUpdateClientId(), _stateFilePath(), _myClientId(0), _mutex(), _valid(true) {
unique_lock<mutex> rhsLock(rhs._mutex);
unique_lock<mutex> lock(_mutex);
_knownVersions = std::move(rhs._knownVersions);
_lastUpdateClientId = std::move(rhs._lastUpdateClientId);
_stateFilePath = std::move(rhs._stateFilePath);
_myClientId = rhs._myClientId;
rhs._valid = false;
}
KnownBlockVersions::~KnownBlockVersions() {
unique_lock<mutex> lock(_mutex);
if (_valid) {
_saveStateFile();
}
}
bool KnownBlockVersions::checkAndUpdateVersion(uint32_t clientId, const BlockId &blockId, uint64_t version) {
unique_lock<mutex> lock(_mutex);
ASSERT(clientId != CLIENT_ID_FOR_DELETED_BLOCK, "This is not a valid client id");
ASSERT(version > 0, "Version has to be >0"); // Otherwise we wouldn't handle notexisting entries correctly.
ASSERT(_valid, "Object not valid due to a std::move");
uint64_t &found = _knownVersions[{clientId, blockId}]; // If the entry doesn't exist, this creates it with value 0.
if (found > version) {
// This client already published a newer block version. Rollbacks are not allowed.
return false;
}
uint32_t &lastUpdateClientId = _lastUpdateClientId[blockId]; // If entry doesn't exist, this creates it with value 0. However, in this case, found == 0 (and version > 0), which means found != version.
if (found == version && lastUpdateClientId != clientId) {
// This is a roll back to the "newest" block of client [clientId], which was since then superseded by a version from client _lastUpdateClientId[blockId].
// This is not allowed.
return false;
}
found = version;
lastUpdateClientId = clientId;
return true;
}
uint64_t KnownBlockVersions::incrementVersion(const BlockId &blockId) {
unique_lock<mutex> lock(_mutex);
uint64_t &found = _knownVersions[{_myClientId, blockId}]; // If the entry doesn't exist, this creates it with value 0.
uint64_t newVersion = found + 1;
if (newVersion == std::numeric_limits<uint64_t>::max()) {
// It's *very* unlikely we ever run out of version numbers in 64bit...but just to be sure...
throw std::runtime_error("Version overflow");
}
found = newVersion;
_lastUpdateClientId[blockId] = _myClientId;
return found;
}
void KnownBlockVersions::_loadStateFile() {
optional<Data> file = Data::LoadFromFile(_stateFilePath);
if (file == none) {
// File doesn't exist means we loaded empty state.
return;
}
Deserializer deserializer(&*file);
if (HEADER != deserializer.readString()) {
throw std::runtime_error("Invalid local state: Invalid integrity file header.");
}
_deserializeKnownVersions(&deserializer);
_deserializeLastUpdateClientIds(&deserializer);
deserializer.finished();
};
void KnownBlockVersions::_saveStateFile() const {
Serializer serializer(
Serializer::StringSize(HEADER) +
sizeof(uint64_t) + _knownVersions.size() * (sizeof(uint32_t) + BlockId::BINARY_LENGTH + sizeof(uint64_t)) +
sizeof(uint64_t) + _lastUpdateClientId.size() * (BlockId::BINARY_LENGTH + sizeof(uint32_t)));
serializer.writeString(HEADER);
_serializeKnownVersions(&serializer);
_serializeLastUpdateClientIds(&serializer);
serializer.finished().StoreToFile(_stateFilePath);
}
void KnownBlockVersions::_deserializeKnownVersions(Deserializer *deserializer) {
uint64_t numEntries = deserializer->readUint64();
_knownVersions.clear();
_knownVersions.reserve(static_cast<uint64_t>(1.2 * numEntries)); // Reserve for factor 1.2 more, so the file system doesn't immediately have to resize it on the first new block.
for (uint64_t i = 0 ; i < numEntries; ++i) {
auto entry = _deserializeKnownVersionsEntry(deserializer);
_knownVersions.insert(entry);
}
}
void KnownBlockVersions::_serializeKnownVersions(Serializer *serializer) const {
uint64_t numEntries = _knownVersions.size();
serializer->writeUint64(numEntries);
for (const auto &entry : _knownVersions) {
_serializeKnownVersionsEntry(serializer, entry);
}
}
pair<ClientIdAndBlockId, uint64_t> KnownBlockVersions::_deserializeKnownVersionsEntry(Deserializer *deserializer) {
uint32_t clientId = deserializer->readUint32();
BlockId blockId(deserializer->readFixedSizeData<BlockId::BINARY_LENGTH>());
uint64_t version = deserializer->readUint64();
return {{clientId, blockId}, version};
};
void KnownBlockVersions::_serializeKnownVersionsEntry(Serializer *serializer, const pair<ClientIdAndBlockId, uint64_t> &entry) {
serializer->writeUint32(entry.first.clientId);
serializer->writeFixedSizeData<BlockId::BINARY_LENGTH>(entry.first.blockId.data());
serializer->writeUint64(entry.second);
}
void KnownBlockVersions::_deserializeLastUpdateClientIds(Deserializer *deserializer) {
uint64_t numEntries = deserializer->readUint64();
_lastUpdateClientId.clear();
_lastUpdateClientId.reserve(static_cast<uint64_t>(1.2 * numEntries)); // Reserve for factor 1.2 more, so the file system doesn't immediately have to resize it on the first new block.
for (uint64_t i = 0 ; i < numEntries; ++i) {
auto entry = _deserializeLastUpdateClientIdEntry(deserializer);
_lastUpdateClientId.insert(entry);
}
}
void KnownBlockVersions::_serializeLastUpdateClientIds(Serializer *serializer) const {
uint64_t numEntries = _lastUpdateClientId.size();
serializer->writeUint64(numEntries);
for (const auto &entry : _lastUpdateClientId) {
_serializeLastUpdateClientIdEntry(serializer, entry);
}
}
pair<BlockId, uint32_t> KnownBlockVersions::_deserializeLastUpdateClientIdEntry(Deserializer *deserializer) {
BlockId blockId(deserializer->readFixedSizeData<BlockId::BINARY_LENGTH>());
uint32_t clientId = deserializer->readUint32();
return {blockId, clientId};
};
void KnownBlockVersions::_serializeLastUpdateClientIdEntry(Serializer *serializer, const pair<BlockId, uint32_t> &entry) {
serializer->writeFixedSizeData<BlockId::BINARY_LENGTH>(entry.first.data());
serializer->writeUint32(entry.second);
}
uint32_t KnownBlockVersions::myClientId() const {
return _myClientId;
}
uint64_t KnownBlockVersions::getBlockVersion(uint32_t clientId, const BlockId &blockId) const {
unique_lock<mutex> lock(_mutex);
return _knownVersions.at({clientId, blockId});
}
void KnownBlockVersions::markBlockAsDeleted(const BlockId &blockId) {
_lastUpdateClientId[blockId] = CLIENT_ID_FOR_DELETED_BLOCK;
}
bool KnownBlockVersions::blockShouldExist(const BlockId &blockId) const {
auto found = _lastUpdateClientId.find(blockId);
if (found == _lastUpdateClientId.end()) {
// We've never seen (i.e. loaded) this block. So we can't say it has to exist.
return false;
}
// We've seen the block before. If we didn't delete it, it should exist (only works for single-client scenario).
return found->second != CLIENT_ID_FOR_DELETED_BLOCK;
}
std::unordered_set<BlockId> KnownBlockVersions::existingBlocks() const {
std::unordered_set<BlockId> result;
for (const auto &entry : _lastUpdateClientId) {
if (entry.second != CLIENT_ID_FOR_DELETED_BLOCK) {
result.insert(entry.first);
}
}
return result;
}
const bf::path &KnownBlockVersions::path() const {
return _stateFilePath;
}
}
}

74
src/blockstore/implementations/integrity/KnownBlockVersions.h Normal file
View File

@ -0,0 +1,74 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_KNOWNBLOCKVERSIONS_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_INTEGRITY_KNOWNBLOCKVERSIONS_H_
#include <cpp-utils/macros.h>
#include <blockstore/utils/BlockId.h>
#include <boost/filesystem/path.hpp>
#include <boost/optional.hpp>
#include "ClientIdAndBlockId.h"
#include <cpp-utils/data/Deserializer.h>
#include <cpp-utils/data/Serializer.h>
#include <mutex>
#include <unordered_set>
namespace blockstore {
namespace integrity {
class KnownBlockVersions final {
public:
KnownBlockVersions(const boost::filesystem::path &stateFilePath, uint32_t myClientId);
KnownBlockVersions(KnownBlockVersions &&rhs); // NOLINT (intentionally not noexcept)
~KnownBlockVersions();
WARN_UNUSED_RESULT
bool checkAndUpdateVersion(uint32_t clientId, const BlockId &blockId, uint64_t version);
uint64_t incrementVersion(const BlockId &blockId);
void markBlockAsDeleted(const BlockId &blockId);
bool blockShouldExist(const BlockId &blockId) const;
std::unordered_set<BlockId> existingBlocks() const;
uint64_t getBlockVersion(uint32_t clientId, const BlockId &blockId) const;
uint32_t myClientId() const;
const boost::filesystem::path &path() const;
static constexpr uint32_t CLIENT_ID_FOR_DELETED_BLOCK = 0;
private:
std::unordered_map<ClientIdAndBlockId, uint64_t> _knownVersions;
std::unordered_map<BlockId, uint32_t> _lastUpdateClientId; // The client who last updated the block
boost::filesystem::path _stateFilePath;
uint32_t _myClientId;
mutable std::mutex _mutex;
bool _valid;
static const std::string HEADER;
void _loadStateFile();
void _saveStateFile() const;
void _deserializeKnownVersions(cpputils::Deserializer *deserializer);
void _serializeKnownVersions(cpputils::Serializer *serializer) const;
static std::pair<ClientIdAndBlockId, uint64_t> _deserializeKnownVersionsEntry(cpputils::Deserializer *deserializer);
static void _serializeKnownVersionsEntry(cpputils::Serializer *serializer, const std::pair<ClientIdAndBlockId, uint64_t> &entry);
void _deserializeLastUpdateClientIds(cpputils::Deserializer *deserializer);
void _serializeLastUpdateClientIds(cpputils::Serializer *serializer) const;
static std::pair<BlockId, uint32_t> _deserializeLastUpdateClientIdEntry(cpputils::Deserializer *deserializer);
static void _serializeLastUpdateClientIdEntry(cpputils::Serializer *serializer, const std::pair<BlockId, uint32_t> &entry);
DISALLOW_COPY_AND_ASSIGN(KnownBlockVersions);
};
}
}
#endif

83
src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.cpp Normal file
View File

@ -0,0 +1,83 @@
#include "LowToHighLevelBlock.h"
using boost::optional;
using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::Data;
namespace DataUtils = cpputils::DataUtils;
using std::unique_lock;
using std::mutex;
namespace blockstore {
namespace lowtohighlevel {
optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::TryCreateNew(BlockStore2 *baseBlockStore, const BlockId &blockId, Data data) {
bool success = baseBlockStore->tryCreate(blockId, data);
if (!success) {
return none;
}
return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(data), baseBlockStore);
}
unique_ref<LowToHighLevelBlock> LowToHighLevelBlock::Overwrite(BlockStore2 *baseBlockStore, const BlockId &blockId, Data data) {
baseBlockStore->store(blockId, data); // TODO Does it make sense to not store here, but only write back in the destructor of LowToHighLevelBlock? Also: What about tryCreate?
return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(data), baseBlockStore);
}
optional<unique_ref<LowToHighLevelBlock>> LowToHighLevelBlock::Load(BlockStore2 *baseBlockStore, const BlockId &blockId) {
optional<Data> loadedData = baseBlockStore->load(blockId);
if (loadedData == none) {
return none;
}
return make_unique_ref<LowToHighLevelBlock>(blockId, std::move(*loadedData), baseBlockStore);
}
LowToHighLevelBlock::LowToHighLevelBlock(const BlockId &blockId, Data data, BlockStore2 *baseBlockStore)
:Block(blockId),
_baseBlockStore(baseBlockStore),
_data(std::move(data)),
_dataChanged(false),
_mutex() {
}
LowToHighLevelBlock::~LowToHighLevelBlock() {
unique_lock<mutex> lock(_mutex);
_storeToBaseBlock();
}
const void *LowToHighLevelBlock::data() const {
return _data.data();
}
void LowToHighLevelBlock::write(const void *source, uint64_t offset, uint64_t count) {
ASSERT(offset <= size() && offset + count <= size(), "Write outside of valid area"); //Also check offset < size() because of possible overflow in the addition
std::memcpy(_data.dataOffset(offset), source, count);
_dataChanged = true;
}
void LowToHighLevelBlock::flush() {
unique_lock<mutex> lock(_mutex);
_storeToBaseBlock();
}
size_t LowToHighLevelBlock::size() const {
return _data.size();
}
void LowToHighLevelBlock::resize(size_t newSize) {
_data = DataUtils::resize(_data, newSize);
_dataChanged = true;
}
void LowToHighLevelBlock::_storeToBaseBlock() {
if (_dataChanged) {
_baseBlockStore->store(blockId(), _data);
_dataChanged = false;
}
}
}
}

54
src/blockstore/implementations/low2highlevel/LowToHighLevelBlock.h Normal file
View File

@ -0,0 +1,54 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCK_H_
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include "../../interface/BlockStore.h"
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
#include <memory>
#include <iostream>
#include <boost/optional.hpp>
#include <cpp-utils/crypto/symmetric/Cipher.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/data/DataUtils.h>
#include <mutex>
#include <cpp-utils/logging/logging.h>
#include "LowToHighLevelBlockStore.h"
namespace blockstore {
namespace lowtohighlevel {
class LowToHighLevelBlock final: public Block {
public:
static boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> TryCreateNew(BlockStore2 *baseBlockStore, const BlockId &blockId, cpputils::Data data);
static cpputils::unique_ref<LowToHighLevelBlock> Overwrite(BlockStore2 *baseBlockStore, const BlockId &blockId, cpputils::Data data);
static boost::optional<cpputils::unique_ref<LowToHighLevelBlock>> Load(BlockStore2 *baseBlockStore, const BlockId &blockId);
LowToHighLevelBlock(const BlockId &blockId, cpputils::Data data, BlockStore2 *baseBlockStore);
~LowToHighLevelBlock();
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t count) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
BlockStore2 *_baseBlockStore;
cpputils::Data _data;
bool _dataChanged;
std::mutex _mutex;
void _storeToBaseBlock();
DISALLOW_COPY_AND_ASSIGN(LowToHighLevelBlock);
};
}
}
#endif

70
src/blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.cpp Normal file
View File

@ -0,0 +1,70 @@
#include <unordered_set>
#include "LowToHighLevelBlockStore.h"
#include "LowToHighLevelBlock.h"
using cpputils::unique_ref;
using cpputils::Data;
using boost::none;
using boost::optional;
using std::string;
namespace blockstore {
namespace lowtohighlevel {
LowToHighLevelBlockStore::LowToHighLevelBlockStore(unique_ref<BlockStore2> baseBlockStore)
: _baseBlockStore(std::move(baseBlockStore)) {
}
BlockId LowToHighLevelBlockStore::createBlockId() {
// TODO Is this the right way?
return BlockId::Random();
}
optional<unique_ref<Block>> LowToHighLevelBlockStore::tryCreate(const BlockId &blockId, Data data) {
//TODO Easier implementation? This is only so complicated because of the cast LowToHighLevelBlock -> Block
auto result = LowToHighLevelBlock::TryCreateNew(_baseBlockStore.get(), blockId, std::move(data));
if (result == none) {
return none;
}
return unique_ref<Block>(std::move(*result));
}
unique_ref<Block> LowToHighLevelBlockStore::overwrite(const BlockId &blockId, Data data) {
return unique_ref<Block>(
LowToHighLevelBlock::Overwrite(_baseBlockStore.get(), blockId, std::move(data))
);
}
optional<unique_ref<Block>> LowToHighLevelBlockStore::load(const BlockId &blockId) {
auto result = optional<unique_ref<Block>>(LowToHighLevelBlock::Load(_baseBlockStore.get(), blockId));
if (result == none) {
return none;
}
return unique_ref<Block>(std::move(*result));
}
void LowToHighLevelBlockStore::remove(const BlockId &blockId) {
bool success = _baseBlockStore->remove(blockId);
if (!success) {
throw std::runtime_error("Couldn't delete block with id " + blockId.ToString());
}
}
uint64_t LowToHighLevelBlockStore::numBlocks() const {
return _baseBlockStore->numBlocks();
}
uint64_t LowToHighLevelBlockStore::estimateNumFreeBytes() const {
return _baseBlockStore->estimateNumFreeBytes();
}
uint64_t LowToHighLevelBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
void LowToHighLevelBlockStore::forEachBlock(std::function<void (const BlockId &)> callback) const {
_baseBlockStore->forEachBlock(std::move(callback));
}
}
}

42
src/blockstore/implementations/low2highlevel/LowToHighLevelBlockStore.h Normal file
View File

@ -0,0 +1,42 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_LOWTOHIGHLEVEL_LOWTOHIGHLEVELBLOCKSTORE_H_
#include "../../interface/BlockStore.h"
#include "../../interface/BlockStore2.h"
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/cast.h>
#include <iostream>
// TODO Think each function through and make sure it's as performant
// to use LowToHighLevelBlockStore<OnDiskBlockStore2> as to use
// OnDiskBlockStore directly (i.e. no additional stores/loads from the disk)
// (same for other base block stores)
namespace blockstore {
namespace lowtohighlevel {
class LowToHighLevelBlockStore final: public BlockStore {
public:
LowToHighLevelBlockStore(cpputils::unique_ref<BlockStore2> baseBlockStore);
BlockId createBlockId() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
cpputils::unique_ref<Block> overwrite(const blockstore::BlockId &blockId, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
void remove(const BlockId &blockId) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
cpputils::unique_ref<BlockStore2> _baseBlockStore;
DISALLOW_COPY_AND_ASSIGN(LowToHighLevelBlockStore);
};
}
}
#endif

18
src/blockstore/implementations/mock/MockBlock.cpp Normal file
View File

@ -0,0 +1,18 @@
#include "MockBlock.h"
#include "MockBlockStore.h"
namespace blockstore {
namespace mock {
void MockBlock::write(const void *source, uint64_t offset, uint64_t size) {
_blockStore->_increaseNumWrittenBlocks(blockId());
return _baseBlock->write(source, offset, size);
}
void MockBlock::resize(size_t newSize) {
_blockStore->_increaseNumResizedBlocks(blockId());
return _baseBlock->resize(newSize);
}
}
}

50
src/blockstore/implementations/mock/MockBlock.h Normal file
View File

@ -0,0 +1,50 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCK_H_
#include <blockstore/interface/Block.h>
#include <cpp-utils/pointer/unique_ref.h>
namespace blockstore {
namespace mock {
class MockBlockStore;
class MockBlock final : public blockstore::Block {
public:
MockBlock(cpputils::unique_ref<blockstore::Block> baseBlock, MockBlockStore *blockStore)
:Block(baseBlock->blockId()), _baseBlock(std::move(baseBlock)), _blockStore(blockStore) {
}
const void *data() const override {
return _baseBlock->data();
}
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override {
return _baseBlock->flush();
}
size_t size() const override {
return _baseBlock->size();
}
void resize(size_t newSize) override;
cpputils::unique_ref<blockstore::Block> releaseBaseBlock() {
return std::move(_baseBlock);
}
private:
cpputils::unique_ref<blockstore::Block> _baseBlock;
MockBlockStore *_blockStore;
DISALLOW_COPY_AND_ASSIGN(MockBlock);
};
}
}
#endif

1
src/blockstore/implementations/mock/MockBlockStore.cpp Normal file
View File

@ -0,0 +1 @@
#include "MockBlockStore.h"

157
src/blockstore/implementations/mock/MockBlockStore.h Normal file
View File

@ -0,0 +1,157 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_MOCK_MOCKBLOCKSTORE_H_
#include <blockstore/implementations/testfake/FakeBlockStore.h>
#include <mutex>
#include "MockBlock.h"
namespace blockstore {
namespace mock {
/**
* This is a blockstore that counts the number of loaded, resized, written, ... blocks.
* It is used for testing that operations only access few blocks (performance tests).
*/
class MockBlockStore final : public BlockStore {
public:
MockBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore = cpputils::make_unique_ref<testfake::FakeBlockStore>())
: _mutex(), _baseBlockStore(std::move(baseBlockStore)), _loadedBlocks(), _createdBlocks(0), _writtenBlocks(), _resizedBlocks(), _removedBlocks() {
}
BlockId createBlockId() override {
return _baseBlockStore->createBlockId();
}
boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override {
_increaseNumCreatedBlocks();
auto base = _baseBlockStore->tryCreate(blockId, std::move(data));
if (base == boost::none) {
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(cpputils::make_unique_ref<MockBlock>(std::move(*base), this));
}
boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override {
_increaseNumLoadedBlocks(blockId);
auto base = _baseBlockStore->load(blockId);
if (base == boost::none) {
return boost::none;
}
return boost::optional<cpputils::unique_ref<Block>>(cpputils::make_unique_ref<MockBlock>(std::move(*base), this));
}
cpputils::unique_ref<Block> overwrite(const BlockId &blockId, cpputils::Data data) override {
_increaseNumWrittenBlocks(blockId);
return _baseBlockStore->overwrite(blockId, std::move(data));
}
void remove(const BlockId &blockId) override {
_increaseNumRemovedBlocks(blockId);
return _baseBlockStore->remove(blockId);
}
uint64_t numBlocks() const override {
return _baseBlockStore->numBlocks();
}
uint64_t estimateNumFreeBytes() const override {
return _baseBlockStore->estimateNumFreeBytes();
}
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override {
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
void forEachBlock(std::function<void(const BlockId &)> callback) const override {
return _baseBlockStore->forEachBlock(callback);
}
void remove(cpputils::unique_ref<Block> block) override {
_increaseNumRemovedBlocks(block->blockId());
auto mockBlock = cpputils::dynamic_pointer_move<MockBlock>(block);
ASSERT(mockBlock != boost::none, "Wrong block type");
return _baseBlockStore->remove((*mockBlock)->releaseBaseBlock());
}
void resetCounters() {
_loadedBlocks = {};
_createdBlocks = 0;
_removedBlocks = {};
_resizedBlocks = {};
_writtenBlocks = {};
}
uint64_t createdBlocks() const {
return _createdBlocks;
}
const std::vector<BlockId> &loadedBlocks() const {
return _loadedBlocks;
}
const std::vector<BlockId> &removedBlocks() const {
return _removedBlocks;
}
const std::vector<BlockId> &resizedBlocks() const {
return _resizedBlocks;
}
const std::vector<BlockId> &writtenBlocks() const {
return _writtenBlocks;
}
std::vector<BlockId> distinctWrittenBlocks() const {
std::vector<BlockId> result(_writtenBlocks);
std::sort(result.begin(), result.end(), [](const BlockId &lhs, const BlockId &rhs) {
return std::memcmp(lhs.data().data(), rhs.data().data(), lhs.BINARY_LENGTH) < 0;
});
result.erase(std::unique(result.begin(), result.end() ), result.end());
return result;
}
private:
void _increaseNumCreatedBlocks() {
std::unique_lock<std::mutex> lock(_mutex);
_createdBlocks += 1;
}
void _increaseNumLoadedBlocks(const BlockId &blockId) {
std::unique_lock<std::mutex> lock(_mutex);
_loadedBlocks.push_back(blockId);
}
void _increaseNumRemovedBlocks(const BlockId &blockId) {
std::unique_lock<std::mutex> lock(_mutex);
_removedBlocks.push_back(blockId);
}
void _increaseNumResizedBlocks(const BlockId &blockId) {
std::unique_lock<std::mutex> lock(_mutex);
_resizedBlocks.push_back(blockId);
}
void _increaseNumWrittenBlocks(const BlockId &blockId) {
std::unique_lock<std::mutex> lock(_mutex);
_writtenBlocks.push_back(blockId);
}
friend class MockBlock;
std::mutex _mutex;
cpputils::unique_ref<BlockStore> _baseBlockStore;
std::vector<BlockId> _loadedBlocks;
uint64_t _createdBlocks;
std::vector<BlockId> _writtenBlocks;
std::vector<BlockId> _resizedBlocks;
std::vector<BlockId> _removedBlocks;
DISALLOW_COPY_AND_ASSIGN(MockBlockStore);
};
}
}
#endif

175
src/blockstore/implementations/ondisk/OnDiskBlock.cpp
View File

@ -1,175 +0,0 @@
#include <cstring>
#include <fstream>
#include <boost/filesystem.hpp>
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include "../../utils/FileDoesntExistException.h"
#include <cpp-utils/data/DataUtils.h>
#include <cpp-utils/assert/assert.h>
#include <cpp-utils/logging/logging.h>
using std::istream;
using std::ostream;
using std::ifstream;
using std::ofstream;
using std::ios;
using std::string;
using cpputils::Data;
using cpputils::make_unique_ref;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
namespace bf = boost::filesystem;
using namespace cpputils::logging;
namespace blockstore {
namespace ondisk {
const string OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
const string OnDiskBlock::FORMAT_VERSION_HEADER = OnDiskBlock::FORMAT_VERSION_HEADER_PREFIX + "0";
OnDiskBlock::OnDiskBlock(const Key &key, const bf::path &filepath, Data data)
: Block(key), _filepath(filepath), _data(std::move(data)), _dataChanged(false), _mutex() {
}
OnDiskBlock::~OnDiskBlock() {
flush();
}
const void *OnDiskBlock::data() const {
return _data.data();
}
void OnDiskBlock::write(const void *source, uint64_t offset, uint64_t size) {
ASSERT(offset <= _data.size() && offset + size <= _data.size(), "Write outside of valid area"); //Also check offset < _data->size() because of possible overflow in the addition
std::memcpy(_data.dataOffset(offset), source, size);
_dataChanged = true;
}
size_t OnDiskBlock::size() const {
return _data.size();
}
void OnDiskBlock::resize(size_t newSize) {
_data = cpputils::DataUtils::resize(std::move(_data), newSize);
_dataChanged = true;
}
bf::path OnDiskBlock::_getFilepath(const bf::path &rootdir, const Key &key) {
string keyStr = key.ToString();
return rootdir / keyStr.substr(0,3) / keyStr.substr(3);
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::LoadFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = _getFilepath(rootdir, key);
try {
boost::optional<Data> data = _loadFromDisk(filepath);
if (data == none) {
return none;
}
return make_unique_ref<OnDiskBlock>(key, filepath, std::move(*data));
} catch (const FileDoesntExistException &e) {
return none;
}
}
optional<unique_ref<OnDiskBlock>> OnDiskBlock::CreateOnDisk(const bf::path &rootdir, const Key &key, Data data) {
auto filepath = _getFilepath(rootdir, key);
bf::create_directory(filepath.parent_path());
if (bf::exists(filepath)) {
return none;
}
auto block = make_unique_ref<OnDiskBlock>(key, filepath, std::move(data));
block->_storeToDisk();
return std::move(block);
}
void OnDiskBlock::RemoveFromDisk(const bf::path &rootdir, const Key &key) {
auto filepath = _getFilepath(rootdir, key);
ASSERT(bf::is_regular_file(filepath), "Block not found on disk");
bool retval = bf::remove(filepath);
if (!retval) {
LOG(ERROR, "Couldn't find block {} to remove", key.ToString());
}
if (bf::is_empty(filepath.parent_path())) {
bf::remove(filepath.parent_path());
}
}
void OnDiskBlock::_storeToDisk() const {
std::ofstream file(_filepath.c_str(), std::ios::binary | std::ios::trunc);
if (!file.good()) {
throw std::runtime_error("Could not open file for writing");
}
file.write(FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
if (!file.good()) {
throw std::runtime_error("Error writing block header");
}
_data.StoreToStream(file);
if (!file.good()) {
throw std::runtime_error("Error writing block data");
}
}
optional<Data> OnDiskBlock::_loadFromDisk(const bf::path &filepath) {
//If it isn't a file, ifstream::good() would return false. We still need this extra check
//upfront, because ifstream::good() doesn't crash if we give it the path of a directory
//instead the path of a file.
if(!bf::is_regular_file(filepath)) {
return none;
}
ifstream file(filepath.c_str(), ios::binary);
if (!file.good()) {
return none;
}
_checkHeader(&file);
Data result = Data::LoadFromStream(file);
//TODO With newer compilers, "return result;" would be enough
return boost::optional<Data>(std::move(result));
}
void OnDiskBlock::_checkHeader(istream *str) {
Data header(formatVersionHeaderSize());
str->read(reinterpret_cast<char*>(header.data()), formatVersionHeaderSize());
if (!_isAcceptedCryfsHeader(header)) {
if (_isOtherCryfsHeader(header)) {
throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
} else {
throw std::runtime_error("This is not a valid block.");
}
}
}
bool OnDiskBlock::_isAcceptedCryfsHeader(const Data &data) {
ASSERT(data.size() == formatVersionHeaderSize(), "We extracted the wrong header size from the block.");
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
}
bool OnDiskBlock::_isOtherCryfsHeader(const Data &data) {
ASSERT(data.size() >= FORMAT_VERSION_HEADER_PREFIX.size(), "We extracted the wrong header size from the block.");
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
}
unsigned int OnDiskBlock::formatVersionHeaderSize() {
return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
}
void OnDiskBlock::flush() {
std::unique_lock<std::mutex> lock(_mutex);
if (_dataChanged) {
_storeToDisk();
_dataChanged = false;
}
}
uint64_t OnDiskBlock::blockSizeFromPhysicalBlockSize(uint64_t blockSize) {
if(blockSize <= formatVersionHeaderSize()) {
return 0;
}
return blockSize - formatVersionHeaderSize();
}
}
}

61
src/blockstore/implementations/ondisk/OnDiskBlock.h
View File

@ -1,61 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCK_H_
#include <boost/filesystem/path.hpp>
#include "../../interface/Block.h"
#include <cpp-utils/data/Data.h>
#include <iostream>
#include <cpp-utils/pointer/unique_ref.h>
#include <mutex>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore;
class OnDiskBlock final: public Block {
public:
OnDiskBlock(const Key &key, const boost::filesystem::path &filepath, cpputils::Data data);
~OnDiskBlock();
static const std::string FORMAT_VERSION_HEADER_PREFIX;
static const std::string FORMAT_VERSION_HEADER;
static unsigned int formatVersionHeaderSize();
static uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> LoadFromDisk(const boost::filesystem::path &rootdir, const Key &key);
static boost::optional<cpputils::unique_ref<OnDiskBlock>> CreateOnDisk(const boost::filesystem::path &rootdir, const Key &key, cpputils::Data data);
static void RemoveFromDisk(const boost::filesystem::path &rootdir, const Key &key);
const void *data() const override;
void write(const void *source, uint64_t offset, uint64_t size) override;
void flush() override;
size_t size() const override;
void resize(size_t newSize) override;
private:
static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
static bool _isOtherCryfsHeader(const cpputils::Data &data);
static void _checkHeader(std::istream *str);
static boost::filesystem::path _getFilepath(const boost::filesystem::path &rootdir, const Key &key);
const boost::filesystem::path _filepath;
cpputils::Data _data;
bool _dataChanged;
static boost::optional<cpputils::Data> _loadFromDisk(const boost::filesystem::path &filepath);
void _storeToDisk() const;
std::mutex _mutex;
DISALLOW_COPY_AND_ASSIGN(OnDiskBlock);
};
}
}
#endif

99
src/blockstore/implementations/ondisk/OnDiskBlockStore.cpp
View File

@ -1,99 +0,0 @@
#include "OnDiskBlock.h"
#include "OnDiskBlockStore.h"
#include <sys/statvfs.h>
using std::string;
using cpputils::Data;
using cpputils::unique_ref;
using boost::optional;
using boost::none;
using std::vector;
namespace bf = boost::filesystem;
namespace blockstore {
namespace ondisk {
OnDiskBlockStore::OnDiskBlockStore(const boost::filesystem::path &rootdir)
: _rootdir(rootdir) {
if (!bf::exists(rootdir)) {
throw std::runtime_error("Base directory not found");
}
if (!bf::is_directory(rootdir)) {
throw std::runtime_error("Base directory is not a directory");
}
//TODO Test for read access, write access, enter (x) access, and throw runtime_error in case
#ifndef CRYFS_NO_COMPATIBILITY
_migrateBlockStore();
#endif
}
#ifndef CRYFS_NO_COMPATIBILITY
void OnDiskBlockStore::_migrateBlockStore() {
vector<string> blocksToMigrate;
for (auto entry = bf::directory_iterator(_rootdir); entry != bf::directory_iterator(); ++entry) {
if (bf::is_regular_file(entry->path()) && _isValidBlockKey(entry->path().filename().native())) {
blocksToMigrate.push_back(entry->path().filename().native());
}
}
if (blocksToMigrate.size() != 0) {
std::cout << "Migrating CryFS filesystem..." << std::flush;
for (auto key : blocksToMigrate) {
Key::FromString(key); // Assert that it can be parsed as a key
string dir = key.substr(0, 3);
string file = key.substr(3);
bf::create_directory(_rootdir / dir);
bf::rename(_rootdir / key, _rootdir / dir / file);
}
std::cout << "done" << std::endl;
}
}
bool OnDiskBlockStore::_isValidBlockKey(const string &key) {
return key.size() == 32 && key.find_first_not_of("0123456789ABCDEF") == string::npos;
}
#endif
//TODO Do I have to lock tryCreate/remove and/or load? Or does ParallelAccessBlockStore take care of that?
optional<unique_ref<Block>> OnDiskBlockStore::tryCreate(const Key &key, Data data) {
//TODO Easier implementation? This is only so complicated because of the cast OnDiskBlock -> Block
auto result = OnDiskBlock::CreateOnDisk(_rootdir, key, std::move(data));
if (result == boost::none) {
return boost::none;
}
return unique_ref<Block>(std::move(*result));
}
optional<unique_ref<Block>> OnDiskBlockStore::load(const Key &key) {
return optional<unique_ref<Block>>(OnDiskBlock::LoadFromDisk(_rootdir, key));
}
void OnDiskBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
cpputils::destruct(std::move(block));
OnDiskBlock::RemoveFromDisk(_rootdir, key);
}
uint64_t OnDiskBlockStore::numBlocks() const {
uint64_t count = 0;
for (auto entry = bf::directory_iterator(_rootdir); entry != bf::directory_iterator(); ++entry) {
if (bf::is_directory(entry->path())) {
count += std::distance(bf::directory_iterator(entry->path()), bf::directory_iterator());
}
}
return count;
}
uint64_t OnDiskBlockStore::estimateNumFreeBytes() const {
struct statvfs stat;
::statvfs(_rootdir.c_str(), &stat);
return stat.f_bsize*stat.f_bavail;
}
uint64_t OnDiskBlockStore::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
return OnDiskBlock::blockSizeFromPhysicalBlockSize(blockSize);
}
}
}

38
src/blockstore/implementations/ondisk/OnDiskBlockStore.h
View File

@ -1,38 +0,0 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE_H_
#include <boost/filesystem.hpp>
#include "../../interface/helpers/BlockStoreWithRandomKeys.h"
#include <cpp-utils/macros.h>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore final: public BlockStoreWithRandomKeys {
public:
explicit OnDiskBlockStore(const boost::filesystem::path &rootdir);
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
//TODO Can we make this faster by allowing to delete blocks by only having theiy Key? So we wouldn't have to load it first?
void remove(cpputils::unique_ref<Block> block) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
private:
const boost::filesystem::path _rootdir;
#ifndef CRYFS_NO_COMPATIBILITY
void _migrateBlockStore();
bool _isValidBlockKey(const std::string &key);
#endif
DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore);
};
}
}
#endif

126
src/blockstore/implementations/ondisk/OnDiskBlockStore2.cpp Normal file
View File

@ -0,0 +1,126 @@
#include "OnDiskBlockStore2.h"
#include <boost/filesystem.hpp>
#include <cpp-utils/system/diskspace.h>
using std::string;
using boost::optional;
using boost::none;
using cpputils::Data;
namespace blockstore {
namespace ondisk {
const string OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX = "cryfs;block;";
const string OnDiskBlockStore2::FORMAT_VERSION_HEADER = OnDiskBlockStore2::FORMAT_VERSION_HEADER_PREFIX + "0";
boost::filesystem::path OnDiskBlockStore2::_getFilepath(const BlockId &blockId) const {
std::string blockIdStr = blockId.ToString();
return _rootDir / blockIdStr.substr(0,3) / blockIdStr.substr(3);
}
Data OnDiskBlockStore2::_checkAndRemoveHeader(const Data &data) {
if (!_isAcceptedCryfsHeader(data)) {
if (_isOtherCryfsHeader(data)) {
throw std::runtime_error("This block is not supported yet. Maybe it was created with a newer version of CryFS?");
} else {
throw std::runtime_error("This is not a valid block.");
}
}
Data result(data.size() - formatVersionHeaderSize());
std::memcpy(result.data(), data.dataOffset(formatVersionHeaderSize()), result.size());
return result;
}
bool OnDiskBlockStore2::_isAcceptedCryfsHeader(const Data &data) {
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
}
bool OnDiskBlockStore2::_isOtherCryfsHeader(const Data &data) {
return 0 == std::memcmp(data.data(), FORMAT_VERSION_HEADER_PREFIX.c_str(), FORMAT_VERSION_HEADER_PREFIX.size());
}
unsigned int OnDiskBlockStore2::formatVersionHeaderSize() {
return FORMAT_VERSION_HEADER.size() + 1; // +1 because of the null byte
}
OnDiskBlockStore2::OnDiskBlockStore2(const boost::filesystem::path& path)
: _rootDir(path) {}
bool OnDiskBlockStore2::tryCreate(const BlockId &blockId, const Data &data) {
auto filepath = _getFilepath(blockId);
if (boost::filesystem::exists(filepath)) {
return false;
}
store(blockId, data);
return true;
}
bool OnDiskBlockStore2::remove(const BlockId &blockId) {
auto filepath = _getFilepath(blockId);
if (!boost::filesystem::is_regular_file(filepath)) { // TODO Is this branch necessary?
return false;
}
bool retval = boost::filesystem::remove(filepath);
if (!retval) {
cpputils::logging::LOG(cpputils::logging::ERR, "Couldn't find block {} to remove", blockId.ToString());
return false;
}
if (boost::filesystem::is_empty(filepath.parent_path())) {
boost::filesystem::remove(filepath.parent_path());
}
return true;
}
optional<Data> OnDiskBlockStore2::load(const BlockId &blockId) const {
auto fileContent = Data::LoadFromFile(_getFilepath(blockId));
if (fileContent == none) {
return boost::none;
}
return _checkAndRemoveHeader(*fileContent);
}
void OnDiskBlockStore2::store(const BlockId &blockId, const Data &data) {
Data fileContent(formatVersionHeaderSize() + data.size());
std::memcpy(fileContent.data(), FORMAT_VERSION_HEADER.c_str(), formatVersionHeaderSize());
std::memcpy(fileContent.dataOffset(formatVersionHeaderSize()), data.data(), data.size());
auto filepath = _getFilepath(blockId);
boost::filesystem::create_directory(filepath.parent_path()); // TODO Instead create all of them once at fs creation time?
fileContent.StoreToFile(filepath);
}
uint64_t OnDiskBlockStore2::numBlocks() const {
uint64_t count = 0;
for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
if (boost::filesystem::is_directory(prefixDir->path())) {
count += std::distance(boost::filesystem::directory_iterator(prefixDir->path()), boost::filesystem::directory_iterator());
}
}
return count;
}
uint64_t OnDiskBlockStore2::estimateNumFreeBytes() const {
return cpputils::free_disk_space_in_bytes(_rootDir);
}
uint64_t OnDiskBlockStore2::blockSizeFromPhysicalBlockSize(uint64_t blockSize) const {
if(blockSize <= formatVersionHeaderSize()) {
return 0;
}
return blockSize - formatVersionHeaderSize();
}
void OnDiskBlockStore2::forEachBlock(std::function<void (const BlockId &)> callback) const {
for (auto prefixDir = boost::filesystem::directory_iterator(_rootDir); prefixDir != boost::filesystem::directory_iterator(); ++prefixDir) {
if (boost::filesystem::is_directory(prefixDir->path())) {
std::string blockIdPrefix = prefixDir->path().filename().string();
for (auto block = boost::filesystem::directory_iterator(prefixDir->path()); block != boost::filesystem::directory_iterator(); ++block) {
std::string blockIdPostfix = block->path().filename().string();
callback(BlockId::FromString(blockIdPrefix + blockIdPostfix));
}
}
}
}
}
}

45
src/blockstore/implementations/ondisk/OnDiskBlockStore2.h Normal file
View File

@ -0,0 +1,45 @@
#pragma once
#ifndef MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
#define MESSMER_BLOCKSTORE_IMPLEMENTATIONS_ONDISK_ONDISKBLOCKSTORE2_H_
#include "../../interface/BlockStore2.h"
#include <boost/filesystem/path.hpp>
#include <cpp-utils/macros.h>
#include <cpp-utils/pointer/unique_ref.h>
#include <cpp-utils/logging/logging.h>
namespace blockstore {
namespace ondisk {
class OnDiskBlockStore2 final: public BlockStore2 {
public:
explicit OnDiskBlockStore2(const boost::filesystem::path& path);
bool tryCreate(const BlockId &blockId, const cpputils::Data &data) override;
bool remove(const BlockId &blockId) override;
boost::optional<cpputils::Data> load(const BlockId &blockId) const override;
void store(const BlockId &blockId, const cpputils::Data &data) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
boost::filesystem::path _rootDir;
static const std::string FORMAT_VERSION_HEADER_PREFIX;
static const std::string FORMAT_VERSION_HEADER;
boost::filesystem::path _getFilepath(const BlockId &blockId) const;
static cpputils::Data _checkAndRemoveHeader(const cpputils::Data &data);
static bool _isAcceptedCryfsHeader(const cpputils::Data &data);
static bool _isOtherCryfsHeader(const cpputils::Data &data);
static unsigned int formatVersionHeaderSize();
DISALLOW_COPY_AND_ASSIGN(OnDiskBlockStore2);
};
}
}
#endif

6
src/blockstore/implementations/parallelaccess/BlockRef.h
View File

@ -11,10 +11,10 @@ namespace blockstore {
namespace parallelaccess {
class ParallelAccessBlockStore;
class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key>::ResourceRefBase {
class BlockRef final: public Block, public parallelaccessstore::ParallelAccessStore<Block, BlockRef, BlockId>::ResourceRefBase {
public:
//TODO Unneccessarily storing Key twice here (in parent class and in _baseBlock).
explicit BlockRef(Block *baseBlock): Block(baseBlock->key()), _baseBlock(baseBlock) {}
//TODO Unneccessarily storing BlockId twice here (in parent class and in _baseBlock).
explicit BlockRef(Block *baseBlock): Block(baseBlock->blockId()), _baseBlock(baseBlock) {}
const void *data() const override {
return _baseBlock->data();

44
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.cpp
View File

@ -6,12 +6,12 @@
#include <cpp-utils/assert/assert.h>
using std::string;
using std::promise;
using cpputils::dynamic_pointer_move;
using cpputils::make_unique_ref;
using boost::none;
using cpputils::unique_ref;
using cpputils::make_unique_ref;
using cpputils::Data;
using boost::optional;
using boost::none;
@ -22,34 +22,52 @@ ParallelAccessBlockStore::ParallelAccessBlockStore(unique_ref<BlockStore> baseBl
: _baseBlockStore(std::move(baseBlockStore)), _parallelAccessStore(make_unique_ref<ParallelAccessBlockStoreAdapter>(_baseBlockStore.get())) {
}
Key ParallelAccessBlockStore::createKey() {
return _baseBlockStore->createKey();
BlockId ParallelAccessBlockStore::createBlockId() {
return _baseBlockStore->createBlockId();
}
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const Key &key, cpputils::Data data) {
ASSERT(!_parallelAccessStore.isOpened(key), ("Key "+key.ToString()+"already exists").c_str());
auto block = _baseBlockStore->tryCreate(key, std::move(data));
optional<unique_ref<Block>> ParallelAccessBlockStore::tryCreate(const BlockId &blockId, Data data) {
if (_parallelAccessStore.isOpened(blockId)) {
return none; // block already exists
}
auto block = _baseBlockStore->tryCreate(blockId, std::move(data));
if (block == none) {
//TODO Test this code branch
return none;
}
return unique_ref<Block>(_parallelAccessStore.add(key, std::move(*block)));
return unique_ref<Block>(_parallelAccessStore.add(blockId, std::move(*block)));
}
optional<unique_ref<Block>> ParallelAccessBlockStore::load(const Key &key) {
auto block = _parallelAccessStore.load(key);
optional<unique_ref<Block>> ParallelAccessBlockStore::load(const BlockId &blockId) {
auto block = _parallelAccessStore.load(blockId);
if (block == none) {
return none;
}
return unique_ref<Block>(std::move(*block));
}
unique_ref<Block> ParallelAccessBlockStore::overwrite(const BlockId &blockId, Data data) {
auto onExists = [&data] (BlockRef *block) {
if (block->size() != data.size()) {
block->resize(data.size());
}
block->write(data.data(), 0, data.size());
};
auto onAdd = [this, blockId, &data] {
return _baseBlockStore->overwrite(blockId, data.copy()); // TODO Without copy?
};
return _parallelAccessStore.loadOrAdd(blockId, onExists, onAdd); // NOLINT (workaround https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82481 )
}
void ParallelAccessBlockStore::remove(unique_ref<Block> block) {
Key key = block->key();
BlockId blockId = block->blockId();
auto block_ref = dynamic_pointer_move<BlockRef>(block);
ASSERT(block_ref != none, "Block is not a BlockRef");
return _parallelAccessStore.remove(key, std::move(*block_ref));
return _parallelAccessStore.remove(blockId, std::move(*block_ref));
}
void ParallelAccessBlockStore::remove(const BlockId &blockId) {
return _parallelAccessStore.remove(blockId);
}
uint64_t ParallelAccessBlockStore::numBlocks() const {
@ -64,5 +82,9 @@ uint64_t ParallelAccessBlockStore::blockSizeFromPhysicalBlockSize(uint64_t block
return _baseBlockStore->blockSizeFromPhysicalBlockSize(blockSize);
}
void ParallelAccessBlockStore::forEachBlock(std::function<void (const BlockId &)> callback) const {
return _baseBlockStore->forEachBlock(callback);
}
}
}

13
src/blockstore/implementations/parallelaccess/ParallelAccessBlockStore.h
View File

@ -15,17 +15,20 @@ class ParallelAccessBlockStore final: public BlockStore {
public:
explicit ParallelAccessBlockStore(cpputils::unique_ref<BlockStore> baseBlockStore);
Key createKey() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const Key &key, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const Key &key) override;
void remove(cpputils::unique_ref<Block> block) override;
BlockId createBlockId() override;
boost::optional<cpputils::unique_ref<Block>> tryCreate(const BlockId &blockId, cpputils::Data data) override;
boost::optional<cpputils::unique_ref<Block>> load(const BlockId &blockId) override;
cpputils::unique_ref<Block> overwrite(const BlockId &blockId, cpputils::Data data) override;
void remove(const BlockId &blockId) override;
void remove(cpputils::unique_ref<Block> node) override;
uint64_t numBlocks() const override;
uint64_t estimateNumFreeBytes() const override;
uint64_t blockSizeFromPhysicalBlockSize(uint64_t blockSize) const override;
void forEachBlock(std::function<void (const BlockId &)> callback) const override;
private:
cpputils::unique_ref<BlockStore> _baseBlockStore;
parallelaccessstore::ParallelAccessStore<Block, BlockRef, Key> _parallelAccessStore;
parallelaccessstore::ParallelAccessStore<Block, BlockRef, BlockId> _parallelAccessStore;
DISALLOW_COPY_AND_ASSIGN(ParallelAccessBlockStore);
};

Some files were not shown because too many files have changed in this diff Show More